Release 2021.9.2

ServeControlStream accidentally became non-blocking in the last quic
change causing stream to not be returned until a SIGTERM was received.
This change makes ServeControlStream be non-blocking for QUIC streams.

.github/workflows/check.yaml

@@ -0,0 +1,20 @@
+on: [push, pull_request]
+name: Check
+jobs:
+  check:
+    strategy:
+      matrix:
+        go-version: [1.17.x]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Install go-sumtype
+      run: go get github.com/sudarshan-reddy/go-sumtype
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Test
+      run: make test

.gitignore

@@ -1,17 +1,18 @@
-.GOPATH/
-bin/
-tmp/
-guide/public
-/.GOPATH
+/tmp
 /bin
 .idea
+.build
 .vscode
 \#*\#
 cscope.*
-cloudflared
-cloudflared.pkg
-cloudflared.exe
-!cmd/cloudflared/
+/cloudflared
+/cloudflared.pkg
+/cloudflared.exe
+/cloudflared.msi
+/cloudflared-x86-64*
+/packaging
 .DS_Store
 *-session.log
 ssh_server_tests/.env
+/.cover
+built_artifacts/

.teamcity/build-macos.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-set -euo pipefail
-
 if [[ "$(uname)" != "Darwin" ]] ; then
     echo "This should be run on macOS"
     exit 1
@@ -13,9 +11,177 @@ export GO111MODULE=on
 # build 'cloudflared-darwin-amd64.tgz'
 mkdir -p artifacts
 FILENAME="$(pwd)/artifacts/cloudflared-darwin-amd64.tgz"
+PKGNAME="$(pwd)/artifacts/cloudflared-amd64.pkg"
+TARGET_DIRECTORY=".build"
+BINARY_NAME="cloudflared"
+VERSION=$(git describe --tags --always --dirty="-dev")
+PRODUCT="cloudflared"
+CODE_SIGN_PRIV="code_sign.p12"
+CODE_SIGN_CERT="code_sign.cer"
+INSTALLER_PRIV="installer.p12"
+INSTALLER_CERT="installer.cer"
+BUNDLE_ID="com.cloudflare.cloudflared"
+SEC_DUP_MSG="security: SecKeychainItemImport: The specified item already exists in the keychain."
 export PATH="$PATH:/usr/local/bin"
 mkdir -p ../src/github.com/cloudflare/    
 cp -r . ../src/github.com/cloudflare/cloudflared
 cd ../src/github.com/cloudflare/cloudflared 
 GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared
-tar czf "$FILENAME" cloudflared
+
+# Add code signing private key to the key chain
+if [[ ! -z "$CFD_CODE_SIGN_KEY" ]]; then
+  if [[ ! -z "$CFD_CODE_SIGN_PASS" ]]; then
+    # write private key to disk and then import it keychain
+    echo -n -e ${CFD_CODE_SIGN_KEY} | base64 -D > ${CODE_SIGN_PRIV}
+    out=$(security import ${CODE_SIGN_PRIV} -A -P "${CFD_CODE_SIGN_PASS}" 2>&1)
+    exitcode=$?
+    if [ -n "$out" ]; then
+      if [ $exitcode -eq 0 ]; then
+          echo "$out"
+      else
+          if [ "$out" != "${SEC_DUP_MSG}" ]; then
+              echo "$out" >&2
+              exit $exitcode
+          fi
+      fi
+    fi
+    rm ${CODE_SIGN_PRIV}
+  fi
+fi
+
+# Add code signing certificate to the key chain
+if [[ ! -z "$CFD_CODE_SIGN_CERT" ]]; then
+  # write certificate to disk and then import it keychain
+  echo -n -e ${CFD_CODE_SIGN_CERT} | base64 -D > ${CODE_SIGN_CERT}
+  out1=$(security import ${CODE_SIGN_CERT} -A 2>&1)
+  exitcode1=$?
+  if [ -n "$out1" ]; then
+    if [ $exitcode1 -eq 0 ]; then
+        echo "$out1"
+    else
+        if [ "$out1" != "${SEC_DUP_MSG}" ]; then
+            echo "$out1" >&2
+            exit $exitcode1
+        else 
+            echo "already imported code signing certificate"
+        fi
+    fi
+  fi
+  rm ${CODE_SIGN_CERT}
+fi
+
+# Add package signing private key to the key chain
+if [[ ! -z "$CFD_INSTALLER_KEY" ]]; then
+  if [[ ! -z "$CFD_INSTALLER_PASS" ]]; then
+    # write private key to disk and then import it into the keychain
+    echo -n -e ${CFD_INSTALLER_KEY} | base64 -D > ${INSTALLER_PRIV}
+    out2=$(security import ${INSTALLER_PRIV} -A -P "${CFD_INSTALLER_PASS}" 2>&1)
+    exitcode2=$?
+    if [ -n "$out2" ]; then
+      if [ $exitcode2 -eq 0 ]; then
+          echo "$out2"
+      else
+          if [ "$out2" != "${SEC_DUP_MSG}" ]; then
+              echo "$out2" >&2
+              exit $exitcode2
+          fi
+      fi
+    fi
+    rm ${INSTALLER_PRIV}
+  fi
+fi
+
+# Add package signing certificate to the key chain
+if [[ ! -z "$CFD_INSTALLER_CERT" ]]; then
+  # write certificate to disk and then import it keychain
+  echo -n -e ${CFD_INSTALLER_CERT} | base64 -D > ${INSTALLER_CERT}
+  out3=$(security import ${INSTALLER_CERT} -A 2>&1)
+  exitcode3=$?
+  if [ -n "$out3" ]; then
+    if [ $exitcode3 -eq 0 ]; then
+        echo "$out3"
+    else
+        if [ "$out3" != "${SEC_DUP_MSG}" ]; then
+            echo "$out3" >&2
+            exit $exitcode3
+        else 
+            echo "already imported installer certificate"
+        fi
+    fi
+  fi
+  rm ${INSTALLER_CERT}
+fi
+
+# get the code signing certificate name
+if [[ ! -z "$CFD_CODE_SIGN_NAME" ]]; then
+  CODE_SIGN_NAME="${CFD_CODE_SIGN_NAME}"
+else
+  if [[ -n "$(security find-certificate -c "Developer ID Application" | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)" ]]; then
+    CODE_SIGN_NAME=$(security find-certificate -c "Developer ID Application" | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)
+  else
+    CODE_SIGN_NAME=""
+  fi
+fi
+
+# get the package signing certificate name
+if [[ ! -z "$CFD_INSTALLER_NAME" ]]; then
+  PKG_SIGN_NAME="${CFD_INSTALLER_NAME}"
+else
+  if [[ -n "$(security find-certificate -c "Developer ID Installer" | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)" ]]; then
+    PKG_SIGN_NAME=$(security find-certificate -c "Developer ID Installer" | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)
+  else
+    PKG_SIGN_NAME=""
+  fi
+fi
+
+# sign the cloudflared binary
+if [[ ! -z "$CODE_SIGN_NAME" ]]; then
+  codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME}
+  
+  # notarize the binary
+  if [[ ! -z "$CFD_NOTE_PASSWORD" ]]; then
+    zip "${BINARY_NAME}.zip" ${BINARY_NAME} 
+    xcrun altool --notarize-app -f "${BINARY_NAME}.zip" -t osx -u ${CFD_NOTE_USERNAME} -p ${CFD_NOTE_PASSWORD} --primary-bundle-id ${BUNDLE_ID}
+  fi
+fi
+
+
+# creating build directory
+rm -rf $TARGET_DIRECTORY
+mkdir "${TARGET_DIRECTORY}"
+mkdir "${TARGET_DIRECTORY}/contents"
+cp -r ".mac_resources/scripts" "${TARGET_DIRECTORY}/scripts"
+
+# copy cloudflared into the build directory
+cp ${BINARY_NAME} "${TARGET_DIRECTORY}/contents/${PRODUCT}"
+
+# compress cloudflared into a tar and gzipped file
+tar czf "$FILENAME" "${BINARY_NAME}"
+
+# build the installer package
+if [[ ! -z "$PKG_SIGN_NAME" ]]; then
+  pkgbuild --identifier com.cloudflare.${PRODUCT} \
+      --version ${VERSION} \
+      --scripts ${TARGET_DIRECTORY}/scripts \
+      --root ${TARGET_DIRECTORY}/contents \
+      --install-location /usr/local/bin \
+      --sign "${PKG_SIGN_NAME}" \
+      ${PKGNAME}
+
+      # notarize the package
+      if [[ ! -z "$CFD_NOTE_PASSWORD" ]]; then
+        xcrun altool --notarize-app -f ${PKGNAME} -t osx -u ${CFD_NOTE_USERNAME} -p ${CFD_NOTE_PASSWORD} --primary-bundle-id ${BUNDLE_ID}
+        xcrun stapler staple ${PKGNAME}
+      fi
+else
+    pkgbuild --identifier com.cloudflare.${PRODUCT} \
+      --version ${VERSION} \
+      --scripts ${TARGET_DIRECTORY}/scripts \
+      --root ${TARGET_DIRECTORY}/contents \
+      --install-location /usr/local/bin \
+      ${PKGNAME}
+fi
+
+
+# cleaning up the build directory
+rm -rf $TARGET_DIRECTORY

.teamcity/update-homebrew.sh

@@ -43,7 +43,7 @@ git checkout -f master
 git reset --hard origin/master
 
 # modify cloudflared.rb
-URL="https://developers.cloudflare.com/argo-tunnel/dl/cloudflared-$VERSION-darwin-amd64.tgz"
+URL="https://packages.argotunnel.com/dl/cloudflared-$VERSION-darwin-amd64.tgz"
 tee cloudflared.rb <<EOF
 class Cloudflared < Formula
   desc 'Argo Tunnel'

CHANGES.md

@@ -0,0 +1,191 @@
+**Experimental**: This is a new format for release notes. The format and availability is subject to change.
+
+## 2021.8.4
+### Improvements
+- Temporary tunnels (those hosted on trycloudflare.com that do not require a Cloudflare login) now run as Named Tunnels
+underneath. We recall that these tunnels should not be relied upon for production usage as they come with no guarantee
+of uptime. Previous cloudflared versions will soon be unable to run legacy temporary tunnels and will require an update
+(to this version or more recent).
+
+## 2021.8.2
+### Improvements
+- Because Equinox os shutting down, all cloudflared releases are now present [here](https://github.com/cloudflare/cloudflared/releases).
+[Equinox](https://dl.equinox.io/cloudflare/cloudflared/stable) will no longer receive updates. 
+
+## 2021.8.0
+### Bug fixes
+- Prevents tunnel from accidentally running when only proxy-dns should run. 
+
+### Improvements
+- If auto protocol transport lookup fails, we now default to a transport instead of not connecting.
+
+## 2021.6.0
+### Bug Fixes
+- Fixes a http2 transport (the new default for Named Tunnels) to work with unix socket origins.
+
+
+## 2021.5.10
+### Bug Fixes
+- Fixes a memory leak in h2mux transport that connects cloudflared to Cloudflare edge.
+
+
+## 2021.5.9
+### New Features
+- Uses new Worker based login helper service to facilitate token exchange in cloudflared flows.
+
+### Bug Fixes
+- Fixes Centos-7 builds.
+
+## 2021.5.8
+### New Features
+- When creating a DNS record to point a hostname at a tunnel, you can now use --overwrite-dns to overwrite any existing
+  DNS records with that hostname. This works both when using the CLI to provision DNS, as well as when starting an adhoc
+  named tunnel, e.g.:
+  - `cloudflared tunnel route dns --overwrite-dns foo-tunnel foo.example.com`
+  - `cloudflared tunnel --overwrite-dns --name foo-tunnel --hostname foo.example.com`
+
+## 2021.5.7
+### New Features
+- Named Tunnels will automatically select the protocol to connect to Cloudflare's edge network.
+
+## 2021.5.0
+
+### New Features
+- It is now possible to run the same tunnel using more than one `cloudflared` instance. This is a server-side change and
+  is compatible with any client version that uses Named Tunnels.
+
+  To get started, visit our [developer documentation](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/run-tunnel/deploy-cloudflared-replicas).
+- `cloudflared tunnel ingress validate` will now warn about unused keys in your config file. This is helpful for
+  detecting typos in your config.
+- If `cloudflared` detects it is running inside a Linux container, it will limit itself to use only the number of CPUs
+  the pod has been granted, instead of trying to use every CPU available.
+
+## 2021.4.0
+
+### Bug Fixes
+
+- Fixed proxying of websocket requests to avoid possibility of losing initial frames that were sent in the same TCP
+  packet as response headers [#345](https://github.com/cloudflare/cloudflared/issues/345).
+- `proxy-dns` option now works in conjunction with running a named tunnel [#346](https://github.com/cloudflare/cloudflared/issues/346).
+
+## 2021.3.6
+
+### Bug Fixes
+
+- Reverted 2021.3.5 improvement to use HTTP/2 in a best-effort manner between cloudflared and origin services because
+  it was found to break in some cases.
+
+## 2021.3.5
+
+### Improvements
+
+ - HTTP/2 transport is now always chosen if origin server supports it and the service url scheme is HTTPS.
+   This was previously done in a best attempt manner.
+
+### Bug Fixes
+
+ - The MacOS binaries were not successfully released in 2021.3.3 and 2021.3.4. This release is aimed at addressing that.
+
+## 2021.3.3
+
+### Improvements
+
+- Tunnel create command, as well as, running ad-hoc tunnels using `cloudflared tunnel -name NAME`, will not overwrite
+  existing files when writing tunnel credentials.
+
+### Bug Fixes
+
+- Tunnel create and delete commands no longer use path to credentials from the configuration file.
+  If you need ot place tunnel credentials file at a specific location, you must use `--credentials-file` flag.
+- Access ssh-gen creates properly named keys for SSH short lived certs.
+
+
+## 2021.3.2
+
+### New Features
+
+- It is now possible to obtain more detailed information about the cloudflared connectors to Cloudflare Edge via
+  `cloudflared tunnel info <name/uuid>`. It is possible to sort the output as well as output in different formats,
+  such as: `cloudflared tunnel info --sort-by version --invert-sort --output json <name/uuid>`.
+  You can obtain more information via `cloudflared tunnel info --help`.
+
+### Bug Fixes
+
+- Don't look for configuration file in default paths when `--config FILE` flag is present after `tunnel` subcommand.
+- cloudflared access token command now functions correctly with the new token-per-app change from 2021.3.0.
+
+
+## 2021.3.0
+
+### New Features
+
+- [Cloudflare One Routing](https://developers.cloudflare.com/cloudflare-one/tutorials/warp-to-tunnel) specific commands
+  now show up in the `cloudflared tunnel route --help` output.
+- There is a new ingress type that allows cloudflared to proxy SOCKS5 as a bastion. You can use it with an ingress
+  rule by adding `service: socks-proxy`. Traffic is routed to any destination specified by the SOCKS5 packet but only
+  if allowed by a rule. In the following example we allow proxying to a certain CIDR but explicitly forbid one address
+  within it:
+```
+ingress:
+  - hostname: socks.example.com
+    service: socks-proxy
+    originRequest:
+      ipRules:
+        - prefix: 192.168.1.8/32
+          allow: false
+        - prefix: 192.168.1.0/24
+          ports: [80, 443]
+          allow: true
+```
+
+
+### Improvements
+
+- Nested commands, such as `cloudflared tunnel run`, now consider CLI arguments even if they appear earlier on the
+  command. For instance, `cloudflared --config config.yaml tunnel run` will now behave the same as
+  `cloudflared tunnel --config config.yaml run`
+- Warnings are now shown in the output logs whenever cloudflared is running without the most recent version and
+  `no-autoupdate` is `true`.
+- Access tokens are now stored per Access App instead of per request path. This decreases the number of times that the
+  user is required to authenticate with an Access policy redundantly.
+
+### Bug Fixes
+
+- GitHub [PR #317](https://github.com/cloudflare/cloudflared/issues/317) was broken in 2021.2.5 and is now fixed again.
+
+## 2021.2.5
+
+### New Features
+
+- We introduce [Cloudflare One Routing](https://developers.cloudflare.com/cloudflare-one/tutorials/warp-to-tunnel) in
+  beta mode. Cloudflare customer can now connect users and private networks with RFC 1918 IP addresses via the
+  Cloudflare edge network. Users running Cloudflare WARP client in the same organization can connect to the services
+  made available by Argo Tunnel IP routes. Please share your feedback in the GitHub issue tracker.
+
+## 2021.2.4
+
+### Bug Fixes
+
+- Reverts the Improvement released in 2021.2.3 for CLI arguments as it introduced a regression where cloudflared failed
+  to read URLs in configuration files.
+- cloudflared now logs the reason for failed connections if the error is recoverable.
+
+## 2021.2.3
+
+### Backward Incompatible Changes
+
+- Removes db-connect. The Cloudflare Workers product will continue to support db-connect implementations with versions
+  of cloudflared that predate this release and include support for db-connect.
+
+### New Features
+
+- Introduces support for proxy configurations with websockets in arbitrary TCP connections (#318).
+
+### Improvements
+
+- (reverted) Nested command line argument handling.
+
+### Bug Fixes
+
+- The maximum number of upstream connections is now limited by default which should fix reported issues of cloudflared
+  exhausting CPU usage when faced with connectivity issues.

Dockerfile

@@ -1,7 +1,7 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.13.3 as builder
+FROM golang:1.17.1 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \

Makefile

@@ -1,4 +1,23 @@
-VERSION       := $(shell git describe --tags --always --dirty="-dev")
+VERSION       := $(shell git describe --tags --always --dirty="-dev" --match "[0-9][0-9][0-9][0-9].*.*")
+MSI_VERSION   := $(shell git tag -l --sort=v:refname | grep "w" | tail -1 | cut -c2-)
+#MSI_VERSION expects the format of the tag to be: (wX.X.X). Starts with the w character to not break cfsetup.
+#e.g. w3.0.1 or w4.2.10. It trims off the w character when creating the MSI.
+
+ifeq ($(FIPS), true)
+	GO_BUILD_TAGS := $(GO_BUILD_TAGS) fips
+endif
+
+ifneq ($(GO_BUILD_TAGS),)
+	GO_BUILD_TAGS := -tags $(GO_BUILD_TAGS)
+endif
+
+ifeq ($(NIGHTLY), true)
+	DEB_PACKAGE_NAME := cloudflared-nightly
+	NIGHTLY_FLAGS := --conflicts cloudflared --replaces cloudflared
+else
+	DEB_PACKAGE_NAME := cloudflared
+endif
+
 DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
 VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
 
@@ -7,25 +26,21 @@ PACKAGE_DIR   := $(CURDIR)/packaging
 INSTALL_BINDIR := /usr/bin/
 MAN_DIR := /usr/share/man/man1/
 
-EQUINOX_FLAGS = --version="$(VERSION)" \
-	--platforms="$(EQUINOX_BUILD_PLATFORMS)" \
-	--app="$(EQUINOX_APP_ID)" \
-	--token="$(EQUINOX_TOKEN)" \
-	--channel="$(EQUINOX_CHANNEL)"
-
-ifeq ($(EQUINOX_IS_DRAFT), true)
-	EQUINOX_FLAGS := --draft $(EQUINOX_FLAGS)
-endif
-
 LOCAL_ARCH ?= $(shell uname -m)
 ifneq ($(GOARCH),)
     TARGET_ARCH ?= $(GOARCH)
 else ifeq ($(LOCAL_ARCH),x86_64)
     TARGET_ARCH ?= amd64
+else ifeq ($(LOCAL_ARCH),amd64)
+    TARGET_ARCH ?= amd64
+else ifeq ($(LOCAL_ARCH),i686)
+    TARGET_ARCH ?= amd64
 else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 5),armv8)
     TARGET_ARCH ?= arm64
 else ifeq ($(LOCAL_ARCH),aarch64)
     TARGET_ARCH ?= arm64
+else ifeq ($(LOCAL_ARCH),arm64)
+    TARGET_ARCH ?= arm64
 else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 4),armv)
     TARGET_ARCH ?= arm
 else
@@ -39,6 +54,8 @@ else ifeq ($(LOCAL_OS),darwin)
     TARGET_OS ?= darwin
 else ifeq ($(LOCAL_OS),windows)
     TARGET_OS ?= windows
+else ifeq ($(LOCAL_OS),freebsd)
+    TARGET_OS ?= freebsd
 else
     $(error This system's OS $(LOCAL_OS) isn't supported)
 endif
@@ -51,8 +68,6 @@ endif
 
 ifeq ($(FLAVOR), centos-7)
 	TARGET_PUBLIC_REPO ?= el7
-else ifeq ($(FLAVOR), centos-6)
-	TARGET_PUBLIC_REPO ?= el6
 else
 	TARGET_PUBLIC_REPO ?= $(FLAVOR)
 endif
@@ -65,8 +80,18 @@ clean:
 	go clean
 
 .PHONY: cloudflared
-cloudflared: tunnel-deps
-	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
+cloudflared: 
+ifeq ($(FIPS), true)
+	$(info Building cloudflared with go-fips)
+	-test -f fips/fips.go && mv fips/fips.go fips/fips.go.linux-amd64
+	mv fips/fips.go.linux-amd64 fips/fips.go
+endif
+
+	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
+
+ifeq ($(FIPS), true)
+	mv fips/fips.go fips/fips.go.linux-amd64
+endif
 
 .PHONY: container
 container:
@@ -74,16 +99,23 @@ container:
 
 .PHONY: test
 test: vet
+ifndef CI
 	go test -v -mod=vendor -race $(VERSION_FLAGS) ./...
+else
+	@mkdir -p .cover
+	go test -v -mod=vendor -race $(VERSION_FLAGS) -coverprofile=".cover/c.out" ./...
+	go tool cover -html ".cover/c.out" -o .cover/all.html
+endif
 
 .PHONY: test-ssh-server
 test-ssh-server:
 	docker-compose -f ssh_server_tests/docker-compose.yml up
 
 define publish_package
+	chmod 664 cloudflared*.$(1); \
 	for HOST in $(CF_PKG_HOSTS); do \
 		ssh-keyscan -t rsa $$HOST >> ~/.ssh/known_hosts; \
-		scp -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
+		scp -p -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
 	done
 endef
 
@@ -99,13 +131,13 @@ define build_package
 	mkdir -p $(PACKAGE_DIR)
 	cp cloudflared $(PACKAGE_DIR)/cloudflared
 	cat cloudflared_man_template | sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' > $(PACKAGE_DIR)/cloudflared.1
-	fakeroot fpm -C $(PACKAGE_DIR) -s dir -t $(1) --$(1)-compression bzip2 \
+	fakeroot fpm -C $(PACKAGE_DIR) -s dir -t $(1) \
 		--description 'Cloudflare Argo tunnel daemon' \
 		--vendor 'Cloudflare' \
 		--license 'Cloudflare Service Agreement' \
 		--url 'https://github.com/cloudflare/cloudflared' \
 		-m 'Cloudflare <support@cloudflare.com>' \
-		-a $(TARGET_ARCH) -v $(VERSION) -n cloudflared --after-install postinst.sh --after-remove postrm.sh \
+		-a $(TARGET_ARCH) -v $(VERSION) -n $(DEB_PACKAGE_NAME) $(NIGHTLY_FLAGS) --after-install postinst.sh --after-remove postrm.sh \
 		cloudflared=$(INSTALL_BINDIR) cloudflared.1=$(MAN_DIR)
 endef
 
@@ -117,11 +149,71 @@ cloudflared-deb: cloudflared
 cloudflared-rpm: cloudflared
 	$(call build_package,rpm)
 
+.PHONY: cloudflared-pkg
+cloudflared-pkg: cloudflared
+	$(call build_package,osxpkg)
+
+.PHONY: cloudflared-msi
+cloudflared-msi: cloudflared
+	wixl --define Version=$(VERSION) --define Path=$(EXECUTABLE_PATH) --output cloudflared-$(VERSION)-$(TARGET_ARCH).msi cloudflared.wxs
+
 .PHONY: cloudflared-darwin-amd64.tgz
 cloudflared-darwin-amd64.tgz: cloudflared
 	tar czf cloudflared-darwin-amd64.tgz cloudflared
 	rm cloudflared
 
+.PHONY: cloudflared-junos
+cloudflared-junos: cloudflared jetez-certificate.pem jetez-key.pem
+	jetez --source . \
+		  -j jet.yaml \
+		  --key jetez-key.pem \
+		  --cert jetez-certificate.pem \
+		  --version $(VERSION)
+	rm jetez-*.pem
+
+jetez-certificate.pem:
+ifndef JETEZ_CERT
+	$(error JETEZ_CERT not defined)
+endif
+	@echo "Writing JetEZ certificate"
+	@echo "$$JETEZ_CERT" > jetez-certificate.pem
+
+jetez-key.pem:
+ifndef JETEZ_KEY
+	$(error JETEZ_KEY not defined)
+endif
+	@echo "Writing JetEZ key"
+	@echo "$$JETEZ_KEY" > jetez-key.pem
+
+.PHONY: publish-cloudflared-junos
+publish-cloudflared-junos: cloudflared-junos cloudflared-x86-64.latest.s3
+ifndef S3_ENDPOINT
+	$(error S3_HOST not defined)
+endif
+ifndef S3_URI
+	$(error S3_URI not defined)
+endif
+ifndef S3_ACCESS_KEY
+	$(error S3_ACCESS_KEY not defined)
+endif
+ifndef S3_SECRET_KEY
+	$(error S3_SECRET_KEY not defined)
+endif
+	sha256sum cloudflared-x86-64-$(VERSION).tgz | awk '{printf $$1}' > cloudflared-x86-64-$(VERSION).tgz.shasum
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+		put cloudflared-x86-64-$(VERSION).tgz $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+		put cloudflared-x86-64-$(VERSION).tgz.shasum $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz.shasum
+	dpkg --compare-versions "$(VERSION)" gt "$(shell cat cloudflared-x86-64.latest.s3)" && \
+		echo -n "$(VERSION)" > cloudflared-x86-64.latest && \
+		s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+			put cloudflared-x86-64.latest $(S3_URI)/cloudflared-x86-64.latest || \
+		echo "Latest version not updated"
+
+cloudflared-x86-64.latest.s3:
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force \
+		get $(S3_URI)/cloudflared-x86-64.latest cloudflared-x86-64.latest.s3
+
 .PHONY: homebrew-upload
 homebrew-upload: cloudflared-darwin-amd64.tgz
 	aws s3 --endpoint-url $(S3_ENDPOINT) cp --acl public-read $$^ $(S3_URI)/cloudflared-$$(VERSION)-$1.tgz
@@ -131,28 +223,44 @@ homebrew-upload: cloudflared-darwin-amd64.tgz
 homebrew-release: homebrew-upload
 	./publish-homebrew-formula.sh cloudflared-darwin-amd64.tgz $(VERSION) homebrew-cloudflare
 
-.PHONY: release
-release: bin/equinox
-	bin/equinox release $(EQUINOX_FLAGS) -- $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
-
 .PHONY: github-release
 github-release: cloudflared
 	python3 github_release.py --path $(EXECUTABLE_PATH) --release-version $(VERSION)
 
-bin/equinox:
-	mkdir -p bin
-	curl -s https://bin.equinox.io/c/75JtLRTsJ3n/release-tool-beta-$(EQUINOX_PLATFORM).tgz | tar xz -C bin/
+.PHONY: github-release-built-pkgs
+github-release-built-pkgs:
+	python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
 
-.PHONY: tunnel-deps
-tunnel-deps: tunnelrpc/tunnelrpc.capnp.go
+.PHONY: github-message
+github-message:
+	python3 github_message.py --release-version $(VERSION)
 
-tunnelrpc/tunnelrpc.capnp.go: tunnelrpc/tunnelrpc.capnp
+.PHONY: github-mac-upload
+github-mac-upload:
+	python3 github_release.py --path artifacts/cloudflared-darwin-amd64.tgz --release-version $(VERSION) --name cloudflared-darwin-amd64.tgz
+	python3 github_release.py --path artifacts/cloudflared-amd64.pkg --release-version $(VERSION) --name cloudflared-amd64.pkg
+
+.PHONY: tunnelrpc-deps
+tunnelrpc-deps:
 	which capnp  # https://capnproto.org/install.html
 	which capnpc-go  # go get zombiezen.com/go/capnproto2/capnpc-go
 	capnp compile -ogo tunnelrpc/tunnelrpc.capnp
 
+.PHONY: quic-deps
+quic-deps: 
+	which capnp 
+	which capnpc-go
+	capnp compile -ogo quic/schema/quic_metadata_protocol.capnp
+
 .PHONY: vet
 vet:
 	go vet -mod=vendor ./...
-	which go-sumtype  # go get github.com/BurntSushi/go-sumtype
+	# go get github.com/sudarshan-reddy/go-sumtype (don't do this in build directory or this will cause vendor issues)
+	# Note: If you have github.com/BurntSushi/go-sumtype then you might have to use the repo above instead
+	# for now because it uses an older version of golang.org/x/tools.
+	which go-sumtype  
 	go-sumtype $$(go list -mod=vendor ./...)
+
+.PHONY: goimports
+goimports:
+	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done

README.md

@@ -1,9 +1,41 @@
 # Argo Tunnel client
 
-Contains the command-line client and its libraries for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network.
+Contains the command-line client for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network. Extensive documentation can be found in the [Argo Tunnel section](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps) of the Cloudflare Docs.
 
-## Getting started
+## Before you get started
 
-    go install github.com/cloudflare/cloudflared/cmd/cloudflared
+Before you use Argo Tunnel, you'll need to complete a few steps in the Cloudflare dashboard. The website you add to Cloudflare will be used to route traffic to your Tunnel.
 
-User documentation for Argo Tunnel can be found at https://developers.cloudflare.com/argo-tunnel/
+1. [Add a website to Cloudflare](https://support.cloudflare.com/hc/en-us/articles/201720164-Creating-a-Cloudflare-account-and-adding-a-website)
+2. [Change your domain nameservers to Cloudflare](https://support.cloudflare.com/hc/en-us/articles/205195708)
+
+## Installing `cloudflared`
+
+Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases here on the `cloudflared` GitHub repository.
+
+* You can [install on macOS](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
+* Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#linux)
+* A Docker image of `cloudflared` is [available on DockerHub](https://hub.docker.com/r/cloudflare/cloudflared)
+* You can install on Windows machines with the [steps here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#windows)
+
+User documentation for Argo Tunnel can be found at https://developers.cloudflare.com/cloudflare-one/connections/connect-apps
+
+## Creating Tunnels and routing traffic
+
+Once installed, you can authenticate `cloudflared` into your Cloudflare account and begin creating Tunnels that serve traffic for hostnames in your account.
+
+* Create a Tunnel with [these instructions](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/create-tunnel)
+* Route traffic to that Tunnel with [DNS records in Cloudflare](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/dns) or with a [Cloudflare Load Balancer](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/lb)
+
+## TryCloudflare
+
+Want to test Argo Tunnel before adding a website to Cloudflare? You can do so with TryCloudflare using the documentation [available here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/run-tunnel/trycloudflare).
+
+## Deprecated versions
+
+Cloudflare currently supports versions of `cloudflared` 2020.5.1 and later. Breaking changes unrelated to feature availability may be introduced that will impact versions released prior to 2020.5.1. You can read more about upgrading `cloudflared` in our [developer documentation](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#updating-cloudflared).
+
+| Version(s) | Deprecation status |
+|---|---|
+| 2020.5.1 and later | Supported |
+| Versions prior to 2020.5.1 | No longer supported |

RELEASE_NOTES

@@ -1,3 +1,543 @@
+2021.9.2
+- 2021-09-21 TUN-5129: Use go 1.17 and copy .git folder to docker build to compute version
+- 2021-09-21 TUN-5128: Enforce maximum grace period
+- 2021-09-22 TUN-5141: Make sure websocket pinger returns before streaming returns
+- 2021-09-24 TUN-5142: Add asynchronous servecontrolstream for QUIC
+- 2021-09-24 TUN-5142: defer close rpcconn inside unregister instead of ServeControlStream
+- 2021-09-27 TUN-5160: Set request.ContentLength when this value is in request header
+
+2021.9.1
+- 2021-09-21 TUN-5118: Quic connection now detects duplicate connections similar to http2
+- 2021-09-15 Fix TryCloudflare link
+
+2021.9.0
+- 2021-09-02 Fix broken TryCloudflare link
+- 2021-09-03 Add support for taking named tunnel credentials from an environment variable
+- 2021-08-30 TUN-5012: Use patched go-sumtype
+- 2021-08-31 TUN-5011: Use the region parameter in fallback SRV lookup
+- 2021-08-31 TUN-5029: Do not strip cf- prefixed headers
+- 2021-08-29 TUN-5009: Updated github action to use go 1.17.x for checks
+- 2021-08-28 TUN-5010: --region should be a string flag
+- 2021-08-10 Allow building on arm64 platforms
+- 2021-06-09 Update README.md
+- 2021-05-31 🖌️ Allow providing TokenID and TokenSecret as env vars when calling cloudflared access
+- 2021-05-31 🎨 Prefix env var parameters with TUNNEL
+
+2021.8.7
+- 2021-08-28 Revert "TUN-4926: Implement --region configuration option"
+
+2021.8.6
+- 2021-08-27 TUN-5000: De-flake logging to dir component test in Windows by increasing to buffer to cope with more logging
+- 2021-08-27 TUN-5003: Fix cfsetup for non-FIPS golang version
+
+2021.8.5
+- 2021-08-27 TUN-4961: Update quic-go to latest
+- 2021-08-27 Release 2021.8.4
+
+2021.8.4
+- 2021-08-26 TUN-4974: Fix regression where we were debug logging by accident
+- 2021-08-26 TUN-4970: Only default to http2 for warp-routing if protocol is h2mux
+- 2021-08-26 TUN-4981: Improve readability of prepareTunnelConfig method
+- 2021-08-26 TUN-4926: Implement --region configuration option
+- 2021-07-09 TUN-4821: Make quick tunnels the default in cloudflared
+
+2021.8.3
+- 2021-08-23 TUN-4889: Add back appendtagheaders function
+- 2021-08-21 TUN-4940: Fix cloudflared not picking up correct NextProtos for quic
+- 2021-08-21 TUN-4613: Add a no-op protocol version slot
+- 2021-08-13 TUN-4922: Downgrade quic-go library to 0.20.0
+- 2021-08-17 TUN-4866: Add Control Stream for QUIC
+- 2021-08-17 TUN-4927: Parameterize region in edge discovery code
+- 2021-08-06 TUN-4602: Added UDP resolves to Edge discovery
+
+2021.8.2
+- 2021-08-03 TUN-4597: Added HTTPProxy for QUIC
+- 2021-08-04 TUN-4795: Remove Equinox releases
+- 2021-08-09 TUN-4911: Append Environment variable to Path instead of overwriting it
+
+2021.8.1
+- 2021-08-02 TUN-4855: Added CHANGES.md for release 2021.8.0
+- 2021-08-03 TUN-4597: Add a QUIC server skeleton
+- 2021-08-03 TUN-4873: Disable unix domain socket test for windows unit tests
+- 2021-08-04 TUN-4875: Added amd64-linux builds back to releases
+
+2021.8.0
+- 2021-07-30 TUN-4847: Allow to list tunnels by prefix name or exclusion prefix name
+- 2021-07-30 TUN-4772: Release built executables with packages
+- 2021-07-30 TUN-4851: Component tests to smoke test that Proxy DNS and Tunnel are only run when expected
+- 2021-07-28 TUN-4811: Publish quick tunnels' hostname in /metrics under `userHostname` for backwards-compatibility
+- 2021-07-29 TUN-4832: Prevent tunnel from running accidentally when only proxy-dns should run
+- 2021-07-28 TUN-4819: Tolerate protocol TXT record lookup failing
+
+2021.7.4
+- 2021-07-28 TUN-4814: Revert "TUN-4699: Make quick tunnels the default in cloudflared"
+- 2021-07-28 TUN-4812: Disable CGO for cloudflared builds
+
+2021.7.3
+- 2021-07-27 TUN-4799: Build deb, msi and rpm packages with fips
+
+2021.7.2
+- 2021-07-27 Fixed a syntax error with python logging.
+
+2021.7.1
+- 2021-07-21 TUN-4755: Add a windows msi release option to Make
+- 2021-07-22 TUN-4761: Added a build-all-packages target to cfsetup
+- 2021-07-26 TUN-4771: Upload deb, rpm and msi packages to github
+- 2021-07-14 TUN-4714: Name nightly package cloudflared-nightly to avoid apt conflict
+- 2021-07-16 TUN-4701: Split Proxy into ProxyHTTP and ProxyTCP
+- 2021-07-08 TUN-4596: Add QUIC application protocol for QUIC stream handshake
+- 2021-07-09 TUN-4699: Make quick tunnels the default in cloudflared
+
+2021.7.0
+- 2021-07-01 TUN-4626: Proxy non-stream based origin websockets with http Roundtrip.
+- 2021-07-01 TUN-4655: ingress.StreamBasedProxy.EstablishConnection takes dest input
+- 2021-07-09 TUN-4698: Add cloudflared metrics endpoint to serve quick tunnel hostname
+- 2021-06-21 TUN-4521: Modify cloudflared to use zoneless-tunnels-worker for free tunnels
+- 2021-04-05 AUTH-3475: Updated GetAppInfo error message
+
+2021.6.0
+- 2021-06-21 TUN-4571: Changelog for 2021.6.0
+- 2021-06-18 TUN-4571: Fix proxying to unix sockets when using HTTP2 transport to Cloudflare Edge
+- 2021-06-07 TUN-4502: Make `cloudflared tunnel route` subcommands described consistently
+- 2021-06-08 TUN-4504: Fix component tests in windows
+- 2021-05-27 TUN-4461: Log resulting DNS hostname if one is received from Cloudflare API
+
+2021.5.10
+- 2021-05-25 TUN-4456: Replaced instances of Tick() with Ticker() in h2mux paths
+
+2021.5.9
+- 2021-05-20 TUN-4426: Fix centos builds
+- 2021-05-20 Update changelog
+- 2021-04-30 AUTH-3426: Point to new transfer service URL and eliminate PUT /ok
+
+2021.5.8
+- 2021-05-14 TUN-4419: Improve error message when cloudflared cannot reach origin
+- 2021-05-19 TUN-4425: --overwrite-dns flag for in adhoc and route dns cmds
+
+2021.5.7
+- 2021-05-17 Fix typo in Changes.md
+- 2021-05-17 TUN-4421: Named Tunnels will automatically select the protocol to connect to Cloudflare's edge network
+
+2021.5.6
+- 2021-05-14 TUN-4418: Downgrade to Go 1.16.3
+
+2021.5.5
+
+
+2021.5.4
+- Fix release pipeline
+
+2021.5.1
+- 2021-05-10 TUN-4342: Fix false positive warning about unused hostname property
+- 2021-05-10 Release 2021.5.0
+
+2021.5.0
+- 2021-05-10 TUN-4384: Silence log from automaxprocs
+- 2021-05-10 AUTH-3537: AUDs in JWTs are now always arrays
+- 2021-05-10 Update changelog for 2021.5.0
+- 2021-05-03 TUN-4343: Fix broken build by setting debug field correctly
+- 2021-05-06 TUN-4356: Set AUTOMAXPROCS to the CPU limit when running in a Linux container
+- 2021-05-06 TUN-4357: Bump Go to 1.16
+- 2021-05-06 TUN-4359: Warn about unused keys in 'tunnel ingress validate'
+- 2021-04-30 debug: log host / path
+- 2021-04-20 AUTH-3513: Checks header for app info in case response is a 403/401 from the edge
+- 2021-04-29 TUN-4000: Release notes for cloudflared replica model
+- 2021-04-09 TUN-2853: rename STDIN-CONTROL env var to STDIN_CONTROL
+- 2021-04-09 TUN-4206: Better error message when user is only using one ingress rule
+
+2021.4.0
+- 2021-04-05 TUN-4178: Fix component test for running as a service in MacOS to not assume a named tunnel
+- 2021-04-05 TUN-4177: Running with proxy-dns should not prevent running Named Tunnels
+- 2021-04-02 TUN-4168: Transparently proxy websocket connections using stdlib HTTP client instead of gorilla/websocket; move websocket client code into carrier package since it's only used by access subcommands now (#345).
+- 2021-04-07 Publish change log for 2021.4.0
+
+2021.3.6
+- 2021-03-30 TUN-4150: Only show the connector table in 'tunnel info' if there are connectors. Don't show rows with zero connections.
+- 2021-03-31 TUN-4153: Revert best-effort HTTP2 usage when talking to origins
+- 2021-03-26 TUN-4141: Better error messages for tunnel info subcommand.
+- 2021-03-29 TUN-4146: Unhide and document grace-period
+- 2021-03-25 TUN-3863: Consolidate header handling logic in the connection package; move headers definitions from h2mux to packages that manage them; cleanup header conversions
+
+2021.3.5
+- 2021-03-26 TUN-3896: http-service and tunnelstore client use http2 transport.
+- 2021-03-25 TUN-4125: Change component tests to run in CI with its own dedicated resources
+- 2021-03-26 Publish change log for 2021.3.5
+
+2021.3.4
+
+
+2021.3.3
+- 2021-03-23 TUN-4111: Warn the user if both properties "tunnel" and "hostname" are used
+- 2021-03-23 TUN-4082: Test logging when running as a service
+- 2021-03-23 TUN-4112: Skip testing graceful shutdown with SIGINT on Windows
+- 2021-03-23 TUN-4116: Ingore credentials-file setting in configuration file during tunnel create and delete opeations.
+- 2021-03-23 TUN-4118: Don't overwrite existing file with tunnel credentials. For ad-hoc tunnels, this means tunnel won't start if there's a file in the way.
+- 2021-03-24 TUN-4123: Don't capture output in reconnect componet test
+- 2021-03-23 TUN-4067: Reformat code for consistent import order, grouping, and fix formatting. Added goimports target to the Makefile to make this easier in the future.
+- 2021-03-24 AUTH-3455: Generate short-lived ssh cert per hostname
+- 2021-03-25 Update changelog 2021.3.3
+
+2021.3.2
+- 2021-03-23 TUN-4042: Capture cloudflared output to debug component tests
+- 2021-03-23 Publish changelog for 2021.3.2
+- 2021-03-16 TUN-4089: Address flakiness in component tests for termination
+- 2021-03-16 TUN-4060: Fix Go Vet warnings (new with go 1.16) where t.Fatalf is called from a test goroutine
+- 2021-03-16 TUN-4091: Use flaky decorator to rerun reconnect component tests when they fail
+- 2021-03-12 TUN-4081: Update log severities to use Zerolog's levels
+- 2021-03-16 TUN-4094: Don't read configuration file for access commands
+- 2021-03-15 TUN-3993: New `cloudflared tunnel info` to obtain details about the active connectors for a tunnel
+- 2021-03-17 TUN-3715: Apply input source to the correct context
+- 2021-03-17 AUTH-3394: Ensure scheme on token command
+- 2021-03-18 TUN-4096: Reduce tunnel not connected assertion backoff to address flaky termination tests
+- 2021-03-19 TUN-3998: Allow to cleanup the connections of a tunnel limited to a single client
+- 2021-02-04 TUN-3715: Only read config file once, right before invoking the command
+
+2021.3.1
+- 2021-03-11 TUN-4051: Add component-tests to test graceful shutdown
+- 2021-03-12 TUN-4052: Add component tests to assert service mode behavior
+
+2021.3.0
+- 2021-03-10 TUN-4075: Dedup test should not compare order of list
+- 2021-03-10 Revert "AUTH-3394: Creates a token per app instead of per path"
+- 2021-03-11 TUN-4066: Remove unnecessary chmod during package publish to CF_PKG_HOSTS
+- 2021-03-11 TUN-4066: Set permissions in build agent before 'scp'-ing to machine hosting package repo
+- 2021-03-11 TUN-4050: Add component tests to assert reconnect behavior
+- 2021-03-10 AUTH-3394: Creates a token per app instead of per path - with fix for free tunnels
+- 2021-03-15 Publish change log for 2021.3.0
+- 2021-03-01 Issue #285 - Makefile does not detect TARGET_ARCH correctly on FreeBSD  (#325)
+- 2021-03-01 TUN-3988: Log why it cannot check if origin cert exists
+- 2021-03-02 TUN-3995: Optional --features flag for tunnel run.
+- 2021-03-02 TUN-3994: Log client_id when running a named tunnel
+- 2021-03-04 TUN-4026: Fix regression where HTTP2 edge transport was no longer propagating control plane errors
+- 2021-03-05 TUN-4055: Skeleton for component tests
+- 2021-03-08 TUN-4047: Add cfsetup target to run component test
+- 2021-03-08 TUN-4016: Delegate decision to update for Worker service
+- 2021-03-02 TUN-3905: Cannot run go mod vendor in cloudflared due to fips
+- 2021-03-08 TUN-4063: Cleanup dependencies between packages.
+- 2021-03-09 Allow partial reads from a GorillaConn; add SetDeadline (from net.Conn) (#330)
+- 2021-03-09 TUN-4069: Fix regression on support for websocket over proxy
+- 2021-03-02 AUTH-3394: Creates a token per app instead of per path
+- 2021-03-01 TUN-4017: Add support for using cloudflared as a full socks proxy.
+- 2021-03-08 TUN-4062: Read component tests config from yaml file
+- 2021-03-08 TUN-4049: Add component tests to assert logging behavior when running from terminal
+- 2021-02-23 TUN-3963: Repoint urfave/cli/v2 library at patched branch at github.com/ipostelnik/cli/v2@fixed which correctly handles reading flags declared at multiple levels of subcommands.
+- 2021-02-25 TUN-3970: Route ip show has alias route ip list
+- 2021-02-26 TUN-3978: Unhide teamnet commands and improve their help
+- 2021-02-26 TUN-3983: Renew CA certs in cloudflared
+- 2021-02-28 TUN-3989: Check in with Updater service in more situations and convey messages to user
+- 2021-02-11 TUN-3819: Remove client-side check that deleted tunnels have no connections
+
+2021.2.5
+- 2021-02-23 Publish change notes for 2021.2.5
+- 2021-02-11 TUN-3838: ResponseWriter no longer reads and origin error tests
+- 2021-02-10 TUN-3895: Tests for socks stream handler
+- 2021-02-19 TUN-3939: Add logging that shows that Warp-routing is enabled
+- 2021-02-02 TUN-3817: Adds tests for websocket based streaming regression
+- 2021-02-04 TUN-3799: extended the Stream interface to take a logger and added debug logs for io.Copy errors
+- 2021-02-03 TUN-3855: Add ability to override target of 'access ssh' command to a different host for testing
+- 2021-02-04 TUN-3853: Respond with ws headers from the origin service rather than generating our own
+- 2021-02-08 TUN-3889: Move host header override logic to httpService
+- 2021-02-05 TUN-3868: Refactor singleTCPService and bridgeService to tcpOverWSService and rawTCPService
+- 2021-01-21 TUN-3753: Select http2 protocol when warp routing is enabled
+- 2021-01-26 TUN-3809: Allow routes ip show to output as JSON or YAML
+- 2021-01-11 TUN-3615: added support to  proxy tcp streams
+- 2021-01-17 TUN-3725: Warp-routing is independent of ingress
+- 2021-01-15 TUN-3764: Actively flush data for TCP streams
+- 2020-12-09 TUN-3617: Separate service from client, and implement different client for http vs. tcp origins
+
+2021.2.4
+- 2021-02-22 TUN-3948: Log error when retrying connection
+- 2021-02-23 TUN-3964: Revert "TUN-3922: Repoint urfave/cli/v2 library at patched branch at github.com/ipostelnik/cli/v2@fixed which correctly handles reading flags declared at multiple levels of subcommands."
+- 2021-02-23 Publish release notes for 2021.2.4
+
+2021.2.3
+- 2021-02-23 Publish release notes for 2021.2.3
+- 2021-02-10 TUN-3902: Add jitter to backoffhandler
+- 2021-02-11 TUN-3913: Help gives wrong exit code for autoupdate
+- 2021-02-12 Add max upstream connections dns-proxy option (#290)
+- 2021-02-16 TUN-3924: Removed db-connect command. Added a placeholder handler for this command that informs users that command is no longer supported.
+- 2021-02-12 TUN-3922: Repoint urfave/cli/v2 library at patched branch at github.com/ipostelnik/cli/v2@fixed which correctly handles reading flags declared at multiple levels of subcommands.
+- 2021-02-19 Added support for proxy (#318)
+- 2021-02-19 TUN-3945: Fix runApp signature for generic service
+- 2021-02-09 Update README.md
+- 2021-02-09 Update the TryCloudflare link
+
+2021.2.2
+- 2021-02-04 TUN-3864: Users can choose where credentials file is written after creating a tunnel
+- 2021-02-04 TUN-3869: Improve reliability of graceful shutdown.
+- 2021-02-07 TUN-3878: Do not supply -tags when none are specified
+- 2021-02-04 TUN-3635: Send event when unregistering tunnel for gracful shutdown so /ready endpoint reports down status befoe connections finish handling pending requests.
+- 2021-02-08 TUN-3890: Code coverage for cloudflared in CI
+- 2021-02-09 AUTH-3375 exchangeOrgToken deleted cookie fix
+- 2020-11-18 Update error message to use login command
+
+2021.2.1
+- 2021-02-04 TUN-3858: Do not suffix cloudflared version with -fips
+
+2021.2.0
+- 2021-02-01 TUN-3837: Remove automation_email from cloudflared status page test
+- 2021-02-03 TUN-3848: Use transport logger for h2mux
+- 2021-02-03 TUN-3854: cloudflared tunnel list flags to sort output
+- 2021-01-21 TUN-3195: Don't colorize console logs when stderr is not a terminal
+- 2021-01-20 Fixed connection error handling by removing duplicated errors, standardizing on non-pointer error types
+- 2021-01-20 TUN-3118: Changed graceful shutdown to immediately unregister tunnel from the edge, keep the connection open until the edge drops it or grace period expires
+- 2021-01-25 TUN-3165: Add reference to Argo Tunnel documentation in the help output
+- 2021-01-25 TUN-3806: Use a .dockerignore
+- 2021-01-21 TUN-3795: Use RFC-3339 style date format for logs, produce timestamp in UTC
+- 2021-01-26 TUN-3795: Removed errant test
+- 2021-01-25 TUN-3792: Handle graceful shutdown correctly when running as a windows service. Only expose one shutdown channel globally, which now triggers the graceful shutdown sequence across all modes. Removed separate handling of zero-duration grace period, instead it's checked only when we need to wait for exit.
+- 2021-01-27 TUN-3811: Better error reporting on http2 connection termination. Registration errors from control loop are now propagated out of the connection server code. Unified error handling between h2mux and http2 connections so we log and retry errors the same way, regardless of underlying transport.
+- 2021-01-28 TUN-3830: Use Go 1.15.7
+- 2021-01-28 TUN-3826: Use go-fips when building cloudflared for linux/amd64
+- 2021-01-19 TUN-3777: Fix /ready endpoint for classic tunnels
+- 2021-01-19 TUN-3773: Add back pprof endpoints
+
+2021.1.5
+- 2021-01-15 TUN-3594: Log ingress response at debug level
+- 2021-01-15 TUN-3765: Fix doubly nested log output by `logfile` option
+- 2021-01-16 TUN-3767: Tolerate logging errors
+- 2021-01-17 TUN-3768: Reuse file loggers
+- 2021-01-14 TUN-3738: Refactor observer to avoid potential of blocking on tunnel notifications
+- 2021-01-15 TUN-3766: Print flags defined at all levels of command hierarchy, not just locally defined flags for a command. This fixes output of overriden settings for  subcommand.
+
+2021.1.4
+- 2021-01-14 TUN-3759: Single file logging output should always append
+
+2021.1.3
+- 2021-01-14 TUN-3756: File logging output must consider the directory
+- 2021-01-14 TUN-3757: Fix legacy Uint flags that are incorrectly handled by ufarve library
+
+2021.1.2
+- 2021-01-13 TUN-3747: Fix logging in Windows
+
+2021.1.1
+- 2021-01-13 TUN-3744: Fix compilation error in windows service
+
+2021.1.0
+- 2021-01-11 TUN-3670: Update Teamnet API gateway prefixes
+- 2021-01-13 TUN-3738: Consume UI events even when UI is disabled
+- 2021-01-06 TUN-3722: Teamnet API paths include /network
+- 2021-01-05 TUN-3688: Subcommand for users to check which route an IP proxies through
+- 2021-01-08 TUN-3691: Edit Teamnet help text
+- 2020-12-30 TUN-3706: Quit if any origin service fails to start
+- 2020-12-31 TUN-3708: Better info message about system root certpool on Windows
+- 2020-12-21 TUN-3669: Teamnet commands to add/show Teamnet routes.
+- 2020-12-29 TUN-3689: Delete routes via cloudflared CLI
+- 2020-12-28 TUN-3471: Add structured log context to logs
+- 2020-12-15 TUN-3650: Remove unused awsuploader package
+- 2020-12-03 Update to add deprecated version note (#271)
+- 2020-12-02 TUN-3472: Set up rolling logger with zerolog and lumberjack
+- 2020-12-08 TUN-3607: Set up single-file logger with zerolog
+- 2020-12-03 Update to add deprecated version note (#271)
+- 2020-11-25 TUN-3470: Replace in-house logger calls with zerolog
+
+2020.12.0
+- 2020-12-04 TUN-3599: improved delete if credentials isnt found.
+- 2020-12-04 TUN-3612: Upgrade to Go 1.15.6
+- 2020-11-30 TUN-3593: /ready endpoint for k8s readiness. Move tunnel events out of UI package, into connection package.
+- 2020-11-27 TUN-3594: Log response status at debug level
+
+2020.11.11
+- 2020-11-20 TUN-3578: cloudflared tunnel route dns should allow wildcard subdomains
+- 2020-11-21 EDGEPLAT-2958 remove deb-compression, defaulting to gzip
+- 2020-11-23 TUN-3581: Tunnels can be run by name using only --credentials-file, no origin cert necessary.
+- 2020-11-15 TUN-3561: Unified logger configuration
+- 2020-11-08 AUTH-3221: Saves org token to disk and uses it to refresh the app token
+
+2020.11.10
+- 2020-11-20 TUN-3562: Fix panic when using bastion mode ingress rule
+- 2020-11-20 EDGEPLAT-2958 build cloudflared for Bullseye
+
+2020.11.9
+- 2020-11-18 TUN-3557: Detect SSE if content-type starts with text/event-stream
+- 2020-11-18 TUN-3559: Share response meta header with other packages
+- 2020-11-18 DEVTOOLS-7936: Remove redundant chgrp from publish
+- 2020-11-18 TUN-3558: cloudflared allows empty config files
+- 2020-11-18 TUN-3544: Upgrade to Go 1.15.5
+
+2020.11.8
+- 2020-11-17 TUN-3555: Single origin service should default to localhost:8080
+
+2020.11.7
+- 2020-11-13 TUN-3514: Stop setting --is-autoupdated flag after autoupdate because it can break named tunnel running in k8s
+- 2020-11-15 TUN-3548, TUN-3547: Bastion mode can be specified as a service, doesn't require URL.
+- 2020-11-16 TUN-3549: Use a separate handler for each websocket proxy
+
+2020.11.6
+- 2020-11-14 TUN-3546: Fix panic in tlsconfig.LoadOriginCA
+
+2020.11.5
+- 2020-11-12 TUN-3540: Better copy in ingress rules error messages
+- 2020-11-12 DEVTOOLS-7936: Set permissions on public packages
+- 2020-11-13 TUN-3543: ProxyAddress not using default in single-origin mode
+
+2020.11.4
+- 2020-11-11 TUN-3534: Specific error message when credentials file is a .pem not .json
+- 2020-11-02 TUN-3500: Integrate replace h2mux by http2 work with multiple origin support
+- 2020-11-09 TUN-3514: Transport logger write to UI when UI is enabled
+- 2020-10-30 TUN-3490: Make sure OriginClient implementation doesn't write after Proxy return
+- 2020-10-20 TUN-3403: Unit test for origin/proxy to test serving HTTP and Websocket
+- 2020-10-23 TUN-3480: Support SSE with http2 connection, and add SSE handler to hello-world server
+- 2020-10-27 TUN-3489: Add unit tests to cover proxy logic in connection package of cloudflared
+- 2020-10-16 TUN-3467: Serialize cf-cloudflared-response-meta during package initialization using jsoniter
+- 2020-10-14 TUN-3456: New protocol option auto to automatically select between http2 and h2mux
+- 2020-10-14 TUN-3458: Upgrade to http2 when available, fallback to h2mux when we reach max retries
+- 2020-10-08 TUN-3449: Use flag to select transport protocol implementation
+- 2020-10-08 TUN-3462: Refactor cloudflared to separate origin from connection
+- 2020-09-21 TUN-3406: Proxy websocket requests over Go http2
+- 2020-09-25 TUN-3420: Establish control plane and send RPC over control plane
+- 2020-09-11 TUN-3400: Use Go HTTP2 library as transport to connect with the edge
+
+2020.11.3
+- 2020-11-11 TUN-3533: Set config for single origin ingress
+
+2020.11.2
+
+
+2020.11.1
+- 2020-11-10 TUN-3527: More specific error for invalid YAML/JSON
+- 2020-11-06 Update README.md (#256)
+
+2020.11.0
+- 2020-11-04 TUN-3484: OriginService that responds with configured HTTP status
+- 2020-11-05 TUN-3505: Response body for status code origin returns EOF on Read
+- 2020-11-04 TUN-3503: Matching ingress rule should not take port into account
+- 2020-11-05 TUN-3506: OriginService needs to set request host and scheme for websocket requests
+- 2020-11-09 TUN-3516: Better error message when parsing invalid YAML config
+- 2020-11-09 TUN-3522: ingress validate checks that the config file exists
+- 2020-11-09 TUN-3524: Don't ignore errors from app-level action handler (#248)
+- 2020-11-09 TUN-3461: Show all origin services in the UI
+- 2020-10-30 TUN-3494: Proceed to create tunnel if at least one edge address can be resolved
+- 2020-10-30 TUN-3492: Refactor OriginService, shrink its interface
+- 2020-10-22 TUN-3478: Increase download timeout to 60s
+- 2020-10-15 TUN-2640: Users can configure per-origin config. Unify single-rule CLI flow with multi-rule config file code.
+
+2020.10.2
+- 2020-10-21 Release 2020.10.1
+- 2020-10-21 AUTH-3185 fixed indention error
+- 2020-10-19 TUN-3459: Make service install on linux use named tunnels
+
+2020.10.1
+- 2020-10-20 Split out typed config from legacy command-line switches; refactor ingress commands and fix tests
+- 2020-10-20 Move raw ingress rules to config package
+- 2020-10-21 TUN-3476: Fix conversion to string and int slice
+- 2020-10-12 TUN-3441:  Multiple-origin routing via ingress rules
+- 2020-10-15 TUN-3464: Newtype to wrap []ingress.Rule
+- 2020-10-15 TUN-3465: Use Go 1.15.3
+- 2020-10-15 TUN-3463: Let users run a named tunnel via config file setting
+- 2020-10-19 TUN-3475: Unify config file handling with typed config for new fields
+- 2020-10-19 TUN-3459: Make service install on linux use named tunnels
+- 2020-10-06 AUTH-3148 fixed cloudflared copy and match all the files in the checksum upload
+- 2020-10-06 TUN-3436, TUN-3437: Parse ingress from YAML, ensure last rule catches everything
+- 2020-10-06 TUN-3446: Use go 1.15.2 and add a step to build cloudflared in the dev Dockerfile
+- 2020-10-07 TUN-3439: 'tunnel validate' command to check ingress rules
+- 2020-10-07 TUN-3440: 'tunnel rule' command to test ingress rules
+- 2020-10-08 TUN-3451: Cloudflared tunnel ingress command
+- 2020-10-09 TUN-3452: Fix loading of flags from config file for tunnel run subcommand. This change also cleans up building of tunnel subcommand list, hides deprecated subcommands and improves help.
+- 2020-10-08 TUN-3438: move ingress into own package, read into TunnelConfig
+
+2020.10.0
+- 2020-10-02 AUTH-2993 cleaned up worker service tests
+- 2020-10-02 TUN-3443: Decode as v4api response on non-200 status
+- 2020-09-24 TRAFFIC-448: allow the user to specify the proxy address and port to bind to, falling back to 127.0.0.1 and random port if not specified
+- 2020-09-28 TUN-3427: Define a struct that only implements RegistrationServer in tunnelpogs
+- 2020-09-29 TUN-3430: Copy flags to configure proxy to run subcommand, print relevant tunnel flags in help
+- 2020-08-12 AUTH-2993 added workers updater logic
+
+2020.9.3
+- 2020-09-22 TRAFFIC-448: build cloudflare for junos and publish to s3
+- 2020-09-22 TUN-3410: Request the v1 Tunnelstore API
+- 2020-09-23 Release 2020.9.2
+- 2020-09-17 updater service exit code should be 11
+- 2020-09-18 AUTH-3109 upload the checksum to workers kv on github releases
+
+2020.9.2
+- 2020-09-22 TRAFFIC-448: build cloudflare for junos and publish to s3
+- 2020-09-22 TUN-3410: Request the v1 Tunnelstore API
+- 2020-09-17 AUTH-3103 CI build fixes
+- 2020-09-18 AUTH-3110-use-cfsetup-precache
+- 2020-09-17 TUN-3295: Show route command results
+- 2020-09-16 TUN-3291: cloudflared tunnel run -h explains how to use flags from parent command
+- 2020-09-18 AUTH-3109 upload the checksum to workers kv on github releases
+- 2020-09-17 updater service exit code should be 11
+- 2020-09-01 TUN-3216: UI improvements
+- 2020-08-25 Rebased and passed TunnelEventChan to LogServerInfo in new ReconnectTunnel function
+- 2020-08-25 TUN-3321: Add box around logs on UI
+- 2020-08-26 TUN-3328: Filter out free tunnel has started log from UI
+- 2020-08-27 TUN-3333: Add text to UI explaining how to exit
+- 2020-08-27 TUN-3335: Dynamically set connection table size for UI
+- 2020-08-10 TUN-3238: Update UI when connection re-connects
+- 2020-08-17 TUN-3261: Display connections on UI for free classic tunnels
+- 2020-07-24 TUN-3201: Create base cloudflared UI structure
+- 2020-07-29 TUN-3200: Add connection information to UI
+- 2020-07-24 TUN-3255: Update UI to display URL instead of hostname
+- 2020-07-29 TUN-3198: Handle errors while running tunnel UI
+
+2020.9.1
+- 2020-09-14 TUN-3395: Unhide named tunnel subcommands, tweak help
+- 2020-09-15 TUN-3395: Improve help for list command
+- 2020-09-14 TUN-3294: Perform basic validation on arguments of route command; remove default pool name which wasn't valid
+- 2020-09-15 TUN-3395: Improve help for list command
+- 2020-09-16 Use Go 1.15.2
+
+2020.9.0
+- 2020-09-11 TUN-3293: Try to use error information from the body of a failed tunnelstore reresponse if available
+- 2020-09-04 AUTH-2653 renabled signing
+- 2020-09-04 TUN-3377: Tunnel route should check dns/lb before checking tunnel ID
+- 2020-09-04 AUTH-2653 changed to proper file extension
+- 2020-09-04 AUTH-2653 handle duplicate key import errors
+- 2020-09-04 TUN-3345: tunnel run accepts name of tunnel as argument
+- 2020-09-08 AUTH-2653 disble error pipe to see what is failing
+- 2020-09-08 AUTH-2653 search for the certificate and not the identity
+- 2020-09-09 TUN-3284: Use cloudflared/<version> as user agent of tunnelstore client
+- 2020-09-09 TUN-3375: Upgrade x/text and gorilla websocket deps
+- 2020-09-09 TUN-3375: Upgrade coredns and prometheus dependencies
+- 2020-09-09 AUTH-2653 add notarization to mac build
+- 2020-09-08 TUN-3292: Mention cleanup in tunnel run help.
+- 2020-08-20 AUTH-2016 fixed variable fail
+- 2020-08-12 TUN-3352 extra debug logging for websockets
+
+2020.8.2
+- 2020-08-20 AUTH-3021 fixed the git version call by using the older flag
+- 2020-08-18 TUN-3268: Each connection has its own event digest to reconnect
+
+2020.8.1
+- 2020-08-14 AUTH-2975 don't check /etc on windows
+- 2020-08-14 AUTH-2977 log file protection
+- 2020-08-18 TUN-3286: Use either ID or name in Named Tunnel subcommands.
+- 2020-08-18 AUTH-2712 fixed the mac build script
+- 2020-08-19 AUTH-2653 disabling signing until we can get the keys
+- 2020-08-05 TUN-3233: List tunnels support filtering by deleted, name, existed at and id
+- 2020-08-05 TUN-3237: By default, don't show connections that are pending reconnect
+- 2020-08-06 TUN-3242: Build with go 1.14
+- 2020-08-07 TUN-3243: Refactor tunnel subcommands to allow commands to compose better
+- 2020-08-07 AUTH-2864 - add macos build to github release
+- 2020-07-31 AUTH-2857 update homebrew script to use new url
+- 2020-07-30 TUN-3213: Create, route and run named tunnels in one command
+- 2020-07-07 AUTH-2712 added MSI build for a windows agent
+
+2020.8.0
+- 2020-07-30 TUN-3220: tunnel route reports created route
+- 2020-07-31 TUN-3221: ConnectionOptions tracks numPreviousAttempts.
+- 2020-07-20 TUN-3190: Initialize logger using command line flags in tunnels subcommands
+- 2020-07-21 TUN-3192: Use zone ID in tunnelstore request path; improve debug logging
+- 2020-07-23 TUN-3194: Don't render log output when level is not enabled
+- 2020-07-22 AUTH-2016 adds sha256 hashes to releases
+- 2020-07-27 Removes centos 6 build
+- 2020-07-27 TUN-3209: Add benchmark for header serialization
+- 2020-07-24 TUN-3209: improve performance and reduce allocations during user header serialization from h1 to h2
+- 2020-07-26 TUN-3208: Add benchmark for large response write
+- 2020-07-27 TUN-3208: Reduce copies and allocations on h2mux write path. Pre-allocate 16KB write buffer on the first write if possible. Use explicit byte array for chunks on write thread to avoid copying through intermediate buffer due to io.CopyN.
+- 2020-07-28 AUTH-2714: Adds arm64 cloudflared build
+- 2020-07-29 AUTH-2927 run message update after all github builds are done
+- 2020-07-17 AUTH-2902 redirect with just the root host on curl commands
+
+2020.7.4
+- 2020-07-20 Build cloudflared for arm64 on native agents
+- 2020-07-10 TUN-3048: Handle error when user tries to delete active tunnel
+- 2020-07-14 AUTH-2890: adds error handler to cli actions
+- 2020-07-06 TUN-3156: Add route subcommand under tunnel
+
 2020.7.3
 - 2020-07-13 Change scp command to use file glob that matches both cloudflared rpms and debs
 

awsuploader/directory_upload_manager.go

@@ -1,106 +0,0 @@
-package awsuploader
-
-import (
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/cloudflare/cloudflared/logger"
-)
-
-// DirectoryUploadManager is used to manage file uploads on an interval from a directory
-type DirectoryUploadManager struct {
-	logger        logger.Service
-	uploader      Uploader
-	rootDirectory string
-	sweepInterval time.Duration
-	ticker        *time.Ticker
-	shutdownC     chan struct{}
-	workQueue     chan string
-}
-
-// NewDirectoryUploadManager create a new DirectoryUploadManager
-// uploader is an Uploader to use as an actual uploading engine
-// directory is the directory to sweep for files to upload
-// sweepInterval is how often to iterate the directory and upload the files within
-func NewDirectoryUploadManager(logger logger.Service, uploader Uploader, directory string, sweepInterval time.Duration, shutdownC chan struct{}) *DirectoryUploadManager {
-	workerCount := 10
-	manager := &DirectoryUploadManager{
-		logger:        logger,
-		uploader:      uploader,
-		rootDirectory: directory,
-		sweepInterval: sweepInterval,
-		shutdownC:     shutdownC,
-		workQueue:     make(chan string, workerCount),
-	}
-
-	//start workers
-	for i := 0; i < workerCount; i++ {
-		go manager.worker()
-	}
-
-	return manager
-}
-
-// Upload a file using the uploader
-// This is useful for "out of band" uploads that need to be triggered immediately instead of waiting for the sweep
-func (m *DirectoryUploadManager) Upload(filepath string) error {
-	return m.uploader.Upload(filepath)
-}
-
-// Start the upload ticker to walk the directories
-func (m *DirectoryUploadManager) Start() {
-	m.ticker = time.NewTicker(m.sweepInterval)
-	go m.run()
-}
-
-func (m *DirectoryUploadManager) run() {
-	for {
-		select {
-		case <-m.shutdownC:
-			m.ticker.Stop()
-			return
-		case <-m.ticker.C:
-			m.sweep()
-		}
-	}
-}
-
-// sweep the directory and kick off uploads
-func (m *DirectoryUploadManager) sweep() {
-	filepath.Walk(m.rootDirectory, func(path string, info os.FileInfo, err error) error {
-		if err != nil || info.IsDir() {
-			return nil
-		}
-		//30 days ago
-		retentionTime := 30 * (time.Hour * 24)
-		checkTime := time.Now().Add(-time.Duration(retentionTime))
-
-		//delete the file it is stale
-		if info.ModTime().Before(checkTime) {
-			os.Remove(path)
-			return nil
-		}
-		//add the upload to the work queue
-		go func() {
-			m.workQueue <- path
-		}()
-		return nil
-	})
-}
-
-// worker handles upload requests
-func (m *DirectoryUploadManager) worker() {
-	for {
-		select {
-		case <-m.shutdownC:
-			return
-		case filepath := <-m.workQueue:
-			if err := m.Upload(filepath); err != nil {
-				m.logger.Errorf("Cannot upload file to s3 bucket: %s", err)
-			} else {
-				os.Remove(filepath)
-			}
-		}
-	}
-}

awsuploader/directory_upload_manager_test.go

@@ -1,137 +0,0 @@
-package awsuploader
-
-import (
-	"errors"
-	"io/ioutil"
-	"math/rand"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/cloudflare/cloudflared/logger"
-)
-
-type MockUploader struct {
-	shouldFail bool
-}
-
-func (m *MockUploader) Upload(filepath string) error {
-	if m.shouldFail {
-		return errors.New("upload set to fail")
-	}
-	return nil
-}
-
-func NewMockUploader(shouldFail bool) Uploader {
-	return &MockUploader{shouldFail: shouldFail}
-}
-
-func getDirectoryPath(t *testing.T) string {
-	dir, err := os.Getwd()
-	if err != nil {
-		t.Fatal("couldn't create the test directory!", err)
-	}
-	return filepath.Join(dir, "uploads")
-}
-
-func setupTestDirectory(t *testing.T) string {
-	path := getDirectoryPath(t)
-	os.RemoveAll(path)
-	time.Sleep(100 * time.Millisecond) //short way to wait for the OS to delete the folder
-	err := os.MkdirAll(path, os.ModePerm)
-	if err != nil {
-		t.Fatal("couldn't create the test directory!", err)
-	}
-	return path
-}
-
-func createUploadManager(t *testing.T, shouldFailUpload bool) *DirectoryUploadManager {
-	rootDirectory := setupTestDirectory(t)
-	uploader := NewMockUploader(shouldFailUpload)
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
-	shutdownC := make(chan struct{})
-	return NewDirectoryUploadManager(logger, uploader, rootDirectory, 1*time.Second, shutdownC)
-}
-
-func createFile(t *testing.T, fileName string) (*os.File, string) {
-	path := filepath.Join(getDirectoryPath(t), fileName)
-	f, err := os.Create(path)
-	if err != nil {
-		t.Fatal("upload to create file for sweep test", err)
-	}
-	return f, path
-}
-
-func TestUploadSuccess(t *testing.T) {
-	manager := createUploadManager(t, false)
-	path := filepath.Join(getDirectoryPath(t), "test_file")
-	if err := manager.Upload(path); err != nil {
-		t.Fatal("the upload request method failed", err)
-	}
-}
-
-func TestUploadFailure(t *testing.T) {
-	manager := createUploadManager(t, true)
-	path := filepath.Join(getDirectoryPath(t), "test_file")
-	if err := manager.Upload(path); err == nil {
-		t.Fatal("the upload request method should have failed and didn't", err)
-	}
-}
-
-func TestSweepSuccess(t *testing.T) {
-	manager := createUploadManager(t, false)
-	f, path := createFile(t, "test_file")
-	defer f.Close()
-
-	manager.Start()
-	time.Sleep(2 * time.Second)
-	if _, err := os.Stat(path); os.IsExist(err) {
-		//the file should have been deleted
-		t.Fatal("the manager failed to delete the file", err)
-	}
-}
-
-func TestSweepFailure(t *testing.T) {
-	manager := createUploadManager(t, true)
-	f, path := createFile(t, "test_file")
-	defer f.Close()
-
-	manager.Start()
-	time.Sleep(2 * time.Second)
-	_, serr := f.Stat()
-	if serr != nil {
-		//the file should still exist
-		os.Remove(path)
-		t.Fatal("the manager failed to delete the file", serr)
-	}
-}
-
-func TestHighLoad(t *testing.T) {
-	manager := createUploadManager(t, false)
-	for i := 0; i < 30; i++ {
-		f, _ := createFile(t, randomString(6))
-		defer f.Close()
-	}
-	manager.Start()
-	time.Sleep(4 * time.Second)
-
-	directory := getDirectoryPath(t)
-	files, err := ioutil.ReadDir(directory)
-	if err != nil || len(files) > 0 {
-		t.Fatalf("the manager failed to upload all the files: %s files left: %d", err, len(files))
-	}
-}
-
-// LowerCase [a-z]
-const randSet = "abcdefghijklmnopqrstuvwxyz"
-
-// String returns a string of length 'n' from a set of letters 'lset'
-func randomString(n int) string {
-	b := make([]byte, n)
-	lsetLen := len(randSet)
-	for i := range b {
-		b[i] = randSet[rand.Intn(lsetLen)]
-	}
-	return string(b)
-}

awsuploader/file_uploader.go

@@ -1,62 +0,0 @@
-package awsuploader
-
-import (
-	"context"
-	"os"
-
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3"
-)
-
-//FileUploader aws compliant bucket upload
-type FileUploader struct {
-	storage    *s3.S3
-	bucketName string
-	clientID   string
-	secretID   string
-}
-
-// NewFileUploader creates a new S3 compliant bucket uploader
-func NewFileUploader(bucketName, region, accessKeyID, secretID, token, s3Host string) (*FileUploader, error) {
-	sess, err := session.NewSession(&aws.Config{
-		Region:      aws.String(region),
-		Credentials: credentials.NewStaticCredentials(accessKeyID, secretID, token),
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	var storage *s3.S3
-	if s3Host != "" {
-		storage = s3.New(sess, &aws.Config{Endpoint: aws.String(s3Host)})
-	} else {
-		storage = s3.New(sess)
-	}
-
-	return &FileUploader{
-		storage:    storage,
-		bucketName: bucketName,
-	}, nil
-}
-
-// Upload a file to the bucket
-func (u *FileUploader) Upload(filepath string) error {
-	info, err := os.Stat(filepath)
-	if err != nil {
-		return err
-	}
-	file, err := os.Open(filepath)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	_, serr := u.storage.PutObjectWithContext(context.Background(), &s3.PutObjectInput{
-		Bucket: aws.String(u.bucketName),
-		Key:    aws.String(info.Name()),
-		Body:   file,
-	})
-	return serr
-}

awsuploader/upload_manager.go

@@ -1,7 +0,0 @@
-package awsuploader
-
-// UploadManager is used to manage file uploads on an interval
-type UploadManager interface {
-	Upload(string) error
-	Start()
-}

awsuploader/uploader.go

@@ -1,7 +0,0 @@
-package awsuploader
-
-// Uploader the functions required to upload to a bucket
-type Uploader interface {
-	//Upload a file to the bucket
-	Upload(string) error
-}

build-packages.sh

@@ -0,0 +1,40 @@
+VERSION=$(git describe --tags --always --dirty="-dev" --match "[0-9][0-9][0-9][0-9].*.*")
+echo $VERSION
+export CGO_ENABLED=0
+# This controls the directory the built artifacts go into
+export ARTIFACT_DIR=built_artifacts/
+mkdir -p $ARTIFACT_DIR
+windowsArchs=("amd64" "386")
+export TARGET_OS=windows
+for arch in ${windowsArchs[@]}; do 
+    export TARGET_ARCH=$arch
+    make cloudflared-msi
+    mv ./cloudflared.exe $ARTIFACT_DIR/cloudflared-windows-$arch.exe
+    mv cloudflared-$VERSION-$arch.msi $ARTIFACT_DIR/cloudflared-windows-$arch.msi
+done
+
+
+export FIPS=true
+linuxArchs=("amd64" "386" "arm" "arm64")
+export TARGET_OS=linux
+for arch in ${linuxArchs[@]}; do 
+    export TARGET_ARCH=$arch
+    make cloudflared-deb
+    mv cloudflared\_$VERSION\_$arch.deb $ARTIFACT_DIR/cloudflared-linux-$arch.deb
+
+    # rpm packages invert the - and _ and use x86_64 instead of amd64. 
+    RPMVERSION=$(echo $VERSION|sed -r 's/-/_/g')
+    RPMARCH=$arch
+    if [ $arch == "amd64" ];then
+        RPMARCH="x86_64"
+    fi
+    if [ $arch == "arm64" ]; then
+        RPMARCH="aarch64"
+    fi
+    make cloudflared-rpm
+    mv cloudflared-$RPMVERSION-1.$RPMARCH.rpm $ARTIFACT_DIR/cloudflared-linux-$RPMARCH.rpm
+
+    # finally move the linux binary as well.
+    mv ./cloudflared $ARTIFACT_DIR/cloudflared-linux-$arch
+done
+

carrier/carrier.go

@@ -4,36 +4,44 @@
 package carrier
 
 import (
+	"crypto/tls"
+	"fmt"
 	"io"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"strings"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/token"
+)
+
+const (
+	LogFieldOriginURL       = "originURL"
+	CFAccessTokenHeader     = "Cf-Access-Token"
+	cfJumpDestinationHeader = "Cf-Access-Jump-Destination"
 )
 
 type StartOptions struct {
-	OriginURL string
-	Headers   http.Header
+	AppInfo         *token.AppInfo
+	OriginURL       string
+	Headers         http.Header
+	Host            string
+	TLSClientConfig *tls.Config
 }
 
 // Connection wraps up all the needed functions to forward over the tunnel
 type Connection interface {
 	// ServeStream is used to forward data from the client to the edge
 	ServeStream(*StartOptions, io.ReadWriter) error
-
-	// StartServer is used to listen for incoming connections from the edge to the origin
-	StartServer(net.Listener, string, <-chan struct{}) error
 }
 
 // StdinoutStream is empty struct for wrapping stdin/stdout
 // into a single ReadWriter
-type StdinoutStream struct {
-}
+type StdinoutStream struct{}
 
 // Read will read from Stdin
 func (c *StdinoutStream) Read(p []byte) (int, error) {
@@ -49,7 +57,7 @@ func (c *StdinoutStream) Write(p []byte) (int, error) {
 // Helper to allow defering the response close with a check that the resp is not nil
 func closeRespBody(resp *http.Response) {
 	if resp != nil {
-		resp.Body.Close()
+		_ = resp.Body.Close()
 	}
 }
 
@@ -103,7 +111,7 @@ func Serve(remoteConn Connection, listener net.Listener, shutdownC <-chan struct
 // serveConnection handles connections for the Serve() call
 func serveConnection(remoteConn Connection, c net.Conn, options *StartOptions) {
 	defer c.Close()
-	remoteConn.ServeStream(options, c)
+	_ = remoteConn.ServeStream(options, c)
 }
 
 // IsAccessResponse checks the http Response to see if the url location
@@ -117,7 +125,7 @@ func IsAccessResponse(resp *http.Response) bool {
 	if err != nil || location == nil {
 		return false
 	}
-	if strings.HasPrefix(location.Path, "/cdn-cgi/access/login") {
+	if strings.HasPrefix(location.Path, token.AccessLoginWorkerPath) {
 		return true
 	}
 
@@ -125,13 +133,13 @@ func IsAccessResponse(resp *http.Response) bool {
 }
 
 // BuildAccessRequest builds an HTTP request with the Access token set
-func BuildAccessRequest(options *StartOptions, logger logger.Service) (*http.Request, error) {
+func BuildAccessRequest(options *StartOptions, log *zerolog.Logger) (*http.Request, error) {
 	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	token, err := token.FetchTokenWithRedirect(req.URL, logger)
+	token, err := token.FetchTokenWithRedirect(req.URL, options.AppInfo, log)
 	if err != nil {
 		return nil, err
 	}
@@ -142,7 +150,7 @@ func BuildAccessRequest(options *StartOptions, logger logger.Service) (*http.Req
 	if err != nil {
 		return nil, err
 	}
-	originRequest.Header.Set(h2mux.CFAccessTokenHeader, token)
+	originRequest.Header.Set(CFAccessTokenHeader, token)
 
 	for k, v := range options.Headers {
 		if len(v) >= 1 {
@@ -152,3 +160,26 @@ func BuildAccessRequest(options *StartOptions, logger logger.Service) (*http.Req
 
 	return originRequest, nil
 }
+
+func SetBastionDest(header http.Header, destination string) {
+	if destination != "" {
+		header.Set(cfJumpDestinationHeader, destination)
+	}
+}
+
+func ResolveBastionDest(r *http.Request) (string, error) {
+	jumpDestination := r.Header.Get(cfJumpDestinationHeader)
+	if jumpDestination == "" {
+		return "", fmt.Errorf("Did not receive final destination from client. The --destination flag is likely not set on the client side")
+	}
+	// Strip scheme and path set by client. Without a scheme
+	// Parsing a hostname and path without scheme might not return an error due to parsing ambiguities
+	if jumpURL, err := url.Parse(jumpDestination); err == nil && jumpURL.Host != "" {
+		return removePath(jumpURL.Host), nil
+	}
+	return removePath(jumpDestination), nil
+}
+
+func removePath(dest string) string {
+	return strings.SplitN(dest, "/", 2)[0]
+}

carrier/carrier_test.go

@@ -9,8 +9,8 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/logger"
 	ws "github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -43,8 +43,8 @@ func (s *testStreamer) Write(p []byte) (int, error) {
 
 func TestStartClient(t *testing.T) {
 	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
-	wsConn := NewWSConnection(logger, false)
+	log := zerolog.Nop()
+	wsConn := NewWSConnection(&log)
 	ts := newTestWebSocketServer()
 	defer ts.Close()
 
@@ -55,10 +55,10 @@ func TestStartClient(t *testing.T) {
 	}
 	err := StartClient(wsConn, buf, options)
 	assert.NoError(t, err)
-	buf.Write([]byte(message))
+	_, _ = buf.Write([]byte(message))
 
 	readBuffer := make([]byte, len(message))
-	buf.Read(readBuffer)
+	_, _ = buf.Read(readBuffer)
 	assert.Equal(t, message, string(readBuffer))
 }
 
@@ -68,9 +68,9 @@ func TestStartServer(t *testing.T) {
 		t.Fatalf("Error starting listener: %v", err)
 	}
 	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
+	log := zerolog.Nop()
 	shutdownC := make(chan struct{})
-	wsConn := NewWSConnection(logger, false)
+	wsConn := NewWSConnection(&log)
 	ts := newTestWebSocketServer()
 	defer ts.Close()
 	options := &StartOptions{
@@ -81,15 +81,16 @@ func TestStartServer(t *testing.T) {
 	go func() {
 		err := Serve(wsConn, listener, shutdownC, options)
 		if err != nil {
-			t.Fatalf("Error running server: %v", err)
+			t.Errorf("Error running server: %v", err)
+			return
 		}
 	}()
 
 	conn, err := net.Dial("tcp", listener.Addr().String())
-	conn.Write([]byte(message))
+	_, _ = conn.Write([]byte(message))
 
 	readBuffer := make([]byte, len(message))
-	conn.Read(readBuffer)
+	_, _ = conn.Read(readBuffer)
 	assert.Equal(t, string(readBuffer), message)
 }
 
@@ -155,3 +156,99 @@ func testRequest(t *testing.T, url string, stream io.ReadWriter) *http.Request {
 
 	return req
 }
+
+func TestBastionDestination(t *testing.T) {
+	tests := []struct {
+		name         string
+		header       http.Header
+		expectedDest string
+		wantErr      bool
+	}{
+		{
+			name: "hostname destination",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"localhost"},
+			},
+			expectedDest: "localhost",
+		},
+		{
+			name: "hostname destination with port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"localhost:9000"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "hostname destination with scheme and port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"ssh://localhost:9000"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "full hostname url",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"ssh://localhost:9000/metrics"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "hostname destination with port and path",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"localhost:9000/metrics"},
+			},
+			expectedDest: "localhost:9000",
+		},
+		{
+			name: "ip destination",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"127.0.0.1"},
+			},
+			expectedDest: "127.0.0.1",
+		},
+		{
+			name: "ip destination with port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"127.0.0.1:9000"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "ip destination with port and path",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"127.0.0.1:9000/metrics"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "ip destination with schem and port",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"tcp://127.0.0.1:9000"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name: "full ip url",
+			header: http.Header{
+				cfJumpDestinationHeader: []string{"ssh://127.0.0.1:9000/metrics"},
+			},
+			expectedDest: "127.0.0.1:9000",
+		},
+		{
+			name:    "no destination",
+			wantErr: true,
+		},
+	}
+	for _, test := range tests {
+		r := &http.Request{
+			Header: test.header,
+		}
+		dest, err := ResolveBastionDest(r)
+		if test.wantErr {
+			assert.Error(t, err, "Test %s expects error", test.name)
+		} else {
+			assert.NoError(t, err, "Test %s expects no error, got error %v", test.name, err)
+			assert.Equal(t, test.expectedDest, dest, "Test %s expect dest %s, got %s", test.name, test.expectedDest, dest)
+		}
+	}
+}

carrier/websocket.go

@@ -1,93 +1,83 @@
 package carrier
 
 import (
-	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/http/httputil"
+	"net/url"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/socks"
-	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
 	"github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/token"
+	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
 )
 
 // Websocket is used to carry data via WS binary frames over the tunnel from client to the origin
 // This implements the functions for glider proxy (sock5) and the carrier interface
 type Websocket struct {
-	logger  logger.Service
+	log     *zerolog.Logger
 	isSocks bool
 }
 
-type wsdialer struct {
-	conn *cfwebsocket.Conn
-}
-
-func (d *wsdialer) Dial(address string) (io.ReadWriteCloser, *socks.AddrSpec, error) {
-	local, ok := d.conn.LocalAddr().(*net.TCPAddr)
-	if !ok {
-		return nil, nil, fmt.Errorf("not a tcp connection")
-	}
-
-	addr := socks.AddrSpec{IP: local.IP, Port: local.Port}
-	return d.conn, &addr, nil
-}
-
 // NewWSConnection returns a new connection object
-func NewWSConnection(logger logger.Service, isSocks bool) Connection {
+func NewWSConnection(log *zerolog.Logger) Connection {
 	return &Websocket{
-		logger:  logger,
-		isSocks: isSocks,
+		log: log,
 	}
 }
 
 // ServeStream will create a Websocket client stream connection to the edge
 // it blocks and writes the raw data from conn over the tunnel
 func (ws *Websocket) ServeStream(options *StartOptions, conn io.ReadWriter) error {
-	wsConn, err := createWebsocketStream(options, ws.logger)
+	wsConn, err := createWebsocketStream(options, ws.log)
 	if err != nil {
-		ws.logger.Errorf("failed to connect to %s with error: %s", options.OriginURL, err)
+		ws.log.Err(err).Str(LogFieldOriginURL, options.OriginURL).Msg("failed to connect to origin")
 		return err
 	}
 	defer wsConn.Close()
 
-	if ws.isSocks {
-		dialer := &wsdialer{conn: wsConn}
-		requestHandler := socks.NewRequestHandler(dialer)
-		socksServer := socks.NewConnectionHandler(requestHandler)
-
-		socksServer.Serve(conn)
-	} else {
-		cfwebsocket.Stream(wsConn, conn)
-	}
+	cfwebsocket.Stream(wsConn, conn, ws.log)
 	return nil
 }
 
-// StartServer creates a Websocket server to listen for connections.
-// This is used on the origin (tunnel) side to take data from the muxer and send it to the origin
-func (ws *Websocket) StartServer(listener net.Listener, remote string, shutdownC <-chan struct{}) error {
-	return cfwebsocket.StartProxyServer(ws.logger, listener, remote, shutdownC, cfwebsocket.DefaultStreamHandler)
-}
-
 // createWebsocketStream will create a WebSocket connection to stream data over
 // It also handles redirects from Access and will present that flow if
 // the token is not present on the request
-func createWebsocketStream(options *StartOptions, logger logger.Service) (*cfwebsocket.Conn, error) {
+func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebsocket.GorillaConn, error) {
 	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
 	if err != nil {
 		return nil, err
 	}
 	req.Header = options.Headers
+	if options.Host != "" {
+		req.Host = options.Host
+	}
 
 	dump, err := httputil.DumpRequest(req, false)
-	logger.Debugf("Websocket request: %s", string(dump))
+	log.Debug().Msgf("Websocket request: %s", string(dump))
 
-	wsConn, resp, err := cfwebsocket.ClientConnect(req, nil)
+	dialer := &websocket.Dialer{
+		TLSClientConfig: options.TLSClientConfig,
+		Proxy:           http.ProxyFromEnvironment,
+	}
+	wsConn, resp, err := clientConnect(req, dialer)
 	defer closeRespBody(resp)
+
 	if err != nil && IsAccessResponse(resp) {
-		wsConn, err = createAccessAuthenticatedStream(options, logger)
+		// Only get Access app info if we know the origin is protected by Access
+		originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		appInfo, err := token.GetAppInfo(originReq.URL)
+		if err != nil {
+			return nil, err
+		}
+		options.AppInfo = appInfo
+
+		wsConn, err = createAccessAuthenticatedStream(options, log)
 		if err != nil {
 			return nil, err
 		}
@@ -95,7 +85,64 @@ func createWebsocketStream(options *StartOptions, logger logger.Service) (*cfweb
 		return nil, err
 	}
 
-	return &cfwebsocket.Conn{Conn: wsConn}, nil
+	return &cfwebsocket.GorillaConn{Conn: wsConn}, nil
+}
+
+var stripWebsocketHeaders = []string{
+	"Upgrade",
+	"Connection",
+	"Sec-Websocket-Key",
+	"Sec-Websocket-Version",
+	"Sec-Websocket-Extensions",
+}
+
+// the gorilla websocket library sets its own Upgrade, Connection, Sec-WebSocket-Key,
+// Sec-WebSocket-Version and Sec-Websocket-Extensions headers.
+// https://github.com/gorilla/websocket/blob/master/client.go#L189-L194.
+func websocketHeaders(req *http.Request) http.Header {
+	wsHeaders := make(http.Header)
+	for key, val := range req.Header {
+		wsHeaders[key] = val
+	}
+	// Assume the header keys are in canonical format.
+	for _, header := range stripWebsocketHeaders {
+		wsHeaders.Del(header)
+	}
+	wsHeaders.Set("Host", req.Host) // See TUN-1097
+	return wsHeaders
+}
+
+// clientConnect creates a WebSocket client connection for provided request. Caller is responsible for closing
+// the connection. The response body may not contain the entire response and does
+// not need to be closed by the application.
+func clientConnect(req *http.Request, dialler *websocket.Dialer) (*websocket.Conn, *http.Response, error) {
+	req.URL.Scheme = changeRequestScheme(req.URL)
+	wsHeaders := websocketHeaders(req)
+	if dialler == nil {
+		dialler = &websocket.Dialer{
+			Proxy: http.ProxyFromEnvironment,
+		}
+	}
+	conn, response, err := dialler.Dial(req.URL.String(), wsHeaders)
+	if err != nil {
+		return nil, response, err
+	}
+	return conn, response, nil
+}
+
+// changeRequestScheme is needed as the gorilla websocket library requires the ws scheme.
+// (even though it changes it back to http/https, but ¯\_(ツ)_/¯.)
+func changeRequestScheme(reqURL *url.URL) string {
+	switch reqURL.Scheme {
+	case "https":
+		return "wss"
+	case "http":
+		return "ws"
+	case "":
+		return "ws"
+	default:
+		return reqURL.Scheme
+	}
 }
 
 // createAccessAuthenticatedStream will try load a token from storage and make
@@ -103,8 +150,8 @@ func createWebsocketStream(options *StartOptions, logger logger.Service) (*cfweb
 // this probably means the token in storage is invalid (expired/revoked). If that
 // happens it deletes the token and runs the connection again, so the user can
 // login again and generate a new one.
-func createAccessAuthenticatedStream(options *StartOptions, logger logger.Service) (*websocket.Conn, error) {
-	wsConn, resp, err := createAccessWebSocketStream(options, logger)
+func createAccessAuthenticatedStream(options *StartOptions, log *zerolog.Logger) (*websocket.Conn, error) {
+	wsConn, resp, err := createAccessWebSocketStream(options, log)
 	defer closeRespBody(resp)
 	if err == nil {
 		return wsConn, nil
@@ -115,14 +162,10 @@ func createAccessAuthenticatedStream(options *StartOptions, logger logger.Servic
 	}
 
 	// Access Token is invalid for some reason. Go through regen flow
-	originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
-	if err != nil {
+	if err := token.RemoveTokenIfExists(options.AppInfo); err != nil {
 		return nil, err
 	}
-	if err := token.RemoveTokenIfExists(originReq.URL); err != nil {
-		return nil, err
-	}
-	wsConn, resp, err = createAccessWebSocketStream(options, logger)
+	wsConn, resp, err = createAccessWebSocketStream(options, log)
 	defer closeRespBody(resp)
 	if err != nil {
 		return nil, err
@@ -132,14 +175,25 @@ func createAccessAuthenticatedStream(options *StartOptions, logger logger.Servic
 }
 
 // createAccessWebSocketStream builds an Access request and makes a connection
-func createAccessWebSocketStream(options *StartOptions, logger logger.Service) (*websocket.Conn, *http.Response, error) {
-	req, err := BuildAccessRequest(options, logger)
+func createAccessWebSocketStream(options *StartOptions, log *zerolog.Logger) (*websocket.Conn, *http.Response, error) {
+	req, err := BuildAccessRequest(options, log)
 	if err != nil {
 		return nil, nil, err
 	}
 
 	dump, err := httputil.DumpRequest(req, false)
-	logger.Debugf("Access Websocket request: %s", string(dump))
+	log.Debug().Msgf("Access Websocket request: %s", string(dump))
 
-	return cfwebsocket.ClientConnect(req, nil)
+	conn, resp, err := clientConnect(req, nil)
+
+	if resp != nil {
+		r, err := httputil.DumpResponse(resp, true)
+		if r != nil {
+			log.Debug().Msgf("Websocket response: %q", r)
+		} else if err != nil {
+			log.Debug().Msgf("Websocket response error: %v", err)
+		}
+	}
+
+	return conn, resp, err
 }

carrier/websocket_test.go

@@ -0,0 +1,123 @@
+package carrier
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"math/rand"
+	"testing"
+	"time"
+
+	gws "github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/websocket"
+
+	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/tlsconfig"
+	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
+)
+
+func websocketClientTLSConfig(t *testing.T) *tls.Config {
+	certPool := x509.NewCertPool()
+	helloCert, err := tlsconfig.GetHelloCertificateX509()
+	assert.NoError(t, err)
+	certPool.AddCert(helloCert)
+	assert.NotNil(t, certPool)
+	return &tls.Config{RootCAs: certPool}
+}
+
+func TestWebsocketHeaders(t *testing.T) {
+	req := testRequest(t, "http://example.com", nil)
+	wsHeaders := websocketHeaders(req)
+	for _, header := range stripWebsocketHeaders {
+		assert.Empty(t, wsHeaders[header])
+	}
+	assert.Equal(t, "curl/7.59.0", wsHeaders.Get("User-Agent"))
+}
+
+func TestServe(t *testing.T) {
+	log := zerolog.Nop()
+	shutdownC := make(chan struct{})
+	errC := make(chan error)
+	listener, err := hello.CreateTLSListener("localhost:1111")
+	assert.NoError(t, err)
+	defer listener.Close()
+
+	go func() {
+		errC <- hello.StartHelloWorldServer(&log, listener, shutdownC)
+	}()
+
+	req := testRequest(t, "https://localhost:1111/ws", nil)
+
+	tlsConfig := websocketClientTLSConfig(t)
+	assert.NotNil(t, tlsConfig)
+	d := gws.Dialer{TLSClientConfig: tlsConfig}
+	conn, resp, err := clientConnect(req, &d)
+	assert.NoError(t, err)
+	assert.Equal(t, "websocket", resp.Header.Get("Upgrade"))
+
+	for i := 0; i < 1000; i++ {
+		messageSize := rand.Int()%2048 + 1
+		clientMessage := make([]byte, messageSize)
+		// rand.Read always returns len(clientMessage) and a nil error
+		rand.Read(clientMessage)
+		err = conn.WriteMessage(websocket.BinaryFrame, clientMessage)
+		assert.NoError(t, err)
+
+		messageType, message, err := conn.ReadMessage()
+		assert.NoError(t, err)
+		assert.Equal(t, websocket.BinaryFrame, messageType)
+		assert.Equal(t, clientMessage, message)
+	}
+
+	_ = conn.Close()
+	close(shutdownC)
+	<-errC
+}
+
+func TestWebsocketWrapper(t *testing.T) {
+	listener, err := hello.CreateTLSListener("localhost:0")
+	require.NoError(t, err)
+
+	serverErrorChan := make(chan error)
+	helloSvrCtx, cancelHelloSvr := context.WithCancel(context.Background())
+	defer func() { <-serverErrorChan }()
+	defer cancelHelloSvr()
+	go func() {
+		log := zerolog.Nop()
+		serverErrorChan <- hello.StartHelloWorldServer(&log, listener, helloSvrCtx.Done())
+	}()
+
+	tlsConfig := websocketClientTLSConfig(t)
+	d := gws.Dialer{TLSClientConfig: tlsConfig, HandshakeTimeout: time.Minute}
+	testAddr := fmt.Sprintf("https://%s/ws", listener.Addr().String())
+	req := testRequest(t, testAddr, nil)
+	conn, resp, err := clientConnect(req, &d)
+	require.NoError(t, err)
+	assert.Equal(t, "websocket", resp.Header.Get("Upgrade"))
+
+	// Websocket now connected to test server so lets check our wrapper
+	wrapper := cfwebsocket.GorillaConn{Conn: conn}
+	buf := make([]byte, 100)
+	wrapper.Write([]byte("abc"))
+	n, err := wrapper.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, n, 3)
+	require.Equal(t, "abc", string(buf[:n]))
+
+	// Test partial read, read 1 of 3 bytes in one read and the other 2 in another read
+	wrapper.Write([]byte("abc"))
+	buf = buf[:1]
+	n, err = wrapper.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, n, 1)
+	require.Equal(t, "a", string(buf[:n]))
+	buf = buf[:cap(buf)]
+	n, err = wrapper.Read(buf)
+	require.NoError(t, err)
+	require.Equal(t, n, 2)
+	require.Equal(t, "bc", string(buf[:n]))
+}

cfsetup.yaml

@@ -1,8 +1,20 @@
-pinned_go: &pinned_go go=1.12.9-1
+pinned_go: &pinned_go go=1.17-1
+pinned_go_fips: &pinned_go_fips go-boring=1.16.6-6
+
 build_dir: &build_dir /cfsetup_build
-default-flavor: stretch
+default-flavor: buster
 stretch: &stretch
   build:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go_fips
+      - build-essential
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - export FIPS=true
+      - make cloudflared
+  build-non-fips:  # helpful to catch problems with non-fips (only used for releasing non-linux artifacts) before releases
     build_dir: *build_dir
     builddeps:
       - *pinned_go
@@ -11,21 +23,81 @@ stretch: &stretch
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared
-  build-deb:
+  build-all-packages: #except osxpkg
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+      - rpm
+      - wget
+      # libmsi and libgcab are libraries the wixl binary depends on.
+      - libmsi-dev
+      - libgcab-dev
+    pre-cache:
+      # TODO: https://jira.cfops.it/browse/TUN-4792 Replace this wixl with the official one once msitools supports
+      # environment.
+      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
+      - chmod a+x /usr/local/bin/wixl
+    post-cache:
+      - export FIPS=true
+      - ./build-packages.sh
+  github-release-pkgs:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go_fips
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+      - rpm
+      - wget
+      # libmsi and libgcab are libraries the wixl binary depends on.
+      - libmsi-dev
+      - libgcab-dev
+      - python3-setuptools
+      - python3-pip
+    pre-cache:
+      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
+      - chmod a+x /usr/local/bin/wixl
+      - pip3 install pygithub
+    post-cache:
+      # build all packages and move them to /cfsetup/built_artifacts
+      - ./build-packages.sh
+      # release the packages built and moved to /cfsetup/built_artifacts
+      - make github-release-built-pkgs
+  build-deb:
+    build_dir: *build_dir
+    builddeps: &build_deb_deps
+      - *pinned_go_fips
       - build-essential
       - fakeroot
       - rubygem-fpm
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
+      - make cloudflared-deb
+  build-deb-nightly:
+    build_dir: *build_dir
+    builddeps: *build_deb_deps
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - export FIPS=true
+      - export NIGHTLY=true
+      - make cloudflared-deb
+  build-deb-arm64:
+    build_dir: *build_dir
+    builddeps: *build_deb_deps
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=arm64
       - make cloudflared-deb
   publish-deb:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
       - fakeroot
       - rubygem-fpm
@@ -33,182 +105,112 @@ stretch: &stretch
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
+      - export FIPS=true
       - make publish-deb
-  release-linux-amd64:
+  github-release-macos-amd64:
     build_dir: *build_dir
     builddeps:
       - *pinned_go
-      - build-essential
-    post-cache:
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - make release
-  github-release-linux-amd64:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - build-essential
       - python3-setuptools
       - python3-pip
-    post-cache:
+    pre-cache: &install_pygithub
       - pip3 install pygithub
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - make github-release
-  release-linux-armv6:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - crossbuild-essential-armhf
-      - gcc-arm-linux-gnueabihf
     post-cache:
-      - export GOOS=linux
-      - export GOARCH=arm
-      - export CC=arm-linux-gnueabihf-gcc
-      - make release
-  github-release-linux-armv6:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - crossbuild-essential-armhf
-      - gcc-arm-linux-gnueabihf
-      - python3-setuptools
-      - python3-pip
-    post-cache:
-      - pip3 install pygithub
-      - export GOOS=linux
-      - export GOARCH=arm
-      - export CC=arm-linux-gnueabihf-gcc
-      - make github-release
-  release-linux-386:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - gcc-multilib
-    post-cache:
-      - export GOOS=linux
-      - export GOARCH=386
-      - make release
-  github-release-linux-386:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - gcc-multilib
-      - python3-setuptools
-      - python3-pip
-    post-cache:
-      - pip3 install pygithub
-      - export GOOS=linux
-      - export GOARCH=386
-      - make github-release
-  release-windows-amd64:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - gcc-mingw-w64
-    post-cache:
-      - export GOOS=windows
-      - export GOARCH=amd64
-      - export CC=x86_64-w64-mingw32-gcc
-      - make release
-  github-release-windows-amd64:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - gcc-mingw-w64
-      - python3-setuptools
-      - python3-pip
-    post-cache:
-      - pip3 install pygithub
-      - export GOOS=windows
-      - export GOARCH=amd64
-      - export CC=x86_64-w64-mingw32-gcc
-      - make github-release
-  release-windows-386:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - gcc-mingw-w64
-    post-cache:
-      - export GOOS=windows
-      - export GOARCH=386
-      - export CC=i686-w64-mingw32-gcc-win32
-      - make release
-  github-release-windows-386:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - gcc-mingw-w64
-      - python3-setuptools
-      - python3-pip
-    post-cache:
-      - pip3 install pygithub
-      - export GOOS=windows
-      - export GOARCH=386
-      - export CC=i686-w64-mingw32-gcc-win32
-      - make github-release
+      - make github-mac-upload
   test:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
+      - gotest-to-teamcity
+    pre-cache:
+      - go get golang.org/x/tools/cmd/goimports
+      - go get github.com/sudarshan-reddy/go-sumtype@v0.0.0-20210827105221-82eca7e5abb1
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
-      # cd to a non-module directory: https://github.com/golang/go/issues/24250
-      - (cd / && go get github.com/BurntSushi/go-sumtype)
+      - export FIPS=true
       - export PATH="$HOME/go/bin:$PATH"
-      - make test
+      - ./fmt-check.sh
+      - make test | gotest-to-teamcity
+  component-test:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go_fips
+      - python3.7
+      - python3-pip
+      - python3-setuptools
+      # procps installs the ps command which is needed in test_sysv_service because the init script
+      # uses ps pid to determine if the agent is running
+      - procps
+    pre-cache-copy-paths:
+      - component-tests/requirements.txt
+    pre-cache:
+      - sudo pip3 install --upgrade -r component-tests/requirements.txt
+    post-cache:
+      # Creates and routes a Named Tunnel for this build. Also constructs config file from env vars.
+      - python3 component-tests/setup.py --type create
+      - pytest component-tests
+      # The Named Tunnel is deleted and its route unprovisioned here.
+      - python3 component-tests/setup.py --type cleanup
   update-homebrew:
     builddeps:
       - openssh-client
       - s3cmd
     post-cache:
       - .teamcity/update-homebrew.sh
-
-jessie: *stretch
+  github-message-release:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - python3-setuptools
+      - python3-pip
+    pre-cache: *install_pygithub
+    post-cache:
+      - make github-message
+  build-junos:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - python3
+      - genisoimage
+      - jetez
+    pre-cache:
+      - ln -s /usr/bin/genisoimage /usr/bin/mkisofs
+    post-cache:
+      - export GOOS=freebsd
+      - export GOARCH=amd64
+      - make cloudflared-junos
+  publish-junos:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - python3
+      - genisoimage
+      - jetez
+      - s4cmd
+    pre-cache:
+      - ln -s /usr/bin/genisoimage /usr/bin/mkisofs
+    post-cache:
+      - export GOOS=freebsd
+      - export GOARCH=amd64
+      - make publish-cloudflared-junos
 
 buster: *stretch
-
-centos-6:
-  build-rpm:
-    build_dir: *build_dir
-    builddeps: &el6_builddeps
-      - https://dl.fedoraproject.org/pub/epel/epel-release-latest-6.noarch.rpm
-    post-cache:
-      - source /usr/local/rvm/scripts/rvm
-      - rvm use 1.9
-      - sudo yum install -y fakeroot
-      - wget https://golang.org/dl/go1.12.9.linux-amd64.tar.gz -P /tmp/
-      - sudo tar -C /usr/local -xzf /tmp/go1.12.9.linux-amd64.tar.gz
-      - export PATH=$PATH:/usr/local/go/bin
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - make cloudflared-rpm
-  publish-rpm:
-    build_dir: *build_dir
-    builddeps: *el6_builddeps
-    post-cache:
-      - source /usr/local/rvm/scripts/rvm
-      - rvm use 1.9
-      - sudo yum install -y fakeroot
-      - wget https://golang.org/dl/go1.12.9.linux-amd64.tar.gz -P /tmp/
-      - sudo tar -C /usr/local -xzf /tmp/go1.12.9.linux-amd64.tar.gz
-      - export PATH=$PATH:/usr/local/go/bin
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - make publish-rpm
-
-
+bullseye: *stretch
 centos-7:
   publish-rpm:
     build_dir: *build_dir
     builddeps: &el7_builddeps
       - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+    pre-cache:
+      - yum install -y fakeroot
+      - yum upgrade -y binutils-2.27-44.base.el7.x86_64
+      - wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.16.3.linux-amd64.tar.gz
     post-cache:
-      - sudo yum install -y fakeroot
-      - wget https://golang.org/dl/go1.12.9.linux-amd64.tar.gz -P /tmp/
-      - sudo tar -C /usr/local -xzf /tmp/go1.12.9.linux-amd64.tar.gz
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
       - export GOARCH=amd64
@@ -216,22 +218,13 @@ centos-7:
   build-rpm:
     build_dir: *build_dir
     builddeps: *el7_builddeps
+    pre-cache:
+      - yum install -y fakeroot
+      - yum upgrade -y binutils-2.27-44.base.el7.x86_64
+      - wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.16.3.linux-amd64.tar.gz
     post-cache:
-      - sudo yum install -y fakeroot
-      - wget https://golang.org/dl/go1.12.9.linux-amd64.tar.gz -P /tmp/
-      - sudo tar -C /usr/local -xzf /tmp/go1.12.9.linux-amd64.tar.gz
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared-rpm
-
-# cfsetup compose
-default-stack: test_dbconnect
-test_dbconnect:
-  compose:
-    up-args:
-      - --renew-anon-volumes
-      - --abort-on-container-exit
-      - --exit-code-from=cloudflared
-    files:
-      - dbconnect_tests/dbconnect.yaml

cloudflared.wxs

@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+
+<?if $(var.Platform)="x86"?>
+    <?define Program_Files="ProgramFilesFolder"?>
+<?else?>
+    <?define Program_Files="ProgramFiles64Folder"?>
+<?endif?>
+<?ifndef var.Version?>
+    <?error Undefined Version variable?>
+<?endif?>
+<?ifndef var.Path?>
+    <?error Undefined Path variable?>
+<?endif?>
+
+<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
+    <Product Id="35e5e858-9372-4449-bf73-1cd6f7267128" 
+        UpgradeCode="23f90fdd-9328-47ea-ab52-5380855a4b12"
+        Name="cloudflared"
+        Version="$(var.Version)"
+        Manufacturer="cloudflare"
+        Language="1033">
+
+      <Package InstallerVersion="200" Compressed="yes" Comments="Windows Installer Package" InstallScope="perMachine"/>
+
+      <Media Id="1" Cabinet="product.cab" EmbedCab="yes"/>
+
+      <Upgrade Id="23f90fdd-9328-47ea-ab52-5380855a4b12">
+         <UpgradeVersion Minimum="$(var.Version)" OnlyDetect="yes" Property="NEWERVERSIONDETECTED"/>
+         <UpgradeVersion Minimum="2020.8.0" Maximum="$(var.Version)" IncludeMinimum="yes" IncludeMaximum="no"
+                         Property="OLDERVERSIONBEINGUPGRADED"/>
+      </Upgrade>
+      <Condition Message="A newer version of this software is already installed.">NOT NEWERVERSIONDETECTED</Condition>
+
+    <Directory Id="TARGETDIR" Name="SourceDir">
+        <!--This specifies where the cloudflared.exe is moved to in the windows Operation System-->
+        <Directory Id="$(var.Program_Files)">
+            <Directory Id="INSTALLDIR" Name="cloudflared">
+                <Component Id="ApplicationFiles" Guid="35e5e858-9372-4449-bf73-1cd6f7267128">
+                    <File Id="ApplicationFile0" Source="$(var.Path)"/>
+                </Component>
+            </Directory>
+        </Directory>
+        <Component Id="ENVS" Guid="6bb74449-d10d-4f4a-933e-6fc9fa006eae">
+        <!--Set the cloudflared bin location to the Path Environment Variable-->
+         <Environment Id="ENV0"
+           Name="PATH"
+           Value="[INSTALLDIR]."
+           Permanent="no"
+           Part="last"
+           Action="create"
+           System="yes" />
+        </Component>
+    </Directory>
+
+
+    <Feature Id='Complete' Level='1'>
+      <ComponentRef Id="ENVS"/>
+      <ComponentRef Id='ApplicationFiles' />
+    </Feature>
+
+    </Product>
+</Wix>

cmd/cloudflared/access/carrier.go

@@ -1,45 +1,45 @@
 package access
 
 import (
+	"crypto/tls"
+	"fmt"
 	"net/http"
-	"net/url"
 	"strings"
 
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+
 	"github.com/cloudflare/cloudflared/carrier"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/validation"
-	"github.com/pkg/errors"
-	cli "gopkg.in/urfave/cli.v2"
+)
+
+const (
+	LogFieldHost               = "host"
+	cfAccessClientIDHeader     = "Cf-Access-Client-Id"
+	cfAccessClientSecretHeader = "Cf-Access-Client-Secret"
 )
 
 // StartForwarder starts a client side websocket forward
-func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, logger logger.Service) error {
-	validURLString, err := validation.ValidateUrl(forwarder.Listener)
+func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, log *zerolog.Logger) error {
+	validURL, err := validation.ValidateUrl(forwarder.Listener)
 	if err != nil {
 		return errors.Wrap(err, "error validating origin URL")
 	}
 
-	validURL, err := url.Parse(validURLString)
-	if err != nil {
-		return errors.Wrap(err, "error parsing origin URL")
-	}
-
 	// get the headers from the config file and add to the request
 	headers := make(http.Header)
 	if forwarder.TokenClientID != "" {
-		headers.Set(h2mux.CFAccessClientIDHeader, forwarder.TokenClientID)
+		headers.Set(cfAccessClientIDHeader, forwarder.TokenClientID)
 	}
 
 	if forwarder.TokenSecret != "" {
-		headers.Set(h2mux.CFAccessClientSecretHeader, forwarder.TokenSecret)
+		headers.Set(cfAccessClientSecretHeader, forwarder.TokenSecret)
 	}
 
-	if forwarder.Destination != "" {
-		headers.Add(h2mux.CFJumpDestinationHeader, forwarder.Destination)
-	}
+	carrier.SetBastionDest(headers, forwarder.Destination)
 
 	options := &carrier.StartOptions{
 		OriginURL: forwarder.URL,
@@ -47,9 +47,9 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, logger
 	}
 
 	// we could add a cmd line variable for this bool if we want the SOCK5 server to be on the client side
-	wsConn := carrier.NewWSConnection(logger, false)
+	wsConn := carrier.NewWSConnection(log)
 
-	logger.Infof("Start Websocket listener on: %s", validURL.Host)
+	log.Info().Str(LogFieldHost, validURL.Host).Msg("Start Websocket listener")
 	return carrier.StartForwarder(wsConn, validURL.Host, shutdown, options)
 }
 
@@ -58,22 +58,7 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, logger
 // useful for proxying other protocols (like ssh) over websockets
 // (which you can put Access in front of)
 func ssh(c *cli.Context) error {
-	logDirectory, logLevel := config.FindLogSettings()
-
-	flagLogDirectory := c.String(sshLogDirectoryFlag)
-	if flagLogDirectory != "" {
-		logDirectory = flagLogDirectory
-	}
-
-	flagLogLevel := c.String(sshLogLevelFlag)
-	if flagLogLevel != "" {
-		logLevel = flagLogLevel
-	}
-
-	logger, err := logger.New(logger.DefaultFile(logDirectory), logger.LogLevelString(logLevel))
-	if err != nil {
-		return cliutil.PrintLoggerSetupError("error setting up logger", err)
-	}
+	log := logger.CreateSSHLoggerFromContext(c, logger.EnableTerminalLog)
 
 	// get the hostname from the cmdline and error out if its not provided
 	rawHostName := c.String(sshHostnameFlag)
@@ -86,41 +71,52 @@ func ssh(c *cli.Context) error {
 	// get the headers from the cmdline and add them
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
-		headers.Set(h2mux.CFAccessClientIDHeader, c.String(sshTokenIDFlag))
+		headers.Set(cfAccessClientIDHeader, c.String(sshTokenIDFlag))
 	}
 	if c.IsSet(sshTokenSecretFlag) {
-		headers.Set(h2mux.CFAccessClientSecretHeader, c.String(sshTokenSecretFlag))
+		headers.Set(cfAccessClientSecretHeader, c.String(sshTokenSecretFlag))
 	}
 
-	destination := c.String(sshDestinationFlag)
-	if destination != "" {
-		headers.Add(h2mux.CFJumpDestinationHeader, destination)
-	}
+	carrier.SetBastionDest(headers, c.String(sshDestinationFlag))
 
 	options := &carrier.StartOptions{
 		OriginURL: originURL,
 		Headers:   headers,
+		Host:      hostname,
+	}
+
+	if connectTo := c.String(sshConnectTo); connectTo != "" {
+		parts := strings.Split(connectTo, ":")
+		switch len(parts) {
+		case 1:
+			options.OriginURL = fmt.Sprintf("https://%s", parts[0])
+		case 2:
+			options.OriginURL = fmt.Sprintf("https://%s:%s", parts[0], parts[1])
+		case 3:
+			options.OriginURL = fmt.Sprintf("https://%s:%s", parts[2], parts[1])
+			options.TLSClientConfig = &tls.Config{
+				InsecureSkipVerify: true,
+				ServerName:         parts[0],
+			}
+			log.Warn().Msgf("Using insecure SSL connection because SNI overridden to %s", parts[0])
+		default:
+			return fmt.Errorf("invalid connection override: %s", connectTo)
+		}
 	}
 
 	// we could add a cmd line variable for this bool if we want the SOCK5 server to be on the client side
-	wsConn := carrier.NewWSConnection(logger, false)
+	wsConn := carrier.NewWSConnection(log)
 
 	if c.NArg() > 0 || c.IsSet(sshURLFlag) {
-		localForwarder, err := config.ValidateUrl(c, true)
+		forwarder, err := config.ValidateUrl(c, true)
 		if err != nil {
-			logger.Errorf("Error validating origin URL: %s", err)
+			log.Err(err).Msg("Error validating origin URL")
 			return errors.Wrap(err, "error validating origin URL")
 		}
-		forwarder, err := url.Parse(localForwarder)
-		if err != nil {
-			logger.Errorf("Error validating origin URL: %s", err)
-			return errors.Wrap(err, "error validating origin URL")
-		}
-
-		logger.Infof("Start Websocket listener on: %s", forwarder.Host)
+		log.Info().Str(LogFieldHost, forwarder.Host).Msg("Start Websocket listener")
 		err = carrier.StartForwarder(wsConn, forwarder.Host, shutdownC, options)
 		if err != nil {
-			logger.Errorf("Error on Websocket listener: %s", err)
+			log.Err(err).Msg("Error on Websocket listener")
 		}
 		return err
 	}

cmd/cloudflared/access/cmd.go

@@ -2,44 +2,44 @@ package access
 
 import (
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"os"
+	"os/exec"
 	"strings"
 	"text/template"
 	"time"
 
-	"github.com/cloudflare/cloudflared/carrier"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/sshgen"
-	"github.com/cloudflare/cloudflared/validation"
+	"github.com/getsentry/raven-go"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
 	"golang.org/x/net/idna"
 
-	"github.com/getsentry/raven-go"
-	"gopkg.in/urfave/cli.v2"
+	"github.com/cloudflare/cloudflared/carrier"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/sshgen"
+	"github.com/cloudflare/cloudflared/token"
+	"github.com/cloudflare/cloudflared/validation"
 )
 
 const (
-	sshHostnameFlag     = "hostname"
-	sshDestinationFlag  = "destination"
-	sshURLFlag          = "url"
-	sshHeaderFlag       = "header"
-	sshTokenIDFlag      = "service-token-id"
-	sshTokenSecretFlag  = "service-token-secret"
-	sshGenCertFlag      = "short-lived-cert"
-	sshLogDirectoryFlag = "log-directory"
-	sshLogLevelFlag     = "log-level"
-	sshConfigTemplate   = `
+	sshHostnameFlag    = "hostname"
+	sshDestinationFlag = "destination"
+	sshURLFlag         = "url"
+	sshHeaderFlag      = "header"
+	sshTokenIDFlag     = "service-token-id"
+	sshTokenSecretFlag = "service-token-secret"
+	sshGenCertFlag     = "short-lived-cert"
+	sshConnectTo       = "connect-to"
+	sshConfigTemplate  = `
 Add to your {{.Home}}/.ssh/config:
 
 Host {{.Hostname}}
 {{- if .ShortLivedCerts}}
-  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1' 
+  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1'
 
 Host cfpipe-{{.Hostname}}
   HostName {{.Hostname}}
@@ -55,13 +55,12 @@ Host cfpipe-{{.Hostname}}
 const sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b@sentry.io/189878"
 
 var (
-	shutdownC      chan struct{}
-	graceShutdownC chan struct{}
+	shutdownC chan struct{}
 )
 
 // Init will initialize and store vars from the main program
-func Init(s, g chan struct{}) {
-	shutdownC, graceShutdownC = s, g
+func Init(shutdown chan struct{}) {
+	shutdownC = shutdown
 }
 
 // Flags return the global flags for Access related commands (hopefully none)
@@ -77,30 +76,24 @@ func Commands() []*cli.Command {
 			Aliases:  []string{"forward"},
 			Category: "Access",
 			Usage:    "access <subcommand>",
-			Description: `Cloudflare Access protects internal resources by securing, authenticating and monitoring access 
-			per-user and by application. With Cloudflare Access, only authenticated users with the required permissions are 
-			able to reach sensitive resources. The commands provided here allow you to interact with Access protected 
+			Description: `Cloudflare Access protects internal resources by securing, authenticating and monitoring access
+			per-user and by application. With Cloudflare Access, only authenticated users with the required permissions are
+			able to reach sensitive resources. The commands provided here allow you to interact with Access protected
 			applications from the command line.`,
 			Subcommands: []*cli.Command{
 				{
 					Name:   "login",
-					Action: cliutil.ErrorHandler(login),
+					Action: cliutil.Action(login),
 					Usage:  "login <url of access application>",
 					Description: `The login subcommand initiates an authentication flow with your identity provider.
 					The subcommand will launch a browser. For headless systems, a url is provided.
 					Once authenticated with your identity provider, the login command will generate a JSON Web Token (JWT)
-					scoped to your identity, the application you intend to reach, and valid for a session duration set by your 
+					scoped to your identity, the application you intend to reach, and valid for a session duration set by your
 					administrator. cloudflared stores the token in local storage.`,
-					Flags: []cli.Flag{
-						&cli.StringFlag{
-							Name:   "url",
-							Hidden: true,
-						},
-					},
 				},
 				{
 					Name:   "curl",
-					Action: cliutil.ErrorHandler(curl),
+					Action: cliutil.Action(curl),
 					Usage:  "curl [--allow-request, -ar] <url> [<curl args>...]",
 					Description: `The curl subcommand wraps curl and automatically injects the JWT into a cf-access-token
 					header when using curl to reach an application behind Access.`,
@@ -109,7 +102,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "token",
-					Action:      cliutil.ErrorHandler(generateToken),
+					Action:      cliutil.Action(generateToken),
 					Usage:       "token -app=<url of access application>",
 					ArgsUsage:   "url of Access application",
 					Description: `The token subcommand produces a JWT which can be used to authenticate requests.`,
@@ -121,7 +114,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "tcp",
-					Action:      cliutil.ErrorHandler(ssh),
+					Action:      cliutil.Action(ssh),
 					Aliases:     []string{"rdp", "ssh", "smb"},
 					Usage:       "",
 					ArgsUsage:   "",
@@ -150,27 +143,34 @@ func Commands() []*cli.Command {
 							Name:    sshTokenIDFlag,
 							Aliases: []string{"id"},
 							Usage:   "specify an Access service token ID you wish to use.",
+							EnvVars: []string{"TUNNEL_SERVICE_TOKEN_ID"},
 						},
 						&cli.StringFlag{
 							Name:    sshTokenSecretFlag,
 							Aliases: []string{"secret"},
 							Usage:   "specify an Access service token secret you wish to use.",
+							EnvVars: []string{"TUNNEL_SERVICE_TOKEN_SECRET"},
 						},
 						&cli.StringFlag{
-							Name:    sshLogDirectoryFlag,
+							Name:    logger.LogSSHDirectoryFlag,
 							Aliases: []string{"logfile"}, //added to match the tunnel side
 							Usage:   "Save application log to this directory for reporting issues.",
 						},
 						&cli.StringFlag{
-							Name:    sshLogLevelFlag,
+							Name:    logger.LogSSHLevelFlag,
 							Aliases: []string{"loglevel"}, //added to match the tunnel side
-							Usage:   "Application logging level {fatal, error, info, debug}. ",
+							Usage:   "Application logging level {debug, info, warn, error, fatal}. ",
+						},
+						&cli.StringFlag{
+							Name:   sshConnectTo,
+							Hidden: true,
+							Usage:  "Connect to alternate location for testing, value is host, host:port, or sni:port:host",
 						},
 					},
 				},
 				{
 					Name:        "ssh-config",
-					Action:      cliutil.ErrorHandler(sshConfig),
+					Action:      cliutil.Action(sshConfig),
 					Usage:       "",
 					Description: `Prints an example configuration ~/.ssh/config`,
 					Flags: []cli.Flag{
@@ -186,7 +186,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "ssh-gen",
-					Action:      cliutil.ErrorHandler(sshGen),
+					Action:      cliutil.Action(sshGen),
 					Usage:       "",
 					Description: `Generates a short lived certificate for given hostname`,
 					Flags: []cli.Flag{
@@ -207,24 +207,27 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	args := c.Args()
 	rawURL := ensureURLScheme(args.First())
 	appURL, err := url.Parse(rawURL)
 	if args.Len() < 1 || err != nil {
-		logger.Errorf("Please provide the url of the Access application\n")
-		return err
-	}
-	if err := verifyTokenAtEdge(appURL, c, logger); err != nil {
-		logger.Errorf("Could not verify token: %s", err)
+		log.Error().Msg("Please provide the url of the Access application")
 		return err
 	}
 
-	cfdToken, err := token.GetTokenIfExists(appURL)
+	appInfo, err := token.GetAppInfo(appURL)
+	if err != nil {
+		return err
+	}
+
+	if err := verifyTokenAtEdge(appURL, appInfo, c, log); err != nil {
+		log.Err(err).Msg("Could not verify token")
+		return err
+	}
+
+	cfdToken, err := token.GetAppTokenIfExists(appInfo)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, "Unable to find token for provided application.")
 		return err
@@ -252,39 +255,61 @@ func curl(c *cli.Context) error {
 	if err := raven.SetDSN(sentryDSN); err != nil {
 		return err
 	}
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	args := c.Args()
 	if args.Len() < 1 {
-		logger.Error("Please provide the access app and command you wish to run.")
+		log.Error().Msg("Please provide the access app and command you wish to run.")
 		return errors.New("incorrect args")
 	}
 
 	cmdArgs, allowRequest := parseAllowRequest(args.Slice())
-	appURL, err := getAppURL(cmdArgs, logger)
+	appURL, err := getAppURL(cmdArgs, log)
 	if err != nil {
 		return err
 	}
 
-	tok, err := token.GetTokenIfExists(appURL)
+	appInfo, err := token.GetAppInfo(appURL)
+	if err != nil {
+		return err
+	}
+	tok, err := token.GetAppTokenIfExists(appInfo)
 	if err != nil || tok == "" {
 		if allowRequest {
-			logger.Info("You don't have an Access token set. Please run access token <access application> to fetch one.")
-			return shell.Run("curl", cmdArgs...)
+			log.Info().Msg("You don't have an Access token set. Please run access token <access application> to fetch one.")
+			return run("curl", cmdArgs...)
 		}
-		tok, err = token.FetchToken(appURL, logger)
+		tok, err = token.FetchToken(appURL, appInfo, log)
 		if err != nil {
-			logger.Errorf("Failed to refresh token: %s", err)
+			log.Err(err).Msg("Failed to refresh token")
 			return err
 		}
 	}
 
 	cmdArgs = append(cmdArgs, "-H")
-	cmdArgs = append(cmdArgs, fmt.Sprintf("%s: %s", h2mux.CFAccessTokenHeader, tok))
-	return shell.Run("curl", cmdArgs...)
+	cmdArgs = append(cmdArgs, fmt.Sprintf("%s: %s", carrier.CFAccessTokenHeader, tok))
+	return run("curl", cmdArgs...)
+}
+
+// run kicks off a shell task and pipe the results to the respective std pipes
+func run(cmd string, args ...string) error {
+	c := exec.Command(cmd, args...)
+	stderr, err := c.StderrPipe()
+	if err != nil {
+		return err
+	}
+	go func() {
+		io.Copy(os.Stderr, stderr)
+	}()
+
+	stdout, err := c.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	go func() {
+		io.Copy(os.Stdout, stdout)
+	}()
+	return c.Run()
 }
 
 // token dumps provided token to stdout
@@ -292,14 +317,19 @@ func generateToken(c *cli.Context) error {
 	if err := raven.SetDSN(sentryDSN); err != nil {
 		return err
 	}
-	appURL, err := url.Parse(c.String("app"))
+	appURL, err := url.Parse(ensureURLScheme(c.String("app")))
 	if err != nil || c.NumFlags() < 1 {
 		fmt.Fprintln(os.Stderr, "Please provide a url.")
 		return err
 	}
-	tok, err := token.GetTokenIfExists(appURL)
+
+	appInfo, err := token.GetAppInfo(appURL)
+	if err != nil {
+		return err
+	}
+	tok, err := token.GetAppTokenIfExists(appInfo)
 	if err != nil || tok == "" {
-		fmt.Fprintln(os.Stderr, "Unable to find token for provided application. Please run token command to generate token.")
+		fmt.Fprintln(os.Stderr, "Unable to find token for provided application. Please run login command to generate token.")
 		return err
 	}
 
@@ -331,10 +361,7 @@ func sshConfig(c *cli.Context) error {
 
 // sshGen generates a short lived certificate for provided hostname
 func sshGen(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	// get the hostname from the cmdline and error out if its not provided
 	rawHostName := c.String(sshHostnameFlag)
@@ -351,7 +378,12 @@ func sshGen(c *cli.Context) error {
 	// this fetchToken function mutates the appURL param. We should refactor that
 	fetchTokenURL := &url.URL{}
 	*fetchTokenURL = *originURL
-	cfdToken, err := token.FetchTokenWithRedirect(fetchTokenURL, logger)
+
+	appInfo, err := token.GetAppInfo(fetchTokenURL)
+	if err != nil {
+		return err
+	}
+	cfdToken, err := token.FetchTokenWithRedirect(fetchTokenURL, appInfo, log)
 	if err != nil {
 		return err
 	}
@@ -363,16 +395,16 @@ func sshGen(c *cli.Context) error {
 	return nil
 }
 
-// getAppURL will pull the appURL needed for fetching a user's Access token
-func getAppURL(cmdArgs []string, logger logger.Service) (*url.URL, error) {
+// getAppURL will pull the request URL needed for fetching a user's Access token
+func getAppURL(cmdArgs []string, log *zerolog.Logger) (*url.URL, error) {
 	if len(cmdArgs) < 1 {
-		logger.Error("Please provide a valid URL as the first argument to curl.")
+		log.Error().Msg("Please provide a valid URL as the first argument to curl.")
 		return nil, errors.New("not a valid url")
 	}
 
 	u, err := processURL(cmdArgs[0])
 	if err != nil {
-		logger.Error("Please provide a valid URL as the first argument to curl.")
+		log.Error().Msg("Please provide a valid URL as the first argument to curl.")
 		return nil, err
 	}
 
@@ -438,27 +470,27 @@ func isFileThere(candidate string) bool {
 // verifyTokenAtEdge checks for a token on disk, or generates a new one.
 // Then makes a request to to the origin with the token to ensure it is valid.
 // Returns nil if token is valid.
-func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, logger logger.Service) error {
+func verifyTokenAtEdge(appUrl *url.URL, appInfo *token.AppInfo, c *cli.Context, log *zerolog.Logger) error {
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
-		headers.Add(h2mux.CFAccessClientIDHeader, c.String(sshTokenIDFlag))
+		headers.Add(cfAccessClientIDHeader, c.String(sshTokenIDFlag))
 	}
 	if c.IsSet(sshTokenSecretFlag) {
-		headers.Add(h2mux.CFAccessClientSecretHeader, c.String(sshTokenSecretFlag))
+		headers.Add(cfAccessClientSecretHeader, c.String(sshTokenSecretFlag))
 	}
-	options := &carrier.StartOptions{OriginURL: appUrl.String(), Headers: headers}
+	options := &carrier.StartOptions{AppInfo: appInfo, OriginURL: appUrl.String(), Headers: headers}
 
-	if valid, err := isTokenValid(options, logger); err != nil {
+	if valid, err := isTokenValid(options, log); err != nil {
 		return err
 	} else if valid {
 		return nil
 	}
 
-	if err := token.RemoveTokenIfExists(appUrl); err != nil {
+	if err := token.RemoveTokenIfExists(appInfo); err != nil {
 		return err
 	}
 
-	if valid, err := isTokenValid(options, logger); err != nil {
+	if valid, err := isTokenValid(options, log); err != nil {
 		return err
 	} else if !valid {
 		return errors.New("failed to verify token")
@@ -468,8 +500,8 @@ func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, logger logger.Service) e
 }
 
 // isTokenValid makes a request to the origin and returns true if the response was not a 302.
-func isTokenValid(options *carrier.StartOptions, logger logger.Service) (bool, error) {
-	req, err := carrier.BuildAccessRequest(options, logger)
+func isTokenValid(options *carrier.StartOptions, log *zerolog.Logger) (bool, error) {
+	req, err := carrier.BuildAccessRequest(options, log)
 	if err != nil {
 		return false, errors.Wrap(err, "Could not create access request")
 	}

cmd/cloudflared/app_forward_service.go

@@ -1,9 +1,10 @@
 package main
 
 import (
+	"github.com/rs/zerolog"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/config"
 )
 
 // ForwardServiceType is used to identify what kind of overwatch service this is
@@ -15,12 +16,12 @@ const ForwardServiceType = "forward"
 type ForwarderService struct {
 	forwarder config.Forwarder
 	shutdown  chan struct{}
-	logger    logger.Service
+	log       *zerolog.Logger
 }
 
 // NewForwardService creates a new forwarder service
-func NewForwardService(f config.Forwarder, logger logger.Service) *ForwarderService {
-	return &ForwarderService{forwarder: f, shutdown: make(chan struct{}, 1), logger: logger}
+func NewForwardService(f config.Forwarder, log *zerolog.Logger) *ForwarderService {
+	return &ForwarderService{forwarder: f, shutdown: make(chan struct{}, 1), log: log}
 }
 
 // Name is used to figure out this service is related to the others (normally the addr it binds to)
@@ -46,5 +47,5 @@ func (s *ForwarderService) Shutdown() {
 
 // Run is the run loop that is started by the overwatch service
 func (s *ForwarderService) Run() error {
-	return access.StartForwarder(s.forwarder, s.shutdown, s.logger)
+	return access.StartForwarder(s.forwarder, s.shutdown, s.log)
 }

cmd/cloudflared/app_resolver_service.go

@@ -1,13 +1,20 @@
 package main
 
 import (
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/tunneldns"
 )
 
-// ResolverServiceType is used to identify what kind of overwatch service this is
-const ResolverServiceType = "resolver"
+const (
+	// ResolverServiceType is used to identify what kind of overwatch service this is
+	ResolverServiceType = "resolver"
+
+	LogFieldResolverAddress          = "resolverAddress"
+	LogFieldResolverPort             = "resolverPort"
+	LogFieldResolverMaxUpstreamConns = "resolverMaxUpstreamConns"
+)
 
 // ResolverService is used to wrap the tunneldns package's DNS over HTTP
 // into a service model for the overwatch package.
@@ -15,14 +22,14 @@ const ResolverServiceType = "resolver"
 type ResolverService struct {
 	resolver config.DNSResolver
 	shutdown chan struct{}
-	logger   logger.Service
+	log      *zerolog.Logger
 }
 
 // NewResolverService creates a new resolver service
-func NewResolverService(r config.DNSResolver, logger logger.Service) *ResolverService {
+func NewResolverService(r config.DNSResolver, log *zerolog.Logger) *ResolverService {
 	return &ResolverService{resolver: r,
 		shutdown: make(chan struct{}),
-		logger:   logger,
+		log:      log,
 	}
 }
 
@@ -51,7 +58,7 @@ func (s *ResolverService) Shutdown() {
 func (s *ResolverService) Run() error {
 	// create a listener
 	l, err := tunneldns.CreateListener(s.resolver.AddressOrDefault(), s.resolver.PortOrDefault(),
-		s.resolver.UpstreamsOrDefault(), s.resolver.BootstrapsOrDefault(), s.logger)
+		s.resolver.UpstreamsOrDefault(), s.resolver.BootstrapsOrDefault(), s.resolver.MaxUpstreamConnectionsOrDefault(), s.log)
 	if err != nil {
 		return err
 	}
@@ -60,14 +67,21 @@ func (s *ResolverService) Run() error {
 	readySignal := make(chan struct{})
 	err = l.Start(readySignal)
 	if err != nil {
-		l.Stop()
+		_ = l.Stop()
 		return err
 	}
 	<-readySignal
-	s.logger.Infof("start resolver on: %s:%d", s.resolver.AddressOrDefault(), s.resolver.PortOrDefault())
+
+	resolverLog := s.log.With().
+		Str(LogFieldResolverAddress, s.resolver.AddressOrDefault()).
+		Uint16(LogFieldResolverPort, s.resolver.PortOrDefault()).
+		Int(LogFieldResolverMaxUpstreamConns, s.resolver.MaxUpstreamConnectionsOrDefault()).
+		Logger()
+
+	resolverLog.Info().Msg("Starting resolver")
 
 	// wait for shutdown signal
 	<-s.shutdown
-	s.logger.Infof("shutdown on: %s:%d", s.resolver.AddressOrDefault(), s.resolver.PortOrDefault())
+	resolverLog.Info().Msg("Shutting down resolver")
 	return l.Stop()
 }

cmd/cloudflared/app_service.go

@@ -1,8 +1,9 @@
 package main
 
 import (
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/overwatch"
 )
 
@@ -13,17 +14,17 @@ type AppService struct {
 	serviceManager   overwatch.Manager
 	shutdownC        chan struct{}
 	configUpdateChan chan config.Root
-	logger           logger.Service
+	log              *zerolog.Logger
 }
 
 // NewAppService creates a new AppService with needed supporting services
-func NewAppService(configManager config.Manager, serviceManager overwatch.Manager, shutdownC chan struct{}, logger logger.Service) *AppService {
+func NewAppService(configManager config.Manager, serviceManager overwatch.Manager, shutdownC chan struct{}, log *zerolog.Logger) *AppService {
 	return &AppService{
 		configManager:    configManager,
 		serviceManager:   serviceManager,
 		shutdownC:        shutdownC,
 		configUpdateChan: make(chan config.Root),
-		logger:           logger,
+		log:              log,
 	}
 }
 
@@ -37,6 +38,7 @@ func (s *AppService) Run() error {
 func (s *AppService) Shutdown() error {
 	s.configManager.Shutdown()
 	s.shutdownC <- struct{}{}
+
 	return nil
 }
 
@@ -66,14 +68,14 @@ func (s *AppService) handleConfigUpdate(c config.Root) {
 	// handle the client forward listeners
 	activeServices := map[string]struct{}{}
 	for _, f := range c.Forwarders {
-		service := NewForwardService(f, s.logger)
+		service := NewForwardService(f, s.log)
 		s.serviceManager.Add(service)
 		activeServices[service.Name()] = struct{}{}
 	}
 
 	// handle resolver changes
 	if c.Resolver.Enabled {
-		service := NewResolverService(c.Resolver, s.logger)
+		service := NewResolverService(c.Resolver, s.log)
 		s.serviceManager.Add(service)
 		activeServices[service.Name()] = struct{}{}
 

cmd/cloudflared/buildinfo/build_info.go

@@ -1,9 +1,10 @@
 package buildinfo
 
 import (
+	"fmt"
 	"runtime"
 
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/rs/zerolog"
 )
 
 type BuildInfo struct {
@@ -22,7 +23,11 @@ func GetBuildInfo(cloudflaredVersion string) *BuildInfo {
 	}
 }
 
-func (bi *BuildInfo) Log(logger logger.Service) {
-	logger.Infof("Version %s", bi.CloudflaredVersion)
-	logger.Infof("GOOS: %s, GOVersion: %s, GoArch: %s", bi.GoOS, bi.GoVersion, bi.GoArch)
+func (bi *BuildInfo) Log(log *zerolog.Logger) {
+	log.Info().Msgf("Version %s", bi.CloudflaredVersion)
+	log.Info().Msgf("GOOS: %s, GOVersion: %s, GoArch: %s", bi.GoOS, bi.GoVersion, bi.GoArch)
+}
+
+func (bi *BuildInfo) OSArch() string {
+	return fmt.Sprintf("%s_%s", bi.GoOS, bi.GoArch)
 }

cmd/cloudflared/cliutil/deprecated.go

@@ -0,0 +1,21 @@
+package cliutil
+
+import (
+	"fmt"
+
+	"github.com/urfave/cli/v2"
+)
+
+func RemovedCommand(name string) *cli.Command {
+	return &cli.Command{
+		Name: name,
+		Action: func(context *cli.Context) error {
+			return cli.Exit(
+				fmt.Sprintf("%s command is no longer supported by cloudflared. Consult Argo Tunnel documentation for possible alternative solutions.", name),
+				-1,
+			)
+		},
+		Description: fmt.Sprintf("%s is deprecated", name),
+		Hidden:      true,
+	}
+}

cmd/cloudflared/cliutil/errors.go

@@ -2,11 +2,8 @@ package cliutil
 
 import (
 	"fmt"
-	"log"
 
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/pkg/errors"
-	"gopkg.in/urfave/cli.v2"
+	"github.com/urfave/cli/v2"
 )
 
 type usageError string
@@ -25,31 +22,17 @@ func UsageError(format string, args ...interface{}) error {
 }
 
 // Ensures exit with error code if actionFunc returns an error
-func ErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
+func WithErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
 	return func(ctx *cli.Context) error {
 		err := actionFunc(ctx)
 		if err != nil {
 			if _, ok := err.(usageError); ok {
 				msg := fmt.Sprintf("%s\nSee 'cloudflared %s --help'.", err.Error(), ctx.Command.FullName())
-				return cli.Exit(msg, -1)
+				err = cli.Exit(msg, -1)
+			} else if _, ok := err.(cli.ExitCoder); !ok {
+				err = cli.Exit(err.Error(), 1)
 			}
-			// os.Exits with error code if err is cli.ExitCoder or cli.MultiError
-			cli.HandleExitCoder(err)
-			err = cli.Exit(err.Error(), 1)
 		}
-		logger.SharedWriteManager.Shutdown()
 		return err
 	}
 }
-
-// PrintLoggerSetupError returns an error to stdout to notify when a logger can't start
-func PrintLoggerSetupError(msg string, err error) error {
-	l, le := logger.New()
-	if le != nil {
-		log.Printf("%s: %s", msg, err)
-	} else {
-		l.Errorf("%s: %s", msg, err)
-	}
-
-	return errors.Wrap(err, msg)
-}

cmd/cloudflared/cliutil/handler.go

@@ -0,0 +1,50 @@
+package cliutil
+
+import (
+	"github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2/altsrc"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+)
+
+func Action(actionFunc cli.ActionFunc) cli.ActionFunc {
+	return WithErrorHandler(actionFunc)
+}
+
+func ConfiguredAction(actionFunc cli.ActionFunc) cli.ActionFunc {
+	// Adapt actionFunc to the type signature required by ConfiguredActionWithWarnings
+	f := func(context *cli.Context, _ string) error {
+		return actionFunc(context)
+	}
+
+	return ConfiguredActionWithWarnings(f)
+}
+
+// Just like ConfiguredAction, but accepts a second parameter with configuration warnings.
+func ConfiguredActionWithWarnings(actionFunc func(*cli.Context, string) error) cli.ActionFunc {
+	return WithErrorHandler(func(c *cli.Context) error {
+		warnings, err := setFlagsFromConfigFile(c)
+		if err != nil {
+			return err
+		}
+		return actionFunc(c, warnings)
+	})
+}
+
+func setFlagsFromConfigFile(c *cli.Context) (configWarnings string, err error) {
+	const errorExitCode = 1
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
+	inputSource, warnings, err := config.ReadConfigFile(c, log)
+	if err != nil {
+		if err == config.ErrNoConfigFile {
+			return "", nil
+		}
+		return "", cli.Exit(err, errorExitCode)
+	}
+
+	if err := altsrc.ApplyInputSource(c, inputSource); err != nil {
+		return "", cli.Exit(err, errorExitCode)
+	}
+	return warnings, nil
+}

cmd/cloudflared/config/configuration.go

@@ -1,190 +0,0 @@
-package config
-
-import (
-	"errors"
-	"os"
-	"path/filepath"
-	"runtime"
-
-	homedir "github.com/mitchellh/go-homedir"
-	"gopkg.in/urfave/cli.v2"
-	"gopkg.in/urfave/cli.v2/altsrc"
-	"gopkg.in/yaml.v2"
-
-	"github.com/cloudflare/cloudflared/validation"
-)
-
-var (
-	// DefaultConfigFiles is the file names from which we attempt to read configuration.
-	DefaultConfigFiles = []string{"config.yml", "config.yaml"}
-
-	// DefaultUnixConfigLocation is the primary location to find a config file
-	DefaultUnixConfigLocation = "/usr/local/etc/cloudflared"
-
-	// DefaultUnixLogLocation is the primary location to find log files
-	DefaultUnixLogLocation = "/var/log/cloudflared"
-
-	// Launchd doesn't set root env variables, so there is default
-	// Windows default config dir was ~/cloudflare-warp in documentation; let's keep it compatible
-	DefaultConfigDirs = []string{"~/.cloudflared", "~/.cloudflare-warp", "~/cloudflare-warp", "/etc/cloudflared", DefaultUnixConfigLocation}
-)
-
-const DefaultCredentialFile = "cert.pem"
-
-// DefaultConfigDirectory returns the default directory of the config file
-func DefaultConfigDirectory() string {
-	if runtime.GOOS == "windows" {
-		path := os.Getenv("CFDPATH")
-		if path == "" {
-			path = filepath.Join(os.Getenv("ProgramFiles(x86)"), "cloudflared")
-			if _, err := os.Stat(path); os.IsNotExist(err) { //doesn't exist, so return an empty failure string
-				return ""
-			}
-		}
-		return path
-	}
-	return DefaultUnixConfigLocation
-}
-
-// DefaultLogDirectory returns the default directory for log files
-func DefaultLogDirectory() string {
-	if runtime.GOOS == "windows" {
-		return DefaultConfigDirectory()
-	}
-	return DefaultUnixLogLocation
-}
-
-// DefaultConfigPath returns the default location of a config file
-func DefaultConfigPath() string {
-	dir := DefaultConfigDirectory()
-	if dir == "" {
-		return DefaultConfigFiles[0]
-	}
-	return filepath.Join(dir, DefaultConfigFiles[0])
-}
-
-// FileExists checks to see if a file exist at the provided path.
-func FileExists(path string) (bool, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		if os.IsNotExist(err) {
-			// ignore missing files
-			return false, nil
-		}
-		return false, err
-	}
-	f.Close()
-	return true, nil
-}
-
-// FindInputSourceContext pulls the input source from the config flag.
-func FindInputSourceContext(context *cli.Context) (altsrc.InputSourceContext, error) {
-	if context.String("config") != "" {
-		return altsrc.NewYamlSourceFromFile(context.String("config"))
-	}
-	return nil, nil
-}
-
-// FindDefaultConfigPath returns the first path that contains a config file.
-// If none of the combination of DefaultConfigDirs and DefaultConfigFiles
-// contains a config file, return empty string.
-func FindDefaultConfigPath() string {
-	for _, configDir := range DefaultConfigDirs {
-		for _, configFile := range DefaultConfigFiles {
-			dirPath, err := homedir.Expand(configDir)
-			if err != nil {
-				continue
-			}
-			path := filepath.Join(dirPath, configFile)
-			if ok, _ := FileExists(path); ok {
-				return path
-			}
-		}
-	}
-	return ""
-}
-
-// FindOrCreateConfigPath returns the first path that contains a config file
-// or creates one in the primary default path if it doesn't exist
-func FindOrCreateConfigPath() string {
-	path := FindDefaultConfigPath()
-
-	if path == "" {
-		// create the default directory if it doesn't exist
-		path = DefaultConfigPath()
-		if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
-			return ""
-		}
-
-		// write a new config file out
-		file, err := os.Create(path)
-		if err != nil {
-			return ""
-		}
-		defer file.Close()
-
-		logDir := DefaultLogDirectory()
-		os.MkdirAll(logDir, os.ModePerm) //try and create it. Doesn't matter if it succeed or not, only byproduct will be no logs
-
-		c := Root{
-			LogDirectory: logDir,
-		}
-		if err := yaml.NewEncoder(file).Encode(&c); err != nil {
-			return ""
-		}
-	}
-
-	return path
-}
-
-// FindLogSettings gets the log directory and level from the config file
-func FindLogSettings() (string, string) {
-	configPath := FindOrCreateConfigPath()
-	defaultDirectory := DefaultLogDirectory()
-	defaultLevel := "info"
-
-	file, err := os.Open(configPath)
-	if err != nil {
-		return defaultDirectory, defaultLevel
-	}
-	defer file.Close()
-
-	var config Root
-	if err := yaml.NewDecoder(file).Decode(&config); err != nil {
-		return defaultDirectory, defaultLevel
-	}
-
-	directory := defaultDirectory
-	if config.LogDirectory != "" {
-		directory = config.LogDirectory
-	}
-
-	level := defaultLevel
-	if config.LogLevel != "" {
-		level = config.LogLevel
-	}
-	return directory, level
-}
-
-// ValidateUnixSocket ensures --unix-socket param is used exclusively
-// i.e. it fails if a user specifies both --url and --unix-socket
-func ValidateUnixSocket(c *cli.Context) (string, error) {
-	if c.IsSet("unix-socket") && (c.IsSet("url") || c.NArg() > 0) {
-		return "", errors.New("--unix-socket must be used exclusivly.")
-	}
-	return c.String("unix-socket"), nil
-}
-
-// ValidateUrl will validate url flag correctness. It can be either from --url or argument
-// Notice ValidateUnixSocket, it will enforce --unix-socket is not used with --url or argument
-func ValidateUrl(c *cli.Context, allowFromArgs bool) (string, error) {
-	var url = c.String("url")
-	if allowFromArgs && c.NArg() > 0 {
-		if c.IsSet("url") {
-			return "", errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
-		}
-		url = c.Args().Get(0)
-	}
-	validUrl, err := validation.ValidateUrl(url)
-	return validUrl, err
-}

cmd/cloudflared/generic_service.go

@@ -5,9 +5,9 @@ package main
 import (
 	"os"
 
-	cli "gopkg.in/urfave/cli.v2"
+	cli "github.com/urfave/cli/v2"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Run(os.Args)
 }

cmd/cloudflared/linux_service.go

@@ -7,26 +7,35 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/pkg/errors"
-	cli "gopkg.in/urfave/cli.v2"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
 		Usage: "Manages the Argo Tunnel system service",
 		Subcommands: []*cli.Command{
-			&cli.Command{
+			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a system service",
-				Action: installLinuxService,
+				Action: cliutil.ConfiguredAction(installLinuxService),
+				Flags: []cli.Flag{
+					&cli.BoolFlag{
+						Name:  "legacy",
+						Usage: "Generate service file for non-named tunnels",
+					},
+				},
 			},
-			&cli.Command{
+			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
-				Action: uninstallLinuxService,
+				Action: cliutil.ConfiguredAction(uninstallLinuxService),
 			},
 		},
 	})
@@ -39,6 +48,7 @@ const (
 	serviceConfigDir      = "/etc/cloudflared"
 	serviceConfigFile     = "config.yml"
 	serviceCredentialFile = "cert.pem"
+	serviceConfigPath     = serviceConfigDir + "/" + serviceConfigFile
 )
 
 var systemdTemplates = []ServiceTemplate{
@@ -51,7 +61,7 @@ After=network.target
 [Service]
 TimeoutStartSec=0
 Type=notify
-ExecStart={{ .Path }} --config /etc/cloudflared/config.yml --origincert /etc/cloudflared/cert.pem --no-autoupdate
+ExecStart={{ .Path }} --config /etc/cloudflared/config.yml --no-autoupdate{{ range .ExtraArgs }} {{ . }}{{ end }}
 Restart=on-failure
 RestartSec=5s
 
@@ -66,7 +76,7 @@ Description=Update Argo Tunnel
 After=network.target
 
 [Service]
-ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 64 ]; then systemctl restart cloudflared; exit 0; fi; exit $code'
+ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 11 ]; then systemctl restart cloudflared; exit 0; fi; exit $code'
 `,
 	},
 	{
@@ -101,7 +111,7 @@ var sysvTemplate = ServiceTemplate{
 # Description:       Argo Tunnel agent
 ### END INIT INFO
 name=$(basename $(readlink -f $0))
-cmd="{{.Path}} --config /etc/cloudflared/config.yml --origincert /etc/cloudflared/cert.pem --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s"
+cmd="{{.Path}} --config /etc/cloudflared/config.yml --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s{{ range .ExtraArgs }} {{ . }}{{ end }}"
 pid_file="/var/run/$name.pid"
 stdout_log="/var/log/$name.log"
 stderr_log="/var/log/$name.err"
@@ -180,11 +190,7 @@ func isSystemd() bool {
 	return false
 }
 
-func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile string, logger logger.Service) error {
-	if err := ensureConfigDirExists(serviceConfigDir); err != nil {
-		return err
-	}
-
+func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile string, log *zerolog.Logger) error {
 	srcCredentialPath := filepath.Join(userConfigDir, userCredentialFile)
 	destCredentialPath := filepath.Join(serviceConfigDir, serviceCredentialFile)
 	if srcCredentialPath != destCredentialPath {
@@ -199,71 +205,108 @@ func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile str
 		if err := copyConfig(srcConfigPath, destConfigPath); err != nil {
 			return err
 		}
-		logger.Infof("Copied %s to %s", srcConfigPath, destConfigPath)
+		log.Info().Msgf("Copied %s to %s", srcConfigPath, destConfigPath)
 	}
 
 	return nil
 }
 
 func installLinuxService(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	etPath, err := os.Executable()
 	if err != nil {
 		return fmt.Errorf("error determining executable path: %v", err)
 	}
-	templateArgs := ServiceTemplateArgs{Path: etPath}
+	templateArgs := ServiceTemplateArgs{
+		Path: etPath,
+	}
 
-	userConfigDir := filepath.Dir(c.String("config"))
-	userConfigFile := filepath.Base(c.String("config"))
-	userCredentialFile := config.DefaultCredentialFile
-	if err = copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile, logger); err != nil {
-		logger.Errorf("Failed to copy user configuration: %s. Before running the service, ensure that %s contains two files, %s and %s", err,
-			serviceConfigDir, serviceCredentialFile, serviceConfigFile)
+	if err := ensureConfigDirExists(serviceConfigDir); err != nil {
 		return err
 	}
+	if c.Bool("legacy") {
+		userConfigDir := filepath.Dir(c.String("config"))
+		userConfigFile := filepath.Base(c.String("config"))
+		userCredentialFile := config.DefaultCredentialFile
+		if err = copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile, log); err != nil {
+			log.Err(err).Msgf("Failed to copy user configuration. Before running the service, ensure that %s contains two files, %s and %s",
+				serviceConfigDir, serviceCredentialFile, serviceConfigFile)
+			return err
+		}
+		templateArgs.ExtraArgs = []string{
+			"--origincert", serviceConfigDir + "/" + serviceCredentialFile,
+		}
+	} else {
+		src, _, err := config.ReadConfigFile(c, log)
+		if err != nil {
+			return err
+		}
+
+		// can't use context because this command doesn't define "credentials-file" flag
+		configPresent := func(s string) bool {
+			val, err := src.String(s)
+			return err == nil && val != ""
+		}
+		if src.TunnelID == "" || !configPresent(tunnel.CredFileFlag) {
+			return fmt.Errorf(`Configuration file %s must contain entries for the tunnel to run and its associated credentials:
+tunnel: TUNNEL-UUID
+credentials-file: CREDENTIALS-FILE
+`, src.Source())
+		}
+		if src.Source() != serviceConfigPath {
+			if exists, err := config.FileExists(serviceConfigPath); err != nil || exists {
+				return fmt.Errorf("Possible conflicting configuration in %[1]s and %[2]s. Either remove %[2]s or run `cloudflared --config %[2]s service install`", src.Source(), serviceConfigPath)
+			}
+
+			if err := copyFile(src.Source(), serviceConfigPath); err != nil {
+				return fmt.Errorf("failed to copy %s to %s: %w", src.Source(), serviceConfigPath, err)
+			}
+		}
+
+		templateArgs.ExtraArgs = []string{
+			"tunnel", "run",
+		}
+	}
 
 	switch {
 	case isSystemd():
-		logger.Infof("Using Systemd")
-		return installSystemd(&templateArgs, logger)
+		log.Info().Msgf("Using Systemd")
+		return installSystemd(&templateArgs, log)
 	default:
-		logger.Infof("Using Sysv")
-		return installSysv(&templateArgs, logger)
+		log.Info().Msgf("Using SysV")
+		return installSysv(&templateArgs, log)
 	}
 }
 
-func installSystemd(templateArgs *ServiceTemplateArgs, logger logger.Service) error {
+func installSystemd(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
 	for _, serviceTemplate := range systemdTemplates {
 		err := serviceTemplate.Generate(templateArgs)
 		if err != nil {
-			logger.Errorf("error generating service template: %s", err)
+			log.Err(err).Msg("error generating service template")
 			return err
 		}
 	}
 	if err := runCommand("systemctl", "enable", "cloudflared.service"); err != nil {
-		logger.Errorf("systemctl enable cloudflared.service error: %s", err)
+		log.Err(err).Msg("systemctl enable cloudflared.service error")
 		return err
 	}
 	if err := runCommand("systemctl", "start", "cloudflared-update.timer"); err != nil {
-		logger.Errorf("systemctl start cloudflared-update.timer error: %s", err)
+		log.Err(err).Msg("systemctl start cloudflared-update.timer error")
 		return err
 	}
-	logger.Infof("systemctl daemon-reload")
+	log.Info().Msg("systemctl daemon-reload")
 	return runCommand("systemctl", "daemon-reload")
 }
 
-func installSysv(templateArgs *ServiceTemplateArgs, logger logger.Service) error {
+func installSysv(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
 	confPath, err := sysvTemplate.ResolvePath()
 	if err != nil {
-		logger.Errorf("error resolving system path: %s", err)
+		log.Err(err).Msg("error resolving system path")
 		return err
 	}
 	if err := sysvTemplate.Generate(templateArgs); err != nil {
-		logger.Errorf("error generating system template: %s", err)
+		log.Err(err).Msg("error generating system template")
 		return err
 	}
 	for _, i := range [...]string{"2", "3", "4", "5"} {
@@ -280,43 +323,40 @@ func installSysv(templateArgs *ServiceTemplateArgs, logger logger.Service) error
 }
 
 func uninstallLinuxService(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	switch {
 	case isSystemd():
-		logger.Infof("Using Systemd")
-		return uninstallSystemd(logger)
+		log.Info().Msg("Using Systemd")
+		return uninstallSystemd(log)
 	default:
-		logger.Infof("Using Sysv")
-		return uninstallSysv(logger)
+		log.Info().Msg("Using SysV")
+		return uninstallSysv(log)
 	}
 }
 
-func uninstallSystemd(logger logger.Service) error {
+func uninstallSystemd(log *zerolog.Logger) error {
 	if err := runCommand("systemctl", "disable", "cloudflared.service"); err != nil {
-		logger.Errorf("systemctl disable cloudflared.service error: %s", err)
+		log.Err(err).Msg("systemctl disable cloudflared.service error")
 		return err
 	}
 	if err := runCommand("systemctl", "stop", "cloudflared-update.timer"); err != nil {
-		logger.Errorf("systemctl stop cloudflared-update.timer error: %s", err)
+		log.Err(err).Msg("systemctl stop cloudflared-update.timer error")
 		return err
 	}
 	for _, serviceTemplate := range systemdTemplates {
 		if err := serviceTemplate.Remove(); err != nil {
-			logger.Errorf("error removing service template: %s", err)
+			log.Err(err).Msg("error removing service template")
 			return err
 		}
 	}
-	logger.Infof("Successfully uninstall cloudflared service")
+	log.Info().Msgf("Successfully uninstalled cloudflared service from systemd")
 	return nil
 }
 
-func uninstallSysv(logger logger.Service) error {
+func uninstallSysv(log *zerolog.Logger) error {
 	if err := sysvTemplate.Remove(); err != nil {
-		logger.Errorf("error removing service template: %s", err)
+		log.Err(err).Msg("error removing service template")
 		return err
 	}
 	for _, i := range [...]string{"2", "3", "4", "5"} {
@@ -329,6 +369,6 @@ func uninstallSysv(logger logger.Service) error {
 			continue
 		}
 	}
-	logger.Infof("Successfully uninstall cloudflared service")
+	log.Info().Msgf("Successfully uninstalled cloudflared service from sysv")
 	return nil
 }

cmd/cloudflared/macos_service.go

@@ -6,17 +6,18 @@ import (
 	"fmt"
 	"os"
 
-	"gopkg.in/urfave/cli.v2"
-
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
 	launchdIdentifier = "com.cloudflare.cloudflared"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
 		Usage: "Manages the Argo Tunnel launch agent",
@@ -24,16 +25,16 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as an user launch agent",
-				Action: installLaunchd,
+				Action: cliutil.ConfiguredAction(installLaunchd),
 			},
 			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel launch agent",
-				Action: uninstallLaunchd,
+				Action: cliutil.ConfiguredAction(uninstallLaunchd),
 			},
 		},
 	})
-	app.Run(os.Args)
+	_ = app.Run(os.Args)
 }
 
 func newLaunchdTemplate(installPath, stdoutPath, stderrPath string) *ServiceTemplate {
@@ -106,71 +107,61 @@ func stderrPath() (string, error) {
 }
 
 func installLaunchd(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	if isRootUser() {
-		logger.Infof("Installing Argo Tunnel client as a system launch daemon. " +
+		log.Info().Msg("Installing Argo Tunnel client as a system launch daemon. " +
 			"Argo Tunnel client will run at boot")
 	} else {
-		logger.Infof("Installing Argo Tunnel client as an user launch agent. " +
+		log.Info().Msg("Installing Argo Tunnel client as an user launch agent. " +
 			"Note that Argo Tunnel client will only run when the user is logged in. " +
 			"If you want to run Argo Tunnel client at boot, install with root permission. " +
 			"For more information, visit https://developers.cloudflare.com/argo-tunnel/reference/service/")
 	}
 	etPath, err := os.Executable()
 	if err != nil {
-		logger.Errorf("Error determining executable path: %s", err)
+		log.Err(err).Msg("Error determining executable path")
 		return fmt.Errorf("Error determining executable path: %v", err)
 	}
 	installPath, err := installPath()
 	if err != nil {
-		logger.Errorf("Error determining install path: %s", err)
+		log.Err(err).Msg("Error determining install path")
 		return errors.Wrap(err, "Error determining install path")
 	}
 	stdoutPath, err := stdoutPath()
 	if err != nil {
-		logger.Errorf("error determining stdout path: %s", err)
+		log.Err(err).Msg("error determining stdout path")
 		return errors.Wrap(err, "error determining stdout path")
 	}
 	stderrPath, err := stderrPath()
 	if err != nil {
-		logger.Errorf("error determining stderr path: %s", err)
+		log.Err(err).Msg("error determining stderr path")
 		return errors.Wrap(err, "error determining stderr path")
 	}
 	launchdTemplate := newLaunchdTemplate(installPath, stdoutPath, stderrPath)
-	if err != nil {
-		logger.Errorf("error creating launchd template: %s", err)
-		return errors.Wrap(err, "error creating launchd template")
-	}
 	templateArgs := ServiceTemplateArgs{Path: etPath}
 	err = launchdTemplate.Generate(&templateArgs)
 	if err != nil {
-		logger.Errorf("error generating launchd template: %s", err)
+		log.Err(err).Msg("error generating launchd template")
 		return err
 	}
 	plistPath, err := launchdTemplate.ResolvePath()
 	if err != nil {
-		logger.Errorf("error resolving launchd template path: %s", err)
+		log.Err(err).Msg("error resolving launchd template path")
 		return err
 	}
 
-	logger.Infof("Outputs are logged to %s and %s", stderrPath, stdoutPath)
+	log.Info().Msgf("Outputs are logged to %s and %s", stderrPath, stdoutPath)
 	return runCommand("launchctl", "load", plistPath)
 }
 
 func uninstallLaunchd(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	if isRootUser() {
-		logger.Infof("Uninstalling Argo Tunnel as a system launch daemon")
+		log.Info().Msg("Uninstalling Argo Tunnel as a system launch daemon")
 	} else {
-		logger.Infof("Uninstalling Argo Tunnel as an user launch agent")
+		log.Info().Msg("Uninstalling Argo Tunnel as an user launch agent")
 	}
 	installPath, err := installPath()
 	if err != nil {
@@ -185,20 +176,17 @@ func uninstallLaunchd(c *cli.Context) error {
 		return errors.Wrap(err, "error determining stderr path")
 	}
 	launchdTemplate := newLaunchdTemplate(installPath, stdoutPath, stderrPath)
-	if err != nil {
-		return errors.Wrap(err, "error creating launchd template")
-	}
 	plistPath, err := launchdTemplate.ResolvePath()
 	if err != nil {
-		logger.Errorf("error resolving launchd template path: %s", err)
+		log.Err(err).Msg("error resolving launchd template path")
 		return err
 	}
 	err = runCommand("launchctl", "unload", plistPath)
 	if err != nil {
-		logger.Errorf("error unloading: %s", err)
+		log.Err(err).Msg("error unloading")
 		return err
 	}
 
-	logger.Infof("Outputs are logged to %s and %s", stderrPath, stdoutPath)
+	log.Info().Msgf("Outputs are logged to %s and %s", stderrPath, stdoutPath)
 	return launchdTemplate.Remove()
 }

cmd/cloudflared/main.go

@@ -2,24 +2,26 @@ package main
 
 import (
 	"fmt"
+	"math/rand"
 	"strings"
 	"time"
 
+	"github.com/getsentry/raven-go"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+	"go.uber.org/automaxprocs/maxprocs"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
-	log "github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/overwatch"
 	"github.com/cloudflare/cloudflared/watcher"
-
-	raven "github.com/getsentry/raven-go"
-	homedir "github.com/mitchellh/go-homedir"
-	cli "gopkg.in/urfave/cli.v2"
-
-	"github.com/pkg/errors"
 )
 
 const (
@@ -43,13 +45,12 @@ var (
 )
 
 func main() {
+	rand.Seed(time.Now().UnixNano())
 	metrics.RegisterBuildInfo(BuildTime, Version)
 	raven.SetRelease(Version)
+	maxprocs.Set()
 
-	// Force shutdown channel used by the app. When closed, app must terminate.
-	// Windows service manager closes this channel when it receives shutdown command.
-	shutdownC := make(chan struct{})
-	// Graceful shutdown channel used by the app. When closed, app must terminate.
+	// Graceful shutdown channel used by the app. When closed, app must terminate gracefully.
 	// Windows service manager closes this channel when it receives stop command.
 	graceShutdownC := make(chan struct{})
 
@@ -73,29 +74,51 @@ func main() {
 	app.Version = fmt.Sprintf("%s (built %s)", Version, BuildTime)
 	app.Description = `cloudflared connects your machine or user identity to Cloudflare's global network.
 	You can use it to authenticate a session to reach an API behind Access, route web traffic to this machine,
-	and configure access control.`
+	and configure access control.
+
+	See https://developers.cloudflare.com/argo-tunnel/ for more in-depth documentation.`
 	app.Flags = flags()
-	app.Action = action(Version, shutdownC, graceShutdownC)
-	app.Before = tunnel.Before
+	app.Action = action(graceShutdownC)
 	app.Commands = commands(cli.ShowVersion)
 
-	tunnel.Init(Version, shutdownC, graceShutdownC) // we need this to support the tunnel sub command...
-	access.Init(shutdownC, graceShutdownC)
-	runApp(app, shutdownC, graceShutdownC)
+	tunnel.Init(Version, graceShutdownC) // we need this to support the tunnel sub command...
+	access.Init(graceShutdownC)
+	updater.Init(Version)
+	runApp(app, graceShutdownC)
 }
 
 func commands(version func(c *cli.Context)) []*cli.Command {
 	cmds := []*cli.Command{
 		{
-			Name:      "update",
-			Action:    updater.Update,
-			Usage:     "Update the agent if a new version exists",
-			ArgsUsage: " ",
+			Name:   "update",
+			Action: cliutil.ConfiguredAction(updater.Update),
+			Usage:  "Update the agent if a new version exists",
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:  "beta",
+					Usage: "specify if you wish to update to the latest beta version",
+				},
+				&cli.BoolFlag{
+					Name:   "force",
+					Usage:  "specify if you wish to force an upgrade to the latest version regardless of the current version",
+					Hidden: true,
+				},
+				&cli.BoolFlag{
+					Name:   "staging",
+					Usage:  "specify if you wish to use the staging url for updating",
+					Hidden: true,
+				},
+				&cli.StringFlag{
+					Name:   "version",
+					Usage:  "specify a version you wish to upgrade or downgrade to",
+					Hidden: false,
+				},
+			},
 			Description: `Looks for a new version on the official download server.
 If a new version exists, updates the agent binary and quits.
 Otherwise, does nothing.
 
-To determine if an update happened in a script, check for error code 64.`,
+To determine if an update happened in a script, check for error code 11.`,
 		},
 		{
 			Name: "version",
@@ -108,6 +131,7 @@ To determine if an update happened in a script, check for error code 64.`,
 		},
 	}
 	cmds = append(cmds, tunnel.Commands()...)
+	cmds = append(cmds, proxydns.Command(false))
 	cmds = append(cmds, access.Commands()...)
 	return cmds
 }
@@ -121,24 +145,20 @@ func isEmptyInvocation(c *cli.Context) bool {
 	return c.NArg() == 0 && c.NumFlags() == 0
 }
 
-func action(version string, shutdownC, graceShutdownC chan struct{}) cli.ActionFunc {
-	return func(c *cli.Context) (err error) {
+func action(graceShutdownC chan struct{}) cli.ActionFunc {
+	return cliutil.ConfiguredAction(func(c *cli.Context) (err error) {
 		if isEmptyInvocation(c) {
-			return handleServiceMode(shutdownC)
+			return handleServiceMode(c, graceShutdownC)
 		}
 		tags := make(map[string]string)
 		tags["hostname"] = c.String("hostname")
 		raven.SetTagsContext(tags)
-		raven.CapturePanic(func() { err = tunnel.StartServer(c, version, shutdownC, graceShutdownC, nil) }, nil)
-		exitCode := 0
+		raven.CapturePanic(func() { err = tunnel.TunnelCommand(c) }, nil)
 		if err != nil {
-			handleError(err)
-			exitCode = 1
+			captureError(err)
 		}
-		// we already handle error printing, so we pass an empty string so we
-		// don't have to print again.
-		return cli.Exit("", exitCode)
-	}
+		return err
+	})
 }
 
 func userHomeDir() (string, error) {
@@ -154,7 +174,7 @@ func userHomeDir() (string, error) {
 }
 
 // In order to keep the amount of noise sent to Sentry low, typical network errors can be filtered out here by a substring match.
-func handleError(err error) {
+func captureError(err error) {
 	errorMessage := err.Error()
 	for _, ignoredErrorMessage := range ignoredErrors {
 		if strings.Contains(errorMessage, ignoredErrorMessage) {
@@ -165,41 +185,34 @@ func handleError(err error) {
 }
 
 // cloudflared was started without any flags
-func handleServiceMode(shutdownC chan struct{}) error {
-	defer log.SharedWriteManager.Shutdown()
-	logDirectory, logLevel := config.FindLogSettings()
-
-	logger, err := log.New(log.DefaultFile(logDirectory), log.LogLevelString(logLevel))
-	if err != nil {
-		return cliutil.PrintLoggerSetupError("error setting up logger", err)
-	}
-	logger.Infof("logging to directory: %s", logDirectory)
+func handleServiceMode(c *cli.Context, shutdownC chan struct{}) error {
+	log := logger.CreateLoggerFromContext(c, logger.DisableTerminalLog)
 
 	// start the main run loop that reads from the config file
 	f, err := watcher.NewFile()
 	if err != nil {
-		logger.Errorf("Cannot load config file: %s", err)
+		log.Err(err).Msg("Cannot load config file")
 		return err
 	}
 
 	configPath := config.FindOrCreateConfigPath()
-	configManager, err := config.NewFileManager(f, configPath, logger)
+	configManager, err := config.NewFileManager(f, configPath, log)
 	if err != nil {
-		logger.Errorf("Cannot setup config file for monitoring: %s", err)
+		log.Err(err).Msg("Cannot setup config file for monitoring")
 		return err
 	}
-	logger.Infof("monitoring config file at: %s", configPath)
+	log.Info().Msgf("monitoring config file at: %s", configPath)
 
 	serviceCallback := func(t string, name string, err error) {
 		if err != nil {
-			logger.Errorf("%s service: %s encountered an error: %s", t, name, err)
+			log.Err(err).Msgf("%s service: %s encountered an error", t, name)
 		}
 	}
 	serviceManager := overwatch.NewAppManager(serviceCallback)
 
-	appService := NewAppService(configManager, serviceManager, shutdownC, logger)
+	appService := NewAppService(configManager, serviceManager, shutdownC, log)
 	if err := appService.Run(); err != nil {
-		logger.Errorf("Failed to start app service: %s", err)
+		log.Err(err).Msg("Failed to start app service")
 		return err
 	}
 	return nil

cmd/cloudflared/path/path.go

@@ -1,30 +0,0 @@
-package path
-
-import (
-	"fmt"
-	"net/url"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/mitchellh/go-homedir"
-)
-
-// GenerateFilePathFromURL will return a filepath for given access application url
-func GenerateFilePathFromURL(url *url.URL, suffix string) (string, error) {
-	configPath, err := homedir.Expand(config.DefaultConfigDirs[0])
-	if err != nil {
-		return "", err
-	}
-	ok, err := config.FileExists(configPath)
-	if !ok && err == nil {
-		// create config directory if doesn't already exist
-		err = os.Mkdir(configPath, 0700)
-	}
-	if err != nil {
-		return "", err
-	}
-	name := strings.Replace(fmt.Sprintf("%s%s-%s", url.Hostname(), url.EscapedPath(), suffix), "/", "-", -1)
-	return filepath.Join(configPath, name), nil
-}

cmd/cloudflared/proxydns/cmd.go

@@ -0,0 +1,114 @@
+package proxydns
+
+import (
+	"net"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/metrics"
+	"github.com/cloudflare/cloudflared/tunneldns"
+)
+
+func Command(hidden bool) *cli.Command {
+	return &cli.Command{
+		Name:   "proxy-dns",
+		Action: cliutil.ConfiguredAction(Run),
+
+		Usage: "Run a DNS over HTTPS proxy server.",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "metrics",
+				Value:   "localhost:",
+				Usage:   "Listen address for metrics reporting.",
+				EnvVars: []string{"TUNNEL_METRICS"},
+			},
+			&cli.StringFlag{
+				Name:    "address",
+				Usage:   "Listen address for the DNS over HTTPS proxy server.",
+				Value:   "localhost",
+				EnvVars: []string{"TUNNEL_DNS_ADDRESS"},
+			},
+			// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+			&cli.IntFlag{
+				Name:    "port",
+				Usage:   "Listen on given port for the DNS over HTTPS proxy server.",
+				Value:   53,
+				EnvVars: []string{"TUNNEL_DNS_PORT"},
+			},
+			&cli.StringSliceFlag{
+				Name:    "upstream",
+				Usage:   "Upstream endpoint URL, you can specify multiple endpoints for redundancy.",
+				Value:   cli.NewStringSlice("https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"),
+				EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
+			},
+			&cli.StringSliceFlag{
+				Name:    "bootstrap",
+				Usage:   "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",
+				Value:   cli.NewStringSlice("https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"),
+				EnvVars: []string{"TUNNEL_DNS_BOOTSTRAP"},
+			},
+			&cli.IntFlag{
+				Name:    "max-upstream-conns",
+				Usage:   "Maximum concurrent connections to upstream. Setting to 0 means unlimited.",
+				Value:   tunneldns.MaxUpstreamConnsDefault,
+				EnvVars: []string{"TUNNEL_DNS_MAX_UPSTREAM_CONNS"},
+			},
+		},
+		ArgsUsage: " ", // can't be the empty string or we get the default output
+		Hidden:    hidden,
+	}
+}
+
+// Run implements a foreground runner
+func Run(c *cli.Context) error {
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
+
+	metricsListener, err := net.Listen("tcp", c.String("metrics"))
+	if err != nil {
+		log.Fatal().Err(err).Msg("Failed to open the metrics listener")
+	}
+
+	go metrics.ServeMetrics(metricsListener, nil, nil, "", log)
+
+	listener, err := tunneldns.CreateListener(
+		c.String("address"),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		uint16(c.Int("port")),
+		c.StringSlice("upstream"),
+		c.StringSlice("bootstrap"),
+		c.Int("max-upstream-conns"),
+		log,
+	)
+
+	if err != nil {
+		log.Err(err).Msg("Failed to create the listeners")
+		return err
+	}
+
+	// Try to start the server
+	readySignal := make(chan struct{})
+	err = listener.Start(readySignal)
+	if err != nil {
+		log.Err(err).Msg("Failed to start the listeners")
+		return listener.Stop()
+	}
+	<-readySignal
+
+	// Wait for signal
+	signals := make(chan os.Signal, 10)
+	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
+	defer signal.Stop(signals)
+	<-signals
+
+	// Shut down server
+	err = listener.Stop()
+	if err != nil {
+		log.Err(err).Msg("failed to stop")
+	}
+	return err
+}

cmd/cloudflared/service_template.go

@@ -10,8 +10,9 @@ import (
 	"os/exec"
 	"text/template"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/mitchellh/go-homedir"
+	homedir "github.com/mitchellh/go-homedir"
+
+	"github.com/cloudflare/cloudflared/config"
 )
 
 type ServiceTemplate struct {
@@ -21,7 +22,8 @@ type ServiceTemplate struct {
 }
 
 type ServiceTemplateArgs struct {
-	Path string
+	Path      string
+	ExtraArgs []string
 }
 
 func (st *ServiceTemplate) ResolvePath() (string, error) {
@@ -80,7 +82,7 @@ func runCommand(command string, args ...string) error {
 		return fmt.Errorf("error starting %s: %v", command, err)
 	}
 
-	ioutil.ReadAll(stderr)
+	_, _ = ioutil.ReadAll(stderr)
 	err = cmd.Wait()
 	if err != nil {
 		return fmt.Errorf("%s returned with error: %v", command, err)
@@ -139,6 +141,33 @@ func copyCredential(srcCredentialPath, destCredentialPath string) error {
 	return nil
 }
 
+func copyFile(src, dest string) error {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+
+	destFile, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	ok := false
+	defer func() {
+		destFile.Close()
+		if !ok {
+			_ = os.Remove(dest)
+		}
+	}()
+
+	if _, err := io.Copy(destFile, srcFile); err != nil {
+		return err
+	}
+
+	ok = true
+	return nil
+}
+
 func copyConfig(srcConfigPath, destConfigPath string) error {
 	// Copy or create config
 	destFile, exists, err := openFile(destConfigPath, true)

cmd/cloudflared/shell/shell.go

@@ -1,33 +0,0 @@
-package shell
-
-import (
-	"io"
-	"os"
-	"os/exec"
-)
-
-// OpenBrowser opens the specified URL in the default browser of the user
-func OpenBrowser(url string) error {
-	return getBrowserCmd(url).Start()
-}
-
-// Run will kick off a shell task and pipe the results to the respective std pipes
-func Run(cmd string, args ...string) error {
-	c := exec.Command(cmd, args...)
-	stderr, err := c.StderrPipe()
-	if err != nil {
-		return err
-	}
-	go func() {
-		io.Copy(os.Stderr, stderr)
-	}()
-
-	stdout, err := c.StdoutPipe()
-	if err != nil {
-		return err
-	}
-	go func() {
-		io.Copy(os.Stdout, stdout)
-	}()
-	return c.Run()
-}

cmd/cloudflared/token/token.go

@@ -1,217 +0,0 @@
-package token
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"net/url"
-	"os"
-	"os/signal"
-	"syscall"
-	"time"
-
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/path"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/origin"
-	"github.com/coreos/go-oidc/jose"
-)
-
-const (
-	keyName = "token"
-)
-
-type lock struct {
-	lockFilePath string
-	backoff      *origin.BackoffHandler
-	sigHandler   *signalHandler
-}
-
-type signalHandler struct {
-	sigChannel chan os.Signal
-	signals    []os.Signal
-}
-
-type jwtPayload struct {
-	Aud   []string `json:"aud"`
-	Email string   `json:"email"`
-	Exp   int      `json:"exp"`
-	Iat   int      `json:"iat"`
-	Nbf   int      `json:"nbf"`
-	Iss   string   `json:"iss"`
-	Type  string   `json:"type"`
-	Subt  string   `json:"sub"`
-}
-
-func (p jwtPayload) isExpired() bool {
-	return int(time.Now().Unix()) > p.Exp
-}
-
-func (s *signalHandler) register(handler func()) {
-	s.sigChannel = make(chan os.Signal, 1)
-	signal.Notify(s.sigChannel, s.signals...)
-	go func(s *signalHandler) {
-		for range s.sigChannel {
-			handler()
-		}
-	}(s)
-}
-
-func (s *signalHandler) deregister() {
-	signal.Stop(s.sigChannel)
-	close(s.sigChannel)
-}
-
-func errDeleteTokenFailed(lockFilePath string) error {
-	return fmt.Errorf("failed to acquire a new Access token. Please try to delete %s", lockFilePath)
-}
-
-// newLock will get a new file lock
-func newLock(path string) *lock {
-	lockPath := path + ".lock"
-	return &lock{
-		lockFilePath: lockPath,
-		backoff:      &origin.BackoffHandler{MaxRetries: 7},
-		sigHandler: &signalHandler{
-			signals: []os.Signal{syscall.SIGINT, syscall.SIGTERM},
-		},
-	}
-}
-
-func (l *lock) Acquire() error {
-	// Intercept SIGINT and SIGTERM to release lock before exiting
-	l.sigHandler.register(func() {
-		l.deleteLockFile()
-		os.Exit(0)
-	})
-
-	// Check for a path.lock file
-	// if the lock file exists; start polling
-	// if not, create the lock file and go through the normal flow.
-	// See AUTH-1736 for the reason why we do all this
-	for isTokenLocked(l.lockFilePath) {
-		if l.backoff.Backoff(context.Background()) {
-			continue
-		}
-
-		if err := l.deleteLockFile(); err != nil {
-			return err
-		}
-	}
-
-	// Create a lock file so other processes won't also try to get the token at
-	// the same time
-	if err := ioutil.WriteFile(l.lockFilePath, []byte{}, 0600); err != nil {
-		return err
-	}
-	return nil
-}
-
-func (l *lock) deleteLockFile() error {
-	if err := os.Remove(l.lockFilePath); err != nil && !os.IsNotExist(err) {
-		return errDeleteTokenFailed(l.lockFilePath)
-	}
-	return nil
-}
-
-func (l *lock) Release() error {
-	defer l.sigHandler.deregister()
-	return l.deleteLockFile()
-}
-
-// isTokenLocked checks to see if there is another process attempting to get the token already
-func isTokenLocked(lockFilePath string) bool {
-	exists, err := config.FileExists(lockFilePath)
-	return exists && err == nil
-}
-
-// FetchTokenWithRedirect will either load a stored token or generate a new one
-// it appends a redirect URL to the access cli request if opening the browser
-func FetchTokenWithRedirect(appURL *url.URL, logger logger.Service) (string, error) {
-	return getToken(appURL, true, logger)
-}
-
-// FetchToken will either load a stored token or generate a new one
-// it doesn't append a redirect URL to the access cli request if opening the browser
-func FetchToken(appURL *url.URL, logger logger.Service) (string, error) {
-	return getToken(appURL, false, logger)
-}
-
-// getToken will either load a stored token or generate a new one
-func getToken(appURL *url.URL, shouldRedirect bool, logger logger.Service) (string, error) {
-	if token, err := GetTokenIfExists(appURL); token != "" && err == nil {
-		return token, nil
-	}
-
-	path, err := path.GenerateFilePathFromURL(appURL, keyName)
-	if err != nil {
-		return "", err
-	}
-
-	fileLock := newLock(path)
-
-	err = fileLock.Acquire()
-	if err != nil {
-		return "", err
-	}
-	defer fileLock.Release()
-
-	// check to see if another process has gotten a token while we waited for the lock
-	if token, err := GetTokenIfExists(appURL); token != "" && err == nil {
-		return token, nil
-	}
-
-	// this weird parameter is the resource name (token) and the key/value
-	// we want to send to the transfer service. the key is token and the value
-	// is blank (basically just the id generated in the transfer service)
-	token, err := transfer.Run(appURL, keyName, keyName, "", path, true, shouldRedirect, logger)
-	if err != nil {
-		return "", err
-	}
-
-	return string(token), nil
-}
-
-// GetTokenIfExists will return the token from local storage if it exists and not expired
-func GetTokenIfExists(url *url.URL) (string, error) {
-	path, err := path.GenerateFilePathFromURL(url, keyName)
-	if err != nil {
-		return "", err
-	}
-	content, err := ioutil.ReadFile(path)
-	if err != nil {
-		return "", err
-	}
-	token, err := jose.ParseJWT(string(content))
-	if err != nil {
-		return "", err
-	}
-
-	var payload jwtPayload
-	err = json.Unmarshal(token.Payload, &payload)
-	if err != nil {
-		return "", err
-	}
-
-	if payload.isExpired() {
-		err := os.Remove(path)
-		return "", err
-	}
-
-	return token.Encode(), nil
-}
-
-// RemoveTokenIfExists removes the a token from local storage if it exists
-func RemoveTokenIfExists(url *url.URL) error {
-	path, err := path.GenerateFilePathFromURL(url, keyName)
-	if err != nil {
-		return err
-	}
-	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-
-	return nil
-}

cmd/cloudflared/tunnel/config_test.go

@@ -0,0 +1,13 @@
+package tunnel
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDedup(t *testing.T) {
+	expected := []string{"a", "b"}
+	actual := dedup([]string{"a", "b", "a"})
+	require.ElementsMatch(t, expected, actual)
+}

cmd/cloudflared/tunnel/configuration.go

@@ -1,43 +1,48 @@
 package tunnel
 
 import (
-	"context"
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
-	"net"
-	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
+	"github.com/google/uuid"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/crypto/ssh/terminal"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/validation"
-
-	"github.com/google/uuid"
-	"github.com/mitchellh/go-homedir"
-	"github.com/pkg/errors"
-	"golang.org/x/crypto/ssh/terminal"
-	"gopkg.in/urfave/cli.v2"
 )
 
+const LogFieldOriginCertPath = "originCertPath"
+
 var (
 	developerPortal = "https://developers.cloudflare.com/argo-tunnel"
 	quickStartUrl   = developerPortal + "/quickstart/quickstart/"
 	serviceUrl      = developerPortal + "/reference/service/"
 	argumentsUrl    = developerPortal + "/reference/arguments/"
+
+	LogFieldHostname = "hostname"
 )
 
-// returns the first path that contains a cert.pem file. If none of the DefaultConfigDirs
+// returns the first path that contains a cert.pem file. If none of the DefaultConfigSearchDirectories
 // contains a cert.pem file, return empty string
 func findDefaultOriginCertPath() string {
-	for _, defaultConfigDir := range config.DefaultConfigDirs {
+	for _, defaultConfigDir := range config.DefaultConfigSearchDirectories() {
 		originCertPath, _ := homedir.Expand(filepath.Join(defaultConfigDir, config.DefaultCredentialFile))
 		if ok, _ := config.FileExists(originCertPath); ok {
 			return originCertPath
@@ -46,30 +51,23 @@ func findDefaultOriginCertPath() string {
 	return ""
 }
 
-func generateRandomClientID(logger logger.Service) (string, error) {
+func generateRandomClientID(log *zerolog.Logger) (string, error) {
 	u, err := uuid.NewRandom()
 	if err != nil {
-		logger.Errorf("couldn't create UUID for client ID %s", err)
+		log.Error().Msgf("couldn't create UUID for client ID %s", err)
 		return "", err
 	}
 	return u.String(), nil
 }
 
-func logClientOptions(c *cli.Context, logger logger.Service) {
+func logClientOptions(c *cli.Context, log *zerolog.Logger) {
 	flags := make(map[string]interface{})
-	for _, flag := range c.LocalFlagNames() {
+	for _, flag := range c.FlagNames() {
 		flags[flag] = c.Generic(flag)
 	}
 
-	sliceFlags := []string{"header", "tag", "proxy-dns-upstream", "upstream", "edge"}
-	for _, sliceFlag := range sliceFlags {
-		if len(c.StringSlice(sliceFlag)) > 0 {
-			flags[sliceFlag] = strings.Join(c.StringSlice(sliceFlag), ", ")
-		}
-	}
-
 	if len(flags) > 0 {
-		logger.Infof("Environment variables %v", flags)
+		log.Info().Msgf("Settings: %v", flags)
 	}
 
 	envs := make(map[string]string)
@@ -84,40 +82,39 @@ func logClientOptions(c *cli.Context, logger logger.Service) {
 		}
 	}
 	if len(envs) > 0 {
-		logger.Infof("Environmental variables %v", envs)
+		log.Info().Msgf("Environmental variables %v", envs)
 	}
 }
 
-func dnsProxyStandAlone(c *cli.Context) bool {
-	return c.IsSet("proxy-dns") && (!c.IsSet("hostname") && !c.IsSet("tag") && !c.IsSet("hello-world"))
+func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelConfig) bool {
+	return c.IsSet("proxy-dns") && (!c.IsSet("hostname") && !c.IsSet("tag") && !c.IsSet("hello-world") && namedTunnel == nil)
 }
 
-func findOriginCert(c *cli.Context, logger logger.Service) (string, error) {
-	originCertPath := c.String("origincert")
+func findOriginCert(originCertPath string, log *zerolog.Logger) (string, error) {
 	if originCertPath == "" {
-		logger.Infof("Cannot determine default origin certificate path. No file %s in %v", config.DefaultCredentialFile, config.DefaultConfigDirs)
+		log.Info().Msgf("Cannot determine default origin certificate path. No file %s in %v", config.DefaultCredentialFile, config.DefaultConfigSearchDirectories())
 		if isRunningFromTerminal() {
-			logger.Errorf("You need to specify the origin certificate path with --origincert option, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", argumentsUrl)
-			return "", fmt.Errorf("Client didn't specify origincert path when running from terminal")
+			log.Error().Msgf("You need to specify the origin certificate path with --origincert option, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", argumentsUrl)
+			return "", fmt.Errorf("client didn't specify origincert path when running from terminal")
 		} else {
-			logger.Errorf("You need to specify the origin certificate path by specifying the origincert option in the configuration file, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", serviceUrl)
-			return "", fmt.Errorf("Client didn't specify origincert path")
+			log.Error().Msgf("You need to specify the origin certificate path by specifying the origincert option in the configuration file, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", serviceUrl)
+			return "", fmt.Errorf("client didn't specify origincert path")
 		}
 	}
 	var err error
 	originCertPath, err = homedir.Expand(originCertPath)
 	if err != nil {
-		logger.Errorf("Cannot resolve path %s: %s", originCertPath, err)
-		return "", fmt.Errorf("Cannot resolve path %s", originCertPath)
+		log.Err(err).Msgf("Cannot resolve origin certificate path")
+		return "", fmt.Errorf("cannot resolve path %s", originCertPath)
 	}
 	// Check that the user has acquired a certificate using the login command
 	ok, err := config.FileExists(originCertPath)
 	if err != nil {
-		logger.Errorf("Cannot check if origin cert exists at path %s", originCertPath)
-		return "", fmt.Errorf("Cannot check if origin cert exists at path %s", originCertPath)
+		log.Error().Err(err).Msgf("Cannot check if origin cert exists at path %s", originCertPath)
+		return "", fmt.Errorf("cannot check if origin cert exists at path %s", originCertPath)
 	}
 	if !ok {
-		logger.Errorf(`Cannot find a valid certificate for your origin at the path:
+		log.Error().Msgf(`Cannot find a valid certificate for your origin at the path:
 
     %s
 
@@ -126,29 +123,26 @@ If you don't have a certificate signed by Cloudflare, run the command:
 
 	%s login
 `, originCertPath, os.Args[0])
-		return "", fmt.Errorf("Cannot find a valid certificate at the path %s", originCertPath)
+		return "", fmt.Errorf("cannot find a valid certificate at the path %s", originCertPath)
 	}
 
 	return originCertPath, nil
 }
 
-func readOriginCert(originCertPath string, logger logger.Service) ([]byte, error) {
-	logger.Debugf("Reading origin cert from %s", originCertPath)
-
+func readOriginCert(originCertPath string) ([]byte, error) {
 	// Easier to send the certificate as []byte via RPC than decoding it at this point
 	originCert, err := ioutil.ReadFile(originCertPath)
 	if err != nil {
-		logger.Errorf("Cannot read %s to load origin certificate: %s", originCertPath, err)
-		return nil, fmt.Errorf("Cannot read %s to load origin certificate", originCertPath)
+		return nil, fmt.Errorf("cannot read %s to load origin certificate", originCertPath)
 	}
 	return originCert, nil
 }
 
-func getOriginCert(c *cli.Context, logger logger.Service) ([]byte, error) {
-	if originCertPath, err := findOriginCert(c, logger); err != nil {
+func getOriginCert(originCertPath string, log *zerolog.Logger) ([]byte, error) {
+	if originCertPath, err := findOriginCert(originCertPath, log); err != nil {
 		return nil, err
 	} else {
-		return readOriginCert(originCertPath, logger)
+		return readOriginCert(originCertPath)
 	}
 }
 
@@ -156,156 +150,192 @@ func prepareTunnelConfig(
 	c *cli.Context,
 	buildInfo *buildinfo.BuildInfo,
 	version string,
-	logger logger.Service,
-	transportLogger logger.Service,
-	namedTunnel *origin.NamedTunnelConfig,
-) (*origin.TunnelConfig, error) {
-	compatibilityMode := namedTunnel == nil
+	log, logTransport *zerolog.Logger,
+	observer *connection.Observer,
+	namedTunnel *connection.NamedTunnelConfig,
+) (*origin.TunnelConfig, ingress.Ingress, error) {
+	isNamedTunnel := namedTunnel != nil
 
-	hostname, err := validation.ValidateHostname(c.String("hostname"))
+	configHostname := c.String("hostname")
+	hostname, err := validation.ValidateHostname(configHostname)
 	if err != nil {
-		logger.Errorf("Invalid hostname: %s", err)
-		return nil, errors.Wrap(err, "Invalid hostname")
+		log.Err(err).Str(LogFieldHostname, configHostname).Msg("Invalid hostname")
+		return nil, ingress.Ingress{}, errors.Wrap(err, "Invalid hostname")
 	}
-	isFreeTunnel := hostname == ""
 	clientID := c.String("id")
 	if !c.IsSet("id") {
-		clientID, err = generateRandomClientID(logger)
+		clientID, err = generateRandomClientID(log)
 		if err != nil {
-			return nil, err
+			return nil, ingress.Ingress{}, err
 		}
 	}
 
 	tags, err := NewTagSliceFromCLI(c.StringSlice("tag"))
 	if err != nil {
-		logger.Errorf("Tag parse failure: %s", err)
-		return nil, errors.Wrap(err, "Tag parse failure")
+		log.Err(err).Msg("Tag parse failure")
+		return nil, ingress.Ingress{}, errors.Wrap(err, "Tag parse failure")
 	}
 
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
 
-	originURL, err := config.ValidateUrl(c, compatibilityMode)
-	if err != nil {
-		logger.Errorf("Error validating origin URL: %s", err)
-		return nil, errors.Wrap(err, "Error validating origin URL")
-	}
-
-	var originCert []byte
-	if !isFreeTunnel {
-		originCert, err = getOriginCert(c, logger)
-		if err != nil {
-			return nil, errors.Wrap(err, "Error getting origin cert")
-		}
-	}
-
-	originCertPool, err := tlsconfig.LoadOriginCA(c, logger)
-	if err != nil {
-		logger.Errorf("Error loading cert pool: %s", err)
-		return nil, errors.Wrap(err, "Error loading cert pool")
-	}
-
-	tunnelMetrics := origin.NewTunnelMetrics()
-	httpTransport := &http.Transport{
-		Proxy:                 http.ProxyFromEnvironment,
-		MaxIdleConns:          c.Int("proxy-keepalive-connections"),
-		MaxIdleConnsPerHost:   c.Int("proxy-keepalive-connections"),
-		IdleConnTimeout:       c.Duration("proxy-keepalive-timeout"),
-		TLSHandshakeTimeout:   c.Duration("proxy-tls-timeout"),
-		ExpectContinueTimeout: 1 * time.Second,
-		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: c.IsSet("no-tls-verify")},
-	}
-
-	dialer := &net.Dialer{
-		Timeout:   c.Duration("proxy-connect-timeout"),
-		KeepAlive: c.Duration("proxy-tcp-keepalive"),
-	}
-	if c.Bool("proxy-no-happy-eyeballs") {
-		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
-	}
-	dialContext := dialer.DialContext
-
-	if c.IsSet("unix-socket") {
-		unixSocket, err := config.ValidateUnixSocket(c)
-		if err != nil {
-			logger.Errorf("Error validating --unix-socket: %s", err)
-			return nil, errors.Wrap(err, "Error validating --unix-socket")
-		}
-
-		logger.Infof("Proxying tunnel requests to unix:%s", unixSocket)
-		httpTransport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
-			// if --unix-socket specified, enforce network type "unix"
-			return dialContext(ctx, "unix", unixSocket)
-		}
-	} else {
-		logger.Infof("Proxying tunnel requests to %s", originURL)
-		httpTransport.DialContext = dialContext
-	}
-
-	if !c.IsSet("hello-world") && c.IsSet("origin-server-name") {
-		httpTransport.TLSClientConfig.ServerName = c.String("origin-server-name")
-	}
-	// If tunnel running in bastion mode, a connection to origin will not exist until initiated by the client.
-	if !c.IsSet(bastionFlag) {
-		if err = validation.ValidateHTTPService(originURL, hostname, httpTransport); err != nil {
-			logger.Errorf("unable to connect to the origin: %s", err)
-		}
-	}
-
-	toEdgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c)
-	if err != nil {
-		logger.Errorf("unable to create TLS config to connect with edge: %s", err)
-		return nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
-	}
-
-	if namedTunnel != nil {
+	var (
+		ingressRules  ingress.Ingress
+		classicTunnel *connection.ClassicTunnelConfig
+	)
+	cfg := config.GetConfiguration()
+	if isNamedTunnel {
 		clientUUID, err := uuid.NewRandom()
 		if err != nil {
-			return nil, errors.Wrap(err, "can't generate clientUUID")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate connector UUID")
 		}
+		log.Info().Msgf("Generated Connector ID: %s", clientUUID)
+		features := append(c.StringSlice("features"), origin.FeatureSerializedHeaders)
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],
-			Features: []string{origin.FeatureSerializedHeaders},
+			Features: dedup(features),
 			Version:  version,
-			Arch:     fmt.Sprintf("%s_%s", buildInfo.GoOS, buildInfo.GoArch),
+			Arch:     buildInfo.OSArch(),
 		}
+		ingressRules, err = ingress.ParseIngress(cfg)
+		if err != nil && err != ingress.ErrNoIngressRules {
+			return nil, ingress.Ingress{}, err
+		}
+		if !ingressRules.IsEmpty() && c.IsSet("url") {
+			return nil, ingress.Ingress{}, ingress.ErrURLIncompatibleWithIngress
+		}
+	} else {
+
+		originCertPath := c.String("origincert")
+		originCertLog := log.With().
+			Str(LogFieldOriginCertPath, originCertPath).
+			Logger()
+
+		originCert, err := getOriginCert(originCertPath, &originCertLog)
+		if err != nil {
+			return nil, ingress.Ingress{}, errors.Wrap(err, "Error getting origin cert")
+		}
+
+		classicTunnel = &connection.ClassicTunnelConfig{
+			Hostname:   hostname,
+			OriginCert: originCert,
+			// turn off use of reconnect token and auth refresh when using named tunnels
+			UseReconnectToken: !isNamedTunnel && c.Bool("use-reconnect-token"),
+		}
+	}
+
+	// Convert single-origin configuration into multi-origin configuration.
+	if ingressRules.IsEmpty() {
+		ingressRules, err = ingress.NewSingleOrigin(c, !isNamedTunnel)
+		if err != nil {
+			return nil, ingress.Ingress{}, err
+		}
+	}
+
+	var warpRoutingService *ingress.WarpRoutingService
+	warpRoutingEnabled := isWarpRoutingEnabled(cfg.WarpRouting, isNamedTunnel)
+	if warpRoutingEnabled {
+		warpRoutingService = ingress.NewWarpRoutingService()
+		log.Info().Msgf("Warp-routing is enabled")
+	}
+
+	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), warpRoutingEnabled, namedTunnel, edgediscovery.HTTP2Percentage, origin.ResolveTTL, log)
+	if err != nil {
+		return nil, ingress.Ingress{}, err
+	}
+	log.Info().Msgf("Initial protocol %s", protocolSelector.Current())
+
+	edgeTLSConfigs := make(map[connection.Protocol]*tls.Config, len(connection.ProtocolList))
+	for _, p := range connection.ProtocolList {
+		tlsSettings := p.TLSSettings()
+		if tlsSettings == nil {
+			return nil, ingress.Ingress{}, fmt.Errorf("%s has unknown TLS settings", p)
+		}
+		edgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c, tlsSettings.ServerName)
+		if err != nil {
+			return nil, ingress.Ingress{}, errors.Wrap(err, "unable to create TLS config to connect with edge")
+		}
+		if len(tlsSettings.NextProtos) > 0 {
+			edgeTLSConfig.NextProtos = tlsSettings.NextProtos
+		}
+		edgeTLSConfigs[p] = edgeTLSConfig
+	}
+
+	originProxy := origin.NewOriginProxy(ingressRules, warpRoutingService, tags, log)
+	gracePeriod, err := gracePeriod(c)
+	if err != nil {
+		return nil, ingress.Ingress{}, err
+	}
+	connectionConfig := &connection.Config{
+		OriginProxy:     originProxy,
+		GracePeriod:     gracePeriod,
+		ReplaceExisting: c.Bool("force"),
+	}
+	muxerConfig := &connection.MuxerConfig{
+		HeartbeatInterval: c.Duration("heartbeat-interval"),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		MaxHeartbeats: uint64(c.Int("heartbeat-count")),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		CompressionSetting: h2mux.CompressionSetting(uint64(c.Int("compression-quality"))),
+		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
 	}
 
 	return &origin.TunnelConfig{
-		BuildInfo:          buildInfo,
-		ClientID:           clientID,
-		ClientTlsConfig:    httpTransport.TLSClientConfig,
-		CompressionQuality: c.Uint64("compression-quality"),
-		EdgeAddrs:          c.StringSlice("edge"),
-		GracePeriod:        c.Duration("grace-period"),
-		HAConnections:      c.Int("ha-connections"),
-		HTTPTransport:      httpTransport,
-		HeartbeatInterval:  c.Duration("heartbeat-interval"),
-		Hostname:           hostname,
-		HTTPHostHeader:     c.String("http-host-header"),
-		IncidentLookup:     origin.NewIncidentLookup(),
-		IsAutoupdated:      c.Bool("is-autoupdated"),
-		IsFreeTunnel:       isFreeTunnel,
-		LBPool:             c.String("lb-pool"),
-		Logger:             logger,
-		TransportLogger:    transportLogger,
-		MaxHeartbeats:      c.Uint64("heartbeat-count"),
-		Metrics:            tunnelMetrics,
-		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
-		NoChunkedEncoding:  c.Bool("no-chunked-encoding"),
-		OriginCert:         originCert,
-		OriginUrl:          originURL,
-		ReportedVersion:    version,
-		Retries:            c.Uint("retries"),
-		RunFromTerminal:    isRunningFromTerminal(),
-		Tags:               tags,
-		TlsConfig:          toEdgeTLSConfig,
-		NamedTunnel:        namedTunnel,
-		ReplaceExisting:    c.Bool("force"),
-		// turn off use of reconnect token and auth refresh when using named tunnels
-		UseReconnectToken:  compatibilityMode && c.Bool("use-reconnect-token"),
-	}, nil
+		ConnectionConfig: connectionConfig,
+		OSArch:           buildInfo.OSArch(),
+		ClientID:         clientID,
+		EdgeAddrs:        c.StringSlice("edge"),
+		Region:           c.String("region"),
+		HAConnections:    c.Int("ha-connections"),
+		IncidentLookup:   origin.NewIncidentLookup(),
+		IsAutoupdated:    c.Bool("is-autoupdated"),
+		LBPool:           c.String("lb-pool"),
+		Tags:             tags,
+		Log:              log,
+		LogTransport:     logTransport,
+		Observer:         observer,
+		ReportedVersion:  version,
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		Retries:          uint(c.Int("retries")),
+		RunFromTerminal:  isRunningFromTerminal(),
+		NamedTunnel:      namedTunnel,
+		ClassicTunnel:    classicTunnel,
+		MuxerConfig:      muxerConfig,
+		ProtocolSelector: protocolSelector,
+		EdgeTLSConfigs:   edgeTLSConfigs,
+	}, ingressRules, nil
+}
+
+func gracePeriod(c *cli.Context) (time.Duration, error) {
+	period := c.Duration("grace-period")
+	if period > connection.MaxGracePeriod {
+		return time.Duration(0), fmt.Errorf("grace-period must be equal or less than %v", connection.MaxGracePeriod)
+	}
+	return period, nil
+}
+
+func isWarpRoutingEnabled(warpConfig config.WarpRoutingConfig, isNamedTunnel bool) bool {
+	return warpConfig.Enabled && isNamedTunnel
 }
 
 func isRunningFromTerminal() bool {
 	return terminal.IsTerminal(int(os.Stdout.Fd()))
 }
+
+// Remove any duplicates from the slice
+func dedup(slice []string) []string {
+
+	// Convert the slice into a set
+	set := make(map[string]bool, 0)
+	for _, str := range slice {
+		set[str] = true
+	}
+
+	// Convert the set back into a slice
+	keys := make([]string, len(set))
+	i := 0
+	for str := range set {
+		keys[i] = str
+		i++
+	}
+	return keys
+}

cmd/cloudflared/tunnel/credential_finder.go

@@ -0,0 +1,83 @@
+package tunnel
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/cloudflare/cloudflared/config"
+
+	"github.com/google/uuid"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+)
+
+// CredFinder can find the tunnel credentials file.
+type CredFinder interface {
+	Path() (string, error)
+}
+
+// Implements CredFinder and looks for the credentials file at the given
+// filepath.
+type staticPath struct {
+	filePath string
+	fs       fileSystem
+}
+
+func newStaticPath(filePath string, fs fileSystem) CredFinder {
+	return staticPath{
+		filePath: filePath,
+		fs:       fs,
+	}
+}
+
+func (a staticPath) Path() (string, error) {
+	if a.filePath != "" && a.fs.validFilePath(a.filePath) {
+		return a.filePath, nil
+	}
+	return "", fmt.Errorf("Tunnel credentials file '%s' doesn't exist or is not a file", a.filePath)
+}
+
+// Implements CredFinder and looks for the credentials file in several directories
+// searching for a file named <id>.json
+type searchByID struct {
+	id  uuid.UUID
+	c   *cli.Context
+	log *zerolog.Logger
+	fs  fileSystem
+}
+
+func newSearchByID(id uuid.UUID, c *cli.Context, log *zerolog.Logger, fs fileSystem) CredFinder {
+	return searchByID{
+		id:  id,
+		c:   c,
+		log: log,
+		fs:  fs,
+	}
+}
+
+func (s searchByID) Path() (string, error) {
+	originCertPath := s.c.String("origincert")
+	originCertLog := s.log.With().
+		Str(LogFieldOriginCertPath, originCertPath).
+		Logger()
+
+	// Fallback to look for tunnel credentials in the origin cert directory
+	if originCertPath, err := findOriginCert(originCertPath, &originCertLog); err == nil {
+		originCertDir := filepath.Dir(originCertPath)
+		if filePath, err := tunnelFilePath(s.id, originCertDir); err == nil {
+			if s.fs.validFilePath(filePath) {
+				return filePath, nil
+			}
+		}
+	}
+
+	// Last resort look under default config directories
+	for _, configDir := range config.DefaultConfigSearchDirectories() {
+		if filePath, err := tunnelFilePath(s.id, configDir); err == nil {
+			if s.fs.validFilePath(filePath) {
+				return filePath, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("tunnel credentials file not found")
+}

cmd/cloudflared/tunnel/filesystem.go

@@ -0,0 +1,27 @@
+package tunnel
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+// Abstract away details of reading files, so that SubcommandContext can read
+// from either the real filesystem, or a mock (when running unit tests).
+type fileSystem interface {
+	readFile(filePath string) ([]byte, error)
+	validFilePath(path string) bool
+}
+
+type realFileSystem struct{}
+
+func (fs realFileSystem) validFilePath(path string) bool {
+	fileStat, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+	return !fileStat.IsDir()
+}
+
+func (fs realFileSystem) readFile(filePath string) ([]byte, error) {
+	return ioutil.ReadFile(filePath)
+}

cmd/cloudflared/tunnel/info.go

@@ -0,0 +1,16 @@
+package tunnel
+
+import (
+	"time"
+
+	"github.com/google/uuid"
+
+	"github.com/cloudflare/cloudflared/tunnelstore"
+)
+
+type Info struct {
+	ID         uuid.UUID                   `json:"id"`
+	Name       string                      `json:"name"`
+	CreatedAt  time.Time                   `json:"createdAt"`
+	Connectors []*tunnelstore.ActiveClient `json:"conns"`
+}

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -0,0 +1,119 @@
+package tunnel
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ingress"
+
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+)
+
+func buildIngressSubcommand() *cli.Command {
+	return &cli.Command{
+		Name:      "ingress",
+		Category:  "Tunnel",
+		Usage:     "Validate and test cloudflared tunnel's ingress configuration",
+		UsageText: "cloudflared tunnel [--config FILEPATH] ingress COMMAND [arguments...]",
+		Hidden:    true,
+		Description: ` Cloudflared lets you route traffic from the internet to multiple different addresses on your
+		origin. Multiple-origin routing is configured by a set of rules. Each rule matches traffic
+		by its hostname or path, and routes it to an address. These rules are configured under the
+		'ingress' key of your config.yaml, for example:
+
+		ingress:
+		  - hostname: www.example.com
+		    service: https://localhost:8000
+		  - hostname: *.example.xyz
+		    path: /[a-zA-Z]+.html
+		    service: https://localhost:8001
+		  - hostname: *
+		    service: https://localhost:8002
+
+		To ensure cloudflared can route all incoming requests, the last rule must be a catch-all
+		rule that matches all traffic. You can validate these rules with the 'ingress validate'
+		command, and test which rule matches a particular URL with 'ingress rule <URL>'.
+
+		Multiple-origin routing is incompatible with the --url flag.`,
+		Subcommands: []*cli.Command{buildValidateIngressCommand(), buildTestURLCommand()},
+	}
+}
+
+func buildValidateIngressCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "validate",
+		Action:      cliutil.ConfiguredActionWithWarnings(validateIngressCommand),
+		Usage:       "Validate the ingress configuration ",
+		UsageText:   "cloudflared tunnel [--config FILEPATH] ingress validate",
+		Description: "Validates the configuration file, ensuring your ingress rules are OK.",
+	}
+}
+
+func buildTestURLCommand() *cli.Command {
+	return &cli.Command{
+		Name:      "rule",
+		Action:    cliutil.ConfiguredAction(testURLCommand),
+		Usage:     "Check which ingress rule matches a given request URL",
+		UsageText: "cloudflared tunnel [--config FILEPATH] ingress rule URL",
+		ArgsUsage: "URL",
+		Description: "Check which ingress rule matches a given request URL. " +
+			"Ingress rules match a request's hostname and path. Hostname is " +
+			"optional and is either a full hostname like `www.example.com` or a " +
+			"hostname with a `*` for its subdomains, e.g. `*.example.com`. Path " +
+			"is optional and matches a regular expression, like `/[a-zA-Z0-9_]+.html`",
+	}
+}
+
+// validateIngressCommand check the syntax of the ingress rules in the cloudflared config file
+func validateIngressCommand(c *cli.Context, warnings string) error {
+	conf := config.GetConfiguration()
+	if conf.Source() == "" {
+		fmt.Println("No configuration file was found. Please create one, or use the --config flag to specify its filepath. You can use the help command to learn more about configuration files")
+		return nil
+	}
+	fmt.Println("Validating rules from", conf.Source())
+	if _, err := ingress.ParseIngress(conf); err != nil {
+		return errors.Wrap(err, "Validation failed")
+	}
+	if c.IsSet("url") {
+		return ingress.ErrURLIncompatibleWithIngress
+	}
+	if warnings != "" {
+		fmt.Println("Warning: unused keys detected in your config file. Here is a list of unused keys:")
+		fmt.Println(warnings)
+		return nil
+	}
+	fmt.Println("OK")
+	return nil
+}
+
+// testURLCommand checks which ingress rule matches the given URL.
+func testURLCommand(c *cli.Context) error {
+	requestArg := c.Args().First()
+	if requestArg == "" {
+		return errors.New("cloudflared tunnel rule expects a single argument, the URL to test")
+	}
+
+	requestURL, err := url.Parse(requestArg)
+	if err != nil {
+		return fmt.Errorf("%s is not a valid URL", requestArg)
+	}
+	if requestURL.Hostname() == "" && requestURL.Scheme == "" {
+		return fmt.Errorf("%s doesn't have a hostname, consider adding a scheme", requestArg)
+	}
+
+	conf := config.GetConfiguration()
+	fmt.Println("Using rules from", conf.Source())
+	ing, err := ingress.ParseIngress(conf)
+	if err != nil {
+		return errors.Wrap(err, "Validation failed")
+	}
+
+	_, i := ing.FindMatchingRule(requestURL.Hostname(), requestURL.Path)
+	fmt.Printf("Matched rule #%d\n", i+1)
+	fmt.Println(ing.Rules[i].MultiLineString())
+	return nil
+}

cmd/cloudflared/tunnel/login.go

@@ -2,29 +2,39 @@ package tunnel
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"path/filepath"
 	"syscall"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
-	"github.com/cloudflare/cloudflared/logger"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
-	cli "gopkg.in/urfave/cli.v2"
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/token"
 )
 
 const (
 	baseLoginURL     = "https://dash.cloudflare.com/argotunnel"
-	callbackStoreURL = "https://login.argotunnel.com/"
+	callbackStoreURL = "https://login.cloudflareaccess.org/"
 )
 
-func login(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
+func buildLoginSubcommand(hidden bool) *cli.Command {
+	return &cli.Command{
+		Name:      "login",
+		Action:    cliutil.ConfiguredAction(login),
+		Usage:     "Generate a configuration file with your login details",
+		ArgsUsage: " ",
+		Hidden:    hidden,
 	}
+}
+
+func login(c *cli.Context) error {
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	path, ok, err := checkForExistingCert()
 	if ok {
@@ -40,18 +50,30 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	_, err = transfer.Run(loginURL, "cert", "callback", callbackStoreURL, path, false, true, logger)
+	resourceData, err := token.RunTransfer(
+		loginURL,
+		"cert",
+		"callback",
+		callbackStoreURL,
+		false,
+		false,
+		log,
+	)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Failed to write the certificate due to the following error:\n%v\n\nYour browser will download the certificate instead. You will have to manually\ncopy it to the following path:\n\n%s\n", err, path)
 		return err
 	}
 
+	if err := ioutil.WriteFile(path, resourceData, 0600); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("error writing cert to %s", path))
+	}
+
 	fmt.Fprintf(os.Stdout, "You have successfully logged in.\nIf you wish to copy your credentials to a server, they have been saved to:\n%s\n", path)
 	return nil
 }
 
 func checkForExistingCert() (string, bool, error) {
-	configPath, err := homedir.Expand(config.DefaultConfigDirs[0])
+	configPath, err := homedir.Expand(config.DefaultConfigSearchDirectories()[0])
 	if err != nil {
 		return "", false, err
 	}

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -0,0 +1,122 @@
+package tunnel
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/connection"
+)
+
+const httpTimeout = 15 * time.Second
+
+const disclaimer = "Thank you for trying Cloudflare Tunnel. Doing so, without a Cloudflare account, is a quick way to" +
+	" experiment and try it out. However, be aware that these account-less Tunnels have no uptime guarantee. If you " +
+	"intend to use Tunnels in production you should use a pre-created named tunnel by following: " +
+	"https://developers.cloudflare.com/cloudflare-one/connections/connect-apps"
+
+// RunQuickTunnel requests a tunnel from the specified service.
+// We use this to power quick tunnels on trycloudflare.com, but the
+// service is open-source and could be used by anyone.
+func RunQuickTunnel(sc *subcommandContext) error {
+	sc.log.Info().Msg(disclaimer)
+	sc.log.Info().Msg("Requesting new quick Tunnel on trycloudflare.com...")
+
+	client := http.Client{
+		Transport: &http.Transport{
+			TLSHandshakeTimeout:   httpTimeout,
+			ResponseHeaderTimeout: httpTimeout,
+		},
+		Timeout: httpTimeout,
+	}
+
+	resp, err := client.Post(fmt.Sprintf("%s/tunnel", sc.c.String("quick-service")), "application/json", nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to request quick Tunnel")
+	}
+	defer resp.Body.Close()
+
+	var data QuickTunnelResponse
+	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
+		return errors.Wrap(err, "failed to unmarshal quick Tunnel")
+	}
+
+	tunnelID, err := uuid.Parse(data.Result.ID)
+	if err != nil {
+		return errors.Wrap(err, "failed to parse quick Tunnel ID")
+	}
+
+	credentials := connection.Credentials{
+		AccountTag:   data.Result.AccountTag,
+		TunnelSecret: data.Result.Secret,
+		TunnelID:     tunnelID,
+		TunnelName:   data.Result.Name,
+	}
+
+	url := data.Result.Hostname
+	if !strings.HasPrefix(url, "https://") {
+		url = "https://" + url
+	}
+
+	for _, line := range AsciiBox([]string{
+		"Your quick Tunnel has been created! Visit it at (it may take some time to be reachable):",
+		url,
+	}, 2) {
+		sc.log.Info().Msg(line)
+	}
+
+	return StartServer(
+		sc.c,
+		version,
+		&connection.NamedTunnelConfig{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
+		sc.log,
+		sc.isUIEnabled,
+	)
+}
+
+type QuickTunnelResponse struct {
+	Success bool
+	Result  QuickTunnel
+	Errors  []QuickTunnelError
+}
+
+type QuickTunnelError struct {
+	Code    int
+	Message string
+}
+
+type QuickTunnel struct {
+	ID         string `json:"id"`
+	Name       string `json:"name"`
+	Hostname   string `json:"hostname"`
+	AccountTag string `json:"account_tag"`
+	Secret     []byte `json:"secret"`
+}
+
+// Print out the given lines in a nice ASCII box.
+func AsciiBox(lines []string, padding int) (box []string) {
+	maxLen := maxLen(lines)
+	spacer := strings.Repeat(" ", padding)
+	border := "+" + strings.Repeat("-", maxLen+(padding*2)) + "+"
+	box = append(box, border)
+	for _, line := range lines {
+		box = append(box, "|"+spacer+line+strings.Repeat(" ", maxLen-len(line))+spacer+"|")
+	}
+	box = append(box, border)
+	return
+}
+
+func maxLen(lines []string) int {
+	max := 0
+	for _, line := range lines {
+		if len(line) > max {
+			max = len(line)
+		}
+	}
+	return max
+}

cmd/cloudflared/tunnel/server.go

@@ -1,20 +1,25 @@
 package tunnel
 
 import (
-	"github.com/cloudflare/cloudflared/logger"
+	"fmt"
+
 	"github.com/cloudflare/cloudflared/tunneldns"
 
-	"gopkg.in/urfave/cli.v2"
-
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
 )
 
-func runDNSProxyServer(c *cli.Context, dnsReadySignal, shutdownC chan struct{}, logger logger.Service) error {
+func runDNSProxyServer(c *cli.Context, dnsReadySignal chan struct{}, shutdownC <-chan struct{}, log *zerolog.Logger) error {
 	port := c.Int("proxy-dns-port")
 	if port <= 0 || port > 65535 {
 		return errors.New("The 'proxy-dns-port' must be a valid port number in <1, 65535> range.")
 	}
-	listener, err := tunneldns.CreateListener(c.String("proxy-dns-address"), uint16(port), c.StringSlice("proxy-dns-upstream"), c.StringSlice("proxy-dns-bootstrap"), logger)
+	maxUpstreamConnections := c.Int("proxy-dns-max-upstream-conns")
+	if maxUpstreamConnections < 0 {
+		return fmt.Errorf("'%s' must be 0 or higher", "proxy-dns-max-upstream-conns")
+	}
+	listener, err := tunneldns.CreateListener(c.String("proxy-dns-address"), uint16(port), c.StringSlice("proxy-dns-upstream"), c.StringSlice("proxy-dns-bootstrap"), maxUpstreamConnections, log)
 	if err != nil {
 		close(dnsReadySignal)
 		listener.Stop()
@@ -26,6 +31,7 @@ func runDNSProxyServer(c *cli.Context, dnsReadySignal, shutdownC chan struct{},
 		return errors.Wrap(err, "Cannot start the DNS over HTTPS proxy server")
 	}
 	<-shutdownC
-	listener.Stop()
+	_ = listener.Stop()
+	log.Info().Msg("DNS server stopped")
 	return nil
 }

cmd/cloudflared/tunnel/signal.go

@@ -4,83 +4,20 @@ import (
 	"os"
 	"os/signal"
 	"syscall"
-	"time"
 
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/rs/zerolog"
 )
 
-// waitForSignal notifies all routines to shutdownC immediately by closing the
-// shutdownC when one of the routines in main exits, or when this process receives
-// SIGTERM/SIGINT
-func waitForSignal(errC chan error, shutdownC chan struct{}, logger logger.Service) error {
+// waitForSignal closes graceShutdownC to indicate that we should start graceful shutdown sequence
+func waitForSignal(graceShutdownC chan struct{}, logger *zerolog.Logger) {
 	signals := make(chan os.Signal, 10)
 	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
 	defer signal.Stop(signals)
 
 	select {
-	case err := <-errC:
-		logger.Infof("terminating due to error: %v", err)
-		close(shutdownC)
-		return err
 	case s := <-signals:
-		logger.Infof("terminating due to signal %s", s)
-		close(shutdownC)
-	case <-shutdownC:
-	}
-	return nil
-}
-
-// waitForSignalWithGraceShutdown notifies all routines to shutdown immediately
-// by closing the shutdownC when one of the routines in main exits.
-// When this process recieves SIGTERM/SIGINT, it closes the graceShutdownC to
-// notify certain routines to start graceful shutdown. When grace period is over,
-// or when some routine exits, it notifies the rest of the routines to shutdown
-// immediately by closing shutdownC.
-// In the case of handling commands from Windows Service Manager, closing graceShutdownC
-// initiate graceful shutdown.
-func waitForSignalWithGraceShutdown(errC chan error,
-	shutdownC, graceShutdownC chan struct{},
-	gracePeriod time.Duration,
-	logger logger.Service,
-) error {
-	signals := make(chan os.Signal, 10)
-	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
-	defer signal.Stop(signals)
-
-	select {
-	case err := <-errC:
-		logger.Infof("Initiating graceful shutdown due to %v ...", err)
+		logger.Info().Msgf("Initiating graceful shutdown due to signal %s ...", s)
 		close(graceShutdownC)
-		close(shutdownC)
-		return err
-	case s := <-signals:
-		logger.Infof("Initiating graceful shutdown due to signal %s ...", s)
-		close(graceShutdownC)
-		waitForGracePeriod(signals, errC, shutdownC, gracePeriod, logger)
 	case <-graceShutdownC:
-		waitForGracePeriod(signals, errC, shutdownC, gracePeriod, logger)
-	case <-shutdownC:
-		close(graceShutdownC)
 	}
-
-	return nil
-}
-
-func waitForGracePeriod(signals chan os.Signal,
-	errC chan error,
-	shutdownC chan struct{},
-	gracePeriod time.Duration,
-	logger logger.Service,
-) {
-	// Unregister signal handler early, so the client can send a second SIGTERM/SIGINT
-	// to force shutdown cloudflared
-	signal.Stop(signals)
-	graceTimerTick := time.Tick(gracePeriod)
-	// send close signal via shutdownC when grace period expires or when an
-	// error is encountered.
-	select {
-	case <-graceTimerTick:
-	case <-errC:
-	}
-	close(shutdownC)
 }

cmd/cloudflared/tunnel/signal_test.go

@@ -1,12 +1,15 @@
+// +build !windows
+
 package tunnel
 
 import (
 	"fmt"
+	"sync"
 	"syscall"
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -18,140 +21,85 @@ var (
 	graceShutdownErr = fmt.Errorf("receive grace shutdown")
 )
 
-func testChannelClosed(t *testing.T, c chan struct{}) {
+func channelClosed(c chan struct{}) bool {
 	select {
 	case <-c:
-		return
+		return true
 	default:
-		t.Fatal("Channel should be closed")
+		return false
 	}
 }
 
-func TestWaitForSignal(t *testing.T) {
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
-
-	// Test handling server error
-	errC := make(chan error)
-	shutdownC := make(chan struct{})
-
-	go func() {
-		errC <- serverErr
-	}()
-
-	// received error, shutdownC should be closed
-	err := waitForSignal(errC, shutdownC, logger)
-	assert.Equal(t, serverErr, err)
-	testChannelClosed(t, shutdownC)
+func TestSignalShutdown(t *testing.T) {
+	log := zerolog.Nop()
 
 	// Test handling SIGTERM & SIGINT
 	for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} {
-		errC = make(chan error)
-		shutdownC = make(chan struct{})
-
-		go func(shutdownC chan struct{}) {
-			<-shutdownC
-			errC <- shutdownErr
-		}(shutdownC)
+		graceShutdownC := make(chan struct{})
 
 		go func(sig syscall.Signal) {
 			// sleep for a tick to prevent sending signal before calling waitForSignal
 			time.Sleep(tick)
-			syscall.Kill(syscall.Getpid(), sig)
+			_ = syscall.Kill(syscall.Getpid(), sig)
 		}(sig)
 
-		err = waitForSignal(errC, shutdownC, logger)
-		assert.Equal(t, nil, err)
-		assert.Equal(t, shutdownErr, <-errC)
-		testChannelClosed(t, shutdownC)
+		time.AfterFunc(time.Second, func() {
+			select {
+			case <-graceShutdownC:
+			default:
+				close(graceShutdownC)
+				t.Fatal("waitForSignal timed out")
+			}
+		})
+
+		waitForSignal(graceShutdownC, &log)
+		assert.True(t, channelClosed(graceShutdownC))
 	}
 }
 
-func TestWaitForSignalWithGraceShutdown(t *testing.T) {
-	// Test server returning error
-	errC := make(chan error)
-	shutdownC := make(chan struct{})
-	graceshutdownC := make(chan struct{})
+func TestWaitForShutdown(t *testing.T) {
+	log := zerolog.Nop()
 
+	errC := make(chan error)
+	graceShutdownC := make(chan struct{})
+	const gracePeriod = 5 * time.Second
+
+	contextCancelled := false
+	cancel := func() {
+		contextCancelled = true
+	}
+	var wg sync.WaitGroup
+
+	// on, error stop immediately
+	contextCancelled = false
+	startTime := time.Now()
 	go func() {
 		errC <- serverErr
 	}()
-
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
-
-	// received error, both shutdownC and graceshutdownC should be closed
-	err := waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
+	err := waitToShutdown(&wg, cancel, errC, graceShutdownC, gracePeriod, &log)
 	assert.Equal(t, serverErr, err)
-	testChannelClosed(t, shutdownC)
-	testChannelClosed(t, graceshutdownC)
+	assert.True(t, contextCancelled)
+	assert.False(t, channelClosed(graceShutdownC))
+	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 
-	// shutdownC closed, graceshutdownC should also be closed and no error
-	errC = make(chan error)
-	shutdownC = make(chan struct{})
-	graceshutdownC = make(chan struct{})
-	close(shutdownC)
-	err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
-	assert.NoError(t, err)
-	testChannelClosed(t, shutdownC)
-	testChannelClosed(t, graceshutdownC)
+	// on graceful shutdown, ignore error but stop as soon as an error arrives
+	contextCancelled = false
+	startTime = time.Now()
+	go func() {
+		close(graceShutdownC)
+		time.Sleep(tick)
+		errC <- serverErr
+	}()
+	err = waitToShutdown(&wg, cancel, errC, graceShutdownC, gracePeriod, &log)
+	assert.Nil(t, err)
+	assert.True(t, contextCancelled)
+	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 
-	// graceshutdownC closed, shutdownC should also be closed and no error
-	errC = make(chan error)
-	shutdownC = make(chan struct{})
-	graceshutdownC = make(chan struct{})
-	close(graceshutdownC)
-	err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
-	assert.NoError(t, err)
-	testChannelClosed(t, shutdownC)
-	testChannelClosed(t, graceshutdownC)
-
-	// Test handling SIGTERM & SIGINT
-	for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} {
-		errC := make(chan error)
-		shutdownC = make(chan struct{})
-		graceshutdownC = make(chan struct{})
-
-		go func(shutdownC, graceshutdownC chan struct{}) {
-			<-graceshutdownC
-			<-shutdownC
-			errC <- graceShutdownErr
-		}(shutdownC, graceshutdownC)
-
-		go func(sig syscall.Signal) {
-			// sleep for a tick to prevent sending signal before calling waitForSignalWithGraceShutdown
-			time.Sleep(tick)
-			syscall.Kill(syscall.Getpid(), sig)
-		}(sig)
-
-		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
-		assert.Equal(t, nil, err)
-		assert.Equal(t, graceShutdownErr, <-errC)
-		testChannelClosed(t, shutdownC)
-		testChannelClosed(t, graceshutdownC)
-	}
-
-	// Test handling SIGTERM & SIGINT, server send error before end of grace period
-	for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGINT} {
-		errC := make(chan error)
-		shutdownC = make(chan struct{})
-		graceshutdownC = make(chan struct{})
-
-		go func(shutdownC, graceshutdownC chan struct{}) {
-			<-graceshutdownC
-			errC <- graceShutdownErr
-			<-shutdownC
-			errC <- shutdownErr
-		}(shutdownC, graceshutdownC)
-
-		go func(sig syscall.Signal) {
-			// sleep for a tick to prevent sending signal before calling waitForSignalWithGraceShutdown
-			time.Sleep(tick)
-			syscall.Kill(syscall.Getpid(), sig)
-		}(sig)
-
-		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
-		assert.Equal(t, nil, err)
-		assert.Equal(t, shutdownErr, <-errC)
-		testChannelClosed(t, shutdownC)
-		testChannelClosed(t, graceshutdownC)
-	}
+	// with graceShutdownC closed stop right away without grace period
+	contextCancelled = false
+	startTime = time.Now()
+	err = waitToShutdown(&wg, cancel, errC, graceShutdownC, 0, &log)
+	assert.Nil(t, err)
+	assert.True(t, contextCancelled)
+	assert.True(t, time.Now().Sub(startTime) < time.Second) // check that wait ended early
 }

cmd/cloudflared/tunnel/subcommand_context.go

@@ -0,0 +1,424 @@
+package tunnel
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/certutil"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunnelstore"
+)
+
+type errInvalidJSONCredential struct {
+	err  error
+	path string
+}
+
+func (e errInvalidJSONCredential) Error() string {
+	return "Invalid JSON when parsing tunnel credentials file"
+}
+
+// subcommandContext carries structs shared between subcommands, to reduce number of arguments needed to
+// pass between subcommands, and make sure they are only initialized once
+type subcommandContext struct {
+	c           *cli.Context
+	log         *zerolog.Logger
+	isUIEnabled bool
+	fs          fileSystem
+
+	// These fields should be accessed using their respective Getter
+	tunnelstoreClient tunnelstore.Client
+	userCredential    *userCredential
+}
+
+func newSubcommandContext(c *cli.Context) (*subcommandContext, error) {
+	isUIEnabled := c.IsSet(uiFlag) && c.String("name") != ""
+
+	// If UI is enabled, terminal log output should be disabled -- log should be written into a UI log window instead
+	log := logger.CreateLoggerFromContext(c, isUIEnabled)
+
+	return &subcommandContext{
+		c:           c,
+		log:         log,
+		isUIEnabled: isUIEnabled,
+		fs:          realFileSystem{},
+	}, nil
+}
+
+// Returns something that can find the given tunnel's credentials file.
+func (sc *subcommandContext) credentialFinder(tunnelID uuid.UUID) CredFinder {
+	if path := sc.c.String(CredFileFlag); path != "" {
+		return newStaticPath(path, sc.fs)
+	}
+	return newSearchByID(tunnelID, sc.c, sc.log, sc.fs)
+}
+
+type userCredential struct {
+	cert     *certutil.OriginCert
+	certPath string
+}
+
+func (sc *subcommandContext) client() (tunnelstore.Client, error) {
+	if sc.tunnelstoreClient != nil {
+		return sc.tunnelstoreClient, nil
+	}
+	credential, err := sc.credential()
+	if err != nil {
+		return nil, err
+	}
+	userAgent := fmt.Sprintf("cloudflared/%s", version)
+	client, err := tunnelstore.NewRESTClient(
+		sc.c.String("api-url"),
+		credential.cert.AccountID,
+		credential.cert.ZoneID,
+		credential.cert.ServiceKey,
+		userAgent,
+		sc.log,
+	)
+
+	if err != nil {
+		return nil, err
+	}
+	sc.tunnelstoreClient = client
+	return client, nil
+}
+
+func (sc *subcommandContext) credential() (*userCredential, error) {
+	if sc.userCredential == nil {
+		originCertPath := sc.c.String("origincert")
+		originCertLog := sc.log.With().
+			Str(LogFieldOriginCertPath, originCertPath).
+			Logger()
+
+		originCertPath, err := findOriginCert(originCertPath, &originCertLog)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error locating origin cert")
+		}
+		blocks, err := readOriginCert(originCertPath)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Can't read origin cert from %s", originCertPath)
+		}
+
+		cert, err := certutil.DecodeOriginCert(blocks)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error decoding origin cert")
+		}
+
+		if cert.AccountID == "" {
+			return nil, errors.Errorf(`Origin certificate needs to be refreshed before creating new tunnels.\nDelete %s and run "cloudflared login" to obtain a new cert.`, originCertPath)
+		}
+
+		sc.userCredential = &userCredential{
+			cert:     cert,
+			certPath: originCertPath,
+		}
+	}
+	return sc.userCredential, nil
+}
+
+func (sc *subcommandContext) readTunnelCredentials(credFinder CredFinder) (connection.Credentials, error) {
+	filePath, err := credFinder.Path()
+	if err != nil {
+		return connection.Credentials{}, err
+	}
+	body, err := sc.fs.readFile(filePath)
+	if err != nil {
+		return connection.Credentials{}, errors.Wrapf(err, "couldn't read tunnel credentials from %v", filePath)
+	}
+
+	var credentials connection.Credentials
+	if err = json.Unmarshal(body, &credentials); err != nil {
+		if strings.HasSuffix(filePath, ".pem") {
+			return connection.Credentials{}, fmt.Errorf("The tunnel credentials file should be .json but you gave a .pem. " +
+				"The tunnel credentials file was originally created by `cloudflared tunnel create`. " +
+				"You may have accidentally used the filepath to cert.pem, which is generated by `cloudflared tunnel " +
+				"login`.")
+		}
+		return connection.Credentials{}, errInvalidJSONCredential{path: filePath, err: err}
+	}
+	return credentials, nil
+}
+
+func (sc *subcommandContext) create(name string, credentialsFilePath string) (*tunnelstore.Tunnel, error) {
+	client, err := sc.client()
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't create client to talk to Argo Tunnel backend")
+	}
+
+	tunnelSecret, err := generateTunnelSecret()
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't generate the secret for your new tunnel")
+	}
+
+	tunnel, err := client.CreateTunnel(name, tunnelSecret)
+	if err != nil {
+		return nil, errors.Wrap(err, "Create Tunnel API call failed")
+	}
+
+	credential, err := sc.credential()
+	if err != nil {
+		return nil, err
+	}
+	tunnelCredentials := connection.Credentials{
+		AccountTag:   credential.cert.AccountID,
+		TunnelSecret: tunnelSecret,
+		TunnelID:     tunnel.ID,
+		TunnelName:   name,
+	}
+	usedCertPath := false
+	if credentialsFilePath == "" {
+		originCertDir := filepath.Dir(credential.certPath)
+		credentialsFilePath, err = tunnelFilePath(tunnelCredentials.TunnelID, originCertDir)
+		if err != nil {
+			return nil, err
+		}
+		usedCertPath = true
+	}
+	writeFileErr := writeTunnelCredentials(credentialsFilePath, &tunnelCredentials)
+	if writeFileErr != nil {
+		var errorLines []string
+		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write tunnel credentials to %s.", tunnel.Name, tunnel.ID, credentialsFilePath))
+		errorLines = append(errorLines, fmt.Sprintf("The file-writing error is: %v", writeFileErr))
+		if deleteErr := client.DeleteTunnel(tunnel.ID); deleteErr != nil {
+			errorLines = append(errorLines, fmt.Sprintf("Cloudflared tried to delete the tunnel for you, but encountered an error. You should use `cloudflared tunnel delete %v` to delete the tunnel yourself, because the tunnel can't be run without the tunnelfile.", tunnel.ID))
+			errorLines = append(errorLines, fmt.Sprintf("The delete tunnel error is: %v", deleteErr))
+		} else {
+			errorLines = append(errorLines, fmt.Sprintf("The tunnel was deleted, because the tunnel can't be run without the credentials file"))
+		}
+		errorMsg := strings.Join(errorLines, "\n")
+		return nil, errors.New(errorMsg)
+	}
+
+	if outputFormat := sc.c.String(outputFormatFlag.Name); outputFormat != "" {
+		return nil, renderOutput(outputFormat, &tunnel)
+	}
+
+	fmt.Printf("Tunnel credentials written to %v.", credentialsFilePath)
+	if usedCertPath {
+		fmt.Print(" cloudflared chose this file based on where your origin certificate was found.")
+	}
+	fmt.Println(" Keep this file secret. To revoke these credentials, delete the tunnel.")
+	fmt.Printf("\nCreated tunnel %s with id %s\n", tunnel.Name, tunnel.ID)
+	return tunnel, nil
+}
+
+func (sc *subcommandContext) list(filter *tunnelstore.Filter) ([]*tunnelstore.Tunnel, error) {
+	client, err := sc.client()
+	if err != nil {
+		return nil, err
+	}
+	return client.ListTunnels(filter)
+}
+
+func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
+	forceFlagSet := sc.c.Bool("force")
+
+	client, err := sc.client()
+	if err != nil {
+		return err
+	}
+
+	for _, id := range tunnelIDs {
+		tunnel, err := client.GetTunnel(id)
+		if err != nil {
+			return errors.Wrapf(err, "Can't get tunnel information. Please check tunnel id: %s", tunnel.ID)
+		}
+
+		// Check if tunnel DeletedAt field has already been set
+		if !tunnel.DeletedAt.IsZero() {
+			return fmt.Errorf("Tunnel %s has already been deleted", tunnel.ID)
+		}
+		if forceFlagSet {
+			if err := client.CleanupConnections(tunnel.ID, tunnelstore.NewCleanupParams()); err != nil {
+				return errors.Wrapf(err, "Error cleaning up connections for tunnel %s", tunnel.ID)
+			}
+		}
+
+		if err := client.DeleteTunnel(tunnel.ID); err != nil {
+			return errors.Wrapf(err, "Error deleting tunnel %s", tunnel.ID)
+		}
+
+		credFinder := sc.credentialFinder(id)
+		if tunnelCredentialsPath, err := credFinder.Path(); err == nil {
+			if err = os.Remove(tunnelCredentialsPath); err != nil {
+				sc.log.Info().Msgf("Tunnel %v was deleted, but we could not remove its credentials file  %s: %s. Consider deleting this file manually.", id, tunnelCredentialsPath, err)
+			}
+		}
+	}
+	return nil
+}
+
+// findCredentials will choose the right way to find the credentials file, find it,
+// and add the TunnelID into any old credentials (generated before TUN-3581 added the `TunnelID`
+// field to credentials files)
+func (sc *subcommandContext) findCredentials(tunnelID uuid.UUID) (connection.Credentials, error) {
+	var credentials connection.Credentials
+	var err error
+	if credentialsContents := sc.c.String(CredContentsFlag); credentialsContents != "" {
+		if err = json.Unmarshal([]byte(credentialsContents), &credentials); err != nil {
+			err = errInvalidJSONCredential{path: "TUNNEL_CRED_CONTENTS", err: err}
+		}
+	} else {
+		credFinder := sc.credentialFinder(tunnelID)
+		credentials, err = sc.readTunnelCredentials(credFinder)
+	}
+	// This line ensures backwards compatibility with credentials files generated before
+	// TUN-3581. Those old credentials files don't have a TunnelID field, so we enrich the struct
+	// with the ID, which we have already resolved from the user input.
+	credentials.TunnelID = tunnelID
+	return credentials, err
+}
+
+func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
+	credentials, err := sc.findCredentials(tunnelID)
+	if err != nil {
+		if e, ok := err.(errInvalidJSONCredential); ok {
+			sc.log.Error().Msgf("The credentials file at %s contained invalid JSON. This is probably caused by passing the wrong filepath. Reminder: the credentials file is a .json file created via `cloudflared tunnel create`.", e.path)
+			sc.log.Error().Msgf("Invalid JSON when parsing credentials file: %s", e.err.Error())
+		}
+		return err
+	}
+
+	return StartServer(
+		sc.c,
+		version,
+		&connection.NamedTunnelConfig{Credentials: credentials},
+		sc.log,
+		sc.isUIEnabled,
+	)
+}
+
+func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
+	params := tunnelstore.NewCleanupParams()
+	extraLog := ""
+	if connector := sc.c.String("connector-id"); connector != "" {
+		connectorID, err := uuid.Parse(connector)
+		if err != nil {
+			return errors.Wrapf(err, "%s is not a valid client ID (must be a UUID)", connector)
+		}
+		params.ForClient(connectorID)
+		extraLog = fmt.Sprintf(" for connector-id %s", connectorID.String())
+	}
+
+	client, err := sc.client()
+	if err != nil {
+		return err
+	}
+	for _, tunnelID := range tunnelIDs {
+		sc.log.Info().Msgf("Cleanup connection for tunnel %s%s", tunnelID, extraLog)
+		if err := client.CleanupConnections(tunnelID, params); err != nil {
+			sc.log.Error().Msgf("Error cleaning up connections for tunnel %v, error :%v", tunnelID, err)
+		}
+	}
+	return nil
+}
+
+func (sc *subcommandContext) route(tunnelID uuid.UUID, r tunnelstore.Route) (tunnelstore.RouteResult, error) {
+	client, err := sc.client()
+	if err != nil {
+		return nil, err
+	}
+
+	return client.RouteTunnel(tunnelID, r)
+}
+
+// Query Tunnelstore to find the active tunnel with the given name.
+func (sc *subcommandContext) tunnelActive(name string) (*tunnelstore.Tunnel, bool, error) {
+	filter := tunnelstore.NewFilter()
+	filter.NoDeleted()
+	filter.ByName(name)
+	tunnels, err := sc.list(filter)
+	if err != nil {
+		return nil, false, err
+	}
+	if len(tunnels) == 0 {
+		return nil, false, nil
+	}
+	// There should only be 1 active tunnel for a given name
+	return tunnels[0], true, nil
+}
+
+// findID parses the input. If it's a UUID, return the UUID.
+// Otherwise, assume it's a name, and look up the ID of that tunnel.
+func (sc *subcommandContext) findID(input string) (uuid.UUID, error) {
+	if u, err := uuid.Parse(input); err == nil {
+		return u, nil
+	}
+
+	// Look up name in the credentials file.
+	credFinder := newStaticPath(sc.c.String(CredFileFlag), sc.fs)
+	if credentials, err := sc.readTunnelCredentials(credFinder); err == nil {
+		if credentials.TunnelID != uuid.Nil && input == credentials.TunnelName {
+			return credentials.TunnelID, nil
+		}
+	}
+
+	// Fall back to querying Tunnelstore.
+	if tunnel, found, err := sc.tunnelActive(input); err != nil {
+		return uuid.Nil, err
+	} else if found {
+		return tunnel.ID, nil
+	}
+
+	return uuid.Nil, fmt.Errorf("%s is neither the ID nor the name of any of your tunnels", input)
+}
+
+// findIDs is just like mapping `findID` over a slice, but it only uses
+// one Tunnelstore API call.
+func (sc *subcommandContext) findIDs(inputs []string) ([]uuid.UUID, error) {
+
+	// Shortcut without Tunnelstore call if we find that all inputs are already UUIDs.
+	uuids, err := convertNamesToUuids(inputs, make(map[string]uuid.UUID))
+	if err == nil {
+		return uuids, nil
+	}
+
+	// First, look up all tunnels the user has
+	filter := tunnelstore.NewFilter()
+	filter.NoDeleted()
+	tunnels, err := sc.list(filter)
+	if err != nil {
+		return nil, err
+	}
+	// Do the pure list-processing in its own function, so that it can be
+	// unit tested easily.
+	return findIDs(tunnels, inputs)
+}
+
+func findIDs(tunnels []*tunnelstore.Tunnel, inputs []string) ([]uuid.UUID, error) {
+	// Put them into a dictionary for faster lookups
+	nameToID := make(map[string]uuid.UUID, len(tunnels))
+	for _, tunnel := range tunnels {
+		nameToID[tunnel.Name] = tunnel.ID
+	}
+
+	return convertNamesToUuids(inputs, nameToID)
+}
+
+func convertNamesToUuids(inputs []string, nameToID map[string]uuid.UUID) ([]uuid.UUID, error) {
+	tunnelIDs := make([]uuid.UUID, len(inputs))
+	var badInputs []string
+	for i, input := range inputs {
+		if id, err := uuid.Parse(input); err == nil {
+			tunnelIDs[i] = id
+		} else if id, ok := nameToID[input]; ok {
+			tunnelIDs[i] = id
+		} else {
+			badInputs = append(badInputs, input)
+		}
+	}
+	if len(badInputs) > 0 {
+		msg := "Please specify either the ID or name of a tunnel. The following inputs were neither: %s"
+		return nil, fmt.Errorf(msg, strings.Join(badInputs, ", "))
+	}
+	return tunnelIDs, nil
+}

cmd/cloudflared/tunnel/subcommand_context_teamnet.go

@@ -0,0 +1,43 @@
+package tunnel
+
+import (
+	"net"
+
+	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/teamnet"
+)
+
+const noClientMsg = "error while creating backend client"
+
+func (sc *subcommandContext) listRoutes(filter *teamnet.Filter) ([]*teamnet.DetailedRoute, error) {
+	client, err := sc.client()
+	if err != nil {
+		return nil, errors.Wrap(err, noClientMsg)
+	}
+	return client.ListRoutes(filter)
+}
+
+func (sc *subcommandContext) addRoute(newRoute teamnet.NewRoute) (teamnet.Route, error) {
+	client, err := sc.client()
+	if err != nil {
+		return teamnet.Route{}, errors.Wrap(err, noClientMsg)
+	}
+	return client.AddRoute(newRoute)
+}
+
+func (sc *subcommandContext) deleteRoute(network net.IPNet) error {
+	client, err := sc.client()
+	if err != nil {
+		return errors.Wrap(err, noClientMsg)
+	}
+	return client.DeleteRoute(network)
+}
+
+func (sc *subcommandContext) getRouteByIP(ip net.IP) (teamnet.DetailedRoute, error) {
+	client, err := sc.client()
+	if err != nil {
+		return teamnet.DetailedRoute{}, errors.Wrap(err, noClientMsg)
+	}
+	return client.GetByIP(ip)
+}

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -0,0 +1,397 @@
+package tunnel
+
+import (
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/tunnelstore"
+)
+
+func Test_findIDs(t *testing.T) {
+	type args struct {
+		tunnels []*tunnelstore.Tunnel
+		inputs  []string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []uuid.UUID
+		wantErr bool
+	}{
+		{
+			name: "input not found",
+			args: args{
+				inputs: []string{"asdf"},
+			},
+			wantErr: true,
+		},
+		{
+			name: "only UUID",
+			args: args{
+				inputs: []string{"a8398a0b-876d-48ed-b609-3fcfd67a4950"},
+			},
+			want: []uuid.UUID{uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950")},
+		},
+		{
+			name: "only name",
+			args: args{
+				tunnels: []*tunnelstore.Tunnel{
+					{
+						ID:   uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950"),
+						Name: "tunnel1",
+					},
+				},
+				inputs: []string{"tunnel1"},
+			},
+			want: []uuid.UUID{uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950")},
+		},
+		{
+			name: "both UUID and name",
+			args: args{
+				tunnels: []*tunnelstore.Tunnel{
+					{
+						ID:   uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950"),
+						Name: "tunnel1",
+					},
+					{
+						ID:   uuid.MustParse("bf028b68-744f-466e-97f8-c46161d80aa5"),
+						Name: "tunnel2",
+					},
+				},
+				inputs: []string{"tunnel1", "bf028b68-744f-466e-97f8-c46161d80aa5"},
+			},
+			want: []uuid.UUID{
+				uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950"),
+				uuid.MustParse("bf028b68-744f-466e-97f8-c46161d80aa5"),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := findIDs(tt.args.tunnels, tt.args.inputs)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("findIDs() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("findIDs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+type mockFileSystem struct {
+	rf  func(string) ([]byte, error)
+	vfp func(string) bool
+}
+
+func (fs mockFileSystem) validFilePath(path string) bool {
+	return fs.vfp(path)
+}
+
+func (fs mockFileSystem) readFile(filePath string) ([]byte, error) {
+	return fs.rf(filePath)
+}
+
+func Test_subcommandContext_findCredentials(t *testing.T) {
+	type fields struct {
+		c                 *cli.Context
+		log               *zerolog.Logger
+		isUIEnabled       bool
+		fs                fileSystem
+		tunnelstoreClient tunnelstore.Client
+		userCredential    *userCredential
+	}
+	type args struct {
+		tunnelID uuid.UUID
+	}
+	oldCertPath := "old_cert.json"
+	newCertPath := "new_cert.json"
+	accountTag := "0000d4d14e84bd4ae5a6a02e0000ac63"
+	secret := []byte{211, 79, 177, 245, 179, 194, 152, 127, 140, 71, 18, 46, 183, 209, 10, 24, 192, 150, 55, 249, 211, 16, 167, 30, 113, 51, 152, 168, 72, 100, 205, 144}
+	secretB64 := base64.StdEncoding.EncodeToString(secret)
+	tunnelID := uuid.MustParse("df5ed608-b8b4-4109-89f3-9f2cf199df64")
+	name := "mytunnel"
+
+	fs := mockFileSystem{
+		rf: func(filePath string) ([]byte, error) {
+			if filePath == oldCertPath {
+				// An old credentials file created before TUN-3581 added the new fields
+				return []byte(fmt.Sprintf(`{"AccountTag":"%s","TunnelSecret":"%s"}`, accountTag, secretB64)), nil
+			}
+			if filePath == newCertPath {
+				// A new credentials file created after TUN-3581 with its new fields.
+				return []byte(fmt.Sprintf(`{"AccountTag":"%s","TunnelSecret":"%s","TunnelID":"%s","TunnelName":"%s"}`, accountTag, secretB64, tunnelID, name)), nil
+			}
+			return nil, errors.New("file not found")
+		},
+		vfp: func(string) bool { return true },
+	}
+	log := zerolog.Nop()
+
+	tests := []struct {
+		name    string
+		fields  fields
+		args    args
+		want    connection.Credentials
+		wantErr bool
+	}{
+		{
+			name: "Filepath given leads to old credentials file",
+			fields: fields{
+				log: &log,
+				fs:  fs,
+				c: func() *cli.Context {
+					flagSet := flag.NewFlagSet("test0", flag.PanicOnError)
+					flagSet.String(CredFileFlag, oldCertPath, "")
+					c := cli.NewContext(cli.NewApp(), flagSet, nil)
+					_ = c.Set(CredFileFlag, oldCertPath)
+					return c
+				}(),
+			},
+			args: args{
+				tunnelID: tunnelID,
+			},
+			want: connection.Credentials{
+				AccountTag:   accountTag,
+				TunnelID:     tunnelID,
+				TunnelSecret: secret,
+			},
+		},
+		{
+			name: "Filepath given leads to new credentials file",
+			fields: fields{
+				log: &log,
+				fs:  fs,
+				c: func() *cli.Context {
+					flagSet := flag.NewFlagSet("test0", flag.PanicOnError)
+					flagSet.String(CredFileFlag, newCertPath, "")
+					c := cli.NewContext(cli.NewApp(), flagSet, nil)
+					_ = c.Set(CredFileFlag, newCertPath)
+					return c
+				}(),
+			},
+			args: args{
+				tunnelID: tunnelID,
+			},
+			want: connection.Credentials{
+				AccountTag:   accountTag,
+				TunnelID:     tunnelID,
+				TunnelSecret: secret,
+				TunnelName:   name,
+			},
+		},
+		{
+			name: "TUNNEL_CRED_CONTENTS given contains old credentials contents",
+			fields: fields{
+				log: &log,
+				fs:  fs,
+				c: func() *cli.Context {
+					flagSet := flag.NewFlagSet("test0", flag.PanicOnError)
+					flagSet.String(CredContentsFlag, "", "")
+					c := cli.NewContext(cli.NewApp(), flagSet, nil)
+					_ = c.Set(CredContentsFlag, fmt.Sprintf(`{"AccountTag":"%s","TunnelSecret":"%s"}`, accountTag, secretB64))
+					return c
+				}(),
+			},
+			args: args{
+				tunnelID: tunnelID,
+			},
+			want: connection.Credentials{
+				AccountTag:   accountTag,
+				TunnelID:     tunnelID,
+				TunnelSecret: secret,
+			},
+		},
+		{
+			name: "TUNNEL_CRED_CONTENTS given contains new credentials contents",
+			fields: fields{
+				log: &log,
+				fs:  fs,
+				c: func() *cli.Context {
+					flagSet := flag.NewFlagSet("test0", flag.PanicOnError)
+					flagSet.String(CredContentsFlag, "", "")
+					c := cli.NewContext(cli.NewApp(), flagSet, nil)
+					_ = c.Set(CredContentsFlag, fmt.Sprintf(`{"AccountTag":"%s","TunnelSecret":"%s","TunnelID":"%s","TunnelName":"%s"}`, accountTag, secretB64, tunnelID, name))
+					return c
+				}(),
+			},
+			args: args{
+				tunnelID: tunnelID,
+			},
+			want: connection.Credentials{
+				AccountTag:   accountTag,
+				TunnelID:     tunnelID,
+				TunnelSecret: secret,
+				TunnelName:   name,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sc := &subcommandContext{
+				c:                 tt.fields.c,
+				log:               tt.fields.log,
+				isUIEnabled:       tt.fields.isUIEnabled,
+				fs:                tt.fields.fs,
+				tunnelstoreClient: tt.fields.tunnelstoreClient,
+				userCredential:    tt.fields.userCredential,
+			}
+			got, err := sc.findCredentials(tt.args.tunnelID)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("subcommandContext.findCredentials() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("subcommandContext.findCredentials() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+type deleteMockTunnelStore struct {
+	tunnelstore.Client
+	mockTunnels      map[uuid.UUID]mockTunnelBehaviour
+	deletedTunnelIDs []uuid.UUID
+}
+
+type mockTunnelBehaviour struct {
+	tunnel     tunnelstore.Tunnel
+	deleteErr  error
+	cleanupErr error
+}
+
+func newDeleteMockTunnelStore(tunnels ...mockTunnelBehaviour) *deleteMockTunnelStore {
+	mockTunnels := make(map[uuid.UUID]mockTunnelBehaviour)
+	for _, tunnel := range tunnels {
+		mockTunnels[tunnel.tunnel.ID] = tunnel
+	}
+	return &deleteMockTunnelStore{
+		mockTunnels:      mockTunnels,
+		deletedTunnelIDs: make([]uuid.UUID, 0),
+	}
+}
+
+func (d *deleteMockTunnelStore) GetTunnel(tunnelID uuid.UUID) (*tunnelstore.Tunnel, error) {
+	tunnel, ok := d.mockTunnels[tunnelID]
+	if !ok {
+		return nil, fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
+	}
+	return &tunnel.tunnel, nil
+}
+
+func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
+	tunnel, ok := d.mockTunnels[tunnelID]
+	if !ok {
+		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
+	}
+
+	if tunnel.deleteErr != nil {
+		return tunnel.deleteErr
+	}
+
+	d.deletedTunnelIDs = append(d.deletedTunnelIDs, tunnelID)
+	tunnel.tunnel.DeletedAt = time.Now()
+	delete(d.mockTunnels, tunnelID)
+	return nil
+}
+
+func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID, _ *tunnelstore.CleanupParams) error {
+	tunnel, ok := d.mockTunnels[tunnelID]
+	if !ok {
+		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
+	}
+	return tunnel.cleanupErr
+}
+
+func Test_subcommandContext_Delete(t *testing.T) {
+	type fields struct {
+		c                 *cli.Context
+		log               *zerolog.Logger
+		isUIEnabled       bool
+		fs                fileSystem
+		tunnelstoreClient *deleteMockTunnelStore
+		userCredential    *userCredential
+	}
+	type args struct {
+		tunnelIDs []uuid.UUID
+	}
+	newCertPath := "new_cert.json"
+	tunnelID1 := uuid.MustParse("df5ed608-b8b4-4109-89f3-9f2cf199df64")
+	tunnelID2 := uuid.MustParse("af5ed608-b8b4-4109-89f3-9f2cf199df64")
+	log := zerolog.Nop()
+
+	var tests = []struct {
+		name    string
+		fields  fields
+		args    args
+		want    []uuid.UUID
+		wantErr bool
+	}{
+		{
+			name: "clean up continues if credentials are not found",
+			fields: fields{
+				log: &log,
+				fs: mockFileSystem{
+					rf: func(filePath string) ([]byte, error) {
+						return nil, errors.New("file not found")
+					},
+					vfp: func(string) bool { return true },
+				},
+				c: func() *cli.Context {
+					flagSet := flag.NewFlagSet("test0", flag.PanicOnError)
+					flagSet.String(CredFileFlag, newCertPath, "")
+					c := cli.NewContext(cli.NewApp(), flagSet, nil)
+					_ = c.Set(CredFileFlag, newCertPath)
+					return c
+				}(),
+				tunnelstoreClient: newDeleteMockTunnelStore(
+					mockTunnelBehaviour{
+						tunnel: tunnelstore.Tunnel{ID: tunnelID1},
+					},
+					mockTunnelBehaviour{
+						tunnel: tunnelstore.Tunnel{ID: tunnelID2},
+					},
+				),
+			},
+
+			args: args{
+				tunnelIDs: []uuid.UUID{tunnelID1, tunnelID2},
+			},
+			want: []uuid.UUID{tunnelID1, tunnelID2},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			sc := &subcommandContext{
+				c:                 tt.fields.c,
+				log:               tt.fields.log,
+				isUIEnabled:       tt.fields.isUIEnabled,
+				fs:                tt.fields.fs,
+				tunnelstoreClient: tt.fields.tunnelstoreClient,
+				userCredential:    tt.fields.userCredential,
+			}
+			err := sc.delete(tt.args.tunnelIDs)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("subcommandContext.findCredentials() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			got := tt.fields.tunnelstoreClient.deletedTunnelIDs
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("subcommandContext.findCredentials() = %v, want %v", got, tt.want)
+				return
+			}
+		})
+	}
+}

cmd/cloudflared/tunnel/subcommands_test.go

@@ -1,15 +1,14 @@
 package tunnel
 
 import (
-	"fmt"
 	"path/filepath"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/tunnelstore"
-	"github.com/mitchellh/go-homedir"
-
 	"github.com/google/uuid"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 func Test_fmtConnections(t *testing.T) {
@@ -40,6 +39,19 @@ func Test_fmtConnections(t *testing.T) {
 			},
 			want: "1xDFW",
 		},
+		{
+			name: "with a pending reconnect",
+			args: args{
+				connections: []tunnelstore.Connection{
+					{
+						ColoName:           "DFW",
+						ID:                 uuid.MustParse("ea550130-57fd-4463-aab1-752822231ddd"),
+						IsPendingReconnect: true,
+					},
+				},
+			},
+			want: "",
+		},
 		{
 			name: "many colos",
 			args: args{
@@ -67,7 +79,7 @@ func Test_fmtConnections(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if got := fmtConnections(tt.args.connections); got != tt.want {
+			if got := fmtConnections(tt.args.connections, false); got != tt.want {
 				t.Errorf("fmtConnections() = %v, want %v", got, tt.want)
 			}
 		})
@@ -75,11 +87,93 @@ func Test_fmtConnections(t *testing.T) {
 }
 
 func TestTunnelfilePath(t *testing.T) {
+	tunnelID, err := uuid.Parse("f48d8918-bc23-4647-9d48-082c5b76de65")
+	assert.NoError(t, err)
 	originCertDir := filepath.Dir("~/.cloudflared/cert.pem")
-	actual, err := tunnelFilePath("tunnel", originCertDir)
+	actual, err := tunnelFilePath(tunnelID, originCertDir)
 	assert.NoError(t, err)
 	homeDir, err := homedir.Dir()
 	assert.NoError(t, err)
-	expected := fmt.Sprintf("%s/.cloudflared/tunnel.json", homeDir)
+	expected := filepath.Join(homeDir, ".cloudflared", tunnelID.String()+".json")
 	assert.Equal(t, expected, actual)
 }
+
+func TestValidateName(t *testing.T) {
+	tests := []struct {
+		name string
+		want bool
+	}{
+		{name: "", want: false},
+		{name: "-", want: false},
+		{name: ".", want: false},
+		{name: "a b", want: false},
+		{name: "a+b", want: false},
+		{name: "-ab", want: false},
+
+		{name: "ab", want: true},
+		{name: "ab-c", want: true},
+		{name: "abc.def", want: true},
+		{name: "_ab_c.-d-ef", want: true},
+	}
+	for _, tt := range tests {
+		if got := validateName(tt.name, false); got != tt.want {
+			t.Errorf("validateName() = %v, want %v", got, tt.want)
+		}
+	}
+}
+
+func Test_validateHostname(t *testing.T) {
+	type args struct {
+		s                      string
+		allowWildcardSubdomain bool
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "Normal",
+			args: args{
+				s:                      "example.com",
+				allowWildcardSubdomain: true,
+			},
+			want: true,
+		},
+		{
+			name: "wildcard subdomain for TUN-358",
+			args: args{
+				s:                      "*.ehrig.io",
+				allowWildcardSubdomain: true,
+			},
+			want: true,
+		},
+		{
+			name: "Misplaced wildcard",
+			args: args{
+				s:                      "subdomain.*.ehrig.io",
+				allowWildcardSubdomain: true,
+			},
+		},
+		{
+			name: "Invalid domain",
+			args: args{
+				s:                      "..",
+				allowWildcardSubdomain: true,
+			},
+		},
+		{
+			name: "Invalid domain",
+			args: args{
+				s: "..",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := validateHostname(tt.args.s, tt.args.allowWildcardSubdomain); got != tt.want {
+				t.Errorf("validateHostname() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/cloudflared/tunnel/teamnet_subcommands.go

@@ -0,0 +1,212 @@
+package tunnel
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"text/tabwriter"
+
+	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/teamnet"
+
+	"github.com/urfave/cli/v2"
+)
+
+func buildRouteIPSubcommand() *cli.Command {
+	return &cli.Command{
+		Name:      "ip",
+		Usage:     "Configure and query Cloudflare WARP routing to services or private networks available through this tunnel.",
+		UsageText: "cloudflared tunnel [--config FILEPATH] route COMMAND [arguments...]",
+		Description: `cloudflared can provision private routes from any IP space to origins in your corporate network.
+Users enrolled in your Cloudflare for Teams organization can reach those routes through the
+Cloudflare WARP client. You can also build rules to determine who can reach certain routes.`,
+		Subcommands: []*cli.Command{
+			{
+				Name:      "add",
+				Action:    cliutil.ConfiguredAction(addRouteCommand),
+				Usage:     "Add any new network to the routing table reachable via the tunnel",
+				UsageText: "cloudflared tunnel [--config FILEPATH] route ip add [CIDR] [TUNNEL] [COMMENT?]",
+				Description: `Adds any network route space (represented as a CIDR) to your routing table.
+That network space becomes reachable for requests egressing from a user's machine
+as long as it is using Cloudflare WARP client and is enrolled in the same account
+that is running the tunnel chosen here. Further, those requests will be proxied to
+the specified tunnel, and reach an IP in the given CIDR, as long as that IP is
+reachable from the tunnel.`,
+			},
+			{
+				Name:        "show",
+				Aliases:     []string{"list"},
+				Action:      cliutil.ConfiguredAction(showRoutesCommand),
+				Usage:       "Show the routing table",
+				UsageText:   "cloudflared tunnel [--config FILEPATH] route ip show [flags]",
+				Description: `Shows your organization private routing table. You can use flags to filter the results.`,
+				Flags:       showRoutesFlags(),
+			},
+			{
+				Name:      "delete",
+				Action:    cliutil.ConfiguredAction(deleteRouteCommand),
+				Usage:     "Delete a row from your organization's private routing table",
+				UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [CIDR]",
+				Description: `Deletes the row for a given CIDR from your routing table. That portion
+of your network will no longer be reachable by the WARP clients.`,
+			},
+			{
+				Name:      "get",
+				Action:    cliutil.ConfiguredAction(getRouteByIPCommand),
+				Usage:     "Check which row of the routing table matches a given IP.",
+				UsageText: "cloudflared tunnel [--config FILEPATH] route ip get [IP]",
+				Description: `Checks which row of the routing table will be used to proxy a given IP.
+				This helps check and validate your config.`,
+			},
+		},
+	}
+}
+
+func showRoutesFlags() []cli.Flag {
+	flags := make([]cli.Flag, 0)
+	flags = append(flags, teamnet.FilterFlags...)
+	flags = append(flags, outputFormatFlag)
+	return flags
+}
+
+func showRoutesCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+
+	filter, err := teamnet.NewFromCLI(c)
+	if err != nil {
+		return errors.Wrap(err, "invalid config for routing filters")
+	}
+
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
+	routes, err := sc.listRoutes(filter)
+	if err != nil {
+		return err
+	}
+
+	if outputFormat := c.String(outputFormatFlag.Name); outputFormat != "" {
+		return renderOutput(outputFormat, routes)
+	}
+
+	if len(routes) > 0 {
+		formatAndPrintRouteList(routes)
+	} else {
+		fmt.Println("You have no routes, use 'cloudflared tunnel route ip add' to add a route")
+	}
+
+	return nil
+}
+
+func addRouteCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+	if c.NArg() < 2 {
+		return errors.New("You must supply at least 2 arguments, first the network you wish to route (in CIDR form e.g. 1.2.3.4/32) and then the tunnel ID to proxy with")
+	}
+	args := c.Args()
+	_, network, err := net.ParseCIDR(args.Get(0))
+	if err != nil {
+		return errors.Wrap(err, "Invalid network CIDR")
+	}
+	if network == nil {
+		return errors.New("Invalid network CIDR")
+	}
+	tunnelRef := args.Get(1)
+	tunnelID, err := sc.findID(tunnelRef)
+	if err != nil {
+		return errors.Wrap(err, "Invalid tunnel")
+	}
+	comment := ""
+	if c.NArg() >= 3 {
+		comment = args.Get(2)
+	}
+	_, err = sc.addRoute(teamnet.NewRoute{
+		Comment:  comment,
+		Network:  *network,
+		TunnelID: tunnelID,
+	})
+	if err != nil {
+		return errors.Wrap(err, "API error")
+	}
+	fmt.Printf("Successfully added route for %s over tunnel %s\n", network, tunnelID)
+	return nil
+}
+
+func deleteRouteCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+	if c.NArg() != 1 {
+		return errors.New("You must supply exactly one argument, the network whose route you want to delete (in CIDR form e.g. 1.2.3.4/32)")
+	}
+	_, network, err := net.ParseCIDR(c.Args().First())
+	if err != nil {
+		return errors.Wrap(err, "Invalid network CIDR")
+	}
+	if network == nil {
+		return errors.New("Invalid network CIDR")
+	}
+	if err := sc.deleteRoute(*network); err != nil {
+		return errors.Wrap(err, "API error")
+	}
+	fmt.Printf("Successfully deleted route for %s\n", network)
+	return nil
+}
+
+func getRouteByIPCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+	if c.NArg() != 1 {
+		return errors.New("You must supply exactly one argument, an IP whose route will be queried (e.g. 1.2.3.4 or 2001:0db8:::7334)")
+	}
+
+	ipInput := c.Args().First()
+	ip := net.ParseIP(ipInput)
+	if ip == nil {
+		return fmt.Errorf("Invalid IP %s", ipInput)
+	}
+	route, err := sc.getRouteByIP(ip)
+	if err != nil {
+		return errors.Wrap(err, "API error")
+	}
+	if route.IsZero() {
+		fmt.Printf("No route matches the IP %s\n", ip)
+	} else {
+		formatAndPrintRouteList([]*teamnet.DetailedRoute{&route})
+	}
+	return nil
+}
+
+func formatAndPrintRouteList(routes []*teamnet.DetailedRoute) {
+	const (
+		minWidth = 0
+		tabWidth = 8
+		padding  = 1
+		padChar  = ' '
+		flags    = 0
+	)
+
+	writer := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
+	defer writer.Flush()
+
+	// Print column headers with tabbed columns
+	_, _ = fmt.Fprintln(writer, "NETWORK\tCOMMENT\tTUNNEL ID\tTUNNEL NAME\tCREATED\tDELETED\t")
+
+	// Loop through routes, create formatted string for each, and print using tabwriter
+	for _, route := range routes {
+		formattedStr := route.TableString()
+		_, _ = fmt.Fprintln(writer, formattedStr)
+	}
+}

cmd/cloudflared/ui/launch_ui.go

@@ -0,0 +1,223 @@
+package ui
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/gdamore/tcell"
+	"github.com/rivo/tview"
+	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/ingress"
+)
+
+type connState struct {
+	location string
+}
+
+type uiModel struct {
+	version       string
+	edgeURL       string
+	metricsURL    string
+	localServices []string
+	connections   []connState
+}
+
+type palette struct {
+	url          string
+	connected    string
+	defaultText  string
+	disconnected string
+	reconnecting string
+	unregistered string
+}
+
+func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haConnections int) *uiModel {
+	localServices := make([]string, len(ing.Rules))
+	for i, rule := range ing.Rules {
+		localServices[i] = rule.Service.String()
+	}
+	return &uiModel{
+		version:       version,
+		edgeURL:       hostname,
+		metricsURL:    metricsURL,
+		localServices: localServices,
+		connections:   make([]connState, haConnections),
+	}
+}
+
+func (data *uiModel) Launch(
+	ctx context.Context,
+	log, transportLog *zerolog.Logger,
+) connection.EventSink {
+	// Configure the logger to stream logs into the textview
+
+	// Add TextView as a group to write output to
+	logTextView := NewDynamicColorTextView()
+	// TODO: Format log for UI
+	//log.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+	//transportLog.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+
+	// Construct the UI
+	palette := palette{
+		url:          "lightblue",
+		connected:    "lime",
+		defaultText:  "white",
+		disconnected: "red",
+		reconnecting: "orange",
+		unregistered: "orange",
+	}
+
+	app := tview.NewApplication()
+
+	grid := tview.NewGrid().SetGap(1, 0)
+	frame := tview.NewFrame(grid)
+	header := fmt.Sprintf("cloudflared [::b]%s", data.version)
+
+	frame.AddText(header, true, tview.AlignLeft, tcell.ColorWhite)
+
+	// Create table to store connection info and status
+	connTable := tview.NewTable()
+	// SetColumns takes a value for each column, representing the size of the column
+	// Numbers <= 0 represent proportional widths and positive numbers represent absolute widths
+	grid.SetColumns(20, 0)
+
+	// SetRows takes a value for each row, representing the size of the row
+	grid.SetRows(1, 1, len(data.connections), 1, 0)
+
+	// AddItem takes a primitive tview type, row, column, rowSpan, columnSpan, minGridHeight, minGridWidth, and focus
+	grid.AddItem(tview.NewTextView().SetText("Tunnel:"), 0, 0, 1, 1, 0, 0, false)
+	grid.AddItem(tview.NewTextView().SetText("Status:"), 1, 0, 1, 1, 0, 0, false)
+	grid.AddItem(tview.NewTextView().SetText("Connections:"), 2, 0, 1, 1, 0, 0, false)
+
+	grid.AddItem(tview.NewTextView().SetText("Metrics:"), 3, 0, 1, 1, 0, 0, false)
+
+	tunnelHostText := tview.NewTextView().SetText(data.edgeURL)
+
+	grid.AddItem(tunnelHostText, 0, 1, 1, 1, 0, 0, false)
+	status := fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, strings.Join(data.localServices, ", "))
+	grid.AddItem(NewDynamicColorTextView().SetText(status), 1, 1, 1, 1, 0, 0, false)
+
+	grid.AddItem(connTable, 2, 1, 1, 1, 0, 0, false)
+
+	grid.AddItem(NewDynamicColorTextView().SetText(fmt.Sprintf("Metrics at [%s::b]http://%s/metrics", palette.url, data.metricsURL)), 3, 1, 1, 1, 0, 0, false)
+
+	// Add TextView to stream logs
+	// Logs are displayed in a new grid so a border can be set around them
+	logGrid := tview.NewGrid().SetBorders(true).AddItem(logTextView.SetChangedFunc(handleNewText(app, logTextView)), 0, 0, 5, 2, 0, 0, false)
+	// LogFrame holds the Logs header as well as the grid with the textView for streamed logs
+	logFrame := tview.NewFrame(logGrid).AddText("[::b]Logs:[::-]", true, tview.AlignLeft, tcell.ColorWhite).SetBorders(0, 0, 0, 0, 0, 0)
+	// Footer for log frame
+	logFrame.AddText("[::d]Use Ctrl+C to exit[::-]", false, tview.AlignRight, tcell.ColorWhite)
+	grid.AddItem(logFrame, 4, 0, 5, 2, 0, 0, false)
+
+	go func() {
+		<-ctx.Done()
+		app.Stop()
+		return
+	}()
+
+	go func() {
+		if err := app.SetRoot(frame, true).Run(); err != nil {
+			log.Error().Msgf("Error launching UI: %s", err)
+		}
+	}()
+
+	return connection.EventSinkFunc(func(event connection.Event) {
+		switch event.EventType {
+		case connection.Connected:
+			data.setConnTableCell(event, connTable, palette)
+		case connection.Disconnected, connection.Reconnecting, connection.Unregistering:
+			data.changeConnStatus(event, connTable, log, palette)
+		case connection.SetURL:
+			tunnelHostText.SetText(event.URL)
+			data.edgeURL = event.URL
+		case connection.RegisteringTunnel:
+			if data.edgeURL == "" {
+				tunnelHostText.SetText(fmt.Sprintf("Registering tunnel connection %d...", event.Index))
+			}
+		}
+		app.Draw()
+	})
+}
+
+func NewDynamicColorTextView() *tview.TextView {
+	return tview.NewTextView().SetDynamicColors(true)
+}
+
+// Re-draws application when new logs are streamed to UI
+func handleNewText(app *tview.Application, logTextView *tview.TextView) func() {
+	return func() {
+		app.Draw()
+		// SetFocus to enable scrolling in textview
+		app.SetFocus(logTextView)
+	}
+}
+
+func (data *uiModel) changeConnStatus(event connection.Event, table *tview.Table, log *zerolog.Logger, palette palette) {
+	index := int(event.Index)
+	// Get connection location and state
+	connState := data.getConnState(index)
+	// Check if connection is already displayed in UI
+	if connState == nil {
+		log.Info().Msg("Connection is not in the UI table")
+		return
+	}
+
+	locationState := event.Location
+
+	if event.EventType == connection.Reconnecting {
+		locationState = "Reconnecting..."
+	}
+
+	connectionNum := index + 1
+	// Get table cell
+	cell := table.GetCell(index, 0)
+	// Change dot color in front of text as well as location state
+	text := newCellText(palette, connectionNum, locationState, event.EventType)
+	cell.SetText(text)
+}
+
+// Return connection location and row in UI table
+func (data *uiModel) getConnState(connID int) *connState {
+	if connID < len(data.connections) {
+		return &data.connections[connID]
+	}
+
+	return nil
+}
+
+func (data *uiModel) setConnTableCell(event connection.Event, table *tview.Table, palette palette) {
+	index := int(event.Index)
+	connectionNum := index + 1
+
+	// Update slice to keep track of connection location and state in UI table
+	data.connections[index].location = event.Location
+
+	// Update text in table cell to show disconnected state
+	text := newCellText(palette, connectionNum, event.Location, event.EventType)
+	cell := tview.NewTableCell(text)
+	table.SetCell(index, 0, cell)
+}
+
+func newCellText(palette palette, connectionNum int, location string, connectedStatus connection.Status) string {
+	// HA connection indicator formatted as: "• #<CONNECTION_INDEX>: <COLO>",
+	//  where the left middle dot's color depends on the status of the connection
+	const connFmtString = "[%s]\u2022[%s] #%d: %s"
+
+	var dotColor string
+	switch connectedStatus {
+	case connection.Connected:
+		dotColor = palette.connected
+	case connection.Disconnected:
+		dotColor = palette.disconnected
+	case connection.Reconnecting:
+		dotColor = palette.reconnecting
+	case connection.Unregistering:
+		dotColor = palette.unregistered
+	}
+
+	return fmt.Sprintf(connFmtString, dotColor, palette.defaultText, connectionNum, location)
+}

cmd/cloudflared/updater/check.go

@@ -0,0 +1,49 @@
+package updater
+
+import (
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+)
+
+type VersionWarningChecker struct {
+	warningChan chan string
+}
+
+func StartWarningCheck(c *cli.Context) VersionWarningChecker {
+	checker := VersionWarningChecker{
+		warningChan: make(chan string),
+	}
+
+	go func() {
+		options := updateOptions{
+			updateDisabled:  true,
+			isBeta:          c.Bool("beta"),
+			isStaging:       c.Bool("staging"),
+			isForced:        false,
+			intendedVersion: "",
+		}
+		checkResult, err := CheckForUpdate(options)
+		if err == nil {
+			checker.warningChan <- checkResult.UserMessage()
+		}
+		close(checker.warningChan)
+	}()
+
+	return checker
+}
+
+func (checker VersionWarningChecker) getWarning() string {
+	select {
+	case message := <-checker.warningChan:
+		return message
+	default:
+		// No feedback on time, we don't wait for it, since this is best-effort.
+		return ""
+	}
+}
+
+func (checker VersionWarningChecker) LogWarningIfAny(log *zerolog.Logger) {
+	if warning := checker.getWarning(); warning != "" {
+		log.Warn().Msg(warning)
+	}
+}

cmd/cloudflared/updater/service.go

@@ -0,0 +1,30 @@
+package updater
+
+// CheckResult is the behaviour resulting from checking in with the Update Service
+type CheckResult interface {
+	Apply() error
+	Version() string
+	UserMessage() string
+}
+
+// Service is the functions to get check for new updates
+type Service interface {
+	Check() (CheckResult, error)
+}
+
+const (
+	// OSKeyName is the url parameter key to send to the checkin API for the operating system of the local cloudflared (e.g. windows, darwin, linux)
+	OSKeyName = "os"
+
+	// ArchitectureKeyName is the url parameter key to send to the checkin API for the architecture of the local cloudflared (e.g. amd64, x86)
+	ArchitectureKeyName = "arch"
+
+	// BetaKeyName is the url parameter key to send to the checkin API to signal if the update should be a beta version or not
+	BetaKeyName = "beta"
+
+	// VersionKeyName is the url parameter key to send to the checkin API to specific what version to upgrade or downgrade to
+	VersionKeyName = "version"
+
+	// ClientVersionName is the url parameter key to send the version that this cloudflared is currently running with
+	ClientVersionName = "clientVersion"
+)

cmd/cloudflared/updater/update.go

@@ -8,37 +8,33 @@ import (
 	"runtime"
 	"time"
 
-	"golang.org/x/crypto/ssh/terminal"
-	"gopkg.in/urfave/cli.v2"
-
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/equinox-io/equinox"
 	"github.com/facebookgo/grace/gracenet"
-	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+	"golang.org/x/crypto/ssh/terminal"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
 	DefaultCheckUpdateFreq        = time.Hour * 24
-	appID                         = "app_idCzgxYerVD"
 	noUpdateInShellMessage        = "cloudflared will not automatically update when run from the shell. To enable auto-updates, run cloudflared as a service: https://developers.cloudflare.com/argo-tunnel/reference/service/"
 	noUpdateOnWindowsMessage      = "cloudflared will not automatically update on Windows systems."
 	noUpdateManagedPackageMessage = "cloudflared will not automatically update if installed by a package manager."
 	isManagedInstallFile          = ".installedFromPackageManager"
+	UpdateURL                     = "https://update.argotunnel.com"
+	StagingUpdateURL              = "https://staging-update.argotunnel.com"
+
+	LogFieldVersion = "version"
 )
 
 var (
-	publicKey = []byte(`
------BEGIN ECDSA PUBLIC KEY-----
-MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE4OWZocTVZ8Do/L6ScLdkV+9A0IYMHoOf
-dsCmJ/QZ6aw0w9qkkwEpne1Lmo6+0pGexZzFZOH6w5amShn+RXt7qkSid9iWlzGq
-EKx0BZogHSor9Wy5VztdFaAaVbsJiCbO
------END ECDSA PUBLIC KEY-----
-`)
+	version string
 )
 
 // BinaryUpdated implements ExitCoder interface, the app will exit with status code 11
-// https://pkg.go.dev/gopkg.in/urfave/cli.v2?tab=doc#ExitCoder
+// https://pkg.go.dev/github.com/urfave/cli/v2?tab=doc#ExitCoder
 type statusSuccess struct {
 	newVersion string
 }
@@ -64,56 +60,96 @@ func (e *statusErr) ExitCode() int {
 	return 10
 }
 
+type updateOptions struct {
+	updateDisabled  bool
+	isBeta          bool
+	isStaging       bool
+	isForced        bool
+	intendedVersion string
+}
+
 type UpdateOutcome struct {
-	Updated bool
-	Version string
-	Error   error
+	Updated     bool
+	Version     string
+	UserMessage string
+	Error       error
 }
 
 func (uo *UpdateOutcome) noUpdate() bool {
 	return uo.Error == nil && uo.Updated == false
 }
 
-func checkForUpdateAndApply() UpdateOutcome {
-	var opts equinox.Options
-	if err := opts.SetPublicKeyPEM(publicKey); err != nil {
-		return UpdateOutcome{Error: err}
-	}
-
-	resp, err := equinox.Check(appID, opts)
-	switch {
-	case err == equinox.NotAvailableErr:
-		return UpdateOutcome{}
-	case err != nil:
-		return UpdateOutcome{Error: err}
-	}
-
-	err = resp.Apply()
-	if err != nil {
-		return UpdateOutcome{Error: err}
-	}
-
-	return UpdateOutcome{Updated: true, Version: resp.ReleaseVersion}
+func Init(v string) {
+	version = v
 }
 
-func Update(_ *cli.Context) error {
-	logger, err := logger.New()
+func CheckForUpdate(options updateOptions) (CheckResult, error) {
+	cfdPath, err := os.Executable()
 	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
+		return nil, err
 	}
 
+	url := UpdateURL
+	if options.isStaging {
+		url = StagingUpdateURL
+	}
+
+	s := NewWorkersService(version, url, cfdPath, Options{IsBeta: options.isBeta,
+		IsForced: options.isForced, RequestedVersion: options.intendedVersion})
+
+	return s.Check()
+}
+
+func applyUpdate(options updateOptions, update CheckResult) UpdateOutcome {
+	if update.Version() == "" || options.updateDisabled {
+		return UpdateOutcome{UserMessage: update.UserMessage()}
+	}
+
+	err := update.Apply()
+	if err != nil {
+		return UpdateOutcome{Error: err}
+	}
+
+	return UpdateOutcome{Updated: true, Version: update.Version(), UserMessage: update.UserMessage()}
+}
+
+// Update is the handler for the update command from the command line
+func Update(c *cli.Context) error {
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
+
 	if wasInstalledFromPackageManager() {
-		logger.Error("cloudflared was installed by a package manager. Please update using the same method.")
+		log.Error().Msg("cloudflared was installed by a package manager. Please update using the same method.")
 		return nil
 	}
 
-	updateOutcome := loggedUpdate(logger)
+	isBeta := c.Bool("beta")
+	if isBeta {
+		log.Info().Msg("cloudflared is set to update to the latest beta version")
+	}
+
+	isStaging := c.Bool("staging")
+	if isStaging {
+		log.Info().Msg("cloudflared is set to update from staging")
+	}
+
+	isForced := c.Bool("force")
+	if isForced {
+		log.Info().Msg("cloudflared is set to upgrade to the latest publish version regardless of the current version")
+	}
+
+	updateOutcome := loggedUpdate(log, updateOptions{
+		updateDisabled:  false,
+		isBeta:          isBeta,
+		isStaging:       isStaging,
+		isForced:        isForced,
+		intendedVersion: c.String("version"),
+	})
 	if updateOutcome.Error != nil {
 		return &statusErr{updateOutcome.Error}
 	}
 
 	if updateOutcome.noUpdate() {
-		logger.Infof("cloudflared is up to date (%s)", updateOutcome.Version)
+		log.Info().Str(LogFieldVersion, updateOutcome.Version).Msg("cloudflared is up to date")
 		return nil
 	}
 
@@ -121,13 +157,19 @@ func Update(_ *cli.Context) error {
 }
 
 // Checks for an update and applies it if one is available
-func loggedUpdate(logger logger.Service) UpdateOutcome {
-	updateOutcome := checkForUpdateAndApply()
+func loggedUpdate(log *zerolog.Logger, options updateOptions) UpdateOutcome {
+	checkResult, err := CheckForUpdate(options)
+	if err != nil {
+		log.Err(err).Msg("update check failed")
+		return UpdateOutcome{Error: err}
+	}
+
+	updateOutcome := applyUpdate(options, checkResult)
 	if updateOutcome.Updated {
-		logger.Infof("cloudflared has been updated to version %s", updateOutcome.Version)
+		log.Info().Str(LogFieldVersion, updateOutcome.Version).Msg("cloudflared has been updated")
 	}
 	if updateOutcome.Error != nil {
-		logger.Errorf("update check failed: %s", updateOutcome.Error)
+		log.Err(updateOutcome.Error).Msg("update failed to apply")
 	}
 
 	return updateOutcome
@@ -138,7 +180,7 @@ type AutoUpdater struct {
 	configurable     *configurable
 	listeners        *gracenet.Net
 	updateConfigChan chan *configurable
-	logger           logger.Service
+	log              *zerolog.Logger
 }
 
 // AutoUpdaterConfigurable is the attributes of AutoUpdater that can be reconfigured during runtime
@@ -147,45 +189,53 @@ type configurable struct {
 	freq    time.Duration
 }
 
-func NewAutoUpdater(freq time.Duration, listeners *gracenet.Net, logger logger.Service) *AutoUpdater {
-	updaterConfigurable := &configurable{
-		enabled: true,
-		freq:    freq,
-	}
-	if freq == 0 {
-		updaterConfigurable.enabled = false
-		updaterConfigurable.freq = DefaultCheckUpdateFreq
-	}
+func NewAutoUpdater(updateDisabled bool, freq time.Duration, listeners *gracenet.Net, log *zerolog.Logger) *AutoUpdater {
 	return &AutoUpdater{
-		configurable:     updaterConfigurable,
+		configurable:     createUpdateConfig(updateDisabled, freq, log),
 		listeners:        listeners,
 		updateConfigChan: make(chan *configurable),
-		logger:           logger,
+		log:              log,
+	}
+}
+
+func createUpdateConfig(updateDisabled bool, freq time.Duration, log *zerolog.Logger) *configurable {
+	if isAutoupdateEnabled(log, updateDisabled, freq) {
+		log.Info().Dur("autoupdateFreq", freq).Msg("Autoupdate frequency is set")
+		return &configurable{
+			enabled: true,
+			freq:    freq,
+		}
+	} else {
+		return &configurable{
+			enabled: false,
+			freq:    DefaultCheckUpdateFreq,
+		}
 	}
 }
 
 func (a *AutoUpdater) Run(ctx context.Context) error {
 	ticker := time.NewTicker(a.configurable.freq)
 	for {
-		if a.configurable.enabled {
-			updateOutcome := loggedUpdate(a.logger)
-			if updateOutcome.Updated {
-				os.Args = append(os.Args, "--is-autoupdated=true")
-				if IsSysV() {
-					// SysV doesn't have a mechanism to keep service alive, we have to restart the process
-					a.logger.Info("Restarting service managed by SysV...")
-					pid, err := a.listeners.StartProcess()
-					if err != nil {
-						a.logger.Errorf("Unable to restart server automatically: %s", err)
-						return &statusErr{err: err}
-					}
-					// stop old process after autoupdate. Otherwise we create a new process
-					// after each update
-					a.logger.Infof("PID of the new process is %d", pid)
+		updateOutcome := loggedUpdate(a.log, updateOptions{updateDisabled: !a.configurable.enabled})
+		if updateOutcome.Updated {
+			Init(updateOutcome.Version)
+			if IsSysV() {
+				// SysV doesn't have a mechanism to keep service alive, we have to restart the process
+				a.log.Info().Msg("Restarting service managed by SysV...")
+				pid, err := a.listeners.StartProcess()
+				if err != nil {
+					a.log.Err(err).Msg("Unable to restart server automatically")
+					return &statusErr{err: err}
 				}
-				return &statusSuccess{newVersion: updateOutcome.Version}
+				// stop old process after autoupdate. Otherwise we create a new process
+				// after each update
+				a.log.Info().Msgf("PID of the new process is %d", pid)
 			}
+			return &statusSuccess{newVersion: updateOutcome.Version}
+		} else if updateOutcome.UserMessage != "" {
+			a.log.Warn().Msg(updateOutcome.UserMessage)
 		}
+
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
@@ -200,39 +250,30 @@ func (a *AutoUpdater) Run(ctx context.Context) error {
 }
 
 // Update is the method to pass new AutoUpdaterConfigurable to a running AutoUpdater. It is safe to be called concurrently
-func (a *AutoUpdater) Update(newFreq time.Duration) {
-	newConfigurable := &configurable{
-		enabled: true,
-		freq:    newFreq,
-	}
-	// A ero duration means autoupdate is disabled
-	if newFreq == 0 {
-		newConfigurable.enabled = false
-		newConfigurable.freq = DefaultCheckUpdateFreq
-	}
-	a.updateConfigChan <- newConfigurable
+func (a *AutoUpdater) Update(updateDisabled bool, newFreq time.Duration) {
+	a.updateConfigChan <- createUpdateConfig(updateDisabled, newFreq, a.log)
 }
 
-func IsAutoupdateEnabled(c *cli.Context, l logger.Service) bool {
-	if !SupportAutoUpdate(l) {
+func isAutoupdateEnabled(log *zerolog.Logger, updateDisabled bool, updateFreq time.Duration) bool {
+	if !supportAutoUpdate(log) {
 		return false
 	}
-	return !c.Bool("no-autoupdate") && c.Duration("autoupdate-freq") != 0
+	return !updateDisabled && updateFreq != 0
 }
 
-func SupportAutoUpdate(logger logger.Service) bool {
+func supportAutoUpdate(log *zerolog.Logger) bool {
 	if runtime.GOOS == "windows" {
-		logger.Info(noUpdateOnWindowsMessage)
+		log.Info().Msg(noUpdateOnWindowsMessage)
 		return false
 	}
 
 	if wasInstalledFromPackageManager() {
-		logger.Info(noUpdateManagedPackageMessage)
+		log.Info().Msg(noUpdateManagedPackageMessage)
 		return false
 	}
 
 	if isRunningFromTerminal() {
-		logger.Info(noUpdateInShellMessage)
+		log.Info().Msg(noUpdateInShellMessage)
 		return false
 	}
 	return true

cmd/cloudflared/updater/update_test.go

@@ -2,17 +2,19 @@ package updater
 
 import (
 	"context"
+	"flag"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/facebookgo/grace/gracenet"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+	"github.com/urfave/cli/v2"
 )
 
 func TestDisabledAutoUpdater(t *testing.T) {
 	listeners := &gracenet.Net{}
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
-	autoupdater := NewAutoUpdater(0, listeners, logger)
+	log := zerolog.Nop()
+	autoupdater := NewAutoUpdater(false, 0, listeners, &log)
 	ctx, cancel := context.WithCancel(context.Background())
 	errC := make(chan error)
 	go func() {
@@ -26,3 +28,13 @@ func TestDisabledAutoUpdater(t *testing.T) {
 	// Make sure that autoupdater terminates after canceling the context
 	assert.Equal(t, context.Canceled, <-errC)
 }
+
+func TestCheckInWithUpdater(t *testing.T) {
+	flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
+
+	warningChecker := StartWarningCheck(cliCtx)
+	warning := warningChecker.getWarning()
+	// Assuming this runs either on a release or development version, then the Worker will never have anything to tell us.
+	assert.Empty(t, warning)
+}

cmd/cloudflared/updater/workers_service.go

@@ -0,0 +1,94 @@
+package updater
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"runtime"
+)
+
+// Options are the update options supported by the
+type Options struct {
+	// IsBeta is for beta updates to be installed if available
+	IsBeta bool
+
+	// IsForced is to forcibly download the latest version regardless of the current version
+	IsForced bool
+
+	// RequestedVersion is the specific version to upgrade or downgrade to
+	RequestedVersion string
+}
+
+// VersionResponse is the JSON response from the Workers API endpoint
+type VersionResponse struct {
+	URL          string `json:"url"`
+	Version      string `json:"version"`
+	Checksum     string `json:"checksum"`
+	IsCompressed bool   `json:"compressed"`
+	UserMessage  string `json:"userMessage"`
+	ShouldUpdate bool   `json:"shouldUpdate"`
+	Error        string `json:"error"`
+}
+
+// WorkersService implements Service.
+// It contains everything needed to check in with the WorkersAPI and download and apply the updates
+type WorkersService struct {
+	currentVersion string
+	url            string
+	targetPath     string
+	opts           Options
+}
+
+// NewWorkersService creates a new updater Service object.
+func NewWorkersService(currentVersion, url, targetPath string, opts Options) Service {
+	return &WorkersService{
+		currentVersion: currentVersion,
+		url:            url,
+		targetPath:     targetPath,
+		opts:           opts,
+	}
+}
+
+// Check does a check in with the Workers API to get a new version update
+func (s *WorkersService) Check() (CheckResult, error) {
+	client := &http.Client{
+		Timeout: clientTimeout,
+	}
+
+	req, err := http.NewRequest(http.MethodGet, s.url, nil)
+	q := req.URL.Query()
+	q.Add(OSKeyName, runtime.GOOS)
+	q.Add(ArchitectureKeyName, runtime.GOARCH)
+	q.Add(ClientVersionName, s.currentVersion)
+
+	if s.opts.IsBeta {
+		q.Add(BetaKeyName, "true")
+	}
+
+	if s.opts.RequestedVersion != "" {
+		q.Add(VersionKeyName, s.opts.RequestedVersion)
+	}
+
+	req.URL.RawQuery = q.Encode()
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var v VersionResponse
+	if err := json.NewDecoder(resp.Body).Decode(&v); err != nil {
+		return nil, err
+	}
+
+	if v.Error != "" {
+		return nil, errors.New(v.Error)
+	}
+
+	versionToUpdate := ""
+	if v.ShouldUpdate {
+		versionToUpdate = v.Version
+	}
+
+	return NewWorkersVersion(v.URL, versionToUpdate, v.Checksum, s.targetPath, v.UserMessage, v.IsCompressed), nil
+}

cmd/cloudflared/updater/workers_service_test.go

@@ -0,0 +1,348 @@
+// +build !windows
+
+package updater
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"crypto/sha256"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+var testFilePath = filepath.Join(os.TempDir(), "test")
+
+func respondWithJSON(w http.ResponseWriter, v interface{}, status int) {
+	data, _ := json.Marshal(v)
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	w.Write(data)
+}
+
+func respondWithData(w http.ResponseWriter, b []byte, status int) {
+	w.Header().Set("Content-Type", "application/octet-stream")
+	w.WriteHeader(status)
+	w.Write(b)
+}
+
+const mostRecentVersion = "2021.2.5"
+const mostRecentBetaVersion = "2021.3.0"
+const knownBuggyVersion = "2020.12.0"
+const expectedUserMsg = "This message is expected when running a known buggy version"
+
+func updateHandler(w http.ResponseWriter, r *http.Request) {
+	version := mostRecentVersion
+	host := fmt.Sprintf("http://%s", r.Host)
+	url := host + "/download"
+
+	query := r.URL.Query()
+
+	if query.Get(BetaKeyName) == "true" {
+		version = mostRecentBetaVersion
+		url = host + "/beta"
+	}
+
+	requestedVersion := query.Get(VersionKeyName)
+	if requestedVersion != "" {
+		version = requestedVersion
+		url = fmt.Sprintf("%s?version=%s", url, requestedVersion)
+	}
+
+	if query.Get(ArchitectureKeyName) != runtime.GOARCH || query.Get(OSKeyName) != runtime.GOOS {
+		respondWithJSON(w, VersionResponse{Error: "unsupported os and architecture"}, http.StatusBadRequest)
+		return
+	}
+
+	h := sha256.New()
+	fmt.Fprint(h, version)
+	checksum := fmt.Sprintf("%x", h.Sum(nil))
+
+	var userMessage = ""
+	if query.Get(ClientVersionName) == knownBuggyVersion {
+		userMessage = expectedUserMsg
+	}
+	shouldUpdate := requestedVersion != "" || IsNewerVersion(query.Get(ClientVersionName), version)
+
+	v := VersionResponse{
+		URL: url, Version: version, Checksum: checksum, UserMessage: userMessage, ShouldUpdate: shouldUpdate,
+	}
+	respondWithJSON(w, v, http.StatusOK)
+}
+
+func gzipUpdateHandler(w http.ResponseWriter, r *http.Request) {
+	log.Println("got a request!")
+	version := "2020.09.02"
+	h := sha256.New()
+	fmt.Fprint(h, version)
+	checksum := fmt.Sprintf("%x", h.Sum(nil))
+
+	url := fmt.Sprintf("http://%s/gzip-download.tgz", r.Host)
+	v := VersionResponse{URL: url, Version: version, Checksum: checksum, ShouldUpdate: true}
+	respondWithJSON(w, v, http.StatusOK)
+}
+
+func compressedDownloadHandler(w http.ResponseWriter, r *http.Request) {
+	version := "2020.09.02"
+	buf := new(bytes.Buffer)
+
+	gw := gzip.NewWriter(buf)
+	tw := tar.NewWriter(gw)
+
+	header := &tar.Header{
+		Size: int64(len(version)),
+		Name: "download",
+	}
+	tw.WriteHeader(header)
+	tw.Write([]byte(version))
+
+	tw.Close()
+	gw.Close()
+
+	respondWithData(w, buf.Bytes(), http.StatusOK)
+}
+
+func downloadHandler(w http.ResponseWriter, r *http.Request) {
+	version := mostRecentVersion
+	requestedVersion := r.URL.Query().Get(VersionKeyName)
+	if requestedVersion != "" {
+		version = requestedVersion
+	}
+	respondWithData(w, []byte(version), http.StatusOK)
+}
+
+func betaHandler(w http.ResponseWriter, r *http.Request) {
+	respondWithData(w, []byte(mostRecentBetaVersion), http.StatusOK)
+}
+
+func failureHandler(w http.ResponseWriter, r *http.Request) {
+	respondWithJSON(w, VersionResponse{Error: "unsupported os and architecture"}, http.StatusBadRequest)
+}
+
+func IsNewerVersion(current string, check string) bool {
+	if current == "" || check == "" {
+		return false
+	}
+	if strings.Contains(strings.ToLower(current), "dev") {
+		return false // dev builds shouldn't update
+	}
+
+	cMajor, cMinor, cPatch, err := SemanticParts(current)
+	if err != nil {
+		return false
+	}
+
+	nMajor, nMinor, nPatch, err := SemanticParts(check)
+	if err != nil {
+		return false
+	}
+
+	if nMajor > cMajor {
+		return true
+	}
+
+	if nMajor == cMajor && nMinor > cMinor {
+		return true
+	}
+
+	if nMajor == cMajor && nMinor == cMinor && nPatch > cPatch {
+		return true
+	}
+	return false
+}
+
+func SemanticParts(version string) (major int, minor int, patch int, err error) {
+	major = 0
+	minor = 0
+	patch = 0
+	parts := strings.Split(version, ".")
+	if len(parts) != 3 {
+		err = errors.New("invalid version")
+		return
+	}
+	major, err = strconv.Atoi(parts[0])
+	if err != nil {
+		return
+	}
+
+	minor, err = strconv.Atoi(parts[1])
+	if err != nil {
+		return
+	}
+
+	patch, err = strconv.Atoi(parts[2])
+	if err != nil {
+		return
+	}
+	return
+}
+
+func createServer() *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/updater", updateHandler)
+	mux.HandleFunc("/download", downloadHandler)
+	mux.HandleFunc("/beta", betaHandler)
+	mux.HandleFunc("/fail", failureHandler)
+	mux.HandleFunc("/compressed", gzipUpdateHandler)
+	mux.HandleFunc("/gzip-download.tgz", compressedDownloadHandler)
+	return httptest.NewServer(mux)
+}
+
+func createTestFile(t *testing.T, path string) {
+	f, err := os.Create(path)
+	require.NoError(t, err)
+	fmt.Fprint(f, "2020.08.04")
+	f.Close()
+}
+
+func TestUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+	log.Println("server url: ", ts.URL)
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, v.Version(), mostRecentVersion)
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, string(dat), mostRecentVersion)
+}
+
+func TestBetaUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{IsBeta: true})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, v.Version(), mostRecentBetaVersion)
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, string(dat), mostRecentBetaVersion)
+}
+
+func TestFailUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/fail", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.Error(t, err)
+	require.Nil(t, v)
+}
+
+func TestNoUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService(mostRecentVersion, fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.NotNil(t, v)
+	require.Empty(t, v.Version())
+}
+
+func TestForcedUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService("2020.8.5", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{IsForced: true})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, v.Version(), mostRecentVersion)
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, string(dat), mostRecentVersion)
+}
+
+func TestUpdateSpecificVersionService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+	reqVersion := "2020.9.1"
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{RequestedVersion: reqVersion})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, reqVersion, v.Version())
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, reqVersion, string(dat))
+}
+
+func TestCompressedUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/compressed", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, "2020.09.02", v.Version())
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, "2020.09.02", string(dat))
+}
+
+func TestUpdateWhenRunningKnownBuggyVersion(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService(knownBuggyVersion, fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, v.Version(), mostRecentVersion)
+	require.Equal(t, v.UserMessage(), expectedUserMsg)
+}

cmd/cloudflared/updater/workers_update.go

@@ -0,0 +1,246 @@
+package updater
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"text/template"
+	"time"
+)
+
+const (
+	clientTimeout = time.Second * 60
+	// stop the service
+	// rename cloudflared.exe to cloudflared.exe.old
+	// rename cloudflared.exe.new to cloudflared.exe
+	// delete cloudflared.exe.old
+	// start the service
+	// delete the batch file
+	windowsUpdateCommandTemplate = `@echo off
+sc stop cloudflared >nul 2>&1
+rename "{{.TargetPath}}" {{.OldName}}
+rename "{{.NewPath}}" {{.BinaryName}}
+del "{{.OldPath}}"
+sc start cloudflared >nul 2>&1
+del {{.BatchName}}`
+	batchFileName = "cfd_update.bat"
+)
+
+// Prepare some data to insert into the template.
+type batchData struct {
+	TargetPath string
+	OldName    string
+	NewPath    string
+	OldPath    string
+	BinaryName string
+	BatchName  string
+}
+
+// WorkersVersion implements the Version interface.
+// It contains everything needed to preform a version upgrade
+type WorkersVersion struct {
+	downloadURL  string
+	checksum     string
+	version      string
+	targetPath   string
+	isCompressed bool
+	userMessage  string
+}
+
+// NewWorkersVersion creates a new Version object. This is normally created by a WorkersService JSON checkin response
+// url is where to download the file
+// version is the version of this update
+// checksum is the expected checksum of the downloaded file
+// target path is where the file should be replace. Normally this the running cloudflared's path
+// userMessage is a possible message to convey back to the user after having checked in with the Updater Service
+// isCompressed tells whether the asset to update cloudflared is compressed or not
+func NewWorkersVersion(url, version, checksum, targetPath, userMessage string, isCompressed bool) CheckResult {
+	return &WorkersVersion{
+		downloadURL:  url,
+		version:      version,
+		checksum:     checksum,
+		targetPath:   targetPath,
+		isCompressed: isCompressed,
+		userMessage:  userMessage,
+	}
+}
+
+// Apply does the actual verification and update logic.
+// This includes signature and checksum validation,
+// replacing the binary, etc
+func (v *WorkersVersion) Apply() error {
+	newFilePath := fmt.Sprintf("%s.new", v.targetPath)
+	os.Remove(newFilePath) //remove any failed updates before download
+
+	// download the file
+	if err := download(v.downloadURL, newFilePath, v.isCompressed); err != nil {
+		return err
+	}
+
+	// check that the file is what is expected
+	if err := isValidChecksum(v.checksum, newFilePath); err != nil {
+		return err
+	}
+
+	oldFilePath := fmt.Sprintf("%s.old", v.targetPath)
+	// Windows requires more effort to self update, especially when it is running as a service:
+	// you have to stop the service (if running as one) in order to move/rename the binary
+	// but now the binary isn't running though, so an external process
+	// has to move the old binary out and the new one in then start the service
+	// the easiest way to do this is with a batch file (or with a DLL, but that gets ugly for a cross compiled binary like cloudflared)
+	// a batch file isn't ideal, but it is the simplest path forward for the constraints Windows creates
+	if runtime.GOOS == "windows" {
+		if err := writeBatchFile(v.targetPath, newFilePath, oldFilePath); err != nil {
+			return err
+		}
+		rootDir := filepath.Dir(v.targetPath)
+		batchPath := filepath.Join(rootDir, batchFileName)
+		return runWindowsBatch(batchPath)
+	}
+
+	// now move the current file out, move the new file in and delete the old file
+	if err := os.Rename(v.targetPath, oldFilePath); err != nil {
+		return err
+	}
+
+	if err := os.Rename(newFilePath, v.targetPath); err != nil {
+		//attempt rollback
+		os.Rename(oldFilePath, v.targetPath)
+		return err
+	}
+	os.Remove(oldFilePath)
+
+	return nil
+}
+
+// String returns the version number of this update/release (e.g. 2020.08.05)
+func (v *WorkersVersion) Version() string {
+	return v.version
+}
+
+// String returns a possible message to convey back to user after having checked in with the Updater Service. E.g.
+// it can warn about the need to update the version currently running.
+func (v *WorkersVersion) UserMessage() string {
+	return v.userMessage
+}
+
+// download the file from the link in the json
+func download(url, filepath string, isCompressed bool) error {
+	client := &http.Client{
+		Timeout: clientTimeout,
+	}
+	resp, err := client.Get(url)
+
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	var r io.Reader
+	r = resp.Body
+
+	// compressed macos binary, need to decompress
+	if isCompressed || isCompressedFile(url) {
+		// first the gzip reader
+		gr, err := gzip.NewReader(resp.Body)
+		if err != nil {
+			return err
+		}
+		defer gr.Close()
+
+		// now the tar
+		tr := tar.NewReader(gr)
+
+		// advance the reader pass the header, which will be the single binary file
+		tr.Next()
+
+		r = tr
+	}
+
+	out, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, r)
+	return err
+}
+
+// isCompressedFile is a really simple file extension check to see if this is a macos tar and gzipped
+func isCompressedFile(urlstring string) bool {
+	if strings.HasSuffix(urlstring, ".tgz") {
+		return true
+	}
+
+	u, err := url.Parse(urlstring)
+	if err != nil {
+		return false
+	}
+	return strings.HasSuffix(u.Path, ".tgz")
+}
+
+// checks if the checksum in the json response matches the checksum of the file download
+func isValidChecksum(checksum, filePath string) error {
+	f, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return err
+	}
+
+	hash := fmt.Sprintf("%x", h.Sum(nil))
+
+	if checksum != hash {
+		return errors.New("checksum validation failed")
+	}
+	return nil
+}
+
+// writeBatchFile writes a batch file out to disk
+// see the dicussion on why it has to be done this way
+func writeBatchFile(targetPath string, newPath string, oldPath string) error {
+	os.Remove(batchFileName) //remove any failed updates before download
+	f, err := os.Create(batchFileName)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	cfdName := filepath.Base(targetPath)
+	oldName := filepath.Base(oldPath)
+
+	data := batchData{
+		TargetPath: targetPath,
+		OldName:    oldName,
+		NewPath:    newPath,
+		OldPath:    oldPath,
+		BinaryName: cfdName,
+		BatchName:  batchFileName,
+	}
+
+	t, err := template.New("batch").Parse(windowsUpdateCommandTemplate)
+	if err != nil {
+		return err
+	}
+	return t.Execute(f, data)
+}
+
+// run each OS command for windows
+func runWindowsBatch(batchFile string) error {
+	cmd := exec.Command("cmd", "/c", batchFile)
+	return cmd.Start()
+}

cmd/cloudflared/windows_service.go

@@ -12,14 +12,15 @@ import (
 	"time"
 	"unsafe"
 
-	"github.com/cloudflare/cloudflared/logger"
 	"github.com/pkg/errors"
-	cli "gopkg.in/urfave/cli.v2"
-
+	"github.com/urfave/cli/v2"
 	"golang.org/x/sys/windows"
 	"golang.org/x/sys/windows/svc"
 	"golang.org/x/sys/windows/svc/eventlog"
 	"golang.org/x/sys/windows/svc/mgr"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
@@ -37,22 +38,24 @@ const (
 	// ERROR_FAILED_SERVICE_CONTROLLER_CONNECT
 	// https://docs.microsoft.com/en-us/windows/desktop/debug/system-error-codes--1000-1299-
 	serviceControllerConnectionFailure = 1063
+
+	LogFieldWindowsServiceName = "windowsServiceName"
 )
 
-func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
+func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
 		Usage: "Manages the Argo Tunnel Windows service",
 		Subcommands: []*cli.Command{
-			&cli.Command{
+			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a Windows service",
-				Action: installWindowsService,
+				Action: cliutil.ConfiguredAction(installWindowsService),
 			},
-			&cli.Command{
+			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
-				Action: uninstallWindowsService,
+				Action: cliutil.ConfiguredAction(uninstallWindowsService),
 			},
 		},
 	})
@@ -67,15 +70,11 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 	//     2. get ERROR_FAILED_SERVICE_CONTROLLER_CONNECT
 	// This involves actually trying to start the service.
 
-	logger, err := logger.New()
-	if err != nil {
-		os.Exit(1)
-		return
-	}
+	log := logger.Create(nil)
 
 	isIntSess, err := svc.IsAnInteractiveSession()
 	if err != nil {
-		logger.Fatalf("failed to determine if we are running in an interactive session: %v", err)
+		log.Fatal().Err(err).Msg("failed to determine if we are running in an interactive session")
 	}
 	if isIntSess {
 		app.Run(os.Args)
@@ -85,7 +84,7 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 	// Run executes service name by calling windowsService which is a Handler
 	// interface that implements Execute method.
 	// It will set service status to stop after Execute returns
-	err = svc.Run(windowsServiceName, &windowsService{app: app, shutdownC: shutdownC, graceShutdownC: graceShutdownC})
+	err = svc.Run(windowsServiceName, &windowsService{app: app, graceShutdownC: graceShutdownC})
 	if err != nil {
 		if errno, ok := err.(syscall.Errno); ok && int(errno) == serviceControllerConnectionFailure {
 			// Hack: assume this is a false negative from the IsAnInteractiveSession() check above.
@@ -93,27 +92,22 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 			app.Run(os.Args)
 			return
 		}
-		logger.Fatalf("%s service failed: %v", windowsServiceName, err)
+		log.Fatal().Err(err).Msgf("%s service failed", windowsServiceName)
 	}
 }
 
 type windowsService struct {
 	app            *cli.App
-	shutdownC      chan struct{}
 	graceShutdownC chan struct{}
 }
 
-// called by the package code at the start of the service
+// Execute is called by the service manager when service starts, the state
+// of the service will be set to Stopped when this function returns.
 func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeRequest, statusChan chan<- svc.Status) (ssec bool, errno uint32) {
-	logger, err := logger.New()
-	if err != nil {
-		os.Exit(1)
-		return
-	}
-
+	log := logger.Create(nil)
 	elog, err := eventlog.Open(windowsServiceName)
 	if err != nil {
-		logger.Errorf("Cannot open event log for %s with error: %s", windowsServiceName, err)
+		log.Err(err).Msgf("Cannot open event log for %s", windowsServiceName)
 		return
 	}
 	defer elog.Close()
@@ -136,13 +130,12 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 	}
 	elog.Info(1, fmt.Sprintf("%s service arguments: %v", windowsServiceName, args))
 
-	const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown
 	statusChan <- svc.Status{State: svc.StartPending}
 	errC := make(chan error)
 	go func() {
 		errC <- s.app.Run(args)
 	}()
-	statusChan <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}
+	statusChan <- svc.Status{State: svc.Running, Accepts: svc.AcceptStop | svc.AcceptShutdown}
 
 	for {
 		select {
@@ -150,19 +143,26 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 			switch c.Cmd {
 			case svc.Interrogate:
 				statusChan <- c.CurrentStatus
-			case svc.Stop:
-				close(s.graceShutdownC)
-				statusChan <- svc.Status{State: svc.StopPending}
-			case svc.Shutdown:
-				close(s.shutdownC)
+			case svc.Stop, svc.Shutdown:
+				if s.graceShutdownC != nil {
+					// start graceful shutdown
+					elog.Info(1, "cloudflared starting graceful shutdown")
+					close(s.graceShutdownC)
+					s.graceShutdownC = nil
+					statusChan <- svc.Status{State: svc.StopPending}
+					continue
+				}
+				// repeated attempts at graceful shutdown forces immediate stop
+				elog.Info(1, "cloudflared terminating immediately")
 				statusChan <- svc.Status{State: svc.StopPending}
+				return false, 0
 			default:
 				elog.Error(1, fmt.Sprintf("unexpected control request #%d", c))
 			}
 		case err := <-errC:
-			ssec = true
 			if err != nil {
 				elog.Error(1, fmt.Sprintf("cloudflared terminated with error %v", err))
+				ssec = true
 				errno = 1
 			} else {
 				elog.Info(1, "cloudflared terminated without error")
@@ -174,80 +174,68 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 }
 
 func installWindowsService(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	zeroLogger := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
-	logger.Infof("Installing Argo Tunnel Windows service")
+	zeroLogger.Info().Msg("Installing Argo Tunnel Windows service")
 	exepath, err := os.Executable()
 	if err != nil {
-		logger.Errorf("Cannot find path name that start the process")
-		return err
+		return errors.Wrap(err, "Cannot find path name that start the process")
 	}
 	m, err := mgr.Connect()
 	if err != nil {
-		logger.Errorf("Cannot establish a connection to the service control manager: %s", err)
-		return err
+		return errors.Wrap(err, "Cannot establish a connection to the service control manager")
 	}
 	defer m.Disconnect()
 	s, err := m.OpenService(windowsServiceName)
+	log := zeroLogger.With().Str(LogFieldWindowsServiceName, windowsServiceName).Logger()
 	if err == nil {
 		s.Close()
-		logger.Errorf("service %s already exists", windowsServiceName)
-		return fmt.Errorf("service %s already exists", windowsServiceName)
+		return fmt.Errorf("Service %s already exists", windowsServiceName)
 	}
 	config := mgr.Config{StartType: mgr.StartAutomatic, DisplayName: windowsServiceDescription}
 	s, err = m.CreateService(windowsServiceName, exepath, config)
 	if err != nil {
-		logger.Errorf("Cannot install service %s", windowsServiceName)
-		return err
+		return errors.Wrap(err, "Cannot install service")
 	}
 	defer s.Close()
-	logger.Infof("Argo Tunnel agent service is installed")
+	log.Info().Msg("Argo Tunnel agent service is installed")
 	err = eventlog.InstallAsEventCreate(windowsServiceName, eventlog.Error|eventlog.Warning|eventlog.Info)
 	if err != nil {
 		s.Delete()
-		logger.Errorf("Cannot install event logger: %s", err)
-		return fmt.Errorf("SetupEventLogSource() failed: %s", err)
+		return errors.Wrap(err, "Cannot install event logger")
 	}
 	err = configRecoveryOption(s.Handle)
 	if err != nil {
-		logger.Errorf("Cannot set service recovery actions: %s", err)
-		logger.Infof("See %s to manually configure service recovery actions", windowsServiceUrl)
+		log.Err(err).Msg("Cannot set service recovery actions")
+		log.Info().Msgf("See %s to manually configure service recovery actions", windowsServiceUrl)
 	}
 	return nil
 }
 
 func uninstallWindowsService(c *cli.Context) error {
-	logger, err := logger.New()
-	if err != nil {
-		return errors.Wrap(err, "error setting up logger")
-	}
+	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog).
+		With().
+		Str(LogFieldWindowsServiceName, windowsServiceName).Logger()
 
-	logger.Infof("Uninstalling Argo Tunnel Windows Service")
+	log.Info().Msg("Uninstalling Argo Tunnel Windows Service")
 	m, err := mgr.Connect()
 	if err != nil {
-		logger.Errorf("Cannot establish a connection to the service control manager")
-		return err
+		return errors.Wrap(err, "Cannot establish a connection to the service control manager")
 	}
 	defer m.Disconnect()
 	s, err := m.OpenService(windowsServiceName)
 	if err != nil {
-		logger.Errorf("service %s is not installed", windowsServiceName)
-		return fmt.Errorf("service %s is not installed", windowsServiceName)
+		return fmt.Errorf("Service %s is not installed", windowsServiceName)
 	}
 	defer s.Close()
 	err = s.Delete()
 	if err != nil {
-		logger.Errorf("Cannot delete service %s", windowsServiceName)
-		return err
+		return errors.Wrap(err, "Cannot delete service")
 	}
-	logger.Infof("Argo Tunnel agent service is uninstalled")
+	log.Info().Msg("Argo Tunnel agent service is uninstalled")
 	err = eventlog.Remove(windowsServiceName)
 	if err != nil {
-		logger.Errorf("Cannot remove event logger")
-		return fmt.Errorf("RemoveEventLogSource() failed: %s", err)
+		return errors.Wrap(err, "Cannot remove event logger")
 	}
 	return nil
 }

component-tests/.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+.pytest_cache

component-tests/README.md

@@ -0,0 +1,52 @@
+# Requirements
+1. Python 3.7 or later with packages in the given `requirements.txt`
+   - E.g. with conda:
+   - `conda create -n component-tests python=3.7`  
+   - `conda activate component-tests`
+   - `pip3 install -r requirements.txt`
+
+2. Create a config yaml file, for example:
+```
+cloudflared_binary: "cloudflared"
+tunnel: "3d539f97-cd3a-4d8e-c33b-65e9099c7a8d"
+credentials_file: "/Users/tunnel/.cloudflared/3d539f97-cd3a-4d8e-c33b-65e9099c7a8d.json"
+classic_hostname: "classic-tunnel-component-tests.example.com"
+origincert: "/Users/tunnel/.cloudflared/cert.pem"
+ingress:
+- hostname: named-tunnel-component-tests.example.com
+  service: http_status:200
+- service: http_status:404
+```
+
+3. Route hostname to the tunnel. For the example config above, we can do that via
+```
+   cloudflared tunnel route dns 3d539f97-cd3a-4d8e-c33b-65e9099c7a8d named-tunnel-component-tests.example.com
+```
+
+4. Turn on linter
+If you are using Visual Studio, follow https://code.visualstudio.com/docs/python/linting to turn on linter.
+
+5. Turn on formatter
+If you are using Visual Studio, follow https://code.visualstudio.com/docs/python/editing#_formatting
+to turn on formatter and https://marketplace.visualstudio.com/items?itemName=cbrevik.toggle-format-on-save
+to turn on format on save.
+
+6. If you have cloudflared running as a service on your machine, you can either stop the service or ignore the service tests
+via `--ignore test_service.py`
+
+# How to run
+Specify path to config file via env var `COMPONENT_TESTS_CONFIG`. This is required.
+## All tests
+Run `pytest` inside this(component-tests) folder
+
+## Specific files
+Run `pytest <file 1 name>.py <file 2 name>.py`
+
+## Specific tests
+Run `pytest file.py -k <test 1 name> -k <test 2 name>`
+
+## Live Logging
+Running with `-o log_cli=true` outputs logging to CLI as the tests are. By default the log level is WARN.
+`--log-cli-level` control logging level.
+For example, to log at info level, run `pytest -o log_cli=true --log-cli-level=INFO`.
+See https://docs.pytest.org/en/latest/logging.html#live-logs for more documentation on logging.

component-tests/config.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python
+import copy
+
+from dataclasses import dataclass, InitVar
+
+from constants import METRICS_PORT, PROXY_DNS_PORT
+
+# frozen=True raises exception when assigning to fields. This emulates immutability
+
+
+@dataclass(frozen=True)
+class BaseConfig:
+    cloudflared_binary: str
+    no_autoupdate: bool = True
+    metrics: str = f'localhost:{METRICS_PORT}'
+
+    def merge_config(self, additional):
+        config = copy.copy(additional)
+        config['no-autoupdate'] = self.no_autoupdate
+        config['metrics'] = self.metrics
+        return config
+
+
+@dataclass(frozen=True)
+class NamedTunnelBaseConfig(BaseConfig):
+    # The attributes of the parent class are ordered before attributes in this class,
+    # so we have to use default values here and check if they are set in __post_init__
+    tunnel: str = None
+    credentials_file: str = None
+    ingress: list = None
+
+    def __post_init__(self):
+        if self.tunnel is None:
+            raise TypeError("Field tunnel is not set")
+        if self.credentials_file is None:
+            raise TypeError("Field credentials_file is not set")
+        if self.ingress is None:
+            raise TypeError("Field ingress is not set")
+
+    def merge_config(self, additional):
+        config = super(NamedTunnelBaseConfig, self).merge_config(additional)
+        config['tunnel'] = self.tunnel
+        config['credentials-file'] = self.credentials_file
+        # In some cases we want to override default ingress, such as in config tests
+        if 'ingress' not in config:
+            config['ingress'] = self.ingress
+        return config
+
+
+@dataclass(frozen=True)
+class NamedTunnelConfig(NamedTunnelBaseConfig):
+    full_config: dict = None
+    additional_config: InitVar[dict] = {}
+
+    def __post_init__(self, additional_config):
+        # Cannot call set self.full_config because the class is frozen, instead, we can use __setattr__
+        # https://docs.python.org/3/library/dataclasses.html#frozen-instances
+        object.__setattr__(self, 'full_config',
+                           self.merge_config(additional_config))
+
+    def get_url(self):
+        return "https://" + self.ingress[0]['hostname']
+
+
+@dataclass(frozen=True)
+class ClassicTunnelBaseConfig(BaseConfig):
+    hostname: str = None
+    origincert: str = None
+
+    def __post_init__(self):
+        if self.hostname is None:
+            raise TypeError("Field tunnel is not set")
+        if self.origincert is None:
+            raise TypeError("Field credentials_file is not set")
+
+    def merge_config(self, additional):
+        config = super(ClassicTunnelBaseConfig, self).merge_config(additional)
+        config['hostname'] = self.hostname
+        config['origincert'] = self.origincert
+        return config
+
+
+@dataclass(frozen=True)
+class ClassicTunnelConfig(ClassicTunnelBaseConfig):
+    full_config: dict = None
+    additional_config: InitVar[dict] = {}
+
+    def __post_init__(self, additional_config):
+        # Cannot call set self.full_config because the class is frozen, instead, we can use __setattr__
+        # https://docs.python.org/3/library/dataclasses.html#frozen-instances
+        object.__setattr__(self, 'full_config',
+                           self.merge_config(additional_config))
+
+    def get_url(self):
+        return "https://" + self.hostname
+
+
+@dataclass(frozen=True)
+class ProxyDnsConfig(BaseConfig):
+    full_config = {
+        "port": PROXY_DNS_PORT,
+        "no-autoupdate": True,
+    }
+

component-tests/conftest.py

@@ -0,0 +1,60 @@
+import os
+from enum import Enum, auto
+from time import sleep
+
+import pytest
+import yaml
+
+from config import NamedTunnelConfig, ClassicTunnelConfig, ProxyDnsConfig
+from constants import BACKOFF_SECS, PROXY_DNS_PORT
+from util import LOGGER
+
+
+class CfdModes(Enum):
+    NAMED = auto()
+    CLASSIC = auto()
+    PROXY_DNS = auto()
+
+
+@pytest.fixture(scope="session")
+def component_tests_config():
+    config_file = os.getenv("COMPONENT_TESTS_CONFIG")
+    if config_file is None:
+        raise Exception(
+            "Need to provide path to config file in COMPONENT_TESTS_CONFIG")
+    with open(config_file, 'r') as stream:
+        config = yaml.safe_load(stream)
+        LOGGER.info(f"component tests base config {config}")
+
+        def _component_tests_config(additional_config={}, cfd_mode=CfdModes.NAMED, run_proxy_dns=True):
+            if run_proxy_dns:
+                # Regression test for TUN-4177, running with proxy-dns should not prevent tunnels from running.
+                # So we run all tests with it.
+                additional_config["proxy-dns"] = True
+                additional_config["proxy-dns-port"] = PROXY_DNS_PORT
+            else:
+                additional_config.pop("proxy-dns", None)
+                additional_config.pop("proxy-dns-port", None)
+
+            if cfd_mode is CfdModes.NAMED:
+                return NamedTunnelConfig(additional_config=additional_config,
+                                         cloudflared_binary=config['cloudflared_binary'],
+                                         tunnel=config['tunnel'],
+                                         credentials_file=config['credentials_file'],
+                                         ingress=config['ingress'])
+            elif cfd_mode is CfdModes.CLASSIC:
+                return ClassicTunnelConfig(
+                    additional_config=additional_config, cloudflared_binary=config['cloudflared_binary'],
+                    hostname=config['classic_hostname'], origincert=config['origincert'])
+            elif cfd_mode is CfdModes.PROXY_DNS:
+                return ProxyDnsConfig(cloudflared_binary=config['cloudflared_binary'])
+            else:
+                raise Exception(f"Unknown cloudflared mode {cfd_mode}")
+
+        return _component_tests_config
+
+
+# This fixture is automatically called before each tests to make sure the previous cloudflared has been shutdown
+@pytest.fixture(autouse=True)
+def wait_previous_cloudflared():
+    sleep(BACKOFF_SECS)

component-tests/constants.py

@@ -0,0 +1,5 @@
+METRICS_PORT = 51000
+MAX_RETRIES = 5
+BACKOFF_SECS = 7
+
+PROXY_DNS_PORT = 9053

component-tests/requirements.txt

@@ -0,0 +1,6 @@
+cloudflare==2.8.15
+flaky==3.7.0
+pytest==6.2.2
+pyyaml==5.4.1
+requests==2.25.1
+retrying==1.3.3

component-tests/setup.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+import argparse
+import base64
+import json
+import os
+import subprocess
+import uuid
+
+import CloudFlare
+import yaml
+from retrying import retry
+
+from constants import MAX_RETRIES, BACKOFF_SECS
+from util import LOGGER
+
+
+def get_config_from_env():
+    config_content = base64.b64decode(get_env("COMPONENT_TESTS_CONFIG_CONTENT")).decode('utf-8')
+    return yaml.safe_load(config_content)
+
+
+def get_config_from_file():
+    config_path = get_env("COMPONENT_TESTS_CONFIG")
+    with open(config_path, 'r') as infile:
+        return yaml.safe_load(infile)
+
+
+def persist_config(config):
+    config_path = get_env("COMPONENT_TESTS_CONFIG")
+    with open(config_path, 'w') as outfile:
+        yaml.safe_dump(config, outfile)
+
+
+def persist_origin_cert(config):
+    origincert = get_env("COMPONENT_TESTS_ORIGINCERT")
+    path = config["origincert"]
+    with open(path, 'w') as outfile:
+        outfile.write(origincert)
+    return path
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def create_tunnel(config, origincert_path, random_uuid):
+    # Delete any previous existing credentials file. If the agent keeps files around (that's the case in Windows) then
+    # cloudflared tunnel create will refuse to create the tunnel because it does not want to overwrite credentials
+    # files.
+    credentials_path = config["credentials_file"]
+    try:
+        os.remove(credentials_path)
+    except OSError:
+        pass
+
+    tunnel_name = "cfd_component_test-" + random_uuid
+    create_cmd = [config["cloudflared_binary"], "tunnel", "--origincert", origincert_path, "create",
+                  "--credentials-file", credentials_path, tunnel_name]
+    LOGGER.info(f"Creating tunnel with {create_cmd}")
+    subprocess.run(create_cmd, check=True)
+
+    list_cmd = [config["cloudflared_binary"], "tunnel", "--origincert", origincert_path, "list", "--name",
+                tunnel_name, "--output", "json"]
+    LOGGER.info(f"Listing tunnel with {list_cmd}")
+    cloudflared = subprocess.run(list_cmd, check=True, capture_output=True)
+    return json.loads(cloudflared.stdout)[0]["id"]
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def delete_tunnel(config):
+    credentials_path = config["credentials_file"]
+    delete_cmd = [config["cloudflared_binary"], "tunnel", "--origincert", config["origincert"], "delete",
+                  "--credentials-file", credentials_path, "-f", config["tunnel"]]
+    LOGGER.info(f"Deleting tunnel with {delete_cmd}")
+    subprocess.run(delete_cmd, check=True)
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def create_dns(config, hostname, type, content):
+    cf = CloudFlare.CloudFlare(debug=True, token=get_env("DNS_API_TOKEN"))
+    cf.zones.dns_records.post(
+        config["zone_tag"],
+        data={'name': hostname, 'type': type, 'content': content, 'proxied': True}
+    )
+
+
+def create_classic_dns(config, random_uuid):
+    classic_hostname = "classic-" + random_uuid + "." + config["zone_domain"]
+    create_dns(config, classic_hostname, "AAAA", "fd10:aec2:5dae::")
+    return classic_hostname
+
+
+def create_named_dns(config, random_uuid):
+    hostname = "named-" + random_uuid + "." + config["zone_domain"]
+    create_dns(config, hostname, "CNAME", config["tunnel"] + ".cfargotunnel.com")
+    return hostname
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def delete_dns(config, hostname):
+    cf = CloudFlare.CloudFlare(debug=True, token=get_env("DNS_API_TOKEN"))
+    zone_tag = config["zone_tag"]
+    dns_records = cf.zones.dns_records.get(zone_tag, params={'name': hostname})
+    if len(dns_records) > 0:
+        cf.zones.dns_records.delete(zone_tag, dns_records[0]['id'])
+
+
+def write_file(content, path):
+    with open(path, 'w') as outfile:
+        outfile.write(content)
+
+
+def get_env(env_name):
+    val = os.getenv(env_name)
+    if val is None:
+        raise Exception(f"{env_name} is not set")
+    return val
+
+
+def create():
+    """
+    Creates the necessary resources for the components test to run.
+     - Creates a named tunnel with a random name.
+     - Creates a random CNAME DNS entry for that tunnel.
+     - Creates a random AAAA DNS entry for a classic tunnel.
+
+    Those created resources are added to the config (obtained from an environment variable).
+    The resulting configuration is persisted for the tests to use.
+    """
+    config = get_config_from_env()
+    origincert_path = persist_origin_cert(config)
+
+    random_uuid = str(uuid.uuid4())
+    config["tunnel"] = create_tunnel(config, origincert_path, random_uuid)
+    config["classic_hostname"] = create_classic_dns(config, random_uuid)
+    config["ingress"] = [
+        {
+            "hostname": create_named_dns(config, random_uuid),
+            "service": "hello_world"
+        },
+        {
+            "service": "http_status:404"
+        }
+    ]
+
+    persist_config(config)
+
+
+def cleanup():
+    """
+    Reads the persisted configuration that was created previously.
+    Deletes the resources that were created there.
+    """
+    config = get_config_from_file()
+    delete_tunnel(config)
+    delete_dns(config, config["classic_hostname"])
+    delete_dns(config, config["ingress"][0]["hostname"])
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='setup component tests')
+    parser.add_argument('--type', choices=['create', 'cleanup'], default='create')
+    args = parser.parse_args()
+
+    if args.type == 'create':
+        create()
+    else:
+        cleanup()

component-tests/test_config.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+from util import start_cloudflared
+
+
+class TestConfig:
+    # tmp_path is a fixture provides a temporary directory unique to the test invocation
+    def test_validate_ingress_rules(self, tmp_path, component_tests_config):
+        extra_config = {
+            'ingress': [
+                {
+                    "hostname": "example.com",
+                    "service": "https://localhost:8000",
+                    "originRequest": {
+                        "originServerName": "test.example.com",
+                        "caPool": "/etc/certs/ca.pem"
+                    },
+                },
+                {
+                    "hostname": "api.example.com",
+                    "path": "login",
+                    "service": "https://localhost:9000",
+                },
+                {
+                    "hostname": "wss.example.com",
+                    "service": "wss://localhost:8000",
+                },
+                {
+                    "hostname": "ssh.example.com",
+                    "service": "ssh://localhost:8000",
+                },
+                {"service": "http_status:404"}
+            ],
+        }
+        config = component_tests_config(extra_config)
+        validate_args = ["ingress", "validate"]
+        _ = start_cloudflared(tmp_path, config, validate_args)
+
+        self.match_rule(tmp_path, config,
+                        "http://example.com/index.html", 1)
+        self.match_rule(tmp_path, config,
+                        "https://example.com/index.html", 1)
+        self.match_rule(tmp_path, config,
+                        "https://api.example.com/login", 2)
+        self.match_rule(tmp_path, config,
+                        "https://wss.example.com", 3)
+        self.match_rule(tmp_path, config,
+                        "https://ssh.example.com", 4)
+        self.match_rule(tmp_path, config,
+                        "https://api.example.com", 5)
+
+    # This is used to check that the command tunnel ingress url <url> matches rule number <rule_num>. Note that rule number uses 1-based indexing
+
+    def match_rule(self, tmp_path, config, url, rule_num):
+        args = ["ingress", "rule", url]
+        match_rule = start_cloudflared(tmp_path, config, args)
+
+        assert f"Matched rule #{rule_num}" .encode() in match_rule.stdout

component-tests/test_logging.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+import json
+import os
+
+from util import start_cloudflared, wait_tunnel_ready, send_requests
+
+# Rolling logger rotate log files after 1 MB
+rotate_after_size = 1000 * 1000
+default_log_file = "cloudflared.log"
+expect_message = "Starting Hello"
+
+
+def assert_log_to_terminal(cloudflared):
+    stderr = cloudflared.stderr.read(1500)
+    assert expect_message.encode() in stderr, f"{stderr} doesn't contain {expect_message}"
+
+
+def assert_log_in_file(file):
+    with open(file, "r") as f:
+        log = f.read(2000)
+        assert expect_message in log, f"{log} doesn't contain {expect_message}"
+
+
+def assert_json_log(file):
+    def assert_in_json(j, key):
+        assert key in j, f"{key} is not in j"
+
+    with open(file, "r") as f:
+        line = f.readline()
+        json_log = json.loads(line)
+        assert_in_json(json_log, "level")
+        assert_in_json(json_log, "time")
+        assert_in_json(json_log, "message")
+
+
+def assert_log_to_dir(config, log_dir):
+    max_batches = 3
+    batch_requests = 1000
+    for _ in range(max_batches):
+        send_requests(config.get_url(),
+                      batch_requests, require_ok=False)
+        files = os.listdir(log_dir)
+        if len(files) == 2:
+            current_log_file_index = files.index(default_log_file)
+            current_file = log_dir / files[current_log_file_index]
+            stats = os.stat(current_file)
+            assert stats.st_size > 0
+            assert_json_log(current_file)
+
+            # One file is the current log file, the other is the rotated log file
+            rotated_log_file_index = 0 if current_log_file_index == 1 else 1
+            rotated_file = log_dir / files[rotated_log_file_index]
+            stats = os.stat(rotated_file)
+            assert stats.st_size > rotate_after_size
+            assert_log_in_file(rotated_file)
+            assert_json_log(current_file)
+            return
+
+    raise Exception(
+        f"Log file isn't rotated after sending {max_batches * batch_requests} requests")
+
+
+class TestLogging:
+    def test_logging_to_terminal(self, tmp_path, component_tests_config):
+        config = component_tests_config()
+        with start_cloudflared(tmp_path, config, new_process=True) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            assert_log_to_terminal(cloudflared)
+
+    def test_logging_to_file(self, tmp_path, component_tests_config):
+        log_file = tmp_path / default_log_file
+        extra_config = {
+            # Convert from pathlib.Path to str
+            "logfile": str(log_file),
+        }
+        config = component_tests_config(extra_config)
+        with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            assert_log_in_file(log_file)
+            assert_json_log(log_file)
+
+    def test_logging_to_dir(self, tmp_path, component_tests_config):
+        log_dir = tmp_path / "logs"
+        extra_config = {
+            "loglevel": "debug",
+            # Convert from pathlib.Path to str
+            "log-directory": str(log_dir),
+        }
+        config = component_tests_config(extra_config)
+        with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            assert_log_to_dir(config, log_dir)

component-tests/test_proxy_dns.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+import socket
+from time import sleep
+
+import constants
+from conftest import CfdModes
+from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
+
+
+# Sanity checks that test that we only run Proxy DNS and Tunnel when we really expect them to be there.
+class TestProxyDns:
+    def test_proxy_dns_with_named_tunnel(self, tmp_path, component_tests_config):
+        run_test_scenario(tmp_path, component_tests_config, CfdModes.NAMED, run_proxy_dns=True)
+
+    def test_proxy_dns_with_classic_tunnel(self, tmp_path, component_tests_config):
+        run_test_scenario(tmp_path, component_tests_config, CfdModes.CLASSIC, run_proxy_dns=True)
+
+    def test_proxy_dns_alone(self, tmp_path, component_tests_config):
+        run_test_scenario(tmp_path, component_tests_config, CfdModes.PROXY_DNS, run_proxy_dns=True)
+
+    def test_named_tunnel_alone(self, tmp_path, component_tests_config):
+        run_test_scenario(tmp_path, component_tests_config, CfdModes.NAMED, run_proxy_dns=False)
+
+    def test_classic_tunnel_alone(self, tmp_path, component_tests_config):
+        run_test_scenario(tmp_path, component_tests_config, CfdModes.CLASSIC, run_proxy_dns=False)
+
+
+def run_test_scenario(tmp_path, component_tests_config, cfd_mode, run_proxy_dns):
+    expect_proxy_dns = run_proxy_dns
+    expect_tunnel = False
+
+    if cfd_mode == CfdModes.NAMED:
+        expect_tunnel = True
+        pre_args = ["tunnel"]
+        args = ["run"]
+    elif cfd_mode == CfdModes.CLASSIC:
+        expect_tunnel = True
+        pre_args = []
+        args = []
+    elif cfd_mode == CfdModes.PROXY_DNS:
+        expect_proxy_dns = True
+        pre_args = []
+        args = ["proxy-dns", "--port", str(constants.PROXY_DNS_PORT)]
+    else:
+        assert False, f"Unknown cfd_mode {cfd_mode}"
+
+    config = component_tests_config(cfd_mode=cfd_mode, run_proxy_dns=run_proxy_dns)
+    with start_cloudflared(tmp_path, config, cfd_pre_args=pre_args, cfd_args=args, new_process=True, capture_output=False):
+        if expect_tunnel:
+            wait_tunnel_ready()
+        else:
+            check_tunnel_not_connected()
+        verify_proxy_dns(expect_proxy_dns)
+
+
+def verify_proxy_dns(should_be_running):
+    # Wait for the Proxy DNS listener to come up.
+    sleep(constants.BACKOFF_SECS)
+    had_failure = False
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        sock.connect(('localhost', constants.PROXY_DNS_PORT))
+        sock.send(b"anything")
+    except:
+        if should_be_running:
+            assert False, "Expected Proxy DNS to be running, but it was not."
+        had_failure = True
+    finally:
+        sock.close()
+
+    if not should_be_running and not had_failure:
+        assert False, "Proxy DNS should not have been running, but it was."

component-tests/test_reconnect.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+import copy
+import platform
+from time import sleep
+
+import pytest
+from flaky import flaky
+
+from conftest import CfdModes
+from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
+
+
+@flaky(max_runs=3, min_passes=1)
+class TestReconnect:
+    default_ha_conns = 4
+    default_reconnect_secs = 15
+    extra_config = {
+        "stdin-control": True,
+    }
+
+    @pytest.mark.skipif(platform.system() == "Windows", reason=f"Currently buggy on Windows TUN-4584")
+    def test_named_reconnect(self, tmp_path, component_tests_config):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(tmp_path, config, new_process=True, allow_input=True, capture_output=False) as cloudflared:
+            # Repeat the test multiple times because some issues only occur after multiple reconnects
+            self.assert_reconnect(config, cloudflared, 5)
+
+    def test_classic_reconnect(self, tmp_path, component_tests_config):
+        extra_config = copy.copy(self.extra_config)
+        extra_config["hello-world"] = True
+        config = component_tests_config(additional_config=extra_config, cfd_mode=CfdModes.CLASSIC)
+        with start_cloudflared(tmp_path, config, cfd_args=[], new_process=True, allow_input=True, capture_output=False) as cloudflared:
+            self.assert_reconnect(config, cloudflared, 1)
+
+    def send_reconnect(self, cloudflared, secs):
+        # Although it is recommended to use the Popen.communicate method, we cannot
+        # use it because it blocks on reading stdout and stderr until EOF is reached
+        cloudflared.stdin.write(f"reconnect {secs}s\n".encode())
+        cloudflared.stdin.flush()
+
+    def assert_reconnect(self, config, cloudflared, repeat):
+        wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=self.default_ha_conns)
+        for _ in range(repeat):
+            for i in range(self.default_ha_conns):
+                self.send_reconnect(cloudflared, self.default_reconnect_secs)
+                expect_connections = self.default_ha_conns-i-1
+                if expect_connections > 0:
+                    # Don't check if tunnel returns 200 here because there is a race condition between wait_tunnel_ready
+                    # retrying to get 200 response and reconnecting
+                    wait_tunnel_ready(require_min_connections=expect_connections)
+                else:
+                    check_tunnel_not_connected()
+
+            sleep(self.default_reconnect_secs * 2)
+            wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=self.default_ha_conns)

component-tests/test_service.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python
+import os
+import platform
+import subprocess
+from contextlib import contextmanager
+from pathlib import Path
+
+import pytest
+
+import test_logging
+from conftest import CfdModes
+from util import start_cloudflared, wait_tunnel_ready
+
+
+def select_platform(plat):
+    return pytest.mark.skipif(
+        platform.system() != plat, reason=f"Only runs on {plat}")
+
+
+def default_config_dir():
+    return os.path.join(Path.home(), ".cloudflared")
+
+
+def default_config_file():
+    return os.path.join(default_config_dir(), "config.yml")
+
+
+class TestServiceMode:
+    @select_platform("Darwin")
+    @pytest.mark.skipif(os.path.exists(default_config_file()), reason=f"There is already a config file in default path")
+    def test_launchd_service_log_to_file(self, tmp_path, component_tests_config):
+        log_file = tmp_path / test_logging.default_log_file
+        additional_config = {
+            # On Darwin cloudflared service defaults to run classic tunnel command
+            "hello-world": True,
+            "logfile": str(log_file),
+        }
+        config = component_tests_config(additional_config=additional_config, cfd_mode=CfdModes.CLASSIC)
+
+        def assert_log_file():
+            test_logging.assert_log_in_file(log_file)
+            test_logging.assert_json_log(log_file)
+
+        self.launchd_service_scenario(config, assert_log_file)
+
+    @select_platform("Darwin")
+    @pytest.mark.skipif(os.path.exists(default_config_file()), reason=f"There is already a config file in default path")
+    def test_launchd_service_rotating_log(self, tmp_path, component_tests_config):
+        log_dir = tmp_path / "logs"
+        additional_config = {
+            # On Darwin cloudflared service defaults to run classic tunnel command
+            "hello-world": True,
+            "loglevel": "debug",
+            "log-directory": str(log_dir),
+        }
+        config = component_tests_config(additional_config=additional_config, cfd_mode=CfdModes.CLASSIC)
+
+        def assert_rotating_log():
+            test_logging.assert_log_to_dir(config, log_dir)
+
+        self.launchd_service_scenario(config, assert_rotating_log)
+
+    def launchd_service_scenario(self, config, extra_assertions):
+        with self.run_service(Path(default_config_dir()), config):
+            self.launchctl_cmd("list")
+            self.launchctl_cmd("start")
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            extra_assertions()
+            self.launchctl_cmd("stop")
+
+        os.remove(default_config_file())
+        self.launchctl_cmd("list", success=False)
+
+    @select_platform("Linux")
+    @pytest.mark.skipif(os.path.exists("/etc/cloudflared/config.yml"),
+                        reason=f"There is already a config file in default path")
+    def test_sysv_service_log_to_file(self, tmp_path, component_tests_config):
+        log_file = tmp_path / test_logging.default_log_file
+        additional_config = {
+            "logfile": str(log_file),
+        }
+        config = component_tests_config(additional_config=additional_config)
+
+        def assert_log_file():
+            test_logging.assert_log_in_file(log_file)
+            test_logging.assert_json_log(log_file)
+
+        self.sysv_service_scenario(config, tmp_path, assert_log_file)
+
+    @select_platform("Linux")
+    @pytest.mark.skipif(os.path.exists("/etc/cloudflared/config.yml"),
+                        reason=f"There is already a config file in default path")
+    def test_sysv_service_rotating_log(self, tmp_path, component_tests_config):
+        log_dir = tmp_path / "logs"
+        additional_config = {
+            "loglevel": "debug",
+            "log-directory": str(log_dir),
+        }
+        config = component_tests_config(additional_config=additional_config)
+
+        def assert_rotating_log():
+            # We need the folder to have executable permissions for the "stat" command in the assertions to work.
+            subprocess.check_call(['sudo', 'chmod', 'o+x', log_dir])
+            test_logging.assert_log_to_dir(config, log_dir)
+
+        self.sysv_service_scenario(config, tmp_path, assert_rotating_log)
+
+    def sysv_service_scenario(self, config, tmp_path, extra_assertions):
+        with self.run_service(tmp_path, config, root=True):
+            self.sysv_cmd("start")
+            self.sysv_cmd("status")
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            extra_assertions()
+            self.sysv_cmd("stop")
+
+        # Service install copies config file to /etc/cloudflared/config.yml
+        subprocess.run(["sudo", "rm", "/etc/cloudflared/config.yml"])
+        self.sysv_cmd("status", success=False)
+
+    @contextmanager
+    def run_service(self, tmp_path, config, root=False):
+        try:
+            service = start_cloudflared(
+                tmp_path, config, cfd_args=["service", "install"], cfd_pre_args=[], capture_output=False, root=root)
+            yield service
+        finally:
+            start_cloudflared(
+                tmp_path, config, cfd_args=["service", "uninstall"], cfd_pre_args=[], capture_output=False, root=root)
+
+    def launchctl_cmd(self, action, success=True):
+        cmd = subprocess.run(
+            ["launchctl", action, "com.cloudflare.cloudflared"], check=success)
+        if not success:
+            assert cmd.returncode != 0, f"Expect {cmd.args} to fail, but it succeed"
+
+    def sysv_cmd(self, action, success=True):
+        cmd = subprocess.run(
+            ["sudo", "service", "cloudflared", action], check=success)
+        if not success:
+            assert cmd.returncode != 0, f"Expect {cmd.args} to fail, but it succeed"

component-tests/test_termination.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python
+from contextlib import contextmanager
+import platform
+import signal
+import threading
+import time
+
+import pytest
+import requests
+
+from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
+
+
+def supported_signals():
+    if platform.system() == "Windows":
+        return [signal.SIGTERM]
+    return [signal.SIGTERM, signal.SIGINT]
+
+
+class TestTermination():
+    grace_period = 5
+    timeout = 10
+    extra_config = {
+        "grace-period": f"{grace_period}s",
+    }
+    sse_endpoint = "/sse?freq=1s"
+
+    @pytest.mark.parametrize("signal", supported_signals())
+    def test_graceful_shutdown(self, tmp_path, component_tests_config, signal):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+
+            connected = threading.Condition()
+            in_flight_req = threading.Thread(
+                target=self.stream_request, args=(config, connected, False, ))
+            in_flight_req.start()
+
+            with connected:
+                connected.wait(self.timeout)
+            # Send signal after the SSE connection is established
+            self.terminate_by_signal(cloudflared, signal)
+            self.wait_eyeball_thread(
+                in_flight_req, self.grace_period + self.timeout)
+
+    # test cloudflared terminates before grace period expires when all eyeball
+    # connections are drained
+    @pytest.mark.parametrize("signal", supported_signals())
+    def test_shutdown_once_no_connection(self, tmp_path, component_tests_config, signal):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+
+            connected = threading.Condition()
+            in_flight_req = threading.Thread(
+                target=self.stream_request, args=(config, connected, True, ))
+            in_flight_req.start()
+
+            with connected:
+                connected.wait(self.timeout)
+            with self.within_grace_period():
+                # Send signal after the SSE connection is established
+                self.terminate_by_signal(cloudflared, signal)
+                self.wait_eyeball_thread(in_flight_req, self.grace_period)
+
+    @pytest.mark.parametrize("signal", supported_signals())
+    def test_no_connection_shutdown(self, tmp_path, component_tests_config, signal):
+        config = component_tests_config(self.extra_config)
+        with start_cloudflared(
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
+            wait_tunnel_ready(tunnel_url=config.get_url())
+            with self.within_grace_period():
+                self.terminate_by_signal(cloudflared, signal)
+
+    def terminate_by_signal(self, cloudflared, sig):
+        cloudflared.send_signal(sig)
+        check_tunnel_not_connected()
+        cloudflared.wait()
+
+    def wait_eyeball_thread(self, thread, timeout):
+        thread.join(timeout)
+        assert thread.is_alive() == False, "eyeball thread is still alive"
+
+    # Using this context asserts logic within the context is executed within grace period
+    @contextmanager
+    def within_grace_period(self):
+        try:
+            start = time.time()
+            yield
+        finally:
+            duration = time.time() - start
+            assert duration < self.grace_period
+
+    def stream_request(self, config, connected, early_terminate):
+        expected_terminate_message = "502 Bad Gateway"
+        url = config.get_url() + self.sse_endpoint
+
+        with requests.get(url, timeout=5, stream=True) as resp:
+            with connected:
+                connected.notifyAll()
+            lines = 0
+            for line in resp.iter_lines():
+                if expected_terminate_message.encode() == line:
+                    break
+                lines += 1
+                if early_terminate and lines == 2:
+                    return
+            # /sse returns count followed by 2 new lines
+            assert lines >= (self.grace_period * 2)

component-tests/util.py

@@ -0,0 +1,99 @@
+from contextlib import contextmanager
+import logging
+import requests
+from retrying import retry
+import subprocess
+import yaml
+
+from time import sleep
+
+from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
+
+LOGGER = logging.getLogger(__name__)
+
+
+def write_config(directory, config):
+    config_path = directory / "config.yml"
+    with open(config_path, 'w') as outfile:
+        yaml.dump(config, outfile)
+    return config_path
+
+
+def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False, allow_input=False, capture_output=True, root=False):
+    config_path = write_config(directory, config.full_config)
+    cmd = cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root)
+    if new_process:
+        return run_cloudflared_background(cmd, allow_input, capture_output)
+    # By setting check=True, it will raise an exception if the process exits with non-zero exit code
+    return subprocess.run(cmd, check=True, capture_output=capture_output)
+
+
+def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
+    cmd = []
+    if root:
+        cmd += ["sudo"]
+    cmd += [config.cloudflared_binary]
+    cmd += cfd_pre_args
+    cmd += ["--config", str(config_path)]
+    cmd += cfd_args
+    LOGGER.info(f"Run cmd {cmd} with config {config}")
+    return cmd
+
+
+@contextmanager
+def run_cloudflared_background(cmd, allow_input, capture_output):
+    output = subprocess.PIPE if capture_output else subprocess.DEVNULL
+    stdin = subprocess.PIPE if allow_input else None
+    try:
+        cfd = subprocess.Popen(cmd, stdin=stdin, stdout=output, stderr=output)
+        yield cfd
+    finally:
+        cfd.terminate()
+        if capture_output:
+            LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def wait_tunnel_ready(tunnel_url=None, require_min_connections=1):
+    metrics_url = f'http://localhost:{METRICS_PORT}/ready'
+
+    with requests.Session() as s:
+        resp = send_request(s, metrics_url, True)
+        assert resp.json()[
+            "readyConnections"] >= require_min_connections, f"Ready endpoint returned {resp.json()} but we expect at least {require_min_connections} connections"
+        if tunnel_url is not None:
+            send_request(s, tunnel_url, True)
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES * BACKOFF_SECS, wait_fixed=1000)
+def check_tunnel_not_connected():
+    url = f'http://localhost:{METRICS_PORT}/ready'
+
+    try:
+        resp = requests.get(url, timeout=1)
+        assert resp.status_code == 503, f"Expect {url} returns 503, got {resp.status_code}"
+    # cloudflared might already terminate
+    except requests.exceptions.ConnectionError as e:
+        LOGGER.warning(f"Failed to connect to {url}, error: {e}")
+
+
+# In some cases we don't need to check response status, such as when sending batch requests to generate logs
+def send_requests(url, count, require_ok=True):
+    errors = 0
+    with requests.Session() as s:
+        for _ in range(count):
+            resp = send_request(s, url, require_ok)
+            if resp is None:
+                errors += 1
+            sleep(0.01)
+    if errors > 0:
+        LOGGER.warning(
+            f"{errors} out of {count} requests to {url} return non-200 status")
+
+
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
+def send_request(session, url, require_ok):
+    resp = session.get(url, timeout=BACKOFF_SECS)
+    if require_ok:
+        assert resp.status_code == 200, f"{url} returned {resp}"
+    return resp if resp.status_code == 200 else None

config/configuration.go

@@ -0,0 +1,399 @@
+package config
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"path/filepath"
+	"runtime"
+	"time"
+
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+	yaml "gopkg.in/yaml.v2"
+
+	"github.com/cloudflare/cloudflared/validation"
+)
+
+var (
+	// DefaultConfigFiles is the file names from which we attempt to read configuration.
+	DefaultConfigFiles = []string{"config.yml", "config.yaml"}
+
+	// DefaultUnixConfigLocation is the primary location to find a config file
+	DefaultUnixConfigLocation = "/usr/local/etc/cloudflared"
+
+	// DefaultUnixLogLocation is the primary location to find log files
+	DefaultUnixLogLocation = "/var/log/cloudflared"
+
+	// Launchd doesn't set root env variables, so there is default
+	// Windows default config dir was ~/cloudflare-warp in documentation; let's keep it compatible
+	defaultUserConfigDirs = []string{"~/.cloudflared", "~/.cloudflare-warp", "~/cloudflare-warp"}
+	defaultNixConfigDirs  = []string{"/etc/cloudflared", DefaultUnixConfigLocation}
+
+	ErrNoConfigFile = fmt.Errorf("Cannot determine default configuration path. No file %v in %v", DefaultConfigFiles, DefaultConfigSearchDirectories())
+)
+
+const (
+	DefaultCredentialFile = "cert.pem"
+
+	// BastionFlag is to enable bastion, or jump host, operation
+	BastionFlag = "bastion"
+)
+
+// DefaultConfigDirectory returns the default directory of the config file
+func DefaultConfigDirectory() string {
+	if runtime.GOOS == "windows" {
+		path := os.Getenv("CFDPATH")
+		if path == "" {
+			path = filepath.Join(os.Getenv("ProgramFiles(x86)"), "cloudflared")
+			if _, err := os.Stat(path); os.IsNotExist(err) { //doesn't exist, so return an empty failure string
+				return ""
+			}
+		}
+		return path
+	}
+	return DefaultUnixConfigLocation
+}
+
+// DefaultLogDirectory returns the default directory for log files
+func DefaultLogDirectory() string {
+	if runtime.GOOS == "windows" {
+		return DefaultConfigDirectory()
+	}
+	return DefaultUnixLogLocation
+}
+
+// DefaultConfigPath returns the default location of a config file
+func DefaultConfigPath() string {
+	dir := DefaultConfigDirectory()
+	if dir == "" {
+		return DefaultConfigFiles[0]
+	}
+	return filepath.Join(dir, DefaultConfigFiles[0])
+}
+
+// DefaultConfigSearchDirectories returns the default folder locations of the config
+func DefaultConfigSearchDirectories() []string {
+	dirs := make([]string, len(defaultUserConfigDirs))
+	copy(dirs, defaultUserConfigDirs)
+	if runtime.GOOS != "windows" {
+		dirs = append(dirs, defaultNixConfigDirs...)
+	}
+	return dirs
+}
+
+// FileExists checks to see if a file exist at the provided path.
+func FileExists(path string) (bool, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// ignore missing files
+			return false, nil
+		}
+		return false, err
+	}
+	_ = f.Close()
+	return true, nil
+}
+
+// FindDefaultConfigPath returns the first path that contains a config file.
+// If none of the combination of DefaultConfigSearchDirectories() and DefaultConfigFiles
+// contains a config file, return empty string.
+func FindDefaultConfigPath() string {
+	for _, configDir := range DefaultConfigSearchDirectories() {
+		for _, configFile := range DefaultConfigFiles {
+			dirPath, err := homedir.Expand(configDir)
+			if err != nil {
+				continue
+			}
+			path := filepath.Join(dirPath, configFile)
+			if ok, _ := FileExists(path); ok {
+				return path
+			}
+		}
+	}
+	return ""
+}
+
+// FindOrCreateConfigPath returns the first path that contains a config file
+// or creates one in the primary default path if it doesn't exist
+func FindOrCreateConfigPath() string {
+	path := FindDefaultConfigPath()
+
+	if path == "" {
+		// create the default directory if it doesn't exist
+		path = DefaultConfigPath()
+		if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
+			return ""
+		}
+
+		// write a new config file out
+		file, err := os.Create(path)
+		if err != nil {
+			return ""
+		}
+		defer file.Close()
+
+		logDir := DefaultLogDirectory()
+		_ = os.MkdirAll(logDir, os.ModePerm) //try and create it. Doesn't matter if it succeed or not, only byproduct will be no logs
+
+		c := Root{
+			LogDirectory: logDir,
+		}
+		if err := yaml.NewEncoder(file).Encode(&c); err != nil {
+			return ""
+		}
+	}
+
+	return path
+}
+
+// ValidateUnixSocket ensures --unix-socket param is used exclusively
+// i.e. it fails if a user specifies both --url and --unix-socket
+func ValidateUnixSocket(c *cli.Context) (string, error) {
+	if c.IsSet("unix-socket") && (c.IsSet("url") || c.NArg() > 0) {
+		return "", errors.New("--unix-socket must be used exclusivly.")
+	}
+	return c.String("unix-socket"), nil
+}
+
+// ValidateUrl will validate url flag correctness. It can be either from --url or argument
+// Notice ValidateUnixSocket, it will enforce --unix-socket is not used with --url or argument
+func ValidateUrl(c *cli.Context, allowURLFromArgs bool) (*url.URL, error) {
+	var url = c.String("url")
+	if allowURLFromArgs && c.NArg() > 0 {
+		if c.IsSet("url") {
+			return nil, errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
+		}
+		url = c.Args().Get(0)
+	}
+	validUrl, err := validation.ValidateUrl(url)
+	return validUrl, err
+}
+
+type UnvalidatedIngressRule struct {
+	Hostname      string
+	Path          string
+	Service       string
+	OriginRequest OriginRequestConfig `yaml:"originRequest"`
+}
+
+// OriginRequestConfig is a set of optional fields that users may set to
+// customize how cloudflared sends requests to origin services. It is used to set
+// up general config that apply to all rules, and also, specific per-rule
+// config.
+// Note: To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
+type OriginRequestConfig struct {
+	// HTTP proxy timeout for establishing a new connection
+	ConnectTimeout *time.Duration `yaml:"connectTimeout"`
+	// HTTP proxy timeout for completing a TLS handshake
+	TLSTimeout *time.Duration `yaml:"tlsTimeout"`
+	// HTTP proxy TCP keepalive duration
+	TCPKeepAlive *time.Duration `yaml:"tcpKeepAlive"`
+	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
+	NoHappyEyeballs *bool `yaml:"noHappyEyeballs"`
+	// HTTP proxy maximum keepalive connection pool size
+	KeepAliveConnections *int `yaml:"keepAliveConnections"`
+	// HTTP proxy timeout for closing an idle connection
+	KeepAliveTimeout *time.Duration `yaml:"keepAliveTimeout"`
+	// Sets the HTTP Host header for the local webserver.
+	HTTPHostHeader *string `yaml:"httpHostHeader"`
+	// Hostname on the origin server certificate.
+	OriginServerName *string `yaml:"originServerName"`
+	// Path to the CA for the certificate of your origin.
+	// This option should be used only if your certificate is not signed by Cloudflare.
+	CAPool *string `yaml:"caPool"`
+	// Disables TLS verification of the certificate presented by your origin.
+	// Will allow any certificate from the origin to be accepted.
+	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
+	NoTLSVerify *bool `yaml:"noTLSVerify"`
+	// Disables chunked transfer encoding.
+	// Useful if you are running a WSGI server.
+	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding"`
+	// Runs as jump host
+	BastionMode *bool `yaml:"bastionMode"`
+	// Listen address for the proxy.
+	ProxyAddress *string `yaml:"proxyAddress"`
+	// Listen port for the proxy.
+	ProxyPort *uint `yaml:"proxyPort"`
+	// Valid options are 'socks' or empty.
+	ProxyType *string `yaml:"proxyType"`
+	// IP rules for the proxy service
+	IPRules []IngressIPRule `yaml:"ipRules"`
+}
+
+type IngressIPRule struct {
+	Prefix *string `yaml:"prefix"`
+	Ports  []int   `yaml:"ports"`
+	Allow  bool    `yaml:"allow"`
+}
+
+type Configuration struct {
+	TunnelID      string `yaml:"tunnel"`
+	Ingress       []UnvalidatedIngressRule
+	WarpRouting   WarpRoutingConfig   `yaml:"warp-routing"`
+	OriginRequest OriginRequestConfig `yaml:"originRequest"`
+	sourceFile    string
+}
+
+type WarpRoutingConfig struct {
+	Enabled bool `yaml:"enabled"`
+}
+
+type configFileSettings struct {
+	Configuration `yaml:",inline"`
+	// older settings will be aggregated into the generic map, should be read via cli.Context
+	Settings map[string]interface{} `yaml:",inline"`
+}
+
+func (c *Configuration) Source() string {
+	return c.sourceFile
+}
+
+func (c *configFileSettings) Int(name string) (int, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(int); ok {
+			return v, nil
+		}
+		return 0, fmt.Errorf("expected int found %T for %s", raw, name)
+	}
+	return 0, nil
+}
+
+func (c *configFileSettings) Duration(name string) (time.Duration, error) {
+	if raw, ok := c.Settings[name]; ok {
+		switch v := raw.(type) {
+		case time.Duration:
+			return v, nil
+		case string:
+			return time.ParseDuration(v)
+		}
+		return 0, fmt.Errorf("expected duration found %T for %s", raw, name)
+	}
+	return 0, nil
+}
+
+func (c *configFileSettings) Float64(name string) (float64, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(float64); ok {
+			return v, nil
+		}
+		return 0, fmt.Errorf("expected float found %T for %s", raw, name)
+	}
+	return 0, nil
+}
+
+func (c *configFileSettings) String(name string) (string, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(string); ok {
+			return v, nil
+		}
+		return "", fmt.Errorf("expected string found %T for %s", raw, name)
+	}
+	return "", nil
+}
+
+func (c *configFileSettings) StringSlice(name string) ([]string, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if slice, ok := raw.([]interface{}); ok {
+			strSlice := make([]string, len(slice))
+			for i, v := range slice {
+				str, ok := v.(string)
+				if !ok {
+					return nil, fmt.Errorf("expected string, found %T for %v", i, v)
+				}
+				strSlice[i] = str
+			}
+			return strSlice, nil
+		}
+		return nil, fmt.Errorf("expected string slice found %T for %s", raw, name)
+	}
+	return nil, nil
+}
+
+func (c *configFileSettings) IntSlice(name string) ([]int, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if slice, ok := raw.([]interface{}); ok {
+			intSlice := make([]int, len(slice))
+			for i, v := range slice {
+				str, ok := v.(int)
+				if !ok {
+					return nil, fmt.Errorf("expected int, found %T for %v ", v, v)
+				}
+				intSlice[i] = str
+			}
+			return intSlice, nil
+		}
+		if v, ok := raw.([]int); ok {
+			return v, nil
+		}
+		return nil, fmt.Errorf("expected int slice found %T for %s", raw, name)
+	}
+	return nil, nil
+}
+
+func (c *configFileSettings) Generic(name string) (cli.Generic, error) {
+	return nil, errors.New("option type Generic not supported")
+}
+
+func (c *configFileSettings) Bool(name string) (bool, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(bool); ok {
+			return v, nil
+		}
+		return false, fmt.Errorf("expected boolean found %T for %s", raw, name)
+	}
+	return false, nil
+}
+
+var configuration configFileSettings
+
+func GetConfiguration() *Configuration {
+	return &configuration.Configuration
+}
+
+// ReadConfigFile returns InputSourceContext initialized from the configuration file.
+// On repeat calls returns with the same file, returns without reading the file again; however,
+// if value of "config" flag changes, will read the new config file
+func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (settings *configFileSettings, warnings string, err error) {
+	configFile := c.String("config")
+	if configuration.Source() == configFile || configFile == "" {
+		if configuration.Source() == "" {
+			return nil, "", ErrNoConfigFile
+		}
+		return &configuration, "", nil
+	}
+
+	log.Debug().Msgf("Loading configuration from %s", configFile)
+	file, err := os.Open(configFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			err = ErrNoConfigFile
+		}
+		return nil, "", err
+	}
+	defer file.Close()
+	if err := yaml.NewDecoder(file).Decode(&configuration); err != nil {
+		if err == io.EOF {
+			log.Error().Msgf("Configuration file %s was empty", configFile)
+			return &configuration, "", nil
+		}
+		return nil, "", errors.Wrap(err, "error parsing YAML in config file at "+configFile)
+	}
+	configuration.sourceFile = configFile
+
+	// Parse it again, with strict mode, to find warnings.
+	if file, err := os.Open(configFile); err == nil {
+		decoder := yaml.NewDecoder(file)
+		decoder.SetStrict(true)
+		var unusedConfig configFileSettings
+		if err := decoder.Decode(&unusedConfig); err != nil {
+			warnings = err.Error()
+		}
+	}
+
+	return &configuration, warnings, nil
+}

config/configuration_test.go

@@ -0,0 +1,83 @@
+package config
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	yaml "gopkg.in/yaml.v2"
+)
+
+func TestConfigFileSettings(t *testing.T) {
+	var (
+		firstIngress = UnvalidatedIngressRule{
+			Hostname: "tunnel1.example.com",
+			Path:     "/id",
+			Service:  "https://localhost:8000",
+		}
+		secondIngress = UnvalidatedIngressRule{
+			Hostname: "*",
+			Path:     "",
+			Service:  "https://localhost:8001",
+		}
+		warpRouting = WarpRoutingConfig{
+			Enabled: true,
+		}
+	)
+	rawYAML := `
+tunnel: config-file-test
+ingress:
+ - hostname: tunnel1.example.com
+   path: /id
+   service: https://localhost:8000
+ - hostname: "*"
+   service: https://localhost:8001
+warp-routing: 
+  enabled: true
+retries: 5
+grace-period: 30s
+percentage: 3.14
+hostname: example.com
+tag:
+ - test
+ - central-1
+counters:
+ - 123
+ - 456
+`
+	var config configFileSettings
+	err := yaml.Unmarshal([]byte(rawYAML), &config)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "config-file-test", config.TunnelID)
+	assert.Equal(t, firstIngress, config.Ingress[0])
+	assert.Equal(t, secondIngress, config.Ingress[1])
+	assert.Equal(t, warpRouting, config.WarpRouting)
+
+	retries, err := config.Int("retries")
+	assert.NoError(t, err)
+	assert.Equal(t, 5, retries)
+
+	gracePeriod, err := config.Duration("grace-period")
+	assert.NoError(t, err)
+	assert.Equal(t, time.Second*30, gracePeriod)
+
+	percentage, err := config.Float64("percentage")
+	assert.NoError(t, err)
+	assert.Equal(t, 3.14, percentage)
+
+	hostname, err := config.String("hostname")
+	assert.NoError(t, err)
+	assert.Equal(t, "example.com", hostname)
+
+	tags, err := config.StringSlice("tag")
+	assert.NoError(t, err)
+	assert.Equal(t, "test", tags[0])
+	assert.Equal(t, "central-1", tags[1])
+
+	counters, err := config.IntSlice("counters")
+	assert.NoError(t, err)
+	assert.Equal(t, 123, counters[0])
+	assert.Equal(t, 456, counters[1])
+
+}

config/manager.go

@@ -1,12 +1,14 @@
 package config
 
 import (
-	"errors"
+	"io"
 	"os"
 
-	"github.com/cloudflare/cloudflared/logger"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	yaml "gopkg.in/yaml.v2"
+
 	"github.com/cloudflare/cloudflared/watcher"
-	"gopkg.in/yaml.v2"
 )
 
 // Notifier sends out config updates
@@ -26,16 +28,16 @@ type FileManager struct {
 	watcher    watcher.Notifier
 	notifier   Notifier
 	configPath string
-	logger     logger.Service
-	ReadConfig func(string) (Root, error)
+	log        *zerolog.Logger
+	ReadConfig func(string, *zerolog.Logger) (Root, error)
 }
 
 // NewFileManager creates a config manager
-func NewFileManager(watcher watcher.Notifier, configPath string, logger logger.Service) (*FileManager, error) {
+func NewFileManager(watcher watcher.Notifier, configPath string, log *zerolog.Logger) (*FileManager, error) {
 	m := &FileManager{
 		watcher:    watcher,
 		configPath: configPath,
-		logger:     logger,
+		log:        log,
 		ReadConfig: readConfigFromPath,
 	}
 	err := watcher.Add(configPath)
@@ -59,7 +61,7 @@ func (m *FileManager) Start(notifier Notifier) error {
 
 // GetConfig reads the yaml file from the disk
 func (m *FileManager) GetConfig() (Root, error) {
-	return m.ReadConfig(m.configPath)
+	return m.ReadConfig(m.configPath, m.log)
 }
 
 // Shutdown stops the watcher
@@ -67,7 +69,7 @@ func (m *FileManager) Shutdown() {
 	m.watcher.Shutdown()
 }
 
-func readConfigFromPath(configPath string) (Root, error) {
+func readConfigFromPath(configPath string, log *zerolog.Logger) (Root, error) {
 	if configPath == "" {
 		return Root{}, errors.New("unable to find config file")
 	}
@@ -80,7 +82,11 @@ func readConfigFromPath(configPath string) (Root, error) {
 
 	var config Root
 	if err := yaml.NewDecoder(file).Decode(&config); err != nil {
-		return Root{}, err
+		if err == io.EOF {
+			log.Error().Msgf("Configuration file %s was empty", configPath)
+			return Root{}, nil
+		}
+		return Root{}, errors.Wrap(err, "error parsing YAML in config file at "+configPath)
 	}
 
 	return config, nil
@@ -93,14 +99,14 @@ func readConfigFromPath(configPath string) (Root, error) {
 func (m *FileManager) WatcherItemDidChange(filepath string) {
 	config, err := m.GetConfig()
 	if err != nil {
-		m.logger.Errorf("Failed to read new config: %s", err)
+		m.log.Err(err).Msg("Failed to read new config")
 		return
 	}
-	m.logger.Info("Config file has been updated")
+	m.log.Info().Msg("Config file has been updated")
 	m.notifier.ConfigDidUpdate(config)
 }
 
 // WatcherDidError notifies of errors with the file watcher
 func (m *FileManager) WatcherDidError(err error) {
-	m.logger.Errorf("Config watcher encountered an error: %s", err)
+	m.log.Err(err).Msg("Config watcher encountered an error")
 }

config/manager_test.go

@@ -4,9 +4,10 @@ import (
 	"os"
 	"testing"
 
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/watcher"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/watcher"
 )
 
 type mockNotifier struct {
@@ -45,8 +46,8 @@ func TestConfigChanged(t *testing.T) {
 	f, err := os.Create(filePath)
 	assert.NoError(t, err)
 	defer func() {
-		f.Close()
-		os.Remove(filePath)
+		_ = f.Close()
+		_ = os.Remove(filePath)
 	}()
 	c := &Root{
 		Forwarders: []Forwarder{
@@ -56,15 +57,15 @@ func TestConfigChanged(t *testing.T) {
 			},
 		},
 	}
-	configRead := func(configPath string) (Root, error) {
+	configRead := func(configPath string, log *zerolog.Logger) (Root, error) {
 		return *c, nil
 	}
 	wait := make(chan struct{})
 	w := &mockFileWatcher{path: filePath, ready: wait}
 
-	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
+	log := zerolog.Nop()
 
-	service, err := NewFileManager(w, filePath, logger)
+	service, err := NewFileManager(w, filePath, &log)
 	service.ReadConfig = configRead
 	assert.NoError(t, err)
 

config/model.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"io"
 	"strings"
+
+	"github.com/cloudflare/cloudflared/tunneldns"
 )
 
 // Forwarder represents a client side listener to forward traffic to the edge
@@ -25,11 +27,12 @@ type Tunnel struct {
 
 // DNSResolver represents a client side DNS resolver
 type DNSResolver struct {
-	Enabled    bool     `json:"enabled"`
-	Address    string   `json:"address,omitempty"`
-	Port       uint16   `json:"port,omitempty"`
-	Upstreams  []string `json:"upstreams,omitempty"`
-	Bootstraps []string `json:"bootstraps,omitempty"`
+	Enabled                bool     `json:"enabled"`
+	Address                string   `json:"address,omitempty"`
+	Port                   uint16   `json:"port,omitempty"`
+	Upstreams              []string `json:"upstreams,omitempty"`
+	Bootstraps             []string `json:"bootstraps,omitempty"`
+	MaxUpstreamConnections int      `json:"max_upstream_connections,omitempty"`
 }
 
 // Root is the base options to configure the service
@@ -59,6 +62,7 @@ func (r *DNSResolver) Hash() string {
 	io.WriteString(h, strings.Join(r.Bootstraps, ","))
 	io.WriteString(h, strings.Join(r.Upstreams, ","))
 	io.WriteString(h, fmt.Sprintf("%d", r.Port))
+	io.WriteString(h, fmt.Sprintf("%d", r.MaxUpstreamConnections))
 	io.WriteString(h, fmt.Sprintf("%v", r.Enabled))
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
@@ -99,3 +103,11 @@ func (r *DNSResolver) BootstrapsOrDefault() []string {
 	}
 	return []string{"https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"}
 }
+
+// MaxUpstreamConnectionsOrDefault return the max upstream connections or returns the default if negative
+func (r *DNSResolver) MaxUpstreamConnectionsOrDefault() int {
+	if r.MaxUpstreamConnections >= 0 {
+		return r.MaxUpstreamConnections
+	}
+	return tunneldns.MaxUpstreamConnsDefault
+}

connection/connection.go

@@ -0,0 +1,182 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/google/uuid"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
+)
+
+const (
+	lbProbeUserAgentPrefix = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
+	LogFieldConnIndex      = "connIndex"
+	MaxGracePeriod         = time.Minute * 3
+)
+
+var switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols, http.StatusText(http.StatusSwitchingProtocols))
+
+type Config struct {
+	OriginProxy     OriginProxy
+	GracePeriod     time.Duration
+	ReplaceExisting bool
+}
+
+type NamedTunnelConfig struct {
+	Credentials    Credentials
+	Client         pogs.ClientInfo
+	QuickTunnelUrl string
+}
+
+// Credentials are stored in the credentials file and contain all info needed to run a tunnel.
+type Credentials struct {
+	AccountTag   string
+	TunnelSecret []byte
+	TunnelID     uuid.UUID
+	TunnelName   string
+}
+
+func (c *Credentials) Auth() pogs.TunnelAuth {
+	return pogs.TunnelAuth{
+		AccountTag:   c.AccountTag,
+		TunnelSecret: c.TunnelSecret,
+	}
+}
+
+type ClassicTunnelConfig struct {
+	Hostname   string
+	OriginCert []byte
+	// feature-flag to use new edge reconnect tokens
+	UseReconnectToken bool
+}
+
+// Type indicates the connection type of the  connection.
+type Type int
+
+const (
+	TypeWebsocket Type = iota
+	TypeTCP
+	TypeControlStream
+	TypeHTTP
+)
+
+// ShouldFlush returns whether this kind of connection should actively flush data
+func (t Type) shouldFlush() bool {
+	switch t {
+	case TypeWebsocket, TypeTCP, TypeControlStream:
+		return true
+	default:
+		return false
+	}
+}
+
+func (t Type) String() string {
+	switch t {
+	case TypeWebsocket:
+		return "websocket"
+	case TypeTCP:
+		return "tcp"
+	case TypeControlStream:
+		return "control stream"
+	case TypeHTTP:
+		return "http"
+	default:
+		return fmt.Sprintf("Unknown Type %d", t)
+	}
+}
+
+// OriginProxy is how data flows from cloudflared to the origin services running behind it.
+type OriginProxy interface {
+	ProxyHTTP(w ResponseWriter, req *http.Request, isWebsocket bool) error
+	ProxyTCP(ctx context.Context, rwa ReadWriteAcker, req *TCPRequest) error
+}
+
+// TCPRequest defines the input format needed to perform a TCP proxy.
+type TCPRequest struct {
+	Dest    string
+	CFRay   string
+	LBProbe bool
+}
+
+// ReadWriteAcker is a readwriter with the ability to Acknowledge to the downstream (edge) that the origin has
+// accepted the connection.
+type ReadWriteAcker interface {
+	io.ReadWriter
+	AckConnection() error
+}
+
+// HTTPResponseReadWriteAcker is an HTTP implementation of ReadWriteAcker.
+type HTTPResponseReadWriteAcker struct {
+	r   io.Reader
+	w   ResponseWriter
+	req *http.Request
+}
+
+// NewHTTPResponseReadWriterAcker returns a new instance of HTTPResponseReadWriteAcker.
+func NewHTTPResponseReadWriterAcker(w ResponseWriter, req *http.Request) *HTTPResponseReadWriteAcker {
+	return &HTTPResponseReadWriteAcker{
+		r:   req.Body,
+		w:   w,
+		req: req,
+	}
+}
+
+func (h *HTTPResponseReadWriteAcker) Read(p []byte) (int, error) {
+	return h.r.Read(p)
+}
+
+func (h *HTTPResponseReadWriteAcker) Write(p []byte) (int, error) {
+	return h.w.Write(p)
+}
+
+// AckConnection acks an HTTP connection by sending a switch protocols status code that enables the caller to
+// upgrade to streams.
+func (h *HTTPResponseReadWriteAcker) AckConnection() error {
+	resp := &http.Response{
+		Status:        switchingProtocolText,
+		StatusCode:    http.StatusSwitchingProtocols,
+		ContentLength: -1,
+	}
+
+	if secWebsocketKey := h.req.Header.Get("Sec-WebSocket-Key"); secWebsocketKey != "" {
+		resp.Header = websocket.NewResponseHeader(h.req)
+	}
+
+	return h.w.WriteRespHeaders(resp.StatusCode, resp.Header)
+}
+
+type ResponseWriter interface {
+	WriteRespHeaders(status int, header http.Header) error
+	io.Writer
+}
+
+type ConnectedFuse interface {
+	Connected()
+	IsConnected() bool
+}
+
+func IsServerSentEvent(headers http.Header) bool {
+	if contentType := headers.Get("content-type"); contentType != "" {
+		return strings.HasPrefix(strings.ToLower(contentType), "text/event-stream")
+	}
+	return false
+}
+
+func uint8ToString(input uint8) string {
+	return strconv.FormatUint(uint64(input), 10)
+}
+
+func FindCfRayHeader(req *http.Request) string {
+	return req.Header.Get("Cf-Ray")
+}
+
+func IsLBProbeRequest(req *http.Request) bool {
+	return strings.HasPrefix(req.UserAgent(), lbProbeUserAgentPrefix)
+}

connection/connection_test.go

@@ -0,0 +1,161 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/gobwas/ws/wsutil"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/ingress"
+)
+
+const (
+	largeFileSize = 2 * 1024 * 1024
+)
+
+var (
+	unusedWarpRoutingService = (*ingress.WarpRoutingService)(nil)
+	testConfig               = &Config{
+		OriginProxy: &mockOriginProxy{},
+		GracePeriod: time.Millisecond * 100,
+	}
+	log           = zerolog.Nop()
+	testOriginURL = &url.URL{
+		Scheme: "https",
+		Host:   "connectiontest.argotunnel.com",
+	}
+	testLargeResp = make([]byte, largeFileSize)
+)
+
+type testRequest struct {
+	name           string
+	endpoint       string
+	expectedStatus int
+	expectedBody   []byte
+	isProxyError   bool
+}
+
+type mockOriginProxy struct{}
+
+func (moc *mockOriginProxy) ProxyHTTP(
+	w ResponseWriter,
+	req *http.Request,
+	isWebsocket bool,
+) error {
+	if isWebsocket {
+		return wsEndpoint(w, req)
+	}
+	switch req.URL.Path {
+	case "/ok":
+		originRespEndpoint(w, http.StatusOK, []byte(http.StatusText(http.StatusOK)))
+	case "/large_file":
+		originRespEndpoint(w, http.StatusOK, testLargeResp)
+	case "/400":
+		originRespEndpoint(w, http.StatusBadRequest, []byte(http.StatusText(http.StatusBadRequest)))
+	case "/500":
+		originRespEndpoint(w, http.StatusInternalServerError, []byte(http.StatusText(http.StatusInternalServerError)))
+	case "/error":
+		return fmt.Errorf("Failed to proxy to origin")
+	default:
+		originRespEndpoint(w, http.StatusNotFound, []byte("page not found"))
+	}
+	return nil
+
+}
+
+func (moc *mockOriginProxy) ProxyTCP(
+	ctx context.Context,
+	rwa ReadWriteAcker,
+	r *TCPRequest,
+) error {
+	return nil
+}
+
+type nowriter struct {
+	io.Reader
+}
+
+func (nowriter) Write(p []byte) (int, error) {
+	return 0, fmt.Errorf("Writer not implemented")
+}
+
+func wsEndpoint(w ResponseWriter, r *http.Request) error {
+	resp := &http.Response{
+		StatusCode: http.StatusSwitchingProtocols,
+	}
+	_ = w.WriteRespHeaders(resp.StatusCode, resp.Header)
+	clientReader := nowriter{r.Body}
+	go func() {
+		for {
+			data, err := wsutil.ReadClientText(clientReader)
+			if err != nil {
+				return
+			}
+			if err := wsutil.WriteServerText(w, data); err != nil {
+				return
+			}
+		}
+	}()
+	<-r.Context().Done()
+	return nil
+}
+
+func originRespEndpoint(w ResponseWriter, status int, data []byte) {
+	resp := &http.Response{
+		StatusCode: status,
+	}
+	_ = w.WriteRespHeaders(resp.StatusCode, resp.Header)
+	_, _ = w.Write(data)
+}
+
+type mockConnectedFuse struct{}
+
+func (mcf mockConnectedFuse) Connected() {}
+
+func (mcf mockConnectedFuse) IsConnected() bool {
+	return true
+}
+
+func TestIsEventStream(t *testing.T) {
+	tests := []struct {
+		headers       http.Header
+		isEventStream bool
+	}{
+		{
+			headers:       newHeader("Content-Type", "text/event-stream"),
+			isEventStream: true,
+		},
+		{
+			headers:       newHeader("content-type", "text/event-stream"),
+			isEventStream: true,
+		},
+		{
+			headers:       newHeader("Content-Type", "text/event-stream; charset=utf-8"),
+			isEventStream: true,
+		},
+		{
+			headers:       newHeader("Content-Type", "application/json"),
+			isEventStream: false,
+		},
+		{
+			headers:       http.Header{},
+			isEventStream: false,
+		},
+	}
+	for _, test := range tests {
+		assert.Equal(t, test.isEventStream, IsServerSentEvent(test.headers))
+	}
+}
+
+func newHeader(key, value string) http.Header {
+	header := http.Header{}
+	header.Add(key, value)
+	return header
+}

connection/control.go

@@ -0,0 +1,100 @@
+package connection
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"github.com/rs/zerolog"
+
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// RPCClientFunc derives a named tunnel rpc client that can then be used to register and unregister connections.
+type RPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
+
+type controlStream struct {
+	observer *Observer
+
+	connectedFuse     ConnectedFuse
+	namedTunnelConfig *NamedTunnelConfig
+	connIndex         uint8
+
+	newRPCClientFunc RPCClientFunc
+
+	gracefulShutdownC <-chan struct{}
+	gracePeriod       time.Duration
+	stoppedGracefully bool
+}
+
+// ControlStreamHandler registers connections with origintunneld and initiates graceful shutdown.
+type ControlStreamHandler interface {
+	ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions, shouldWaitForUnregister bool) error
+	IsStopped() bool
+}
+
+// NewControlStream returns a new instance of ControlStreamHandler
+func NewControlStream(
+	observer *Observer,
+	connectedFuse ConnectedFuse,
+	namedTunnelConfig *NamedTunnelConfig,
+	connIndex uint8,
+	newRPCClientFunc RPCClientFunc,
+	gracefulShutdownC <-chan struct{},
+	gracePeriod time.Duration,
+) ControlStreamHandler {
+	if newRPCClientFunc == nil {
+		newRPCClientFunc = newRegistrationRPCClient
+	}
+	return &controlStream{
+		observer:          observer,
+		connectedFuse:     connectedFuse,
+		namedTunnelConfig: namedTunnelConfig,
+		newRPCClientFunc:  newRPCClientFunc,
+		connIndex:         connIndex,
+		gracefulShutdownC: gracefulShutdownC,
+		gracePeriod:       gracePeriod,
+	}
+}
+
+func (c *controlStream) ServeControlStream(
+	ctx context.Context,
+	rw io.ReadWriteCloser,
+	connOptions *tunnelpogs.ConnectionOptions,
+	shouldWaitForUnregister bool,
+) error {
+	rpcClient := c.newRPCClientFunc(ctx, rw, c.observer.log)
+
+	if err := rpcClient.RegisterConnection(ctx, c.namedTunnelConfig, connOptions, c.connIndex, c.observer); err != nil {
+		rpcClient.Close()
+		return err
+	}
+	c.connectedFuse.Connected()
+
+	if shouldWaitForUnregister {
+		c.waitForUnregister(ctx, rpcClient)
+	} else {
+		go c.waitForUnregister(ctx, rpcClient)
+	}
+
+	return nil
+}
+
+func (c *controlStream) waitForUnregister(ctx context.Context, rpcClient NamedTunnelRPCClient) {
+	// wait for connection termination or start of graceful shutdown
+	defer rpcClient.Close()
+	select {
+	case <-ctx.Done():
+		break
+	case <-c.gracefulShutdownC:
+		c.stoppedGracefully = true
+	}
+
+	c.observer.sendUnregisteringEvent(c.connIndex)
+	rpcClient.GracefulShutdown(ctx, c.gracePeriod)
+	c.observer.log.Info().Uint8(LogFieldConnIndex, c.connIndex).Msg("Unregistered tunnel connection")
+}
+
+func (c *controlStream) IsStopped() bool {
+	return c.stoppedGracefully
+}

connection/errors.go

@@ -0,0 +1,68 @@
+package connection
+
+import (
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+const (
+	DuplicateConnectionError = "EDUPCONN"
+)
+
+type DupConnRegisterTunnelError struct{}
+
+var errDuplicationConnection = DupConnRegisterTunnelError{}
+
+func (e DupConnRegisterTunnelError) Error() string {
+	return "already connected to this server, trying another address"
+}
+
+// RegisterTunnel error from server
+type ServerRegisterTunnelError struct {
+	Cause     error
+	Permanent bool
+}
+
+func (e ServerRegisterTunnelError) Error() string {
+	return e.Cause.Error()
+}
+
+func serverRegistrationErrorFromRPC(err error) ServerRegisterTunnelError {
+	if retryable, ok := err.(*tunnelpogs.RetryableError); ok {
+		return ServerRegisterTunnelError{
+			Cause:     retryable.Unwrap(),
+			Permanent: false,
+		}
+	}
+	return ServerRegisterTunnelError{
+		Cause:     err,
+		Permanent: true,
+	}
+}
+
+type muxerShutdownError struct{}
+
+func (e muxerShutdownError) Error() string {
+	return "muxer shutdown"
+}
+
+var errMuxerStopped = muxerShutdownError{}
+
+func isHandshakeErrRecoverable(err error, connIndex uint8, observer *Observer) bool {
+	log := observer.log.With().
+		Uint8(LogFieldConnIndex, connIndex).
+		Err(err).
+		Logger()
+
+	switch err.(type) {
+	case edgediscovery.DialError:
+		log.Error().Msg("Connection unable to dial edge")
+	case h2mux.MuxerHandshakeError:
+		log.Error().Msg("Connection handshake with edge server failed")
+	default:
+		log.Error().Msg("Connection failed")
+		return false
+	}
+	return true
+}

connection/event.go

@@ -0,0 +1,27 @@
+package connection
+
+// Event is something that happened to a connection, e.g. disconnection or registration.
+type Event struct {
+	Index     uint8
+	EventType Status
+	Location  string
+	URL       string
+}
+
+// Status is the status of a connection.
+type Status int
+
+const (
+	// Disconnected means the connection to the edge was broken.
+	Disconnected Status = iota
+	// Connected means the connection to the edge was successfully established.
+	Connected
+	// Reconnecting means the connection to the edge is being re-established.
+	Reconnecting
+	// SetURL means this connection's tunnel was given a URL by the edge. Used for quick tunnels.
+	SetURL
+	// RegisteringTunnel means the non-named tunnel is registering its connection.
+	RegisteringTunnel
+	// We're unregistering tunnel from the edge in preparation for a disconnect
+	Unregistering
+)

connection/h2mux.go

@@ -0,0 +1,262 @@
+package connection
+
+import (
+	"context"
+	"io"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
+)
+
+const (
+	muxerTimeout      = 5 * time.Second
+	openStreamTimeout = 30 * time.Second
+)
+
+type h2muxConnection struct {
+	config      *Config
+	muxerConfig *MuxerConfig
+	muxer       *h2mux.Muxer
+	// connectionID is only used by metrics, and prometheus requires labels to be string
+	connIndexStr string
+	connIndex    uint8
+
+	observer          *Observer
+	gracefulShutdownC <-chan struct{}
+	stoppedGracefully bool
+
+	log *zerolog.Logger
+
+	// newRPCClientFunc allows us to mock RPCs during testing
+	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
+}
+
+type MuxerConfig struct {
+	HeartbeatInterval  time.Duration
+	MaxHeartbeats      uint64
+	CompressionSetting h2mux.CompressionSetting
+	MetricsUpdateFreq  time.Duration
+}
+
+func (mc *MuxerConfig) H2MuxerConfig(h h2mux.MuxedStreamHandler, log *zerolog.Logger) *h2mux.MuxerConfig {
+	return &h2mux.MuxerConfig{
+		Timeout:            muxerTimeout,
+		Handler:            h,
+		IsClient:           true,
+		HeartbeatInterval:  mc.HeartbeatInterval,
+		MaxHeartbeats:      mc.MaxHeartbeats,
+		Log:                log,
+		CompressionQuality: mc.CompressionSetting,
+	}
+}
+
+// NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
+func NewH2muxConnection(
+	config *Config,
+	muxerConfig *MuxerConfig,
+	edgeConn net.Conn,
+	connIndex uint8,
+	observer *Observer,
+	gracefulShutdownC <-chan struct{},
+) (*h2muxConnection, error, bool) {
+	h := &h2muxConnection{
+		config:            config,
+		muxerConfig:       muxerConfig,
+		connIndexStr:      uint8ToString(connIndex),
+		connIndex:         connIndex,
+		observer:          observer,
+		gracefulShutdownC: gracefulShutdownC,
+		newRPCClientFunc:  newRegistrationRPCClient,
+	}
+
+	// Establish a muxed connection with the edge
+	// Client mux handshake with agent server
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig.H2MuxerConfig(h, observer.logTransport), h2mux.ActiveStreams)
+	if err != nil {
+		recoverable := isHandshakeErrRecoverable(err, connIndex, observer)
+		return nil, err, recoverable
+	}
+	h.muxer = muxer
+	return h, nil, false
+}
+
+func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelConfig, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() error {
+		if err := h.registerNamedTunnel(serveCtx, namedTunnel, connOptions); err != nil {
+			return err
+		}
+		connectedFuse.Connected()
+		return nil
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, true)
+		return nil
+	})
+
+	err := errGroup.Wait()
+	if err == errMuxerStopped {
+		if h.stoppedGracefully {
+			return nil
+		}
+		h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unexpected muxer shutdown")
+	}
+	return err
+}
+
+func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel *ClassicTunnelConfig, credentialManager CredentialManager, registrationOptions *tunnelpogs.RegistrationOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() (err error) {
+		defer func() {
+			if err == nil {
+				connectedFuse.Connected()
+			}
+		}()
+		if classicTunnel.UseReconnectToken && connectedFuse.IsConnected() {
+			err := h.reconnectTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
+			if err == nil {
+				return nil
+			}
+			// log errors and proceed to RegisterTunnel
+			h.observer.log.Err(err).
+				Uint8(LogFieldConnIndex, h.connIndex).
+				Msg("Couldn't reconnect connection. Re-registering it instead.")
+		}
+		return h.registerTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, false)
+		return nil
+	})
+
+	err := errGroup.Wait()
+	if err == errMuxerStopped {
+		if h.stoppedGracefully {
+			return nil
+		}
+		h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unexpected muxer shutdown")
+	}
+	return err
+}
+
+func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
+	// All routines should stop when muxer finish serving. When muxer is shutdown
+	// gracefully, it doesn't return an error, so we need to return errMuxerShutdown
+	// here to notify other routines to stop
+	err := h.muxer.Serve(ctx)
+	if err == nil {
+		return errMuxerStopped
+	}
+	return err
+}
+
+func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse ConnectedFuse, isNamedTunnel bool) {
+	updateMetricsTicker := time.NewTicker(h.muxerConfig.MetricsUpdateFreq)
+	defer updateMetricsTicker.Stop()
+	var shutdownCompleted <-chan struct{}
+	for {
+		select {
+		case <-h.gracefulShutdownC:
+			if connectedFuse.IsConnected() {
+				h.unregister(isNamedTunnel)
+			}
+			h.stoppedGracefully = true
+			h.gracefulShutdownC = nil
+			shutdownCompleted = h.muxer.Shutdown()
+
+		case <-shutdownCompleted:
+			return
+
+		case <-ctx.Done():
+			// UnregisterTunnel blocks until the RPC call returns
+			if !h.stoppedGracefully && connectedFuse.IsConnected() {
+				h.unregister(isNamedTunnel)
+			}
+			h.muxer.Shutdown()
+			// don't wait for shutdown to finish when context is closed, this is the hard termination path
+			return
+
+		case <-updateMetricsTicker.C:
+			h.observer.metrics.updateMuxerMetrics(h.connIndexStr, h.muxer.Metrics())
+		}
+	}
+}
+
+func (h *h2muxConnection) newRPCStream(ctx context.Context, rpcName rpcName) (*h2mux.MuxedStream, error) {
+	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer openStreamCancel()
+	stream, err := h.muxer.OpenRPCStream(openStreamCtx)
+	if err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
+
+func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
+	respWriter := &h2muxRespWriter{stream}
+
+	req, reqErr := h.newRequest(stream)
+	if reqErr != nil {
+		respWriter.WriteErrorResponse()
+		return reqErr
+	}
+
+	var sourceConnectionType = TypeHTTP
+	if websocket.IsWebSocketUpgrade(req) {
+		sourceConnectionType = TypeWebsocket
+	}
+
+	err := h.config.OriginProxy.ProxyHTTP(respWriter, req, sourceConnectionType == TypeWebsocket)
+	if err != nil {
+		respWriter.WriteErrorResponse()
+	}
+	return err
+}
+
+func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request, error) {
+	req, err := http.NewRequest("GET", "http://localhost:8080", h2mux.MuxedStreamReader{MuxedStream: stream})
+	if err != nil {
+		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
+	}
+	err = H2RequestHeadersToH1Request(stream.Headers, req)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid request received")
+	}
+	return req, nil
+}
+
+type h2muxRespWriter struct {
+	*h2mux.MuxedStream
+}
+
+func (rp *h2muxRespWriter) WriteRespHeaders(status int, header http.Header) error {
+	headers := H1ResponseToH2ResponseHeaders(status, header)
+	headers = append(headers, h2mux.Header{Name: ResponseMetaHeader, Value: responseMetaHeaderOrigin})
+	return rp.WriteHeaders(headers)
+}
+
+func (rp *h2muxRespWriter) WriteErrorResponse() {
+	_ = rp.WriteHeaders([]h2mux.Header{
+		{Name: ":status", Value: "502"},
+		{Name: ResponseMetaHeader, Value: responseMetaHeaderCfd},
+	})
+	_, _ = rp.Write([]byte("502 Bad Gateway"))
+}

connection/h2mux_test.go

@@ -0,0 +1,301 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/gobwas/ws/wsutil"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+)
+
+var (
+	testMuxerConfig = &MuxerConfig{
+		HeartbeatInterval:  time.Second * 5,
+		MaxHeartbeats:      5,
+		CompressionSetting: 0,
+		MetricsUpdateFreq:  time.Second * 5,
+	}
+)
+
+func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
+	edgeConn, originConn := net.Pipe()
+	edgeMuxChan := make(chan *h2mux.Muxer)
+	go func() {
+		edgeMuxConfig := h2mux.MuxerConfig{
+			Log: &log,
+			Handler: h2mux.MuxedStreamFunc(func(stream *h2mux.MuxedStream) error {
+				// we only expect RPC traffic in client->edge direction, provide minimal support for mocking
+				require.True(t, stream.IsRPCStream())
+				return stream.WriteHeaders([]h2mux.Header{
+					{Name: ":status", Value: "200"},
+				})
+			}),
+		}
+		edgeMux, err := h2mux.Handshake(edgeConn, edgeConn, edgeMuxConfig, h2mux.ActiveStreams)
+		require.NoError(t, err)
+		edgeMuxChan <- edgeMux
+	}()
+	var connIndex = uint8(0)
+	testObserver := NewObserver(&log, &log, false)
+	h2muxConn, err, _ := NewH2muxConnection(testConfig, testMuxerConfig, originConn, connIndex, testObserver, nil)
+	require.NoError(t, err)
+	return h2muxConn, <-edgeMuxChan
+}
+
+func TestServeStreamHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "/ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "/large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "/400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "/500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "/error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		_ = edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	for _, test := range tests {
+		headers := []h2mux.Header{
+			{
+				Name:  ":path",
+				Value: test.endpoint,
+			},
+		}
+		stream, err := edgeMux.OpenStream(ctx, headers, nil)
+		require.NoError(t, err)
+		require.True(t, hasHeader(stream, ":status", strconv.Itoa(test.expectedStatus)))
+
+		if test.isProxyError {
+			assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderCfd))
+		} else {
+			assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
+			body := make([]byte, len(test.expectedBody))
+			_, err = stream.Read(body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, body)
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+func TestServeStreamWS(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: "/ws",
+		},
+		{
+			Name:  "connection",
+			Value: "upgrade",
+		},
+		{
+			Name:  "upgrade",
+			Value: "websocket",
+		},
+	}
+
+	readPipe, writePipe := io.Pipe()
+	stream, err := edgeMux.OpenStream(ctx, headers, readPipe)
+	require.NoError(t, err)
+
+	require.True(t, hasHeader(stream, ":status", strconv.Itoa(http.StatusSwitchingProtocols)))
+	assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientText(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerText(stream)
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	wg.Wait()
+}
+
+func TestGracefulShutdownH2Mux(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	h2muxConn, edgeMux := newH2MuxConnection(t)
+
+	shutdownC := make(chan struct{})
+	unregisteredC := make(chan struct{})
+	h2muxConn.gracefulShutdownC = shutdownC
+	h2muxConn.newRPCClientFunc = func(_ context.Context, _ io.ReadWriteCloser, _ *zerolog.Logger) NamedTunnelRPCClient {
+		return &mockNamedTunnelRPCClient{
+			registered:   nil,
+			unregistered: unregisteredC,
+		}
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(3)
+	go func() {
+		defer wg.Done()
+		_ = edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		_ = h2muxConn.serveMuxer(ctx)
+	}()
+
+	go func() {
+		defer wg.Done()
+		h2muxConn.controlLoop(ctx, &mockConnectedFuse{}, true)
+	}()
+
+	time.Sleep(100 * time.Millisecond)
+	close(shutdownC)
+
+	select {
+	case <-unregisteredC:
+		break // ok
+	case <-time.Tick(time.Second):
+		assert.Fail(t, "timed out waiting for control loop to unregister")
+	}
+
+	cancel()
+	wg.Wait()
+
+	assert.True(t, h2muxConn.stoppedGracefully)
+	assert.Nil(t, h2muxConn.gracefulShutdownC)
+}
+
+func hasHeader(stream *h2mux.MuxedStream, name, val string) bool {
+	for _, header := range stream.Headers {
+		if header.Name == name && header.Value == val {
+			return true
+		}
+	}
+	return false
+}
+
+func benchmarkServeStreamHTTPSimple(b *testing.B, test testRequest) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(b)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(b, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: test.endpoint,
+		},
+	}
+
+	body := make([]byte, len(test.expectedBody))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		stream, openstreamErr := edgeMux.OpenStream(ctx, headers, nil)
+		_, readBodyErr := stream.Read(body)
+		b.StopTimer()
+
+		require.NoError(b, openstreamErr)
+		assert.True(b, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
+		require.True(b, hasHeader(stream, ":status", strconv.Itoa(http.StatusOK)))
+		require.NoError(b, readBodyErr)
+		require.Equal(b, test.expectedBody, body)
+	}
+
+	cancel()
+	wg.Wait()
+}
+
+func BenchmarkServeStreamHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "/ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}
+
+func BenchmarkServeStreamHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "/large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}

connection/header.go

@@ -1,8 +1,7 @@
-package h2mux
+package connection
 
 import (
 	"encoding/base64"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -10,39 +9,56 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/h2mux"
 )
 
-type Header struct {
-	Name, Value string
+var (
+	// h2mux-style special headers
+	RequestUserHeaders  = "cf-cloudflared-request-headers"
+	ResponseUserHeaders = "cf-cloudflared-response-headers"
+	ResponseMetaHeader  = "cf-cloudflared-response-meta"
+
+	// h2mux-style special headers
+	CanonicalResponseUserHeaders = http.CanonicalHeaderKey(ResponseUserHeaders)
+	CanonicalResponseMetaHeader  = http.CanonicalHeaderKey(ResponseMetaHeader)
+)
+
+var (
+	// pre-generate possible values for res
+	responseMetaHeaderCfd    = mustInitRespMetaHeader("cloudflared")
+	responseMetaHeaderOrigin = mustInitRespMetaHeader("origin")
+)
+
+type responseMetaHeader struct {
+	Source string `json:"src"`
+}
+
+func mustInitRespMetaHeader(src string) string {
+	header, err := json.Marshal(responseMetaHeader{Source: src})
+	if err != nil {
+		panic(fmt.Sprintf("Failed to serialize response meta header = %s, err: %v", src, err))
+	}
+	return string(header)
 }
 
 var headerEncoding = base64.RawStdEncoding
 
-const (
-	RequestUserHeadersField  = "cf-cloudflared-request-headers"
-	ResponseUserHeadersField = "cf-cloudflared-response-headers"
-
-	ResponseMetaHeaderField   = "cf-cloudflared-response-meta"
-	ResponseSourceCloudflared = "cloudflared"
-	ResponseSourceOrigin      = "origin"
-
-	CFAccessTokenHeader        = "cf-access-token"
-	CFJumpDestinationHeader    = "CF-Access-Jump-Destination"
-	CFAccessClientIDHeader     = "CF-Access-Client-Id"
-	CFAccessClientSecretHeader = "CF-Access-Client-Secret"
-)
+// note: all h2mux headers should be lower-case (http/2 style)
+const ()
 
 // H2RequestHeadersToH1Request converts the HTTP/2 headers coming from origintunneld
 // to an HTTP/1 Request object destined for the local origin web service.
 // This operation includes conversion of the pseudo-headers into their closest
 // HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3
-func H2RequestHeadersToH1Request(h2 []Header, h1 *http.Request) error {
+func H2RequestHeadersToH1Request(h2 []h2mux.Header, h1 *http.Request) error {
 	for _, header := range h2 {
-		if !IsControlHeader(header.Name) {
+		name := strings.ToLower(header.Name)
+		if !IsControlRequestHeader(name) {
 			continue
 		}
 
-		switch strings.ToLower(header.Name) {
+		switch name {
 		case ":method":
 			h1.Method = header.Value
 		case ":scheme":
@@ -86,114 +102,123 @@ func H2RequestHeadersToH1Request(h2 []Header, h1 *http.Request) error {
 				return fmt.Errorf("unparseable content length")
 			}
 			h1.ContentLength = contentLength
-		case RequestUserHeadersField:
+		case RequestUserHeaders:
 			// Do not forward the serialized headers to the origin -- deserialize them, and ditch the serialized version
 			// Find and parse user headers serialized into a single one
-			userHeaders, err := ParseUserHeaders(RequestUserHeadersField, h2)
+			userHeaders, err := DeserializeHeaders(header.Value)
 			if err != nil {
 				return errors.Wrap(err, "Unable to parse user headers")
 			}
 			for _, userHeader := range userHeaders {
-				h1.Header.Add(http.CanonicalHeaderKey(userHeader.Name), userHeader.Value)
+				h1.Header.Add(userHeader.Name, userHeader.Value)
 			}
 		default:
 			// All other control headers shall just be proxied transparently
-			h1.Header.Add(http.CanonicalHeaderKey(header.Name), header.Value)
+			h1.Header.Add(header.Name, header.Value)
 		}
 	}
 
 	return nil
 }
 
-func ParseUserHeaders(headerNameToParseFrom string, headers []Header) ([]Header, error) {
-	for _, header := range headers {
-		if header.Name == headerNameToParseFrom {
-			return DeserializeHeaders(header.Value)
-		}
-	}
-
-	return nil, fmt.Errorf("%v header not found", RequestUserHeadersField)
-}
-
-func IsControlHeader(headerName string) bool {
-	headerName = strings.ToLower(headerName)
-
+func IsControlRequestHeader(headerName string) bool {
 	return headerName == "content-length" ||
-		headerName == "connection" || headerName == "upgrade" || // Websocket headers
+		headerName == "connection" || headerName == "upgrade" || // Websocket request headers
 		strings.HasPrefix(headerName, ":") ||
 		strings.HasPrefix(headerName, "cf-")
 }
 
-// IsWebsocketClientHeader returns true if the header name is required by the client to upgrade properly
-func IsWebsocketClientHeader(headerName string) bool {
-	headerName = strings.ToLower(headerName)
+func IsControlResponseHeader(headerName string) bool {
+	return headerName == "content-length" ||
+		strings.HasPrefix(headerName, ":") ||
+		strings.HasPrefix(headerName, "cf-int-") ||
+		strings.HasPrefix(headerName, "cf-cloudflared-")
+}
 
+// isWebsocketClientHeader returns true if the header name is required by the client to upgrade properly
+func IsWebsocketClientHeader(headerName string) bool {
 	return headerName == "sec-websocket-accept" ||
 		headerName == "connection" ||
 		headerName == "upgrade"
 }
 
-func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
-	h2 = []Header{
-		{Name: ":status", Value: strconv.Itoa(h1.StatusCode)},
+func H1ResponseToH2ResponseHeaders(status int, h1 http.Header) (h2 []h2mux.Header) {
+	h2 = []h2mux.Header{
+		{Name: ":status", Value: strconv.Itoa(status)},
 	}
-	userHeaders := http.Header{}
-	for header, values := range h1.Header {
-		for _, value := range values {
-			if strings.ToLower(header) == "content-length" {
-				// This header has meaning in HTTP/2 and will be used by the edge,
-				// so it should be sent as an HTTP/2 response header.
+	userHeaders := make(http.Header, len(h1))
+	for header, values := range h1 {
+		h2name := strings.ToLower(header)
+		if h2name == "content-length" {
+			// This header has meaning in HTTP/2 and will be used by the edge,
+			// so it should be sent as an HTTP/2 response header.
 
-				// Since these are http2 headers, they're required to be lowercase
-				h2 = append(h2, Header{Name: strings.ToLower(header), Value: value})
-			} else if !IsControlHeader(header) || IsWebsocketClientHeader(header) {
-				// User headers, on the other hand, must all be serialized so that
-				// HTTP/2 header validation won't be applied to HTTP/1 header values
-				if _, ok := userHeaders[header]; ok {
-					userHeaders[header] = append(userHeaders[header], value)
-				} else {
-					userHeaders[header] = []string{value}
-				}
-			}
+			// Since these are http2 headers, they're required to be lowercase
+			h2 = append(h2, h2mux.Header{Name: "content-length", Value: values[0]})
+		} else if !IsControlResponseHeader(h2name) || IsWebsocketClientHeader(h2name) {
+			// User headers, on the other hand, must all be serialized so that
+			// HTTP/2 header validation won't be applied to HTTP/1 header values
+			userHeaders[header] = values
 		}
 	}
 
 	// Perform user header serialization and set them in the single header
-	h2 = append(h2, CreateSerializedHeaders(ResponseUserHeadersField, userHeaders)...)
-
+	h2 = append(h2, h2mux.Header{Name: ResponseUserHeaders, Value: SerializeHeaders(userHeaders)})
 	return h2
 }
 
 // Serialize HTTP1.x headers by base64-encoding each header name and value,
 // and then joining them in the format of [key:value;]
 func SerializeHeaders(h1Headers http.Header) string {
-	var serializedHeaders []string
+	// compute size of the fully serialized value and largest temp buffer we will need
+	serializedLen := 0
+	maxTempLen := 0
 	for headerName, headerValues := range h1Headers {
 		for _, headerValue := range headerValues {
-			encodedName := make([]byte, headerEncoding.EncodedLen(len(headerName)))
-			headerEncoding.Encode(encodedName, []byte(headerName))
+			nameLen := headerEncoding.EncodedLen(len(headerName))
+			valueLen := headerEncoding.EncodedLen(len(headerValue))
+			const delims = 2
+			serializedLen += delims + nameLen + valueLen
+			if nameLen > maxTempLen {
+				maxTempLen = nameLen
+			}
+			if valueLen > maxTempLen {
+				maxTempLen = valueLen
+			}
+		}
+	}
+	var buf strings.Builder
+	buf.Grow(serializedLen)
 
-			encodedValue := make([]byte, headerEncoding.EncodedLen(len(headerValue)))
-			headerEncoding.Encode(encodedValue, []byte(headerValue))
+	temp := make([]byte, maxTempLen)
+	writeB64 := func(s string) {
+		n := headerEncoding.EncodedLen(len(s))
+		if n > len(temp) {
+			temp = make([]byte, n)
+		}
+		headerEncoding.Encode(temp[:n], []byte(s))
+		buf.Write(temp[:n])
+	}
 
-			serializedHeaders = append(
-				serializedHeaders,
-				strings.Join(
-					[]string{string(encodedName), string(encodedValue)},
-					":",
-				),
-			)
+	for headerName, headerValues := range h1Headers {
+		for _, headerValue := range headerValues {
+			if buf.Len() > 0 {
+				buf.WriteByte(';')
+			}
+			writeB64(headerName)
+			buf.WriteByte(':')
+			writeB64(headerValue)
 		}
 	}
 
-	return strings.Join(serializedHeaders, ";")
+	return buf.String()
 }
 
 // Deserialize headers serialized by `SerializeHeader`
-func DeserializeHeaders(serializedHeaders string) ([]Header, error) {
+func DeserializeHeaders(serializedHeaders string) ([]h2mux.Header, error) {
 	const unableToDeserializeErr = "Unable to deserialize headers"
 
-	var deserialized []Header
+	var deserialized []h2mux.Header
 	for _, serializedPair := range strings.Split(serializedHeaders, ";") {
 		if len(serializedPair) == 0 {
 			continue
@@ -216,7 +241,7 @@ func DeserializeHeaders(serializedHeaders string) ([]Header, error) {
 			return nil, errors.Wrap(err, unableToDeserializeErr)
 		}
 
-		deserialized = append(deserialized, Header{
+		deserialized = append(deserialized, h2mux.Header{
 			Name:  string(deserializedName),
 			Value: string(deserializedValue),
 		})
@@ -224,31 +249,3 @@ func DeserializeHeaders(serializedHeaders string) ([]Header, error) {
 
 	return deserialized, nil
 }
-
-func CreateSerializedHeaders(headersField string, headers ...http.Header) []Header {
-	var serializedHeaderChunks []string
-	for _, headerChunk := range headers {
-		serializedHeaderChunks = append(serializedHeaderChunks, SerializeHeaders(headerChunk))
-	}
-
-	return []Header{{
-		headersField,
-		strings.Join(serializedHeaderChunks, ";"),
-	}}
-}
-
-type ResponseMetaHeader struct {
-	Source string `json:"src"`
-}
-
-func CreateResponseMetaHeader(headerName, source string) Header {
-	jsonResponseMetaHeader, err := json.Marshal(ResponseMetaHeader{Source: source})
-	if err != nil {
-		panic(err)
-	}
-
-	return Header{
-		Name:  headerName,
-		Value: string(jsonResponseMetaHeader),
-	}
-}

connection/header_test.go

@@ -1,4 +1,4 @@
-package h2mux
+package connection
 
 import (
 	"fmt"
@@ -14,9 +14,11 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/h2mux"
 )
 
-type ByName []Header
+type ByName []h2mux.Header
 
 func (a ByName) Len() int      { return len(a) }
 func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
@@ -37,25 +39,33 @@ func TestH2RequestHeadersToH1Request_RegularHeaders(t *testing.T) {
 		"Mock header 2": {"Mock value 2"},
 	}
 
-	headersConversionErr := H2RequestHeadersToH1Request(CreateSerializedHeaders(RequestUserHeadersField, mockHeaders), request)
+	headersConversionErr := H2RequestHeadersToH1Request(createSerializedHeaders(RequestUserHeaders, mockHeaders), request)
 
 	assert.True(t, reflect.DeepEqual(mockHeaders, request.Header))
 	assert.NoError(t, headersConversionErr)
 }
 
+func createSerializedHeaders(headersField string, headers http.Header) []h2mux.Header {
+	return []h2mux.Header{{
+		Name:  headersField,
+		Value: SerializeHeaders(headers),
+	}}
+}
+
 func TestH2RequestHeadersToH1Request_NoHeaders(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
 	assert.NoError(t, err)
 
+	emptyHeaders := make(http.Header)
 	headersConversionErr := H2RequestHeadersToH1Request(
-		[]Header{{
-			RequestUserHeadersField,
-			SerializeHeaders(http.Header{}),
+		[]h2mux.Header{{
+			Name:  RequestUserHeaders,
+			Value: SerializeHeaders(emptyHeaders),
 		}},
 		request,
 	)
 
-	assert.True(t, reflect.DeepEqual(http.Header{}, request.Header))
+	assert.True(t, reflect.DeepEqual(emptyHeaders, request.Header))
 	assert.NoError(t, headersConversionErr)
 }
 
@@ -63,9 +73,9 @@ func TestH2RequestHeadersToH1Request_InvalidHostPath(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
 	assert.NoError(t, err)
 
-	mockRequestHeaders := []Header{
+	mockRequestHeaders := []h2mux.Header{
 		{Name: ":path", Value: "//bad_path/"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 	}
 
 	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -83,9 +93,9 @@ func TestH2RequestHeadersToH1Request_HostPathWithQuery(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
 	assert.NoError(t, err)
 
-	mockRequestHeaders := []Header{
+	mockRequestHeaders := []h2mux.Header{
 		{Name: ":path", Value: "/?query=mock%20value"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 	}
 
 	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -103,9 +113,9 @@ func TestH2RequestHeadersToH1Request_HostPathWithURLEncoding(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
 	assert.NoError(t, err)
 
-	mockRequestHeaders := []Header{
+	mockRequestHeaders := []h2mux.Header{
 		{Name: ":path", Value: "/mock%20path"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 	}
 
 	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -260,9 +270,9 @@ func TestH2RequestHeadersToH1Request_WeirdURLs(t *testing.T) {
 		request, err := http.NewRequest(http.MethodGet, requestURL, nil)
 		assert.NoError(t, err)
 
-		mockRequestHeaders := []Header{
+		mockRequestHeaders := []h2mux.Header{
 			{Name: ":path", Value: testCase.path},
-			{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+			{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
 		}
 
 		headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
@@ -330,12 +340,12 @@ func TestH2RequestHeadersToH1Request_QuickCheck(t *testing.T) {
 				const expectedMethod = "POST"
 				const expectedHostname = "request.hostname.example.com"
 
-				h2 := []Header{
+				h2 := []h2mux.Header{
 					{Name: ":method", Value: expectedMethod},
 					{Name: ":scheme", Value: testScheme},
 					{Name: ":authority", Value: expectedHostname},
 					{Name: ":path", Value: testPath},
-					{Name: RequestUserHeadersField, Value: ""},
+					{Name: RequestUserHeaders, Value: ""},
 				}
 				h1, err := http.NewRequest("GET", testOrigin.url, nil)
 				require.NoError(t, err)
@@ -365,18 +375,18 @@ func randomASCIIPrintableChar(rand *rand.Rand) int {
 // between 1 and `maxLength`.
 func randomASCIIText(rand *rand.Rand, minLength int, maxLength int) string {
 	length := minLength + rand.Intn(maxLength)
-	result := ""
+	var result strings.Builder
 	for i := 0; i < length; i++ {
 		c := randomASCIIPrintableChar(rand)
 
 		// 1/4 chance of using percent encoding when not necessary
 		if c == '%' || rand.Intn(4) == 0 {
-			result += fmt.Sprintf("%%%02X", c)
+			result.WriteString(fmt.Sprintf("%%%02X", c))
 		} else {
-			result += string(c)
+			result.WriteByte(byte(c))
 		}
 	}
-	return result
+	return result.String()
 }
 
 // Calls `randomASCIIText` and ensures the result is a valid URL path,
@@ -417,10 +427,10 @@ func randomHTTP2Path(t *testing.T, rand *rand.Rand) string {
 	return result
 }
 
-func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []Header) {
+func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []h2mux.Header) {
 	for name, values := range headers {
 		for _, value := range values {
-			h2muxHeaders = append(h2muxHeaders, Header{name, value})
+			h2muxHeaders = append(h2muxHeaders, h2mux.Header{Name: name, Value: value})
 		}
 	}
 
@@ -501,48 +511,63 @@ func TestDeserializeMalformed(t *testing.T) {
 	}
 }
 
-func TestParseHeaders(t *testing.T) {
+func TestParseRequestHeaders(t *testing.T) {
 	mockUserHeadersToSerialize := http.Header{
 		"Mock-Header-One":   {"1", "1.5"},
 		"Mock-Header-Two":   {"2"},
 		"Mock-Header-Three": {"3"},
 	}
 
-	mockHeaders := []Header{
-		{Name: "One", Value: "1"},
+	mockHeaders := []h2mux.Header{
+		{Name: "One", Value: "1"}, // will be dropped
 		{Name: "Cf-Two", Value: "cf-value-1"},
 		{Name: "Cf-Two", Value: "cf-value-2"},
-		{Name: RequestUserHeadersField, Value: SerializeHeaders(mockUserHeadersToSerialize)},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(mockUserHeadersToSerialize)},
 	}
 
-	expectedHeaders := []Header{
+	expectedHeaders := []h2mux.Header{
+		{Name: "Cf-Two", Value: "cf-value-1"},
+		{Name: "Cf-Two", Value: "cf-value-2"},
 		{Name: "Mock-Header-One", Value: "1"},
 		{Name: "Mock-Header-One", Value: "1.5"},
 		{Name: "Mock-Header-Two", Value: "2"},
 		{Name: "Mock-Header-Three", Value: "3"},
 	}
-	parsedHeaders, err := ParseUserHeaders(RequestUserHeadersField, mockHeaders)
-	assert.NoError(t, err)
-	assert.ElementsMatch(t, expectedHeaders, parsedHeaders)
-}
-
-func TestParseHeadersNoSerializedHeader(t *testing.T) {
-	mockHeaders := []Header{
-		{Name: "One", Value: "1"},
-		{Name: "Cf-Two", Value: "cf-value-1"},
-		{Name: "Cf-Two", Value: "cf-value-2"},
+	h1 := &http.Request{
+		Header: make(http.Header),
 	}
-
-	_, err := ParseUserHeaders(RequestUserHeadersField, mockHeaders)
-	assert.EqualError(t, err, fmt.Sprintf("%s header not found", RequestUserHeadersField))
+	err := H2RequestHeadersToH1Request(mockHeaders, h1)
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, expectedHeaders, stdlibHeaderToH2muxHeader(h1.Header))
 }
 
-func TestIsControlHeader(t *testing.T) {
-	controlHeaders := []string{
+func TestIsControlRequestHeader(t *testing.T) {
+	controlRequestHeaders := []string{
 		// Anything that begins with cf-
 		"cf-sample-header",
-		"CF-SAMPLE-HEADER",
-		"Cf-Sample-Header",
+
+		// Any http2 pseudoheader
+		":sample-pseudo-header",
+
+		// content-length is a special case, it has to be there
+		// for some requests to work (per the HTTP2 spec)
+		"content-length",
+
+		// Websocket request headers
+		"connection",
+		"upgrade",
+	}
+
+	for _, header := range controlRequestHeaders {
+		assert.True(t, IsControlRequestHeader(header))
+	}
+}
+
+func TestIsControlResponseHeader(t *testing.T) {
+	controlResponseHeaders := []string{
+		// Anything that begins with cf-int- or cf-cloudflared-
+		"cf-int-sample-header",
+		"cf-cloudflared-sample-header",
 
 		// Any http2 pseudoheader
 		":sample-pseudo-header",
@@ -552,39 +577,55 @@ func TestIsControlHeader(t *testing.T) {
 		"content-length",
 	}
 
-	for _, header := range controlHeaders {
-		assert.True(t, IsControlHeader(header))
+	for _, header := range controlResponseHeaders {
+		assert.True(t, IsControlResponseHeader(header))
 	}
 }
 
-func TestIsNotControlHeader(t *testing.T) {
-	notControlHeaders := []string{
-		"Mock-header",
-		"Another-sample-header",
+func TestIsNotControlRequestHeader(t *testing.T) {
+	notControlRequestHeaders := []string{
+		"mock-header",
+		"another-sample-header",
 	}
 
-	for _, header := range notControlHeaders {
-		assert.False(t, IsControlHeader(header))
+	for _, header := range notControlRequestHeaders {
+		assert.False(t, IsControlRequestHeader(header))
+	}
+}
+
+func TestIsNotControlResponseHeader(t *testing.T) {
+	notControlResponseHeaders := []string{
+		"mock-header",
+		"another-sample-header",
+		"upgrade",
+		"connection",
+		"cf-whatever", // On the response path, we only want to filter cf-int- and cf-cloudflared-
+	}
+
+	for _, header := range notControlResponseHeaders {
+		assert.False(t, IsControlResponseHeader(header))
 	}
 }
 
 func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
 	mockHeaders := http.Header{
-		"User-header-one": {""},
-		"User-header-two": {"1", "2"},
-		"cf-header":       {"cf-value"},
-		"Content-Length":  {"123"},
+		"User-header-one":       {""},
+		"User-header-two":       {"1", "2"},
+		"cf-header":             {"cf-value"},
+		"cf-int-header":         {"cf-int-value"},
+		"cf-cloudflared-header": {"cf-cloudflared-value"},
+		"Content-Length":        {"123"},
 	}
 	mockResponse := http.Response{
 		StatusCode: 200,
 		Header:     mockHeaders,
 	}
 
-	headers := H1ResponseToH2ResponseHeaders(&mockResponse)
+	headers := H1ResponseToH2ResponseHeaders(mockResponse.StatusCode, mockResponse.Header)
 
 	serializedHeadersIndex := -1
 	for i, header := range headers {
-		if header.Name == ResponseUserHeadersField {
+		if header.Name == ResponseUserHeaders {
 			serializedHeadersIndex = i
 			break
 		}
@@ -594,7 +635,7 @@ func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
 		headers[:serializedHeadersIndex],
 		headers[serializedHeadersIndex+1:]...,
 	)
-	expectedControlHeaders := []Header{
+	expectedControlHeaders := []h2mux.Header{
 		{Name: ":status", Value: "200"},
 		{Name: "content-length", Value: "123"},
 	}
@@ -602,10 +643,11 @@ func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
 	assert.ElementsMatch(t, expectedControlHeaders, actualControlHeaders)
 
 	actualUserHeaders, err := DeserializeHeaders(headers[serializedHeadersIndex].Value)
-	expectedUserHeaders := []Header{
+	expectedUserHeaders := []h2mux.Header{
 		{Name: "User-header-one", Value: ""},
 		{Name: "User-header-two", Value: "1"},
 		{Name: "User-header-two", Value: "2"},
+		{Name: "cf-header", Value: "cf-value"},
 	}
 	assert.NoError(t, err)
 	assert.ElementsMatch(t, expectedUserHeaders, actualUserHeaders)
@@ -623,7 +665,7 @@ func TestHeaderSize(t *testing.T) {
 		Header:     largeHeaders,
 	}
 
-	serializedHeaders := H1ResponseToH2ResponseHeaders(&mockResponse)
+	serializedHeaders := H1ResponseToH2ResponseHeaders(mockResponse.StatusCode, mockResponse.Header)
 	request, err := http.NewRequest(http.MethodGet, "https://example.com/", nil)
 	assert.NoError(t, err)
 	for _, header := range serializedHeaders {
@@ -631,7 +673,7 @@ func TestHeaderSize(t *testing.T) {
 	}
 
 	for _, header := range serializedHeaders {
-		if header.Name != ResponseUserHeadersField {
+		if header.Name != ResponseUserHeaders {
 			continue
 		}
 
@@ -650,3 +692,26 @@ func randSeq(n int) string {
 	}
 	return string(b)
 }
+
+func BenchmarkH1ResponseToH2ResponseHeaders(b *testing.B) {
+	ser := "eC1mb3J3YXJkZWQtcHJvdG8:aHR0cHM;dXBncmFkZS1pbnNlY3VyZS1yZXF1ZXN0cw:MQ;YWNjZXB0LWxhbmd1YWdl:ZW4tVVMsZW47cT0wLjkscnU7cT0wLjg;YWNjZXB0LWVuY29kaW5n:Z3ppcA;eC1mb3J3YXJkZWQtZm9y:MTczLjI0NS42MC42;dXNlci1hZ2VudA:TW96aWxsYS81LjAgKE1hY2ludG9zaDsgSW50ZWwgTWFjIE9TIFggMTBfMTRfNikgQXBwbGVXZWJLaXQvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lLzg0LjAuNDE0Ny44OSBTYWZhcmkvNTM3LjM2;c2VjLWZldGNoLW1vZGU:bmF2aWdhdGU;Y2RuLWxvb3A:Y2xvdWRmbGFyZQ;c2VjLWZldGNoLWRlc3Q:ZG9jdW1lbnQ;c2VjLWZldGNoLXVzZXI:PzE;c2VjLWZldGNoLXNpdGU:bm9uZQ;Y29va2ll:X19jZmR1aWQ9ZGNkOWZjOGNjNWMxMzE0NTMyYTFkMjhlZDEyOWRhOTYwMTU2OTk1MTYzNDsgX19jZl9ibT1mYzY2MzMzYzAzZmM0MWFiZTZmOWEyYzI2ZDUwOTA0YzIxYzZhMTQ2LTE1OTU2MjIzNDEtMTgwMC1BZTVzS2pIU2NiWGVFM05mMUhrTlNQMG1tMHBLc2pQWkloVnM1Z2g1SkNHQkFhS1UxVDB2b003alBGN3FjMHVSR2NjZGcrWHdhL1EzbTJhQzdDVU4xZ2M9;YWNjZXB0:dGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksaW1hZ2Uvd2VicCxpbWFnZS9hcG5nLCovKjtxPTAuOCxhcHBsaWNhdGlvbi9zaWduZWQtZXhjaGFuZ2U7dj1iMztxPTAuOQ"
+	h2, _ := DeserializeHeaders(ser)
+	h1 := make(http.Header)
+	for _, header := range h2 {
+		h1.Add(header.Name, header.Value)
+	}
+	h1.Add("Content-Length", "200")
+	h1.Add("Cf-Something", "Else")
+	h1.Add("Upgrade", "websocket")
+
+	h1resp := &http.Response{
+		StatusCode: 200,
+		Header:     h1,
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_ = H1ResponseToH2ResponseHeaders(h1resp.StatusCode, h1resp.Header)
+	}
+}

connection/http2.go

@@ -0,0 +1,302 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math"
+	"net"
+	"net/http"
+	"runtime/debug"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"golang.org/x/net/http2"
+
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// note: these constants are exported so we can reuse them in the edge-side code
+const (
+	InternalUpgradeHeader     = "Cf-Cloudflared-Proxy-Connection-Upgrade"
+	InternalTCPProxySrcHeader = "Cf-Cloudflared-Proxy-Src"
+	WebsocketUpgrade          = "websocket"
+	ControlStreamUpgrade      = "control-stream"
+)
+
+var errEdgeConnectionClosed = fmt.Errorf("connection with edge closed")
+
+// HTTP2Connection represents a net.Conn that uses HTTP2 frames to proxy traffic from the edge to cloudflared on the
+// origin.
+type HTTP2Connection struct {
+	conn        net.Conn
+	server      *http2.Server
+	config      *Config
+	connOptions *tunnelpogs.ConnectionOptions
+	observer    *Observer
+	connIndex   uint8
+	// newRPCClientFunc allows us to mock RPCs during testing
+	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
+
+	log                  *zerolog.Logger
+	activeRequestsWG     sync.WaitGroup
+	controlStreamHandler ControlStreamHandler
+	stoppedGracefully    bool
+	controlStreamErr     error // result of running control stream handler
+}
+
+// NewHTTP2Connection returns a new instance of HTTP2Connection.
+func NewHTTP2Connection(
+	conn net.Conn,
+	config *Config,
+	connOptions *tunnelpogs.ConnectionOptions,
+	observer *Observer,
+	connIndex uint8,
+	controlStreamHandler ControlStreamHandler,
+	log *zerolog.Logger,
+) *HTTP2Connection {
+	return &HTTP2Connection{
+		conn: conn,
+		server: &http2.Server{
+			MaxConcurrentStreams: math.MaxUint32,
+		},
+		config:               config,
+		connOptions:          connOptions,
+		observer:             observer,
+		connIndex:            connIndex,
+		newRPCClientFunc:     newRegistrationRPCClient,
+		controlStreamHandler: controlStreamHandler,
+		log:                  log,
+	}
+}
+
+// Serve serves an HTTP2 server that the edge can talk to.
+func (c *HTTP2Connection) Serve(ctx context.Context) error {
+	go func() {
+		<-ctx.Done()
+		c.close()
+	}()
+	c.server.ServeConn(c.conn, &http2.ServeConnOpts{
+		Context: ctx,
+		Handler: c,
+	})
+
+	switch {
+	case c.controlStreamHandler.IsStopped():
+		return nil
+	case c.controlStreamErr != nil:
+		return c.controlStreamErr
+	default:
+		c.observer.log.Info().Uint8(LogFieldConnIndex, c.connIndex).Msg("Lost connection with the edge")
+		return errEdgeConnectionClosed
+	}
+}
+
+func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	c.activeRequestsWG.Add(1)
+	defer c.activeRequestsWG.Done()
+
+	connType := determineHTTP2Type(r)
+	handleMissingRequestParts(connType, r)
+
+	respWriter, err := NewHTTP2RespWriter(r, w, connType)
+	if err != nil {
+		c.observer.log.Error().Msg(err.Error())
+		return
+	}
+
+	switch connType {
+	case TypeControlStream:
+		if err := c.controlStreamHandler.ServeControlStream(r.Context(), respWriter, c.connOptions, true); err != nil {
+			c.controlStreamErr = err
+			c.log.Error().Err(err)
+			respWriter.WriteErrorResponse()
+		}
+
+	case TypeWebsocket, TypeHTTP:
+		stripWebsocketUpgradeHeader(r)
+		if err := c.config.OriginProxy.ProxyHTTP(respWriter, r, connType == TypeWebsocket); err != nil {
+			err := fmt.Errorf("Failed to proxy HTTP: %w", err)
+			c.log.Error().Err(err)
+			respWriter.WriteErrorResponse()
+		}
+
+	case TypeTCP:
+		host, err := getRequestHost(r)
+		if err != nil {
+			err := fmt.Errorf(`cloudflared recieved a warp-routing request with an empty host value: %w`, err)
+			c.log.Error().Err(err)
+			respWriter.WriteErrorResponse()
+		}
+
+		rws := NewHTTPResponseReadWriterAcker(respWriter, r)
+		if err := c.config.OriginProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
+			Dest:    host,
+			CFRay:   FindCfRayHeader(r),
+			LBProbe: IsLBProbeRequest(r),
+		}); err != nil {
+			respWriter.WriteErrorResponse()
+		}
+
+	default:
+		err := fmt.Errorf("Received unknown connection type: %s", connType)
+		c.log.Error().Err(err)
+		respWriter.WriteErrorResponse()
+	}
+}
+
+func (c *HTTP2Connection) close() {
+	// Wait for all serve HTTP handlers to return
+	c.activeRequestsWG.Wait()
+	c.conn.Close()
+}
+
+type http2RespWriter struct {
+	r           io.Reader
+	w           http.ResponseWriter
+	flusher     http.Flusher
+	shouldFlush bool
+}
+
+func NewHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type) (*http2RespWriter, error) {
+	flusher, isFlusher := w.(http.Flusher)
+	if !isFlusher {
+		respWriter := &http2RespWriter{
+			r: r.Body,
+			w: w,
+		}
+		respWriter.WriteErrorResponse()
+		return nil, fmt.Errorf("%T doesn't implement http.Flusher", w)
+	}
+
+	return &http2RespWriter{
+		r:           r.Body,
+		w:           w,
+		flusher:     flusher,
+		shouldFlush: connType.shouldFlush(),
+	}, nil
+}
+
+func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) error {
+	dest := rp.w.Header()
+	userHeaders := make(http.Header, len(header))
+	for name, values := range header {
+		// Since these are http2 headers, they're required to be lowercase
+		h2name := strings.ToLower(name)
+		if h2name == "content-length" {
+			// This header has meaning in HTTP/2 and will be used by the edge,
+			// so it should be sent as an HTTP/2 response header.
+			dest[name] = values
+			// Since these are http2 headers, they're required to be lowercase
+		} else if !IsControlResponseHeader(h2name) || IsWebsocketClientHeader(h2name) {
+			// User headers, on the other hand, must all be serialized so that
+			// HTTP/2 header validation won't be applied to HTTP/1 header values
+			userHeaders[name] = values
+		}
+	}
+
+	// Perform user header serialization and set them in the single header
+	dest.Set(CanonicalResponseUserHeaders, SerializeHeaders(userHeaders))
+	rp.setResponseMetaHeader(responseMetaHeaderOrigin)
+	// HTTP2 removes support for 101 Switching Protocols https://tools.ietf.org/html/rfc7540#section-8.1.1
+	if status == http.StatusSwitchingProtocols {
+		status = http.StatusOK
+	}
+	rp.w.WriteHeader(status)
+	if IsServerSentEvent(header) {
+		rp.shouldFlush = true
+	}
+	if rp.shouldFlush {
+		rp.flusher.Flush()
+	}
+	return nil
+}
+
+func (rp *http2RespWriter) WriteErrorResponse() {
+	rp.setResponseMetaHeader(responseMetaHeaderCfd)
+	rp.w.WriteHeader(http.StatusBadGateway)
+}
+
+func (rp *http2RespWriter) setResponseMetaHeader(value string) {
+	rp.w.Header().Set(CanonicalResponseMetaHeader, value)
+}
+
+func (rp *http2RespWriter) Read(p []byte) (n int, err error) {
+	return rp.r.Read(p)
+}
+
+func (rp *http2RespWriter) Write(p []byte) (n int, err error) {
+	defer func() {
+		// Implementer of OriginClient should make sure it doesn't write to the connection after Proxy returns
+		// Register a recover routine just in case.
+		if r := recover(); r != nil {
+			println(fmt.Sprintf("Recover from http2 response writer panic, error %s", debug.Stack()))
+		}
+	}()
+	n, err = rp.w.Write(p)
+	if err == nil && rp.shouldFlush {
+		rp.flusher.Flush()
+	}
+	return n, err
+}
+
+func (rp *http2RespWriter) Close() error {
+	return nil
+}
+
+func determineHTTP2Type(r *http.Request) Type {
+	switch {
+	case isWebsocketUpgrade(r):
+		return TypeWebsocket
+	case IsTCPStream(r):
+		return TypeTCP
+	case isControlStreamUpgrade(r):
+		return TypeControlStream
+	default:
+		return TypeHTTP
+	}
+}
+
+func handleMissingRequestParts(connType Type, r *http.Request) {
+	if connType == TypeHTTP {
+		// http library has no guarantees that we receive a filled URL. If not, then we fill it, as we reuse the request
+		// for proxying. We use the same values as we used to in h2mux. For proxying they should not matter since we
+		// control the dialer on every egress proxied.
+		if len(r.URL.Scheme) == 0 {
+			r.URL.Scheme = "http"
+		}
+		if len(r.URL.Host) == 0 {
+			r.URL.Host = "localhost:8080"
+		}
+	}
+}
+
+func isControlStreamUpgrade(r *http.Request) bool {
+	return r.Header.Get(InternalUpgradeHeader) == ControlStreamUpgrade
+}
+
+func isWebsocketUpgrade(r *http.Request) bool {
+	return r.Header.Get(InternalUpgradeHeader) == WebsocketUpgrade
+}
+
+// IsTCPStream discerns if the connection request needs a tcp stream proxy.
+func IsTCPStream(r *http.Request) bool {
+	return r.Header.Get(InternalTCPProxySrcHeader) != ""
+}
+
+func stripWebsocketUpgradeHeader(r *http.Request) {
+	r.Header.Del(InternalUpgradeHeader)
+}
+
+// getRequestHost returns the host of the http.Request.
+func getRequestHost(r *http.Request) (string, error) {
+	if r.Host != "" {
+		return r.Host, nil
+	}
+	if r.URL != nil {
+		return r.URL.Host, nil
+	}
+	return "", errors.New("host not set in incoming request")
+}

connection/http2_test.go

@@ -0,0 +1,451 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/gobwas/ws/wsutil"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/http2"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+var (
+	testTransport = http2.Transport{}
+)
+
+func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
+	edgeConn, originConn := net.Pipe()
+	var connIndex = uint8(0)
+	log := zerolog.Nop()
+	obs := NewObserver(&log, &log, false)
+	controlStream := NewControlStream(
+		obs,
+		mockConnectedFuse{},
+		&NamedTunnelConfig{},
+		connIndex,
+		nil,
+		nil,
+		1*time.Second,
+	)
+	return NewHTTP2Connection(
+		originConn,
+		testConfig,
+		&pogs.ConnectionOptions{},
+		obs,
+		connIndex,
+		controlStream,
+		&log,
+	), edgeConn
+}
+
+func TestServeHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	for _, test := range tests {
+		endpoint := fmt.Sprintf("http://localhost:8080/%s", test.endpoint)
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+		require.NoError(t, err)
+
+		resp, err := edgeHTTP2Conn.RoundTrip(req)
+		require.NoError(t, err)
+		require.Equal(t, test.expectedStatus, resp.StatusCode)
+		if test.expectedBody != nil {
+			respBody, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, respBody)
+		}
+		if test.isProxyError {
+			require.Equal(t, responseMetaHeaderCfd, resp.Header.Get(ResponseMetaHeader))
+		} else {
+			require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeader))
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+type mockNamedTunnelRPCClient struct {
+	shouldFail   error
+	registered   chan struct{}
+	unregistered chan struct{}
+}
+
+func (mc mockNamedTunnelRPCClient) RegisterConnection(
+	c context.Context,
+	config *NamedTunnelConfig,
+	options *tunnelpogs.ConnectionOptions,
+	connIndex uint8,
+	observer *Observer,
+) error {
+	if mc.shouldFail != nil {
+		return mc.shouldFail
+	}
+	close(mc.registered)
+	return nil
+}
+
+func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+	close(mc.unregistered)
+}
+
+func (mockNamedTunnelRPCClient) Close() {}
+
+type mockRPCClientFactory struct {
+	shouldFail   error
+	registered   chan struct{}
+	unregistered chan struct{}
+}
+
+func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient {
+	return mockNamedTunnelRPCClient{
+		shouldFail:   mf.shouldFail,
+		registered:   mf.registered,
+		unregistered: mf.unregistered,
+	}
+}
+
+type wsRespWriter struct {
+	*httptest.ResponseRecorder
+	readPipe  *io.PipeReader
+	writePipe *io.PipeWriter
+}
+
+func newWSRespWriter() *wsRespWriter {
+	readPipe, writePipe := io.Pipe()
+	return &wsRespWriter{
+		httptest.NewRecorder(),
+		readPipe,
+		writePipe,
+	}
+}
+
+func (w *wsRespWriter) RespBody() io.ReadWriter {
+	return nowriter{w.readPipe}
+}
+
+func (w *wsRespWriter) Write(data []byte) (n int, err error) {
+	return w.writePipe.Write(data)
+}
+
+func TestServeWS(t *testing.T) {
+	http2Conn, _ := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	respWriter := newWSRespWriter()
+	readPipe, writePipe := io.Pipe()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/ws", readPipe)
+	require.NoError(t, err)
+	req.Header.Set(InternalUpgradeHeader, WebsocketUpgrade)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.ServeHTTP(respWriter, req)
+	}()
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientText(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerText(respWriter.RespBody())
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	resp := respWriter.Result()
+	// http2RespWriter should rewrite status 101 to 200
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeader))
+
+	wg.Wait()
+}
+
+func TestServeControlStream(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+
+	obs := NewObserver(&log, &log, false)
+	controlStream := NewControlStream(
+		obs,
+		mockConnectedFuse{},
+		&NamedTunnelConfig{},
+		1,
+		rpcClientFactory.newMockRPCClient,
+		nil,
+		1*time.Second,
+	)
+	http2Conn.controlStreamHandler = controlStream
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(InternalUpgradeHeader, ControlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		edgeHTTP2Conn.RoundTrip(req)
+	}()
+
+	<-rpcClientFactory.registered
+	cancel()
+	<-rpcClientFactory.unregistered
+	assert.False(t, http2Conn.stoppedGracefully)
+
+	wg.Wait()
+}
+
+func TestFailRegistration(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		shouldFail:   errDuplicationConnection,
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+
+	obs := NewObserver(&log, &log, false)
+	controlStream := NewControlStream(
+		obs,
+		mockConnectedFuse{},
+		&NamedTunnelConfig{},
+		http2Conn.connIndex,
+		rpcClientFactory.newMockRPCClient,
+		nil,
+		1*time.Second,
+	)
+	http2Conn.controlStreamHandler = controlStream
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(InternalUpgradeHeader, ControlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+	resp, err := edgeHTTP2Conn.RoundTrip(req)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusBadGateway, resp.StatusCode)
+
+	assert.NotNil(t, http2Conn.controlStreamErr)
+	cancel()
+	wg.Wait()
+}
+
+func TestGracefulShutdownHTTP2(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+	events := &eventCollectorSink{}
+
+	shutdownC := make(chan struct{})
+	obs := NewObserver(&log, &log, false)
+	obs.RegisterSink(events)
+	controlStream := NewControlStream(
+		obs,
+		mockConnectedFuse{},
+		&NamedTunnelConfig{},
+		http2Conn.connIndex,
+		rpcClientFactory.newMockRPCClient,
+		shutdownC,
+		1*time.Second,
+	)
+
+	http2Conn.controlStreamHandler = controlStream
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(InternalUpgradeHeader, ControlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, _ = edgeHTTP2Conn.RoundTrip(req)
+	}()
+
+	select {
+	case <-rpcClientFactory.registered:
+		break //ok
+	case <-time.Tick(time.Second):
+		t.Fatal("timeout out waiting for registration")
+	}
+
+	// signal graceful shutdown
+	close(shutdownC)
+
+	select {
+	case <-rpcClientFactory.unregistered:
+		break //ok
+	case <-time.Tick(time.Second):
+		t.Fatal("timeout out waiting for unregistered signal")
+	}
+	assert.True(t, controlStream.IsStopped())
+
+	cancel()
+	wg.Wait()
+
+	events.assertSawEvent(t, Event{
+		Index:     http2Conn.connIndex,
+		EventType: Unregistering,
+	})
+}
+
+func benchmarkServeHTTP(b *testing.B, test testRequest) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	endpoint := fmt.Sprintf("http://localhost:8080/%s", test.endpoint)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+	require.NoError(b, err)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(b, err)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		resp, err := edgeHTTP2Conn.RoundTrip(req)
+		b.StopTimer()
+		require.NoError(b, err)
+		require.Equal(b, test.expectedStatus, resp.StatusCode)
+		if test.expectedBody != nil {
+			respBody, err := ioutil.ReadAll(resp.Body)
+			require.NoError(b, err)
+			require.Equal(b, test.expectedBody, respBody)
+		}
+		resp.Body.Close()
+	}
+
+	cancel()
+	wg.Wait()
+}
+
+func BenchmarkServeHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeHTTP(b, test)
+}
+
+func BenchmarkServeHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeHTTP(b, test)
+}

connection/json.go

@@ -0,0 +1,7 @@
+package connection
+
+import (
+	jsoniter "github.com/json-iterator/go"
+)
+
+var json = jsoniter.ConfigFastest

connection/metrics.go

@@ -0,0 +1,418 @@
+package connection
+
+import (
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+)
+
+const (
+	MetricsNamespace = "cloudflared"
+	TunnelSubsystem  = "tunnel"
+	muxerSubsystem   = "muxer"
+)
+
+type muxerMetrics struct {
+	rtt              *prometheus.GaugeVec
+	rttMin           *prometheus.GaugeVec
+	rttMax           *prometheus.GaugeVec
+	receiveWindowAve *prometheus.GaugeVec
+	sendWindowAve    *prometheus.GaugeVec
+	receiveWindowMin *prometheus.GaugeVec
+	receiveWindowMax *prometheus.GaugeVec
+	sendWindowMin    *prometheus.GaugeVec
+	sendWindowMax    *prometheus.GaugeVec
+	inBoundRateCurr  *prometheus.GaugeVec
+	inBoundRateMin   *prometheus.GaugeVec
+	inBoundRateMax   *prometheus.GaugeVec
+	outBoundRateCurr *prometheus.GaugeVec
+	outBoundRateMin  *prometheus.GaugeVec
+	outBoundRateMax  *prometheus.GaugeVec
+	compBytesBefore  *prometheus.GaugeVec
+	compBytesAfter   *prometheus.GaugeVec
+	compRateAve      *prometheus.GaugeVec
+}
+
+type tunnelMetrics struct {
+	timerRetries    prometheus.Gauge
+	serverLocations *prometheus.GaugeVec
+	// locationLock is a mutex for oldServerLocations
+	locationLock sync.Mutex
+	// oldServerLocations stores the last server the tunnel was connected to
+	oldServerLocations map[string]string
+
+	regSuccess *prometheus.CounterVec
+	regFail    *prometheus.CounterVec
+	rpcFail    *prometheus.CounterVec
+
+	muxerMetrics        *muxerMetrics
+	tunnelsHA           tunnelsForHA
+	userHostnamesCounts *prometheus.CounterVec
+}
+
+func newMuxerMetrics() *muxerMetrics {
+	rtt := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt",
+			Help:      "Round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rtt)
+
+	rttMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_min",
+			Help:      "Shortest round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rttMin)
+
+	rttMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_max",
+			Help:      "Longest round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rttMax)
+
+	receiveWindowAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_ave",
+			Help:      "Average receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowAve)
+
+	sendWindowAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_ave",
+			Help:      "Average send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowAve)
+
+	receiveWindowMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_min",
+			Help:      "Smallest receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowMin)
+
+	receiveWindowMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_max",
+			Help:      "Largest receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowMax)
+
+	sendWindowMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_min",
+			Help:      "Smallest send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowMin)
+
+	sendWindowMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_max",
+			Help:      "Largest send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowMax)
+
+	inBoundRateCurr := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_curr",
+			Help:      "Current inbounding bytes per second, 0 if there is no incoming connection",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateCurr)
+
+	inBoundRateMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero inbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateMin)
+
+	inBoundRateMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_max",
+			Help:      "Maximum inbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateMax)
+
+	outBoundRateCurr := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_curr",
+			Help:      "Current outbounding bytes per second, 0 if there is no outgoing traffic",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateCurr)
+
+	outBoundRateMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero outbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateMin)
+
+	outBoundRateMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_max",
+			Help:      "Maximum outbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateMax)
+
+	compBytesBefore := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_before",
+			Help:      "Bytes sent via cross-stream compression, pre compression",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compBytesBefore)
+
+	compBytesAfter := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_after",
+			Help:      "Bytes sent via cross-stream compression, post compression",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compBytesAfter)
+
+	compRateAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_rate_ave",
+			Help:      "Average outbound cross-stream compression ratio",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compRateAve)
+
+	return &muxerMetrics{
+		rtt:              rtt,
+		rttMin:           rttMin,
+		rttMax:           rttMax,
+		receiveWindowAve: receiveWindowAve,
+		sendWindowAve:    sendWindowAve,
+		receiveWindowMin: receiveWindowMin,
+		receiveWindowMax: receiveWindowMax,
+		sendWindowMin:    sendWindowMin,
+		sendWindowMax:    sendWindowMax,
+		inBoundRateCurr:  inBoundRateCurr,
+		inBoundRateMin:   inBoundRateMin,
+		inBoundRateMax:   inBoundRateMax,
+		outBoundRateCurr: outBoundRateCurr,
+		outBoundRateMin:  outBoundRateMin,
+		outBoundRateMax:  outBoundRateMax,
+		compBytesBefore:  compBytesBefore,
+		compBytesAfter:   compBytesAfter,
+		compRateAve:      compRateAve,
+	}
+}
+
+func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
+	m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
+	m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
+	m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
+	m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
+	m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
+	m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
+	m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
+	m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
+	m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
+	m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
+	m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
+	m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
+	m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
+	m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
+	m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
+	m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
+	m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
+	m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
+}
+
+func convertRTTMilliSec(t time.Duration) float64 {
+	return float64(t / time.Millisecond)
+}
+
+// Metrics that can be collected without asking the edge
+func initTunnelMetrics() *tunnelMetrics {
+	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "max_concurrent_requests_per_tunnel",
+			Help:      "Largest number of concurrent requests proxied through each tunnel so far",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
+
+	timerRetries := prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "timer_retries",
+			Help:      "Unacknowledged heart beats count",
+		})
+	prometheus.MustRegister(timerRetries)
+
+	serverLocations := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "server_locations",
+			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
+		},
+		[]string{"connection_id", "location"},
+	)
+	prometheus.MustRegister(serverLocations)
+
+	rpcFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_rpc_fail",
+			Help:      "Count of RPC connection errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(rpcFail)
+
+	registerFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_register_fail",
+			Help:      "Count of tunnel registration errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(registerFail)
+
+	userHostnamesCounts := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "user_hostnames_counts",
+			Help:      "Which user hostnames cloudflared is serving",
+		},
+		[]string{"userHostname"},
+	)
+	prometheus.MustRegister(userHostnamesCounts)
+
+	registerSuccess := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_register_success",
+			Help:      "Count of successful tunnel registrations",
+		},
+		[]string{"rpcName"},
+	)
+	prometheus.MustRegister(registerSuccess)
+
+	return &tunnelMetrics{
+		timerRetries:        timerRetries,
+		serverLocations:     serverLocations,
+		oldServerLocations:  make(map[string]string),
+		muxerMetrics:        newMuxerMetrics(),
+		tunnelsHA:           newTunnelsForHA(),
+		regSuccess:          registerSuccess,
+		regFail:             registerFail,
+		rpcFail:             rpcFail,
+		userHostnamesCounts: userHostnamesCounts,
+	}
+}
+
+func (t *tunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
+	t.muxerMetrics.update(connectionID, metrics)
+}
+
+func (t *tunnelMetrics) registerServerLocation(connectionID, loc string) {
+	t.locationLock.Lock()
+	defer t.locationLock.Unlock()
+	if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
+		return
+	} else if ok {
+		t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
+	}
+	t.serverLocations.WithLabelValues(connectionID, loc).Inc()
+	t.oldServerLocations[connectionID] = loc
+}
+
+var tunnelMetricsInternal struct {
+	sync.Once
+	metrics *tunnelMetrics
+}
+
+func newTunnelMetrics() *tunnelMetrics {
+	tunnelMetricsInternal.Do(func() {
+		tunnelMetricsInternal.metrics = initTunnelMetrics()
+	})
+	return tunnelMetricsInternal.metrics
+}