Release 2020.11.8

require URL.

.docker-images

@@ -0,0 +1,8 @@
+images:
+  - name: cloudflared
+    dockerfile: Dockerfile
+    context: .
+    registries:
+    - name: docker.io/cloudflare
+      user: env:DOCKER_USER
+      password: env:DOCKER_PASSWORD

.gitignore

@@ -5,11 +5,15 @@ guide/public
 /.GOPATH
 /bin
 .idea
+.build
 .vscode
 \#*\#
 cscope.*
 cloudflared
+cloudflared.pkg
 cloudflared.exe
+cloudflared.msi
+cloudflared-x86-64*
 !cmd/cloudflared/
 .DS_Store
 *-session.log

.mac_resources/scripts/postinstall

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# uninstall first in case this is an upgrade
+/usr/local/bin/cloudflared service uninstall
+
+# install the new service using launchctl
+/usr/local/bin/cloudflared service install

.mac_resources/uninstall.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+/usr/local/bin/cloudflared service uninstall
+rm /usr/local/bin/cloudflared
+pkgutil --forget com.cloudflare.cloudflared

.teamcity/build-macos.sh

@@ -0,0 +1,187 @@
+#!/bin/bash
+
+if [[ "$(uname)" != "Darwin" ]] ; then
+    echo "This should be run on macOS"
+    exit 1
+fi
+
+go version
+export GO111MODULE=on
+
+# build 'cloudflared-darwin-amd64.tgz'
+mkdir -p artifacts
+FILENAME="$(pwd)/artifacts/cloudflared-darwin-amd64.tgz"
+PKGNAME="$(pwd)/artifacts/cloudflared-amd64.pkg"
+TARGET_DIRECTORY=".build"
+BINARY_NAME="cloudflared"
+VERSION=$(git describe --tags --always --dirty="-dev")
+PRODUCT="cloudflared"
+CODE_SIGN_PRIV="code_sign.p12"
+CODE_SIGN_CERT="code_sign.cer"
+INSTALLER_PRIV="installer.p12"
+INSTALLER_CERT="installer.cer"
+BUNDLE_ID="com.cloudflare.cloudflared"
+SEC_DUP_MSG="security: SecKeychainItemImport: The specified item already exists in the keychain."
+export PATH="$PATH:/usr/local/bin"
+mkdir -p ../src/github.com/cloudflare/    
+cp -r . ../src/github.com/cloudflare/cloudflared
+cd ../src/github.com/cloudflare/cloudflared 
+GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared
+
+# Add code signing private key to the key chain
+if [[ ! -z "$CFD_CODE_SIGN_KEY" ]]; then
+  if [[ ! -z "$CFD_CODE_SIGN_PASS" ]]; then
+    # write private key to disk and then import it keychain
+    echo -n -e ${CFD_CODE_SIGN_KEY} | base64 -D > ${CODE_SIGN_PRIV}
+    out=$(security import ${CODE_SIGN_PRIV} -A -P "${CFD_CODE_SIGN_PASS}" 2>&1)
+    exitcode=$?
+    if [ -n "$out" ]; then
+      if [ $exitcode -eq 0 ]; then
+          echo "$out"
+      else
+          if [ "$out" != "${SEC_DUP_MSG}" ]; then
+              echo "$out" >&2
+              exit $exitcode
+          fi
+      fi
+    fi
+    rm ${CODE_SIGN_PRIV}
+  fi
+fi
+
+# Add code signing certificate to the key chain
+if [[ ! -z "$CFD_CODE_SIGN_CERT" ]]; then
+  # write certificate to disk and then import it keychain
+  echo -n -e ${CFD_CODE_SIGN_CERT} | base64 -D > ${CODE_SIGN_CERT}
+  out1=$(security import ${CODE_SIGN_CERT} -A 2>&1)
+  exitcode1=$?
+  if [ -n "$out1" ]; then
+    if [ $exitcode1 -eq 0 ]; then
+        echo "$out1"
+    else
+        if [ "$out1" != "${SEC_DUP_MSG}" ]; then
+            echo "$out1" >&2
+            exit $exitcode1
+        else 
+            echo "already imported code signing certificate"
+        fi
+    fi
+  fi
+  rm ${CODE_SIGN_CERT}
+fi
+
+# Add package signing private key to the key chain
+if [[ ! -z "$CFD_INSTALLER_KEY" ]]; then
+  if [[ ! -z "$CFD_INSTALLER_PASS" ]]; then
+    # write private key to disk and then import it into the keychain
+    echo -n -e ${CFD_INSTALLER_KEY} | base64 -D > ${INSTALLER_PRIV}
+    out2=$(security import ${INSTALLER_PRIV} -A -P "${CFD_INSTALLER_PASS}" 2>&1)
+    exitcode2=$?
+    if [ -n "$out2" ]; then
+      if [ $exitcode2 -eq 0 ]; then
+          echo "$out2"
+      else
+          if [ "$out2" != "${SEC_DUP_MSG}" ]; then
+              echo "$out2" >&2
+              exit $exitcode2
+          fi
+      fi
+    fi
+    rm ${INSTALLER_PRIV}
+  fi
+fi
+
+# Add package signing certificate to the key chain
+if [[ ! -z "$CFD_INSTALLER_CERT" ]]; then
+  # write certificate to disk and then import it keychain
+  echo -n -e ${CFD_INSTALLER_CERT} | base64 -D > ${INSTALLER_CERT}
+  out3=$(security import ${INSTALLER_CERT} -A 2>&1)
+  exitcode3=$?
+  if [ -n "$out3" ]; then
+    if [ $exitcode3 -eq 0 ]; then
+        echo "$out3"
+    else
+        if [ "$out3" != "${SEC_DUP_MSG}" ]; then
+            echo "$out3" >&2
+            exit $exitcode3
+        else 
+            echo "already imported installer certificate"
+        fi
+    fi
+  fi
+  rm ${INSTALLER_CERT}
+fi
+
+# get the code signing certificate name
+if [[ ! -z "$CFD_CODE_SIGN_NAME" ]]; then
+  CODE_SIGN_NAME="${CFD_CODE_SIGN_NAME}"
+else
+  if [[ -n "$(security find-certificate -c "Developer ID Application" | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)" ]]; then
+    CODE_SIGN_NAME=$(security find-certificate -c "Developer ID Application" | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)
+  else
+    CODE_SIGN_NAME=""
+  fi
+fi
+
+# get the package signing certificate name
+if [[ ! -z "$CFD_INSTALLER_NAME" ]]; then
+  PKG_SIGN_NAME="${CFD_INSTALLER_NAME}"
+else
+  if [[ -n "$(security find-certificate -c "Developer ID Installer" | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)" ]]; then
+    PKG_SIGN_NAME=$(security find-certificate -c "Developer ID Installer" | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)
+  else
+    PKG_SIGN_NAME=""
+  fi
+fi
+
+# sign the cloudflared binary
+if [[ ! -z "$CODE_SIGN_NAME" ]]; then
+  codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME}
+  
+  # notarize the binary
+  if [[ ! -z "$CFD_NOTE_PASSWORD" ]]; then
+    zip "${BINARY_NAME}.zip" ${BINARY_NAME} 
+    xcrun altool --notarize-app -f "${BINARY_NAME}.zip" -t osx -u ${CFD_NOTE_USERNAME} -p ${CFD_NOTE_PASSWORD} --primary-bundle-id ${BUNDLE_ID}
+  fi
+fi
+
+
+# creating build directory
+rm -rf $TARGET_DIRECTORY
+mkdir "${TARGET_DIRECTORY}"
+mkdir "${TARGET_DIRECTORY}/contents"
+cp -r ".mac_resources/scripts" "${TARGET_DIRECTORY}/scripts"
+
+# copy cloudflared into the build directory
+cp ${BINARY_NAME} "${TARGET_DIRECTORY}/contents/${PRODUCT}"
+
+# compress cloudflared into a tar and gzipped file
+tar czf "$FILENAME" "${BINARY_NAME}"
+
+# build the installer package
+if [[ ! -z "$PKG_SIGN_NAME" ]]; then
+  pkgbuild --identifier com.cloudflare.${PRODUCT} \
+      --version ${VERSION} \
+      --scripts ${TARGET_DIRECTORY}/scripts \
+      --root ${TARGET_DIRECTORY}/contents \
+      --install-location /usr/local/bin \
+      --sign "${PKG_SIGN_NAME}" \
+      ${PKGNAME}
+
+      # notarize the package
+      if [[ ! -z "$CFD_NOTE_PASSWORD" ]]; then
+        xcrun altool --notarize-app -f ${PKGNAME} -t osx -u ${CFD_NOTE_USERNAME} -p ${CFD_NOTE_PASSWORD} --primary-bundle-id ${BUNDLE_ID}
+        xcrun stapler staple ${PKGNAME}
+      fi
+else
+    pkgbuild --identifier com.cloudflare.${PRODUCT} \
+      --version ${VERSION} \
+      --scripts ${TARGET_DIRECTORY}/scripts \
+      --root ${TARGET_DIRECTORY}/contents \
+      --install-location /usr/local/bin \
+      ${PKGNAME}
+fi
+
+
+# cleaning up the build directory
+rm -rf $TARGET_DIRECTORY

.teamcity/update-homebrew.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -euo pipefail
+
+FILENAME="${PWD}/artifacts/cloudflared-darwin-amd64.tgz"
+
+if ! VERSION="$(git describe --tags --exact-match 2>/dev/null)" ; then
+    echo "Skipping public release for an untagged commit."
+    echo "##teamcity[buildStatus status='SUCCESS' text='Skipped due to lack of tag']"
+    exit 0
+fi
+
+if [[ ! -f "$FILENAME" ]] ; then
+    echo "Missing $FILENAME"
+    exit 1
+fi
+
+if [[ "${GITHUB_PRIVATE_KEY:-}" == "" ]] ; then
+    echo "Missing GITHUB_PRIVATE_KEY"
+    exit 1
+fi
+
+# upload to s3 bucket for use by Homebrew formula
+s3cmd \
+    --acl-public --signature-v2 --access_key="$AWS_ACCESS_KEY_ID" --secret_key="$AWS_SECRET_ACCESS_KEY" --host-bucket="%(bucket)s.s3.cfdata.org" \
+    put "$FILENAME" "s3://cftunnel-docs/dl/cloudflared-$VERSION-darwin-amd64.tgz"
+s3cmd \
+    --acl-public --signature-v2 --access_key="$AWS_ACCESS_KEY_ID" --secret_key="$AWS_SECRET_ACCESS_KEY" --host-bucket="%(bucket)s.s3.cfdata.org" \
+    cp "s3://cftunnel-docs/dl/cloudflared-$VERSION-darwin-amd64.tgz" "s3://cftunnel-docs/dl/cloudflared-stable-darwin-amd64.tgz"
+SHA256=$(sha256sum "$FILENAME" | cut -b1-64)
+
+# set up git (note that UserKnownHostsFile is an absolute path so we can cd wherever)
+mkdir -p tmp
+ssh-keyscan -t rsa github.com > tmp/github.txt
+echo "$GITHUB_PRIVATE_KEY" > tmp/private.key
+chmod 0400 tmp/private.key
+export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=$PWD/tmp/github.txt -i $PWD/tmp/private.key -o IdentitiesOnly=yes"
+
+# clone Homebrew repo into tmp/homebrew-cloudflare
+git clone git@github.com:cloudflare/homebrew-cloudflare.git tmp/homebrew-cloudflare    
+cd tmp/homebrew-cloudflare
+git checkout -f master
+git reset --hard origin/master
+
+# modify cloudflared.rb
+URL="https://packages.argotunnel.com/dl/cloudflared-$VERSION-darwin-amd64.tgz"
+tee cloudflared.rb <<EOF
+class Cloudflared < Formula
+  desc 'Argo Tunnel'
+  homepage 'https://developers.cloudflare.com/argo-tunnel/'
+  url '$URL'
+  sha256 '$SHA256'
+  version '$VERSION'
+  def install
+    bin.install 'cloudflared'
+  end
+end
+EOF
+
+# push cloudflared.rb
+git add cloudflared.rb
+git diff
+git config user.name "cloudflare-warp-bot"
+git config user.email "warp-bot@cloudflare.com"
+git commit -m "Release Argo Tunnel $VERSION"
+
+git push -v origin master

Dockerfile

@@ -1,8 +1,11 @@
 # use a builder image for building cloudflare
-FROM golang:1.13.3 as builder
-ENV GO111MODULE=on
-ENV CGO_ENABLED=0
-ENV GOOS=linux
+ARG TARGET_GOOS
+ARG TARGET_GOARCH
+FROM golang:1.15.3 as builder
+ENV GO111MODULE=on \
+    CGO_ENABLED=0 \
+    TARGET_GOOS=${TARGET_GOOS} \
+    TARGET_GOARCH=${TARGET_GOARCH}
 
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
@@ -12,8 +15,6 @@ COPY . .
 # compile cloudflared
 RUN make cloudflared
 
-# ---
-
 # use a distroless base image with glibc
 FROM gcr.io/distroless/base-debian10:nonroot
 

Makefile

@@ -1,23 +1,65 @@
-VERSION       := $(shell git describe --tags --always --dirty="-dev")
+VERSION       := $(shell git describe --tags --always --dirty="-dev" --match "[0-9][0-9][0-9][0-9].*.*")
 DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
 VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
+MSI_VERSION   := $(shell git tag -l --sort=v:refname | grep "w" | tail -1 | cut -c2-)
+#MSI_VERSION expects the format of the tag to be: (wX.X.X). Starts with the w character to not break cfsetup.
+#e.g. w3.0.1 or w4.2.10. It trims off the w character when creating the MSI.
 
 IMPORT_PATH   := github.com/cloudflare/cloudflared
 PACKAGE_DIR   := $(CURDIR)/packaging
-INSTALL_BINDIR := usr/local/bin
+INSTALL_BINDIR := /usr/bin/
+MAN_DIR := /usr/share/man/man1/
 
 EQUINOX_FLAGS = --version="$(VERSION)" \
-				 --platforms="$(EQUINOX_BUILD_PLATFORMS)" \
-				 --app="$(EQUINOX_APP_ID)" \
-				 --token="$(EQUINOX_TOKEN)" \
-				 --channel="$(EQUINOX_CHANNEL)"
+	--platforms="$(EQUINOX_BUILD_PLATFORMS)" \
+	--app="$(EQUINOX_APP_ID)" \
+	--token="$(EQUINOX_TOKEN)" \
+	--channel="$(EQUINOX_CHANNEL)"
 
 ifeq ($(EQUINOX_IS_DRAFT), true)
 	EQUINOX_FLAGS := --draft $(EQUINOX_FLAGS)
 endif
 
-ifeq ($(GOARCH),)
-	GOARCH := amd64
+LOCAL_ARCH ?= $(shell uname -m)
+ifneq ($(GOARCH),)
+    TARGET_ARCH ?= $(GOARCH)
+else ifeq ($(LOCAL_ARCH),x86_64)
+    TARGET_ARCH ?= amd64
+else ifeq ($(LOCAL_ARCH),i686)
+    TARGET_ARCH ?= amd64
+else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 5),armv8)
+    TARGET_ARCH ?= arm64
+else ifeq ($(LOCAL_ARCH),aarch64)
+    TARGET_ARCH ?= arm64
+else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 4),armv)
+    TARGET_ARCH ?= arm
+else
+    $(error This system's architecture $(LOCAL_ARCH) isn't supported)
+endif
+
+LOCAL_OS ?= $(shell go env GOOS)
+ifeq ($(LOCAL_OS),linux)
+    TARGET_OS ?= linux
+else ifeq ($(LOCAL_OS),darwin)
+    TARGET_OS ?= darwin
+else ifeq ($(LOCAL_OS),windows)
+    TARGET_OS ?= windows
+else ifeq ($(LOCAL_OS),freebsd)
+    TARGET_OS ?= freebsd
+else
+    $(error This system's OS $(LOCAL_OS) isn't supported)
+endif
+
+ifeq ($(TARGET_OS), windows)
+	EXECUTABLE_PATH=./cloudflared.exe
+else
+	EXECUTABLE_PATH=./cloudflared
+endif
+
+ifeq ($(FLAVOR), centos-7)
+	TARGET_PUBLIC_REPO ?= el7
+else
+	TARGET_PUBLIC_REPO ?= $(FLAVOR)
 endif
 
 .PHONY: all
@@ -29,11 +71,11 @@ clean:
 
 .PHONY: cloudflared
 cloudflared: tunnel-deps
-	go build -v -mod=vendor $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
+	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
 
 .PHONY: container
 container:
-	docker build -t cloudflare/cloudflared:"$(VERSION)" .
+	docker build --build-arg=TARGET_ARCH=$(TARGET_ARCH) --build-arg=TARGET_OS=$(TARGET_OS) -t cloudflare/cloudflared-$(TARGET_OS)-$(TARGET_ARCH):"$(VERSION)" .
 
 .PHONY: test
 test: vet
@@ -43,18 +85,101 @@ test: vet
 test-ssh-server:
 	docker-compose -f ssh_server_tests/docker-compose.yml up
 
-.PHONY: cloudflared-deb
-cloudflared-deb: cloudflared
+define publish_package
+	for HOST in $(CF_PKG_HOSTS); do \
+		ssh-keyscan -t rsa $$HOST >> ~/.ssh/known_hosts; \
+		scp -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
+		ssh cfsync@$$HOST 'chgrp cf /state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/*.$(1) && chmod g+w /state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/*.$(1)'; \
+	done
+endef
+
+.PHONY: publish-deb
+publish-deb: cloudflared-deb
+	$(call publish_package,deb,apt)
+
+.PHONY: publish-rpm
+publish-rpm: cloudflared-rpm
+	$(call publish_package,rpm,yum)
+
+define build_package
 	mkdir -p $(PACKAGE_DIR)
 	cp cloudflared $(PACKAGE_DIR)/cloudflared
-	fakeroot fpm -C $(PACKAGE_DIR) -s dir -t deb --deb-compression bzip2 \
-		-a $(GOARCH) -v $(VERSION) -n cloudflared cloudflared=/usr/local/bin/
+	cat cloudflared_man_template | sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' > $(PACKAGE_DIR)/cloudflared.1
+	fakeroot fpm -C $(PACKAGE_DIR) -s dir -t $(1) --$(1)-compression bzip2 \
+		--description 'Cloudflare Argo tunnel daemon' \
+		--vendor 'Cloudflare' \
+		--license 'Cloudflare Service Agreement' \
+		--url 'https://github.com/cloudflare/cloudflared' \
+		-m 'Cloudflare <support@cloudflare.com>' \
+		-a $(TARGET_ARCH) -v $(VERSION) -n cloudflared --after-install postinst.sh --after-remove postrm.sh \
+		cloudflared=$(INSTALL_BINDIR) cloudflared.1=$(MAN_DIR)
+endef
+
+.PHONY: cloudflared-deb
+cloudflared-deb: cloudflared
+	$(call build_package,deb)
+
+.PHONY: cloudflared-rpm
+cloudflared-rpm: cloudflared
+	$(call build_package,rpm)
 
 .PHONY: cloudflared-darwin-amd64.tgz
 cloudflared-darwin-amd64.tgz: cloudflared
 	tar czf cloudflared-darwin-amd64.tgz cloudflared
 	rm cloudflared
 
+.PHONY: cloudflared-junos
+cloudflared-junos: cloudflared jetez-certificate.pem jetez-key.pem
+	jetez --source . \
+		  -j jet.yaml \
+		  --key jetez-key.pem \
+		  --cert jetez-certificate.pem \
+		  --version $(VERSION)
+	rm jetez-*.pem
+
+jetez-certificate.pem:
+ifndef JETEZ_CERT
+	$(error JETEZ_CERT not defined)
+endif
+	@echo "Writing JetEZ certificate"
+	@echo "$$JETEZ_CERT" > jetez-certificate.pem
+
+jetez-key.pem:
+ifndef JETEZ_KEY
+	$(error JETEZ_KEY not defined)
+endif
+	@echo "Writing JetEZ key"
+	@echo "$$JETEZ_KEY" > jetez-key.pem
+
+.PHONY: publish-cloudflared-junos
+publish-cloudflared-junos: cloudflared-junos cloudflared-x86-64.latest.s3
+ifndef S3_ENDPOINT
+	$(error S3_HOST not defined)
+endif
+ifndef S3_URI
+	$(error S3_URI not defined)
+endif
+ifndef S3_ACCESS_KEY
+	$(error S3_ACCESS_KEY not defined)
+endif
+ifndef S3_SECRET_KEY
+	$(error S3_SECRET_KEY not defined)
+endif
+	sha256sum cloudflared-x86-64-$(VERSION).tgz | awk '{printf $$1}' > cloudflared-x86-64-$(VERSION).tgz.shasum
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+		put cloudflared-x86-64-$(VERSION).tgz $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+		put cloudflared-x86-64-$(VERSION).tgz.shasum $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz.shasum
+	dpkg --compare-versions "$(VERSION)" gt "$(shell cat cloudflared-x86-64.latest.s3)" && \
+		echo -n "$(VERSION)" > cloudflared-x86-64.latest && \
+		s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+			put cloudflared-x86-64.latest $(S3_URI)/cloudflared-x86-64.latest || \
+		echo "Latest version not updated"
+
+cloudflared-x86-64.latest.s3:
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force \
+		get $(S3_URI)/cloudflared-x86-64.latest cloudflared-x86-64.latest.s3
+
 .PHONY: homebrew-upload
 homebrew-upload: cloudflared-darwin-amd64.tgz
 	aws s3 --endpoint-url $(S3_ENDPOINT) cp --acl public-read $$^ $(S3_URI)/cloudflared-$$(VERSION)-$1.tgz
@@ -68,6 +193,19 @@ homebrew-release: homebrew-upload
 release: bin/equinox
 	bin/equinox release $(EQUINOX_FLAGS) -- $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
 
+.PHONY: github-release
+github-release: cloudflared
+	python3 github_release.py --path $(EXECUTABLE_PATH) --release-version $(VERSION)
+
+.PHONY: github-message
+github-message:
+	python3 github_message.py --release-version $(VERSION)
+
+.PHONY: github-mac-upload
+github-mac-upload:
+	python3 github_release.py --path artifacts/cloudflared-darwin-amd64.tgz --release-version $(VERSION) --name cloudflared-darwin-amd64.tgz
+	python3 github_release.py --path artifacts/cloudflared-amd64.pkg --release-version $(VERSION) --name cloudflared-amd64.pkg
+
 bin/equinox:
 	mkdir -p bin
 	curl -s https://bin.equinox.io/c/75JtLRTsJ3n/release-tool-beta-$(EQUINOX_PLATFORM).tgz | tar xz -C bin/
@@ -85,3 +223,7 @@ vet:
 	go vet -mod=vendor ./...
 	which go-sumtype  # go get github.com/BurntSushi/go-sumtype
 	go-sumtype $$(go list -mod=vendor ./...)
+
+.PHONY: msi
+msi: cloudflared
+	go-msi make --msi cloudflared.msi --version $(MSI_VERSION)

README.md

@@ -1,9 +1,32 @@
 # Argo Tunnel client
 
-Contains the command-line client and its libraries for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network.
+Contains the command-line client for Argo Tunnel, a tunneling daemon that proxies any local webserver through the Cloudflare network. Extensive documentation can be found in the [Argo Tunnel section](https://developers.cloudflare.com/argo-tunnel/) of the Cloudflare Docs.
 
-## Getting started
+## Before you get started
 
-    go install github.com/cloudflare/cloudflared/cmd/cloudflared
+Before you use Argo Tunnel, you'll need to complete a few steps in the Cloudflare dashboard. The website you add to Cloudflare will be used to route traffic to your Tunnel.
+
+1. [Add a website to Cloudflare](https://support.cloudflare.com/hc/en-us/articles/201720164-Creating-a-Cloudflare-account-and-adding-a-website)
+2. [Change your domain nameservers to Cloudflare](https://support.cloudflare.com/hc/en-us/articles/205195708)
+
+## Installing `cloudflared`
+
+Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases here on the `cloudflared` GitHub repository.
+
+* You can [install on macOS](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
+* Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#linux)
+* A Docker image of `cloudflared` is [available on DockerHub](https://hub.docker.com/r/cloudflare/cloudflared)
+* You can install on Windows machines with the [steps here](https://developers.cloudflare.com/argo-tunnel/getting-started/installation#windows)
 
 User documentation for Argo Tunnel can be found at https://developers.cloudflare.com/argo-tunnel/
+
+## Creating Tunnels and routing traffic
+
+Once installed, you can authenticate `cloudflared` into your Cloudflare account and begin creating Tunnels that serve traffic for hostnames in your account.
+
+* Create a Tunnel with [these instructions](https://developers.cloudflare.com/argo-tunnel/create-tunnel)
+* Route traffic to that Tunnel with [DNS records in Cloudflare](https://developers.cloudflare.com/argo-tunnel/routing-to-tunnel/dns) or with a [Cloudflare Load Balancer](https://developers.cloudflare.com/argo-tunnel/routing-to-tunnel/lb)
+
+## TryCloudflare
+
+Want to test Argo Tunnel before adding a website to Cloudflare? You can do so with TryCloudflare using the documentation [available here](https://developers.cloudflare.com/argo-tunnel/learning/trycloudflare).

RELEASE_NOTES

@@ -1,3 +1,290 @@
+2020.11.8
+- 2020-11-17 TUN-3555: Single origin service should default to localhost:8080
+
+2020.11.7
+- 2020-11-13 TUN-3514: Stop setting --is-autoupdated flag after autoupdate because it can break named tunnel running in k8s
+- 2020-11-15 TUN-3548, TUN-3547: Bastion mode can be specified as a service, doesn't require URL.
+- 2020-11-16 TUN-3549: Use a separate handler for each websocket proxy
+
+2020.11.6
+- 2020-11-14 TUN-3546: Fix panic in tlsconfig.LoadOriginCA
+
+2020.11.5
+- 2020-11-12 TUN-3540: Better copy in ingress rules error messages
+- 2020-11-12 DEVTOOLS-7936: Set permissions on public packages
+- 2020-11-13 TUN-3543: ProxyAddress not using default in single-origin mode
+
+2020.11.4
+- 2020-11-11 TUN-3534: Specific error message when credentials file is a .pem not .json
+- 2020-11-02 TUN-3500: Integrate replace h2mux by http2 work with multiple origin support
+- 2020-11-09 TUN-3514: Transport logger write to UI when UI is enabled
+- 2020-10-30 TUN-3490: Make sure OriginClient implementation doesn't write after Proxy return
+- 2020-10-20 TUN-3403: Unit test for origin/proxy to test serving HTTP and Websocket
+- 2020-10-23 TUN-3480: Support SSE with http2 connection, and add SSE handler to hello-world server
+- 2020-10-27 TUN-3489: Add unit tests to cover proxy logic in connection package of cloudflared
+- 2020-10-16 TUN-3467: Serialize cf-cloudflared-response-meta during package initialization using jsoniter
+- 2020-10-14 TUN-3456: New protocol option auto to automatically select between http2 and h2mux
+- 2020-10-14 TUN-3458: Upgrade to http2 when available, fallback to h2mux when we reach max retries
+- 2020-10-08 TUN-3449: Use flag to select transport protocol implementation
+- 2020-10-08 TUN-3462: Refactor cloudflared to separate origin from connection
+- 2020-09-21 TUN-3406: Proxy websocket requests over Go http2
+- 2020-09-25 TUN-3420: Establish control plane and send RPC over control plane
+- 2020-09-11 TUN-3400: Use Go HTTP2 library as transport to connect with the edge
+
+2020.11.3
+- 2020-11-11 TUN-3533: Set config for single origin ingress
+
+2020.11.2
+
+
+2020.11.1
+- 2020-11-10 TUN-3527: More specific error for invalid YAML/JSON
+- 2020-11-06 Update README.md (#256)
+
+2020.11.0
+- 2020-11-04 TUN-3484: OriginService that responds with configured HTTP status
+- 2020-11-05 TUN-3505: Response body for status code origin returns EOF on Read
+- 2020-11-04 TUN-3503: Matching ingress rule should not take port into account
+- 2020-11-05 TUN-3506: OriginService needs to set request host and scheme for websocket requests
+- 2020-11-09 TUN-3516: Better error message when parsing invalid YAML config
+- 2020-11-09 TUN-3522: ingress validate checks that the config file exists
+- 2020-11-09 TUN-3524: Don't ignore errors from app-level action handler (#248)
+- 2020-11-09 TUN-3461: Show all origin services in the UI
+- 2020-10-30 TUN-3494: Proceed to create tunnel if at least one edge address can be resolved
+- 2020-10-30 TUN-3492: Refactor OriginService, shrink its interface
+- 2020-10-22 TUN-3478: Increase download timeout to 60s
+- 2020-10-15 TUN-2640: Users can configure per-origin config. Unify single-rule CLI flow with multi-rule config file code.
+
+2020.10.2
+- 2020-10-21 Release 2020.10.1
+- 2020-10-21 AUTH-3185 fixed indention error
+- 2020-10-19 TUN-3459: Make service install on linux use named tunnels
+
+2020.10.1
+- 2020-10-20 Split out typed config from legacy command-line switches; refactor ingress commands and fix tests
+- 2020-10-20 Move raw ingress rules to config package
+- 2020-10-21 TUN-3476: Fix conversion to string and int slice
+- 2020-10-12 TUN-3441:  Multiple-origin routing via ingress rules
+- 2020-10-15 TUN-3464: Newtype to wrap []ingress.Rule
+- 2020-10-15 TUN-3465: Use Go 1.15.3
+- 2020-10-15 TUN-3463: Let users run a named tunnel via config file setting
+- 2020-10-19 TUN-3475: Unify config file handling with typed config for new fields
+- 2020-10-19 TUN-3459: Make service install on linux use named tunnels
+- 2020-10-06 AUTH-3148 fixed cloudflared copy and match all the files in the checksum upload
+- 2020-10-06 TUN-3436, TUN-3437: Parse ingress from YAML, ensure last rule catches everything
+- 2020-10-06 TUN-3446: Use go 1.15.2 and add a step to build cloudflared in the dev Dockerfile
+- 2020-10-07 TUN-3439: 'tunnel validate' command to check ingress rules
+- 2020-10-07 TUN-3440: 'tunnel rule' command to test ingress rules
+- 2020-10-08 TUN-3451: Cloudflared tunnel ingress command
+- 2020-10-09 TUN-3452: Fix loading of flags from config file for tunnel run subcommand. This change also cleans up building of tunnel subcommand list, hides deprecated subcommands and improves help.
+- 2020-10-08 TUN-3438: move ingress into own package, read into TunnelConfig
+
+2020.10.0
+- 2020-10-02 AUTH-2993 cleaned up worker service tests
+- 2020-10-02 TUN-3443: Decode as v4api response on non-200 status
+- 2020-09-24 TRAFFIC-448: allow the user to specify the proxy address and port to bind to, falling back to 127.0.0.1 and random port if not specified
+- 2020-09-28 TUN-3427: Define a struct that only implements RegistrationServer in tunnelpogs
+- 2020-09-29 TUN-3430: Copy flags to configure proxy to run subcommand, print relevant tunnel flags in help
+- 2020-08-12 AUTH-2993 added workers updater logic
+
+2020.9.3
+- 2020-09-22 TRAFFIC-448: build cloudflare for junos and publish to s3
+- 2020-09-22 TUN-3410: Request the v1 Tunnelstore API
+- 2020-09-23 Release 2020.9.2
+- 2020-09-17 updater service exit code should be 11
+- 2020-09-18 AUTH-3109 upload the checksum to workers kv on github releases
+
+2020.9.2
+- 2020-09-22 TRAFFIC-448: build cloudflare for junos and publish to s3
+- 2020-09-22 TUN-3410: Request the v1 Tunnelstore API
+- 2020-09-17 AUTH-3103 CI build fixes
+- 2020-09-18 AUTH-3110-use-cfsetup-precache
+- 2020-09-17 TUN-3295: Show route command results
+- 2020-09-16 TUN-3291: cloudflared tunnel run -h explains how to use flags from parent command
+- 2020-09-18 AUTH-3109 upload the checksum to workers kv on github releases
+- 2020-09-17 updater service exit code should be 11
+- 2020-09-01 TUN-3216: UI improvements
+- 2020-08-25 Rebased and passed TunnelEventChan to LogServerInfo in new ReconnectTunnel function
+- 2020-08-25 TUN-3321: Add box around logs on UI
+- 2020-08-26 TUN-3328: Filter out free tunnel has started log from UI
+- 2020-08-27 TUN-3333: Add text to UI explaining how to exit
+- 2020-08-27 TUN-3335: Dynamically set connection table size for UI
+- 2020-08-10 TUN-3238: Update UI when connection re-connects
+- 2020-08-17 TUN-3261: Display connections on UI for free classic tunnels
+- 2020-07-24 TUN-3201: Create base cloudflared UI structure
+- 2020-07-29 TUN-3200: Add connection information to UI
+- 2020-07-24 TUN-3255: Update UI to display URL instead of hostname
+- 2020-07-29 TUN-3198: Handle errors while running tunnel UI
+
+2020.9.1
+- 2020-09-14 TUN-3395: Unhide named tunnel subcommands, tweak help
+- 2020-09-15 TUN-3395: Improve help for list command
+- 2020-09-14 TUN-3294: Perform basic validation on arguments of route command; remove default pool name which wasn't valid
+- 2020-09-15 TUN-3395: Improve help for list command
+- 2020-09-16 Use Go 1.15.2
+
+2020.9.0
+- 2020-09-11 TUN-3293: Try to use error information from the body of a failed tunnelstore reresponse if available
+- 2020-09-04 AUTH-2653 renabled signing
+- 2020-09-04 TUN-3377: Tunnel route should check dns/lb before checking tunnel ID
+- 2020-09-04 AUTH-2653 changed to proper file extension
+- 2020-09-04 AUTH-2653 handle duplicate key import errors
+- 2020-09-04 TUN-3345: tunnel run accepts name of tunnel as argument
+- 2020-09-08 AUTH-2653 disble error pipe to see what is failing
+- 2020-09-08 AUTH-2653 search for the certificate and not the identity
+- 2020-09-09 TUN-3284: Use cloudflared/<version> as user agent of tunnelstore client
+- 2020-09-09 TUN-3375: Upgrade x/text and gorilla websocket deps
+- 2020-09-09 TUN-3375: Upgrade coredns and prometheus dependencies
+- 2020-09-09 AUTH-2653 add notarization to mac build
+- 2020-09-08 TUN-3292: Mention cleanup in tunnel run help.
+- 2020-08-20 AUTH-2016 fixed variable fail
+- 2020-08-12 TUN-3352 extra debug logging for websockets
+
+2020.8.2
+- 2020-08-20 AUTH-3021 fixed the git version call by using the older flag
+- 2020-08-18 TUN-3268: Each connection has its own event digest to reconnect
+
+2020.8.1
+- 2020-08-14 AUTH-2975 don't check /etc on windows
+- 2020-08-14 AUTH-2977 log file protection
+- 2020-08-18 TUN-3286: Use either ID or name in Named Tunnel subcommands.
+- 2020-08-18 AUTH-2712 fixed the mac build script
+- 2020-08-19 AUTH-2653 disabling signing until we can get the keys
+- 2020-08-05 TUN-3233: List tunnels support filtering by deleted, name, existed at and id
+- 2020-08-05 TUN-3237: By default, don't show connections that are pending reconnect
+- 2020-08-06 TUN-3242: Build with go 1.14
+- 2020-08-07 TUN-3243: Refactor tunnel subcommands to allow commands to compose better
+- 2020-08-07 AUTH-2864 - add macos build to github release
+- 2020-07-31 AUTH-2857 update homebrew script to use new url
+- 2020-07-30 TUN-3213: Create, route and run named tunnels in one command
+- 2020-07-07 AUTH-2712 added MSI build for a windows agent
+
+2020.8.0
+- 2020-07-30 TUN-3220: tunnel route reports created route
+- 2020-07-31 TUN-3221: ConnectionOptions tracks numPreviousAttempts.
+- 2020-07-20 TUN-3190: Initialize logger using command line flags in tunnels subcommands
+- 2020-07-21 TUN-3192: Use zone ID in tunnelstore request path; improve debug logging
+- 2020-07-23 TUN-3194: Don't render log output when level is not enabled
+- 2020-07-22 AUTH-2016 adds sha256 hashes to releases
+- 2020-07-27 Removes centos 6 build
+- 2020-07-27 TUN-3209: Add benchmark for header serialization
+- 2020-07-24 TUN-3209: improve performance and reduce allocations during user header serialization from h1 to h2
+- 2020-07-26 TUN-3208: Add benchmark for large response write
+- 2020-07-27 TUN-3208: Reduce copies and allocations on h2mux write path. Pre-allocate 16KB write buffer on the first write if possible. Use explicit byte array for chunks on write thread to avoid copying through intermediate buffer due to io.CopyN.
+- 2020-07-28 AUTH-2714: Adds arm64 cloudflared build
+- 2020-07-29 AUTH-2927 run message update after all github builds are done
+- 2020-07-17 AUTH-2902 redirect with just the root host on curl commands
+
+2020.7.4
+- 2020-07-20 Build cloudflared for arm64 on native agents
+- 2020-07-10 TUN-3048: Handle error when user tries to delete active tunnel
+- 2020-07-14 AUTH-2890: adds error handler to cli actions
+- 2020-07-06 TUN-3156: Add route subcommand under tunnel
+
+2020.7.3
+- 2020-07-13 Change scp command to use file glob that matches both cloudflared rpms and debs
+
+2020.7.2
+- 2020-07-02 AUTH-2644: Change install location and add man page
+- 2020-07-02 TUN-3131: Allow user to specify tunnel credentials path, and remove it in tunnel delete command
+- 2020-07-03 TUN-3008: Implement cloudflared tunnel cleanup command
+- 2020-07-02 TUN-3150: cloudflared tunnel list's table should use intelligent column width
+- 2020-07-07 TUN-3169: Move on to the next address when edge returns duplicate connection. There's no point in trying to connect to the same address since it will be hashed to the same metal. Improve logging of errors from serve tunnel loop, hide useless context cancelled error.
+- 2020-07-08 beautify package meta information generated by fpm (#218)
+- 2020-07-06 AUTH-2871: fix rpm builds
+- 2020-07-08 AUTH-2858: Set file to disable autoupdate
+- 2020-07-09 AUTH-2872: Adds centos-6 build
+
+2020.7.1
+- 2020-07-02 DEVTOOLS-7321: Push GitHub homebrew updates to master
+- 2020-07-06 TUN-3161: Upgrade golang.org/x/ deps
+- 2020-06-30 AUTH-2854: Create cloudflared RPMs
+- 2020-06-26 AUTH-2850 log config file path
+
+2020.7.0
+- 2020-06-30 TUN-3140: Add timestamps to terminal log entries
+- 2020-06-30 AUTH-2860: Fix builds
+- 2020-06-25 TUN-3007: Implement named tunnel connection registration and unregistration.
+
+2020.6.6
+- 2020-06-23 AUTH-2685: Adds script to create release
+- 2020-06-25 AUTH-2652: Update cloudflare repo
+- 2020-06-26 AUTH-2718: Add target for publishing deb to pkg.cloudflare repo
+- 2020-06-26 AUTH-2849 all log output to stderr
+- 2020-06-17 TUN-3106: Pass NamedTunnel config to StartServer
+- 2020-06-18 TUN-3107: UnregisterConnection shouldn't wrap nil error as RPC error
+- 2020-06-17 AUTH-2652: Adds .docker-images to push images to docker hub
+- 2020-06-18 AUTH-2712 mac package build script and better config file handling when started as a service
+
+2020.6.5
+- 2020-06-16 DEVTOOLS-7321: Don't skip macOS builds based on tag
+- 2020-06-16 fix for a flaky test
+- 2020-06-16 AUTH-2815 flag check was wrong. stupid oversight
+- 2020-06-16 TUN-3101: Tunnel list command should only show non-deleted, by default
+- 2020-06-16 TUN-3066: Command line action for tunnel run
+- 2020-06-16 TUN-3100 make updater report the right text
+
+2020.6.4
+- 2020-06-11 TUN-3085: Pass connection authentication information using TunnelAuth struct
+- 2020-06-15 TUN-3084: Generate and store tunnel_secret value during tunnel creation
+- 2020-06-16 AUTH-2815 add the log file to support the config.yaml file
+- 2020-06-02 TUN-3015: Add a new cap'n'proto RPC interface for connection registration as well as matching client and server implementations. The old interface extends the new one for backward compatibility.
+
+2020.6.3
+- 2020-06-15 DEVTOOLS-7321: Add openssh-client pkg for missing ssh-keyscan
+- 2020-06-15 AUTH-2813 adds back a single file support a cloudflared log file
+
+2020.6.2
+- 2020-06-11 AUTH-2648 updated usage text
+- 2020-06-11 AUTH-2763 don't redirect from curl command
+- 2020-06-12 TUN-3090: Upgrade crypto dep
+- 2020-06-11 TUN-3038: Add connections to tunnel list table
+- 2020-06-12 AUTH-2810 added warn for backwards compatibility sake
+
+2020.6.1
+- 2020-06-09 AUTH-2796 fixed windows build
+
+2020.6.0
+- 2020-06-05 AUTH-2645 protect against user mistaken flag input
+- 2020-06-05 AUTH-2687 don't copy config unnecessarily
+- 2020-06-05 AUTH-2169 make access login page more generic
+- 2020-06-05 AUTH-2729 added log file and level to cmd flags to match config file settings
+- 2020-06-08 AUTH-2785 service token flag fix and logger fix
+- 2020-05-20 AUTH-2682: Create buster build
+- 2020-05-21 TUN-2928, TUN-2929, TUN-2930: Add tunnel subcommands to interact with tunnel store service
+- 2020-05-29 Adding support for multi-architecture images and binaries (#184)
+- 2020-05-29 TUN-3019: Remove declarative tunnel entry code
+- 2020-05-29 TUN-3020: Remove declarative tunnel related RPC code
+- 2020-05-13 AUTH-2505 added aliases
+- 2020-05-14 AUTH-2529 added deprecation text to db-connect command
+- 2020-05-18 AUTH-2686: Added error handling to tunnel subcommand
+- 2020-05-04 AUTH-2369: RDP Bastion prototype
+- 2020-04-29 AUTH-2596 added new logger package and replaced logrus
+- 2020-04-25 DEVTOOLS-7321: Use SSH key from env for pushing to GitHub
+- 2020-04-25 DEVTOOLS-7321: Push to a test branch instead of to master
+- 2020-03-30 DEVTOOLS-7321: Add scripts for macOS builds and homebrew uploads
+
+2020.5.1
+- 2020-05-07 TUN-2860: Enable quick reconnect feature by default
+- 2020-05-07 AUTH-2564: error handling and minor fixes
+- 2020-05-01 AUTH-2588 add DoH to service mode
+
+2020.5.0
+- 2020-05-01 TUN-2943: Copy certutil from edge into cloudflared
+- 2020-05-05 TUN-2955: Fix connection and goroutine leaks when tunnel conection is terminated on error. Only unregister tunnels that had connected successfully. Close edge connection used to unregister the tunnel. Use buffered channels for error channels where receiver may quit early on context cancellation.
+- 2020-04-30 TUN-2940: Added delay parameter to stdin reconnect command.
+- 2020-04-27 TUN-2921: Rework address selection logic to avoid corner cases
+- 2020-04-28 TUN-2872: Exit with non-0 status code when the binary is updated so launchd will restart the service
+- 2020-04-13 AUTH-2587 add config watcher and reload logic for access client forwarder
+
+2020.4.0
+- 2020-04-10 TUN-2881: Parameterize response meta information header name in the generating function
+- 2020-04-11 TUN-2894: ResponseMetaHeader should be public
+- 2020-04-09 TUN-2880: Return metadata about source of the response from cloudflared
+- 2020-04-04 ARES-899: Fixes DoH client as system resolver. Fixes #91
+- 2020-03-31 AUTH-2394 added socks5 proxy
+- 2020-02-24 AUTH-2235 GetTokenIfExists now parses JWT payload for json expiry field to detect if the cached access token is expired
+
 2020.3.2
 - 2020-03-31 TUN-2854: Quick Reconnects should be an optional supported feature
 - 2020-03-30 TUN-2850: Tunnel stripping Cloudflare headers

awsuploader/directory_upload_manager.go

@@ -5,12 +5,12 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/sirupsen/logrus"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 // DirectoryUploadManager is used to manage file uploads on an interval from a directory
 type DirectoryUploadManager struct {
-	logger        *logrus.Logger
+	logger        logger.Service
 	uploader      Uploader
 	rootDirectory string
 	sweepInterval time.Duration
@@ -23,7 +23,7 @@ type DirectoryUploadManager struct {
 // uploader is an Uploader to use as an actual uploading engine
 // directory is the directory to sweep for files to upload
 // sweepInterval is how often to iterate the directory and upload the files within
-func NewDirectoryUploadManager(logger *logrus.Logger, uploader Uploader, directory string, sweepInterval time.Duration, shutdownC chan struct{}) *DirectoryUploadManager {
+func NewDirectoryUploadManager(logger logger.Service, uploader Uploader, directory string, sweepInterval time.Duration, shutdownC chan struct{}) *DirectoryUploadManager {
 	workerCount := 10
 	manager := &DirectoryUploadManager{
 		logger:        logger,
@@ -97,7 +97,7 @@ func (m *DirectoryUploadManager) worker() {
 			return
 		case filepath := <-m.workQueue:
 			if err := m.Upload(filepath); err != nil {
-				m.logger.WithError(err).Error("Cannot upload file to s3 bucket")
+				m.logger.Errorf("Cannot upload file to s3 bucket: %s", err)
 			} else {
 				os.Remove(filepath)
 			}

awsuploader/directory_upload_manager_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/sirupsen/logrus"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 type MockUploader struct {
@@ -49,7 +49,7 @@ func setupTestDirectory(t *testing.T) string {
 func createUploadManager(t *testing.T, shouldFailUpload bool) *DirectoryUploadManager {
 	rootDirectory := setupTestDirectory(t)
 	uploader := NewMockUploader(shouldFailUpload)
-	logger := logrus.New()
+	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
 	shutdownC := make(chan struct{})
 	return NewDirectoryUploadManager(logger, uploader, rootDirectory, 1*time.Second, shutdownC)
 }

carrier/carrier.go

@@ -11,9 +11,9 @@ import (
 	"strings"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
-	cloudflaredWebsocket "github.com/cloudflare/cloudflared/websocket"
-	"github.com/gorilla/websocket"
-	"github.com/sirupsen/logrus"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/pkg/errors"
 )
 
 type StartOptions struct {
@@ -21,6 +21,15 @@ type StartOptions struct {
 	Headers   http.Header
 }
 
+// Connection wraps up all the needed functions to forward over the tunnel
+type Connection interface {
+	// ServeStream is used to forward data from the client to the edge
+	ServeStream(*StartOptions, io.ReadWriter) error
+
+	// StartServer is used to listen for incoming connections from the edge to the origin
+	StartServer(net.Listener, string, <-chan struct{}) error
+}
+
 // StdinoutStream is empty struct for wrapping stdin/stdout
 // into a single ReadWriter
 type StdinoutStream struct {
@@ -44,86 +53,57 @@ func closeRespBody(resp *http.Response) {
 	}
 }
 
-// StartClient will copy the data from stdin/stdout over a WebSocket connection
-// to the edge (originURL)
-func StartClient(logger *logrus.Logger, stream io.ReadWriter, options *StartOptions) error {
-	return serveStream(logger, stream, options)
-}
-
-// StartServer will setup a listener on a specified address/port and then
+// StartForwarder will setup a listener on a specified address/port and then
 // forward connections to the origin by calling `Serve()`.
-func StartServer(logger *logrus.Logger, address string, shutdownC <-chan struct{}, options *StartOptions) error {
+func StartForwarder(conn Connection, address string, shutdownC <-chan struct{}, options *StartOptions) error {
 	listener, err := net.Listen("tcp", address)
 	if err != nil {
-		logger.WithError(err).Error("failed to start forwarding server")
-		return err
+		return errors.Wrap(err, "failed to start forwarding server")
 	}
-	logger.Info("Started listening on ", address)
-	return Serve(logger, listener, shutdownC, options)
+	return Serve(conn, listener, shutdownC, options)
+}
+
+// StartClient will copy the data from stdin/stdout over a WebSocket connection
+// to the edge (originURL)
+func StartClient(conn Connection, stream io.ReadWriter, options *StartOptions) error {
+	return conn.ServeStream(options, stream)
 }
 
 // Serve accepts incoming connections on the specified net.Listener.
 // Each connection is handled in a new goroutine: its data is copied over a
 // WebSocket connection to the edge (originURL).
 // `Serve` always closes `listener`.
-func Serve(logger *logrus.Logger, listener net.Listener, shutdownC <-chan struct{}, options *StartOptions) error {
+func Serve(remoteConn Connection, listener net.Listener, shutdownC <-chan struct{}, options *StartOptions) error {
 	defer listener.Close()
-	for {
-		select {
-		case <-shutdownC:
-			return nil
-		default:
+	errChan := make(chan error)
+
+	go func() {
+		for {
 			conn, err := listener.Accept()
 			if err != nil {
-				return err
+				// don't block if parent goroutine quit early
+				select {
+				case errChan <- err:
+				default:
+				}
+				return
 			}
-			go serveConnection(logger, conn, options)
+			go serveConnection(remoteConn, conn, options)
 		}
+	}()
+
+	select {
+	case <-shutdownC:
+		return nil
+	case err := <-errChan:
+		return err
 	}
 }
 
 // serveConnection handles connections for the Serve() call
-func serveConnection(logger *logrus.Logger, c net.Conn, options *StartOptions) {
+func serveConnection(remoteConn Connection, c net.Conn, options *StartOptions) {
 	defer c.Close()
-	serveStream(logger, c, options)
-}
-
-// serveStream will serve the data over the WebSocket stream
-func serveStream(logger *logrus.Logger, conn io.ReadWriter, options *StartOptions) error {
-	wsConn, err := createWebsocketStream(options)
-	if err != nil {
-		logger.WithError(err).Errorf("failed to connect to %s\n", options.OriginURL)
-		return err
-	}
-	defer wsConn.Close()
-
-	cloudflaredWebsocket.Stream(wsConn, conn)
-
-	return nil
-}
-
-// createWebsocketStream will create a WebSocket connection to stream data over
-// It also handles redirects from Access and will present that flow if
-// the token is not present on the request
-func createWebsocketStream(options *StartOptions) (*cloudflaredWebsocket.Conn, error) {
-	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header = options.Headers
-
-	wsConn, resp, err := cloudflaredWebsocket.ClientConnect(req, nil)
-	defer closeRespBody(resp)
-	if err != nil && IsAccessResponse(resp) {
-		wsConn, err = createAccessAuthenticatedStream(options)
-		if err != nil {
-			return nil, err
-		}
-	} else if err != nil {
-		return nil, err
-	}
-
-	return &cloudflaredWebsocket.Conn{Conn: wsConn}, nil
+	remoteConn.ServeStream(options, c)
 }
 
 // IsAccessResponse checks the http Response to see if the url location
@@ -144,56 +124,14 @@ func IsAccessResponse(resp *http.Response) bool {
 	return false
 }
 
-// createAccessAuthenticatedStream will try load a token from storage and make
-// a connection with the token set on the request. If it still get redirect,
-// this probably means the token in storage is invalid (expired/revoked). If that
-// happens it deletes the token and runs the connection again, so the user can
-// login again and generate a new one.
-func createAccessAuthenticatedStream(options *StartOptions) (*websocket.Conn, error) {
-	wsConn, resp, err := createAccessWebSocketStream(options)
-	defer closeRespBody(resp)
-	if err == nil {
-		return wsConn, nil
-	}
-
-	if !IsAccessResponse(resp) {
-		return nil, err
-	}
-
-	// Access Token is invalid for some reason. Go through regen flow
-	originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	if err := token.RemoveTokenIfExists(originReq.URL); err != nil {
-		return nil, err
-	}
-	wsConn, resp, err = createAccessWebSocketStream(options)
-	defer closeRespBody(resp)
-	if err != nil {
-		return nil, err
-	}
-
-	return wsConn, nil
-}
-
-// createAccessWebSocketStream builds an Access request and makes a connection
-func createAccessWebSocketStream(options *StartOptions) (*websocket.Conn, *http.Response, error) {
-	req, err := BuildAccessRequest(options)
-	if err != nil {
-		return nil, nil, err
-	}
-	return cloudflaredWebsocket.ClientConnect(req, nil)
-}
-
-// buildAccessRequest builds an HTTP request with the Access token set
-func BuildAccessRequest(options *StartOptions) (*http.Request, error) {
+// BuildAccessRequest builds an HTTP request with the Access token set
+func BuildAccessRequest(options *StartOptions, logger logger.Service) (*http.Request, error) {
 	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
 	if err != nil {
 		return nil, err
 	}
 
-	token, err := token.FetchToken(req.URL)
+	token, err := token.FetchTokenWithRedirect(req.URL, logger)
 	if err != nil {
 		return nil, err
 	}
@@ -204,7 +142,7 @@ func BuildAccessRequest(options *StartOptions) (*http.Request, error) {
 	if err != nil {
 		return nil, err
 	}
-	originRequest.Header.Set("cf-access-token", token)
+	originRequest.Header.Set(h2mux.CFAccessTokenHeader, token)
 
 	for k, v := range options.Headers {
 		if len(v) >= 1 {

carrier/carrier_test.go

@@ -9,8 +9,8 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/cloudflare/cloudflared/logger"
 	ws "github.com/gorilla/websocket"
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -43,7 +43,8 @@ func (s *testStreamer) Write(p []byte) (int, error) {
 
 func TestStartClient(t *testing.T) {
 	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
-	logger := logrus.New()
+	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
+	wsConn := NewWSConnection(logger, false)
 	ts := newTestWebSocketServer()
 	defer ts.Close()
 
@@ -52,7 +53,7 @@ func TestStartClient(t *testing.T) {
 		OriginURL: "http://" + ts.Listener.Addr().String(),
 		Headers:   nil,
 	}
-	err := StartClient(logger, buf, options)
+	err := StartClient(wsConn, buf, options)
 	assert.NoError(t, err)
 	buf.Write([]byte(message))
 
@@ -67,8 +68,9 @@ func TestStartServer(t *testing.T) {
 		t.Fatalf("Error starting listener: %v", err)
 	}
 	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
-	logger := logrus.New()
+	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
 	shutdownC := make(chan struct{})
+	wsConn := NewWSConnection(logger, false)
 	ts := newTestWebSocketServer()
 	defer ts.Close()
 	options := &StartOptions{
@@ -77,7 +79,7 @@ func TestStartServer(t *testing.T) {
 	}
 
 	go func() {
-		err := Serve(logger, listener, shutdownC, options)
+		err := Serve(wsConn, listener, shutdownC, options)
 		if err != nil {
 			t.Fatalf("Error running server: %v", err)
 		}

carrier/websocket.go

@@ -0,0 +1,157 @@
+package carrier
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httputil"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/socks"
+	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
+	"github.com/gorilla/websocket"
+)
+
+// Websocket is used to carry data via WS binary frames over the tunnel from client to the origin
+// This implements the functions for glider proxy (sock5) and the carrier interface
+type Websocket struct {
+	logger  logger.Service
+	isSocks bool
+}
+
+type wsdialer struct {
+	conn *cfwebsocket.Conn
+}
+
+func (d *wsdialer) Dial(address string) (io.ReadWriteCloser, *socks.AddrSpec, error) {
+	local, ok := d.conn.LocalAddr().(*net.TCPAddr)
+	if !ok {
+		return nil, nil, fmt.Errorf("not a tcp connection")
+	}
+
+	addr := socks.AddrSpec{IP: local.IP, Port: local.Port}
+	return d.conn, &addr, nil
+}
+
+// NewWSConnection returns a new connection object
+func NewWSConnection(logger logger.Service, isSocks bool) Connection {
+	return &Websocket{
+		logger:  logger,
+		isSocks: isSocks,
+	}
+}
+
+// ServeStream will create a Websocket client stream connection to the edge
+// it blocks and writes the raw data from conn over the tunnel
+func (ws *Websocket) ServeStream(options *StartOptions, conn io.ReadWriter) error {
+	wsConn, err := createWebsocketStream(options, ws.logger)
+	if err != nil {
+		ws.logger.Errorf("failed to connect to %s with error: %s", options.OriginURL, err)
+		return err
+	}
+	defer wsConn.Close()
+
+	if ws.isSocks {
+		dialer := &wsdialer{conn: wsConn}
+		requestHandler := socks.NewRequestHandler(dialer)
+		socksServer := socks.NewConnectionHandler(requestHandler)
+
+		socksServer.Serve(conn)
+	} else {
+		cfwebsocket.Stream(wsConn, conn)
+	}
+	return nil
+}
+
+// StartServer creates a Websocket server to listen for connections.
+// This is used on the origin (tunnel) side to take data from the muxer and send it to the origin
+func (ws *Websocket) StartServer(listener net.Listener, remote string, shutdownC <-chan struct{}) error {
+	return cfwebsocket.StartProxyServer(ws.logger, listener, remote, shutdownC, cfwebsocket.DefaultStreamHandler)
+}
+
+// createWebsocketStream will create a WebSocket connection to stream data over
+// It also handles redirects from Access and will present that flow if
+// the token is not present on the request
+func createWebsocketStream(options *StartOptions, logger logger.Service) (*cfwebsocket.Conn, error) {
+	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header = options.Headers
+
+	dump, err := httputil.DumpRequest(req, false)
+	logger.Debugf("Websocket request: %s", string(dump))
+
+	wsConn, resp, err := cfwebsocket.ClientConnect(req, nil)
+	defer closeRespBody(resp)
+
+	if err != nil && IsAccessResponse(resp) {
+		wsConn, err = createAccessAuthenticatedStream(options, logger)
+		if err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+
+	return &cfwebsocket.Conn{Conn: wsConn}, nil
+}
+
+// createAccessAuthenticatedStream will try load a token from storage and make
+// a connection with the token set on the request. If it still get redirect,
+// this probably means the token in storage is invalid (expired/revoked). If that
+// happens it deletes the token and runs the connection again, so the user can
+// login again and generate a new one.
+func createAccessAuthenticatedStream(options *StartOptions, logger logger.Service) (*websocket.Conn, error) {
+	wsConn, resp, err := createAccessWebSocketStream(options, logger)
+	defer closeRespBody(resp)
+	if err == nil {
+		return wsConn, nil
+	}
+
+	if !IsAccessResponse(resp) {
+		return nil, err
+	}
+
+	// Access Token is invalid for some reason. Go through regen flow
+	originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	if err := token.RemoveTokenIfExists(originReq.URL); err != nil {
+		return nil, err
+	}
+	wsConn, resp, err = createAccessWebSocketStream(options, logger)
+	defer closeRespBody(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	return wsConn, nil
+}
+
+// createAccessWebSocketStream builds an Access request and makes a connection
+func createAccessWebSocketStream(options *StartOptions, logger logger.Service) (*websocket.Conn, *http.Response, error) {
+	req, err := BuildAccessRequest(options, logger)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	dump, err := httputil.DumpRequest(req, false)
+	logger.Debugf("Access Websocket request: %s", string(dump))
+
+	conn, resp, err := cfwebsocket.ClientConnect(req, nil)
+
+	if resp != nil {
+		r, err := httputil.DumpResponse(resp, true)
+		if r != nil {
+			logger.Debugf("Websocket response: %q", r)
+		} else if err != nil {
+			logger.Debugf("Websocket response error: %v", err)
+		}
+	}
+
+	return conn, resp, err
+}

certutil/certutil.go

@@ -0,0 +1,94 @@
+package certutil
+
+import (
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"strings"
+)
+
+type namedTunnelToken struct {
+	ZoneID     string `json:"zoneID"`
+	AccountID  string `json:"accountID"`
+	ServiceKey string `json:"serviceKey"`
+}
+
+type OriginCert struct {
+	PrivateKey interface{}
+	Cert       *x509.Certificate
+	ZoneID     string
+	ServiceKey string
+	AccountID  string
+}
+
+func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
+	if len(blocks) == 0 {
+		return nil, fmt.Errorf("Cannot decode empty certificate")
+	}
+	originCert := OriginCert{}
+	block, rest := pem.Decode(blocks)
+	for {
+		if block == nil {
+			break
+		}
+		switch block.Type {
+		case "PRIVATE KEY":
+			if originCert.PrivateKey != nil {
+				return nil, fmt.Errorf("Found multiple private key in the certificate")
+			}
+			// RSA private key
+			privateKey, err := x509.ParsePKCS8PrivateKey(block.Bytes)
+			if err != nil {
+				return nil, fmt.Errorf("Cannot parse private key")
+			}
+			originCert.PrivateKey = privateKey
+		case "CERTIFICATE":
+			if originCert.Cert != nil {
+				return nil, fmt.Errorf("Found multiple certificates in the certificate")
+			}
+			cert, err := x509.ParseCertificates(block.Bytes)
+			if err != nil {
+				return nil, fmt.Errorf("Cannot parse certificate")
+			} else if len(cert) > 1 {
+				return nil, fmt.Errorf("Found multiple certificates in the certificate")
+			}
+			originCert.Cert = cert[0]
+		case "WARP TOKEN", "ARGO TUNNEL TOKEN":
+			if originCert.ZoneID != "" || originCert.ServiceKey != "" {
+				return nil, fmt.Errorf("Found multiple tokens in the certificate")
+			}
+			// The token is a string,
+			// Try the newer JSON format
+			ntt := namedTunnelToken{}
+			if err := json.Unmarshal(block.Bytes, &ntt); err == nil {
+				originCert.ZoneID = ntt.ZoneID
+				originCert.ServiceKey = ntt.ServiceKey
+				originCert.AccountID = ntt.AccountID
+			} else {
+				// Try the older format, where the zoneID and service key are seperated by
+				// a new line character
+				token := string(block.Bytes)
+				s := strings.Split(token, "\n")
+				if len(s) != 2 {
+					return nil, fmt.Errorf("Cannot parse token")
+				}
+				originCert.ZoneID = s[0]
+				originCert.ServiceKey = s[1]
+			}
+		default:
+			return nil, fmt.Errorf("Unknown block %s in the certificate", block.Type)
+		}
+		block, rest = pem.Decode(rest)
+	}
+
+	if originCert.PrivateKey == nil {
+		return nil, fmt.Errorf("Missing private key in the certificate")
+	} else if originCert.Cert == nil {
+		return nil, fmt.Errorf("Missing certificate in the certificate")
+	} else if originCert.ZoneID == "" || originCert.ServiceKey == "" {
+		return nil, fmt.Errorf("Missing token in the certificate")
+	}
+
+	return &originCert, nil
+}

certutil/certutil_test.go

@@ -0,0 +1,67 @@
+package certutil
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadOriginCert(t *testing.T) {
+	cert, err := DecodeOriginCert([]byte{})
+	assert.Equal(t, fmt.Errorf("Cannot decode empty certificate"), err)
+	assert.Nil(t, cert)
+
+	blocks, err := ioutil.ReadFile("test-cert-no-key.pem")
+	assert.Nil(t, err)
+	cert, err = DecodeOriginCert(blocks)
+	assert.Equal(t, fmt.Errorf("Missing private key in the certificate"), err)
+	assert.Nil(t, cert)
+
+	blocks, err = ioutil.ReadFile("test-cert-two-certificates.pem")
+	assert.Nil(t, err)
+	cert, err = DecodeOriginCert(blocks)
+	assert.Equal(t, fmt.Errorf("Found multiple certificates in the certificate"), err)
+	assert.Nil(t, cert)
+
+	blocks, err = ioutil.ReadFile("test-cert-unknown-block.pem")
+	assert.Nil(t, err)
+	cert, err = DecodeOriginCert(blocks)
+	assert.Equal(t, fmt.Errorf("Unknown block RSA PRIVATE KEY in the certificate"), err)
+	assert.Nil(t, cert)
+
+	blocks, err = ioutil.ReadFile("test-cert.pem")
+	assert.Nil(t, err)
+	cert, err = DecodeOriginCert(blocks)
+	assert.Nil(t, err)
+	assert.NotNil(t, cert)
+	assert.Equal(t, "7b0a4d77dfb881c1a3b7d61ea9443e19", cert.ZoneID)
+	key := "v1.0-58bd4f9e28f7b3c28e05a35ff3e80ab4fd9644ef3fece537eb0d12e2e9258217-183442fbb0bbdb3e571558fec9b5589ebd77aafc87498ee3f09f64a4ad79ffe8791edbae08b36c1d8f1d70a8670de56922dff92b15d214a524f4ebfa1958859e-7ce80f79921312a6022c5d25e2d380f82ceaefe3fbdc43dd13b080e3ef1e26f7"
+	assert.Equal(t, key, cert.ServiceKey)
+}
+
+func TestNewlineArgoTunnelToken(t *testing.T) {
+	ArgoTunnelTokenTest(t, "test-argo-tunnel-cert.pem")
+}
+
+func TestJSONArgoTunnelToken(t *testing.T) {
+	// The given cert's Argo Tunnel Token was generated by base64 encoding this JSON:
+	// {
+	// "zoneID": "7b0a4d77dfb881c1a3b7d61ea9443e19",
+	// "serviceKey": "test-service-key",
+	// "accountID": "abcdabcdabcdabcd1234567890abcdef"
+	// }
+	ArgoTunnelTokenTest(t, "test-argo-tunnel-cert-json.pem")
+}
+
+func ArgoTunnelTokenTest(t *testing.T, path string) {
+	blocks, err := ioutil.ReadFile(path)
+	assert.Nil(t, err)
+	cert, err := DecodeOriginCert(blocks)
+	assert.Nil(t, err)
+	assert.NotNil(t, cert)
+	assert.Equal(t, "7b0a4d77dfb881c1a3b7d61ea9443e19", cert.ZoneID)
+	key := "test-service-key"
+	assert.Equal(t, key, cert.ServiceKey)
+}

certutil/test-argo-tunnel-cert-json.pem

@@ -0,0 +1,57 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
+sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
+1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
+z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
+6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
+x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
+1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
+IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
+xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
+sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
+2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
+sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
+Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
+AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
+m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
+QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
+P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
+pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
+G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
+6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
+LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
+OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
+W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
+M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
+y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
+uQicoQq3yzeQh20wtrtaXzTNmA==
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
+gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
+VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
+cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
+MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
+ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
+aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
+GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
+6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
+tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
+PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
+AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
+HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
+zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
+RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
+YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
+YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
+cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
+K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
+x+Yo/cL8fGfVpPt4UM8=
+-----END CERTIFICATE-----
+-----BEGIN ARGO TUNNEL TOKEN-----
+eyJ6b25lSUQiOiAiN2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkiLCAi
+c2VydmljZUtleSI6ICJ0ZXN0LXNlcnZpY2Uta2V5IiwgImFjY291bnRJRCI6ICJh
+YmNkYWJjZGFiY2RhYmNkMTIzNDU2Nzg5MGFiY2RlZiJ9
+-----END ARGO TUNNEL TOKEN-----

certutil/test-argo-tunnel-cert.pem

@@ -0,0 +1,56 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
+sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
+1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
+z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
+6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
+x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
+1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
+IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
+xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
+sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
+2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
+sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
+Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
+AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
+m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
+QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
+P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
+pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
+G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
+6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
+LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
+OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
+W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
+M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
+y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
+uQicoQq3yzeQh20wtrtaXzTNmA==
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
+gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
+VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
+cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
+MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
+ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
+aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
+GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
+6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
+tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
+PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
+AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
+HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
+zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
+RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
+YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
+YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
+cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
+K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
+x+Yo/cL8fGfVpPt4UM8=
+-----END CERTIFICATE-----
+-----BEGIN ARGO TUNNEL TOKEN-----
+N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdGVzdC1zZXJ2aWNlLWtl
+eQ==
+-----END ARGO TUNNEL TOKEN-----

certutil/test-cert-no-key.pem

@@ -0,0 +1,33 @@
+-----BEGIN CERTIFICATE-----
+MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
+gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
+VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
+cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
+MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
+ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
+aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
+GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
+6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
+tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
+PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
+AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
+HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
+zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
+RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
+YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
+YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
+cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
+K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
+x+Yo/cL8fGfVpPt4UM8=
+-----END CERTIFICATE-----
+-----BEGIN WARP TOKEN-----
+N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
+ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
+MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
+NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
+NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
+OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
+ZWYxZTI2Zjc=
+-----END WARP TOKEN-----

certutil/test-cert-two-certificates.pem

@@ -0,0 +1,85 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
+sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
+1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
+z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
+6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
+x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
+1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
+IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
+xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
+sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
+2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
+sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
+Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
+AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
+m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
+QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
+P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
+pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
+G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
+6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
+LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
+OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
+W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
+M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
+y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
+uQicoQq3yzeQh20wtrtaXzTNmA==
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
+gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
+VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
+cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
+MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
+ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
+aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
+GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
+6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
+tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
+PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
+AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
+HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
+zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
+RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
+YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
+YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
+cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
+K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
+x+Yo/cL8fGfVpPt4UM8=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
+gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
+VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
+cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
+MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
+ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
+aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
+GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
+6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
+tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
+PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
+AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
+HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
+zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
+RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
+YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
+YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
+cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
+K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
+x+Yo/cL8fGfVpPt4UM8=
+-----END CERTIFICATE-----
+-----BEGIN WARP TOKEN-----
+N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
+ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
+MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
+NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
+NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
+OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
+ZWYxZTI2Zjc=
+-----END WARP TOKEN-----

certutil/test-cert-unknown-block.pem

@@ -0,0 +1,89 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
+sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
+1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
+z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
+6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
+x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
+1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
+IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
+xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
+sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
+2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
+sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
+Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
+AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
+m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
+QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
+P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
+pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
+G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
+6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
+LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
+OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
+W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
+M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
+y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
+uQicoQq3yzeQh20wtrtaXzTNmA==
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
+gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
+VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
+cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
+MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
+ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
+aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
+GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
+6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
+tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
+PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
+AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
+HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
+zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
+RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
+YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
+YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
+cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
+K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
+x+Yo/cL8fGfVpPt4UM8=
+-----END CERTIFICATE-----
+-----BEGIN WARP TOKEN-----
+N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
+ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
+MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
+NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
+NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
+OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
+ZWYxZTI2Zjc=
+-----END WARP TOKEN-----
+-----BEGIN RSA PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
+sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
+1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
+z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
+6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
+x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
+1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
+IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
+xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
+sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
+2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
+sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
+Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
+AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
+m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
+QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
+P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
+pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
+G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
+6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
+LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
+OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
+W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
+M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
+y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
+uQicoQq3yzeQh20wtrtaXzTNmA==
+-----END RSA PRIVATE KEY-----

certutil/test-cert.pem

@@ -0,0 +1,61 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
+sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
+1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
+z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
+6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
+x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
+1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
+IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
+xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
+sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
+2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
+sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
+Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
+AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
+m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
+QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
+P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
+pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
+G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
+6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
+LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
+OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
+W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
+M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
+y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
+uQicoQq3yzeQh20wtrtaXzTNmA==
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
+gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
+VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
+cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
+MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
+ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
+aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
+GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
+6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
+tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
+PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
+AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
+HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
+zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
+RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
+YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
+YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
+cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
+K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
+x+Yo/cL8fGfVpPt4UM8=
+-----END CERTIFICATE-----
+-----BEGIN WARP TOKEN-----
+N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
+ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
+MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
+NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
+NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
+OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
+ZWYxZTI2Zjc=
+-----END WARP TOKEN-----

cfsetup.yaml

@@ -1,4 +1,4 @@
-pinned_go: &pinned_go go=1.12.7-1
+pinned_go: &pinned_go go=1.15.3-1
 build_dir: &build_dir /cfsetup_build
 default-flavor: stretch
 stretch: &stretch
@@ -22,6 +22,29 @@ stretch: &stretch
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared-deb
+  build-deb-arm64:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=arm64
+      - make cloudflared-deb
+  publish-deb:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+      - openssh-client
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - make publish-deb
   release-linux-amd64:
     build_dir: *build_dir
     builddeps:
@@ -31,6 +54,19 @@ stretch: &stretch
       - export GOOS=linux
       - export GOARCH=amd64
       - make release
+  github-release-linux-amd64:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - python3-setuptools
+      - python3-pip
+    pre-cache: &install_pygithub
+      - pip3 install pygithub
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - make github-release
   release-linux-armv6:
     build_dir: *build_dir
     builddeps:
@@ -42,6 +78,20 @@ stretch: &stretch
       - export GOARCH=arm
       - export CC=arm-linux-gnueabihf-gcc
       - make release
+  github-release-linux-armv6:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - crossbuild-essential-armhf
+      - gcc-arm-linux-gnueabihf
+      - python3-setuptools
+      - python3-pip
+    pre-cache: *install_pygithub
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=arm
+      - export CC=arm-linux-gnueabihf-gcc
+      - make github-release
   release-linux-386:
     build_dir: *build_dir
     builddeps:
@@ -51,6 +101,18 @@ stretch: &stretch
       - export GOOS=linux
       - export GOARCH=386
       - make release
+  github-release-linux-386:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - gcc-multilib
+      - python3-setuptools
+      - python3-pip
+    pre-cache: *install_pygithub
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=386
+      - make github-release
   release-windows-amd64:
     build_dir: *build_dir
     builddeps:
@@ -61,6 +123,19 @@ stretch: &stretch
       - export GOARCH=amd64
       - export CC=x86_64-w64-mingw32-gcc
       - make release
+  github-release-windows-amd64:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - gcc-mingw-w64
+      - python3-setuptools
+      - python3-pip
+    pre-cache: *install_pygithub
+    post-cache:
+      - export GOOS=windows
+      - export GOARCH=amd64
+      - export CC=x86_64-w64-mingw32-gcc
+      - make github-release
   release-windows-386:
     build_dir: *build_dir
     builddeps:
@@ -71,6 +146,42 @@ stretch: &stretch
       - export GOARCH=386
       - export CC=i686-w64-mingw32-gcc-win32
       - make release
+  github-release-windows-386:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - gcc-mingw-w64
+      - python3-setuptools
+      - python3-pip
+    pre-cache: *install_pygithub
+    post-cache:
+      - export GOOS=windows
+      - export GOARCH=386
+      - export CC=i686-w64-mingw32-gcc-win32
+      - make github-release
+  github-release-linux-arm64:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - crossbuild-essential-armhf
+      - g++-aarch64-linux-gnu
+      - python3-setuptools
+      - python3-pip
+    pre-cache: *install_pygithub
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=arm64
+      - export CC=aarch64-linux-gnu-gcc
+      - make github-release
+  github-release-macos-amd64:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - python3-setuptools
+      - python3-pip
+    pre-cache: *install_pygithub
+    post-cache:
+      - make github-mac-upload
   test:
     build_dir: *build_dir
     builddeps:
@@ -83,9 +194,82 @@ stretch: &stretch
       - (cd / && go get github.com/BurntSushi/go-sumtype)
       - export PATH="$HOME/go/bin:$PATH"
       - make test
+  update-homebrew:
+    builddeps:
+      - openssh-client
+      - s3cmd
+    post-cache:
+      - .teamcity/update-homebrew.sh
+  github-message-release:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - python3-setuptools
+      - python3-pip
+    pre-cache: *install_pygithub
+    post-cache:
+      - make github-message
+  build-junos:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - python3
+      - genisoimage
+      - jetez
+    pre-cache:
+      - ln -s /usr/bin/genisoimage /usr/bin/mkisofs
+    post-cache:
+      - export GOOS=freebsd
+      - export GOARCH=amd64
+      - make cloudflared-junos
+  publish-junos:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - python3
+      - genisoimage
+      - jetez
+      - s4cmd
+    pre-cache:
+      - ln -s /usr/bin/genisoimage /usr/bin/mkisofs
+    post-cache:
+      - export GOOS=freebsd
+      - export GOARCH=amd64
+      - make publish-cloudflared-junos
 
 jessie: *stretch
 
+buster: *stretch
+
+centos-7:
+  publish-rpm:
+    build_dir: *build_dir
+    builddeps: &el7_builddeps
+      - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+    pre-cache:
+      - yum install -y fakeroot
+      - wget https://golang.org/dl/go1.15.3.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.15.3.linux-amd64.tar.gz
+    post-cache:
+      - export PATH=$PATH:/usr/local/go/bin
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - make publish-rpm
+  build-rpm:
+    build_dir: *build_dir
+    builddeps: *el7_builddeps
+    pre-cache:
+      - yum install -y fakeroot
+      - wget https://golang.org/dl/go1.15.3.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.15.3.linux-amd64.tar.gz
+    post-cache:
+      - export PATH=$PATH:/usr/local/go/bin
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - make cloudflared-rpm
+
 # cfsetup compose
 default-stack: test_dbconnect
 test_dbconnect:

cloudflared_man_template

@@ -0,0 +1,6 @@
+.\" Manpage for cloudflared.
+.TH man 1 ${DATE} "${VERSION}" "cloudflared man page"
+.SH NAME
+cloudflared \- creates a connection to the cloudflare edge network
+.SH DESCRIPTION
+cloudflared creates a persistent connection between a local service and the Cloudflare network. Once the daemon is running and the Tunnel has been configured, the local service can be locked down to only allow connections from Cloudflare.

cmd/cloudflared/access/carrier.go

@@ -2,21 +2,73 @@ package access
 
 import (
 	"net/http"
-	"net/url"
 	"strings"
 
 	"github.com/cloudflare/cloudflared/carrier"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/validation"
 	"github.com/pkg/errors"
-	cli "gopkg.in/urfave/cli.v2"
+	cli "github.com/urfave/cli/v2"
 )
 
+// StartForwarder starts a client side websocket forward
+func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, logger logger.Service) error {
+	validURL, err := validation.ValidateUrl(forwarder.Listener)
+	if err != nil {
+		return errors.Wrap(err, "error validating origin URL")
+	}
+
+	// get the headers from the config file and add to the request
+	headers := make(http.Header)
+	if forwarder.TokenClientID != "" {
+		headers.Set(h2mux.CFAccessClientIDHeader, forwarder.TokenClientID)
+	}
+
+	if forwarder.TokenSecret != "" {
+		headers.Set(h2mux.CFAccessClientSecretHeader, forwarder.TokenSecret)
+	}
+
+	if forwarder.Destination != "" {
+		headers.Add(h2mux.CFJumpDestinationHeader, forwarder.Destination)
+	}
+
+	options := &carrier.StartOptions{
+		OriginURL: forwarder.URL,
+		Headers:   headers, //TODO: TUN-2688 support custom headers from config file
+	}
+
+	// we could add a cmd line variable for this bool if we want the SOCK5 server to be on the client side
+	wsConn := carrier.NewWSConnection(logger, false)
+
+	logger.Infof("Start Websocket listener on: %s", validURL.Host)
+	return carrier.StartForwarder(wsConn, validURL.Host, shutdown, options)
+}
+
 // ssh will start a WS proxy server for server mode
 // or copy from stdin/stdout for client mode
 // useful for proxying other protocols (like ssh) over websockets
 // (which you can put Access in front of)
 func ssh(c *cli.Context) error {
+	logDirectory, logLevel := config.FindLogSettings()
+
+	flagLogDirectory := c.String(sshLogDirectoryFlag)
+	if flagLogDirectory != "" {
+		logDirectory = flagLogDirectory
+	}
+
+	flagLogLevel := c.String(sshLogLevelFlag)
+	if flagLogLevel != "" {
+		logLevel = flagLogLevel
+	}
+
+	logger, err := logger.New(logger.DefaultFile(logDirectory), logger.LogLevelString(logLevel))
+	if err != nil {
+		return cliutil.PrintLoggerSetupError("error setting up logger", err)
+	}
+
 	// get the hostname from the cmdline and error out if its not provided
 	rawHostName := c.String(sshHostnameFlag)
 	hostname, err := validation.ValidateHostname(rawHostName)
@@ -28,15 +80,15 @@ func ssh(c *cli.Context) error {
 	// get the headers from the cmdline and add them
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
-		headers.Add("CF-Access-Client-Id", c.String(sshTokenIDFlag))
+		headers.Set(h2mux.CFAccessClientIDHeader, c.String(sshTokenIDFlag))
 	}
 	if c.IsSet(sshTokenSecretFlag) {
-		headers.Add("CF-Access-Client-Secret", c.String(sshTokenSecretFlag))
+		headers.Set(h2mux.CFAccessClientSecretHeader, c.String(sshTokenSecretFlag))
 	}
 
 	destination := c.String(sshDestinationFlag)
 	if destination != "" {
-		headers.Add("CF-Access-SSH-Destination", destination)
+		headers.Add(h2mux.CFJumpDestinationHeader, destination)
 	}
 
 	options := &carrier.StartOptions{
@@ -44,21 +96,25 @@ func ssh(c *cli.Context) error {
 		Headers:   headers,
 	}
 
+	// we could add a cmd line variable for this bool if we want the SOCK5 server to be on the client side
+	wsConn := carrier.NewWSConnection(logger, false)
+
 	if c.NArg() > 0 || c.IsSet(sshURLFlag) {
-		localForwarder, err := config.ValidateUrl(c)
+		forwarder, err := config.ValidateUrl(c, true)
 		if err != nil {
-			logger.WithError(err).Error("Error validating origin URL")
+			logger.Errorf("Error validating origin URL: %s", err)
 			return errors.Wrap(err, "error validating origin URL")
 		}
-		forwarder, err := url.Parse(localForwarder)
+
+		logger.Infof("Start Websocket listener on: %s", forwarder.Host)
+		err = carrier.StartForwarder(wsConn, forwarder.Host, shutdownC, options)
 		if err != nil {
-			logger.WithError(err).Error("Error validating origin URL")
-			return errors.Wrap(err, "error validating origin URL")
+			logger.Errorf("Error on Websocket listener: %s", err)
 		}
-		return carrier.StartServer(logger, forwarder.Host, shutdownC, options)
+		return err
 	}
 
-	return carrier.StartClient(logger, &carrier.StdinoutStream{}, options)
+	return carrier.StartClient(wsConn, &carrier.StdinoutStream{}, options)
 }
 
 func buildRequestHeaders(values []string) http.Header {

cmd/cloudflared/access/cmd.go

@@ -10,27 +10,31 @@ import (
 	"time"
 
 	"github.com/cloudflare/cloudflared/carrier"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/sshgen"
 	"github.com/cloudflare/cloudflared/validation"
 	"github.com/pkg/errors"
 	"golang.org/x/net/idna"
 
-	"github.com/cloudflare/cloudflared/log"
-	raven "github.com/getsentry/raven-go"
-	cli "gopkg.in/urfave/cli.v2"
+	"github.com/getsentry/raven-go"
+	"github.com/urfave/cli/v2"
 )
 
 const (
-	sshHostnameFlag    = "hostname"
-	sshDestinationFlag = "destination"
-	sshURLFlag         = "url"
-	sshHeaderFlag      = "header"
-	sshTokenIDFlag     = "service-token-id"
-	sshTokenSecretFlag = "service-token-secret"
-	sshGenCertFlag     = "short-lived-cert"
-	sshConfigTemplate  = `
+	sshHostnameFlag     = "hostname"
+	sshDestinationFlag  = "destination"
+	sshURLFlag          = "url"
+	sshHeaderFlag       = "header"
+	sshTokenIDFlag      = "service-token-id"
+	sshTokenSecretFlag  = "service-token-secret"
+	sshGenCertFlag      = "short-lived-cert"
+	sshLogDirectoryFlag = "log-directory"
+	sshLogLevelFlag     = "log-level"
+	sshConfigTemplate   = `
 Add to your {{.Home}}/.ssh/config:
 
 Host {{.Hostname}}
@@ -51,7 +55,6 @@ Host cfpipe-{{.Hostname}}
 const sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b@sentry.io/189878"
 
 var (
-	logger         = log.CreateLogger()
 	shutdownC      chan struct{}
 	graceShutdownC chan struct{}
 )
@@ -71,17 +74,17 @@ func Commands() []*cli.Command {
 	return []*cli.Command{
 		{
 			Name:     "access",
-			Category: "Access (BETA)",
+			Aliases:  []string{"forward"},
+			Category: "Access",
 			Usage:    "access <subcommand>",
-			Description: `(BETA) Cloudflare Access protects internal resources by securing, authenticating and monitoring access 
+			Description: `Cloudflare Access protects internal resources by securing, authenticating and monitoring access 
 			per-user and by application. With Cloudflare Access, only authenticated users with the required permissions are 
 			able to reach sensitive resources. The commands provided here allow you to interact with Access protected 
-			applications from the command line. This feature is considered beta. Your feedback is greatly appreciated!
-			https://cfl.re/CLIAuthBeta`,
+			applications from the command line.`,
 			Subcommands: []*cli.Command{
 				{
 					Name:   "login",
-					Action: login,
+					Action: cliutil.ErrorHandler(login),
 					Usage:  "login <url of access application>",
 					Description: `The login subcommand initiates an authentication flow with your identity provider.
 					The subcommand will launch a browser. For headless systems, a url is provided.
@@ -97,7 +100,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:   "curl",
-					Action: curl,
+					Action: cliutil.ErrorHandler(curl),
 					Usage:  "curl [--allow-request, -ar] <url> [<curl args>...]",
 					Description: `The curl subcommand wraps curl and automatically injects the JWT into a cf-access-token
 					header when using curl to reach an application behind Access.`,
@@ -106,7 +109,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "token",
-					Action:      generateToken,
+					Action:      cliutil.ErrorHandler(generateToken),
 					Usage:       "token -app=<url of access application>",
 					ArgsUsage:   "url of Access application",
 					Description: `The token subcommand produces a JWT which can be used to authenticate requests.`,
@@ -117,45 +120,57 @@ func Commands() []*cli.Command {
 					},
 				},
 				{
-					Name:        "ssh",
-					Action:      ssh,
-					Aliases:     []string{"rdp", "tcp"},
+					Name:        "tcp",
+					Action:      cliutil.ErrorHandler(ssh),
+					Aliases:     []string{"rdp", "ssh", "smb"},
 					Usage:       "",
 					ArgsUsage:   "",
-					Description: `The ssh subcommand sends data over a proxy to the Cloudflare edge.`,
+					Description: `The tcp subcommand sends data over a proxy to the Cloudflare edge.`,
 					Flags: []cli.Flag{
 						&cli.StringFlag{
-							Name:  sshHostnameFlag,
-							Usage: "specify the hostname of your application.",
+							Name:    sshHostnameFlag,
+							Aliases: []string{"tunnel-host", "T"},
+							Usage:   "specify the hostname of your application.",
 						},
 						&cli.StringFlag{
 							Name:  sshDestinationFlag,
 							Usage: "specify the destination address of your SSH server.",
 						},
 						&cli.StringFlag{
-							Name:  sshURLFlag,
-							Usage: "specify the host:port to forward data to Cloudflare edge.",
+							Name:    sshURLFlag,
+							Aliases: []string{"listener", "L"},
+							Usage:   "specify the host:port to forward data to Cloudflare edge.",
 						},
 						&cli.StringSliceFlag{
 							Name:    sshHeaderFlag,
 							Aliases: []string{"H"},
 							Usage:   "specify additional headers you wish to send.",
 						},
-						&cli.StringSliceFlag{
+						&cli.StringFlag{
 							Name:    sshTokenIDFlag,
 							Aliases: []string{"id"},
 							Usage:   "specify an Access service token ID you wish to use.",
 						},
-						&cli.StringSliceFlag{
+						&cli.StringFlag{
 							Name:    sshTokenSecretFlag,
 							Aliases: []string{"secret"},
 							Usage:   "specify an Access service token secret you wish to use.",
 						},
+						&cli.StringFlag{
+							Name:    sshLogDirectoryFlag,
+							Aliases: []string{"logfile"}, //added to match the tunnel side
+							Usage:   "Save application log to this directory for reporting issues.",
+						},
+						&cli.StringFlag{
+							Name:    sshLogLevelFlag,
+							Aliases: []string{"loglevel"}, //added to match the tunnel side
+							Usage:   "Application logging level {fatal, error, info, debug}. ",
+						},
 					},
 				},
 				{
 					Name:        "ssh-config",
-					Action:      sshConfig,
+					Action:      cliutil.ErrorHandler(sshConfig),
 					Usage:       "",
 					Description: `Prints an example configuration ~/.ssh/config`,
 					Flags: []cli.Flag{
@@ -171,7 +186,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "ssh-gen",
-					Action:      sshGen,
+					Action:      cliutil.ErrorHandler(sshGen),
 					Usage:       "",
 					Description: `Generates a short lived certificate for given hostname`,
 					Flags: []cli.Flag{
@@ -188,8 +203,15 @@ func Commands() []*cli.Command {
 
 // login pops up the browser window to do the actual login and JWT generation
 func login(c *cli.Context) error {
-	raven.SetDSN(sentryDSN)
-	logger := log.CreateLogger()
+	if err := raven.SetDSN(sentryDSN); err != nil {
+		return err
+	}
+
+	logger, err := logger.New()
+	if err != nil {
+		return errors.Wrap(err, "error setting up logger")
+	}
+
 	args := c.Args()
 	rawURL := ensureURLScheme(args.First())
 	appURL, err := url.Parse(rawURL)
@@ -197,17 +219,20 @@ func login(c *cli.Context) error {
 		logger.Errorf("Please provide the url of the Access application\n")
 		return err
 	}
-	if err := verifyTokenAtEdge(appURL, c); err != nil {
-		logger.WithError(err).Error("Could not verify token")
+	if err := verifyTokenAtEdge(appURL, c, logger); err != nil {
+		logger.Errorf("Could not verify token: %s", err)
 		return err
 	}
 
-	token, err := token.GetTokenIfExists(appURL)
-	if err != nil || token == "" {
+	cfdToken, err := token.GetTokenIfExists(appURL)
+	if err != nil {
 		fmt.Fprintln(os.Stderr, "Unable to find token for provided application.")
 		return err
+	} else if cfdToken == "" {
+		fmt.Fprintln(os.Stderr, "token for provided application was empty.")
+		return errors.New("empty application token")
 	}
-	fmt.Fprintf(os.Stdout, "Successfully fetched your token:\n\n%s\n\n", string(token))
+	fmt.Fprintf(os.Stdout, "Successfully fetched your token:\n\n%s\n\n", cfdToken)
 
 	return nil
 }
@@ -224,8 +249,14 @@ func ensureURLScheme(url string) string {
 
 // curl provides a wrapper around curl, passing Access JWT along in request
 func curl(c *cli.Context) error {
-	raven.SetDSN(sentryDSN)
-	logger := log.CreateLogger()
+	if err := raven.SetDSN(sentryDSN); err != nil {
+		return err
+	}
+	logger, err := logger.New()
+	if err != nil {
+		return errors.Wrap(err, "error setting up logger")
+	}
+
 	args := c.Args()
 	if args.Len() < 1 {
 		logger.Error("Please provide the access app and command you wish to run.")
@@ -233,7 +264,7 @@ func curl(c *cli.Context) error {
 	}
 
 	cmdArgs, allowRequest := parseAllowRequest(args.Slice())
-	appURL, err := getAppURL(cmdArgs)
+	appURL, err := getAppURL(cmdArgs, logger)
 	if err != nil {
 		return err
 	}
@@ -241,24 +272,26 @@ func curl(c *cli.Context) error {
 	tok, err := token.GetTokenIfExists(appURL)
 	if err != nil || tok == "" {
 		if allowRequest {
-			logger.Warn("You don't have an Access token set. Please run access token <access application> to fetch one.")
+			logger.Info("You don't have an Access token set. Please run access token <access application> to fetch one.")
 			return shell.Run("curl", cmdArgs...)
 		}
-		tok, err = token.FetchToken(appURL)
+		tok, err = token.FetchToken(appURL, logger)
 		if err != nil {
-			logger.Error("Failed to refresh token: ", err)
+			logger.Errorf("Failed to refresh token: %s", err)
 			return err
 		}
 	}
 
 	cmdArgs = append(cmdArgs, "-H")
-	cmdArgs = append(cmdArgs, fmt.Sprintf("cf-access-token: %s", tok))
+	cmdArgs = append(cmdArgs, fmt.Sprintf("%s: %s", h2mux.CFAccessTokenHeader, tok))
 	return shell.Run("curl", cmdArgs...)
 }
 
 // token dumps provided token to stdout
 func generateToken(c *cli.Context) error {
-	raven.SetDSN(sentryDSN)
+	if err := raven.SetDSN(sentryDSN); err != nil {
+		return err
+	}
 	appURL, err := url.Parse(c.String("app"))
 	if err != nil || c.NumFlags() < 1 {
 		fmt.Fprintln(os.Stderr, "Please provide a url.")
@@ -298,6 +331,11 @@ func sshConfig(c *cli.Context) error {
 
 // sshGen generates a short lived certificate for provided hostname
 func sshGen(c *cli.Context) error {
+	logger, err := logger.New()
+	if err != nil {
+		return errors.Wrap(err, "error setting up logger")
+	}
+
 	// get the hostname from the cmdline and error out if its not provided
 	rawHostName := c.String(sshHostnameFlag)
 	hostname, err := validation.ValidateHostname(rawHostName)
@@ -313,12 +351,12 @@ func sshGen(c *cli.Context) error {
 	// this fetchToken function mutates the appURL param. We should refactor that
 	fetchTokenURL := &url.URL{}
 	*fetchTokenURL = *originURL
-	token, err := token.FetchToken(fetchTokenURL)
+	cfdToken, err := token.FetchTokenWithRedirect(fetchTokenURL, logger)
 	if err != nil {
 		return err
 	}
 
-	if err := sshgen.GenerateShortLivedCertificate(originURL, token); err != nil {
+	if err := sshgen.GenerateShortLivedCertificate(originURL, cfdToken); err != nil {
 		return err
 	}
 
@@ -326,7 +364,7 @@ func sshGen(c *cli.Context) error {
 }
 
 // getAppURL will pull the appURL needed for fetching a user's Access token
-func getAppURL(cmdArgs []string) (*url.URL, error) {
+func getAppURL(cmdArgs []string, logger logger.Service) (*url.URL, error) {
 	if len(cmdArgs) < 1 {
 		logger.Error("Please provide a valid URL as the first argument to curl.")
 		return nil, errors.New("not a valid url")
@@ -400,17 +438,17 @@ func isFileThere(candidate string) bool {
 // verifyTokenAtEdge checks for a token on disk, or generates a new one.
 // Then makes a request to to the origin with the token to ensure it is valid.
 // Returns nil if token is valid.
-func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context) error {
+func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context, logger logger.Service) error {
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
-		headers.Add("CF-Access-Client-Id", c.String(sshTokenIDFlag))
+		headers.Add(h2mux.CFAccessClientIDHeader, c.String(sshTokenIDFlag))
 	}
 	if c.IsSet(sshTokenSecretFlag) {
-		headers.Add("CF-Access-Client-Secret", c.String(sshTokenSecretFlag))
+		headers.Add(h2mux.CFAccessClientSecretHeader, c.String(sshTokenSecretFlag))
 	}
 	options := &carrier.StartOptions{OriginURL: appUrl.String(), Headers: headers}
 
-	if valid, err := isTokenValid(options); err != nil {
+	if valid, err := isTokenValid(options, logger); err != nil {
 		return err
 	} else if valid {
 		return nil
@@ -420,7 +458,7 @@ func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context) error {
 		return err
 	}
 
-	if valid, err := isTokenValid(options); err != nil {
+	if valid, err := isTokenValid(options, logger); err != nil {
 		return err
 	} else if !valid {
 		return errors.New("failed to verify token")
@@ -430,8 +468,8 @@ func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context) error {
 }
 
 // isTokenValid makes a request to the origin and returns true if the response was not a 302.
-func isTokenValid(options *carrier.StartOptions) (bool, error) {
-	req, err := carrier.BuildAccessRequest(options)
+func isTokenValid(options *carrier.StartOptions, logger logger.Service) (bool, error) {
+	req, err := carrier.BuildAccessRequest(options, logger)
 	if err != nil {
 		return false, errors.Wrap(err, "Could not create access request")
 	}

cmd/cloudflared/app_forward_service.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+)
+
+// ForwardServiceType is used to identify what kind of overwatch service this is
+const ForwardServiceType = "forward"
+
+// ForwarderService is used to wrap the access package websocket forwarders
+// into a service model for the overwatch package.
+// it also holds a reference to the config object that represents its state
+type ForwarderService struct {
+	forwarder config.Forwarder
+	shutdown  chan struct{}
+	logger    logger.Service
+}
+
+// NewForwardService creates a new forwarder service
+func NewForwardService(f config.Forwarder, logger logger.Service) *ForwarderService {
+	return &ForwarderService{forwarder: f, shutdown: make(chan struct{}, 1), logger: logger}
+}
+
+// Name is used to figure out this service is related to the others (normally the addr it binds to)
+// e.g. localhost:78641 or 127.0.0.1:2222 since this is a websocket forwarder
+func (s *ForwarderService) Name() string {
+	return s.forwarder.Listener
+}
+
+// Type is used to identify what kind of overwatch service this is
+func (s *ForwarderService) Type() string {
+	return ForwardServiceType
+}
+
+// Hash is used to figure out if this forwarder is the unchanged or not from the config file updates
+func (s *ForwarderService) Hash() string {
+	return s.forwarder.Hash()
+}
+
+// Shutdown stops the websocket listener
+func (s *ForwarderService) Shutdown() {
+	s.shutdown <- struct{}{}
+}
+
+// Run is the run loop that is started by the overwatch service
+func (s *ForwarderService) Run() error {
+	return access.StartForwarder(s.forwarder, s.shutdown, s.logger)
+}

cmd/cloudflared/app_resolver_service.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunneldns"
+)
+
+// ResolverServiceType is used to identify what kind of overwatch service this is
+const ResolverServiceType = "resolver"
+
+// ResolverService is used to wrap the tunneldns package's DNS over HTTP
+// into a service model for the overwatch package.
+// it also holds a reference to the config object that represents its state
+type ResolverService struct {
+	resolver config.DNSResolver
+	shutdown chan struct{}
+	logger   logger.Service
+}
+
+// NewResolverService creates a new resolver service
+func NewResolverService(r config.DNSResolver, logger logger.Service) *ResolverService {
+	return &ResolverService{resolver: r,
+		shutdown: make(chan struct{}),
+		logger:   logger,
+	}
+}
+
+// Name is used to figure out this service is related to the others (normally the addr it binds to)
+// this is just "resolver" since there can only be one DNS resolver running
+func (s *ResolverService) Name() string {
+	return ResolverServiceType
+}
+
+// Type is used to identify what kind of overwatch service this is
+func (s *ResolverService) Type() string {
+	return ResolverServiceType
+}
+
+// Hash is used to figure out if this forwarder is the unchanged or not from the config file updates
+func (s *ResolverService) Hash() string {
+	return s.resolver.Hash()
+}
+
+// Shutdown stops the tunneldns listener
+func (s *ResolverService) Shutdown() {
+	s.shutdown <- struct{}{}
+}
+
+// Run is the run loop that is started by the overwatch service
+func (s *ResolverService) Run() error {
+	// create a listener
+	l, err := tunneldns.CreateListener(s.resolver.AddressOrDefault(), s.resolver.PortOrDefault(),
+		s.resolver.UpstreamsOrDefault(), s.resolver.BootstrapsOrDefault(), s.logger)
+	if err != nil {
+		return err
+	}
+
+	// start the listener.
+	readySignal := make(chan struct{})
+	err = l.Start(readySignal)
+	if err != nil {
+		l.Stop()
+		return err
+	}
+	<-readySignal
+	s.logger.Infof("start resolver on: %s:%d", s.resolver.AddressOrDefault(), s.resolver.PortOrDefault())
+
+	// wait for shutdown signal
+	<-s.shutdown
+	s.logger.Infof("shutdown on: %s:%d", s.resolver.AddressOrDefault(), s.resolver.PortOrDefault())
+	return l.Stop()
+}

cmd/cloudflared/app_service.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/overwatch"
+)
+
+// AppService is the main service that runs when no command lines flags are passed to cloudflared
+// it manages all the running services such as tunnels, forwarders, DNS resolver, etc
+type AppService struct {
+	configManager    config.Manager
+	serviceManager   overwatch.Manager
+	shutdownC        chan struct{}
+	configUpdateChan chan config.Root
+	logger           logger.Service
+}
+
+// NewAppService creates a new AppService with needed supporting services
+func NewAppService(configManager config.Manager, serviceManager overwatch.Manager, shutdownC chan struct{}, logger logger.Service) *AppService {
+	return &AppService{
+		configManager:    configManager,
+		serviceManager:   serviceManager,
+		shutdownC:        shutdownC,
+		configUpdateChan: make(chan config.Root),
+		logger:           logger,
+	}
+}
+
+// Run starts the run loop to handle config updates and run forwarders, tunnels, etc
+func (s *AppService) Run() error {
+	go s.actionLoop()
+	return s.configManager.Start(s)
+}
+
+// Shutdown kills all the running services
+func (s *AppService) Shutdown() error {
+	s.configManager.Shutdown()
+	s.shutdownC <- struct{}{}
+	return nil
+}
+
+// ConfigDidUpdate is a delegate notification from the config manager
+// it is trigger when the config file has been updated and now the service needs
+// to update its services accordingly
+func (s *AppService) ConfigDidUpdate(c config.Root) {
+	s.configUpdateChan <- c
+}
+
+// actionLoop handles the actions from running processes
+func (s *AppService) actionLoop() {
+	for {
+		select {
+		case c := <-s.configUpdateChan:
+			s.handleConfigUpdate(c)
+		case <-s.shutdownC:
+			for _, service := range s.serviceManager.Services() {
+				service.Shutdown()
+			}
+			return
+		}
+	}
+}
+
+func (s *AppService) handleConfigUpdate(c config.Root) {
+	// handle the client forward listeners
+	activeServices := map[string]struct{}{}
+	for _, f := range c.Forwarders {
+		service := NewForwardService(f, s.logger)
+		s.serviceManager.Add(service)
+		activeServices[service.Name()] = struct{}{}
+	}
+
+	// handle resolver changes
+	if c.Resolver.Enabled {
+		service := NewResolverService(c.Resolver, s.logger)
+		s.serviceManager.Add(service)
+		activeServices[service.Name()] = struct{}{}
+
+	}
+
+	// TODO: TUN-1451 - tunnels
+
+	// remove any services that are no longer active
+	for _, service := range s.serviceManager.Services() {
+		if _, ok := activeServices[service.Name()]; !ok {
+			s.serviceManager.Remove(service.Name())
+		}
+	}
+}

cmd/cloudflared/buildinfo/build_info.go

@@ -3,7 +3,7 @@ package buildinfo
 import (
 	"runtime"
 
-	"github.com/sirupsen/logrus"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 type BuildInfo struct {
@@ -22,7 +22,7 @@ func GetBuildInfo(cloudflaredVersion string) *BuildInfo {
 	}
 }
 
-func (bi *BuildInfo) Log(logger *logrus.Logger) {
+func (bi *BuildInfo) Log(logger logger.Service) {
 	logger.Infof("Version %s", bi.CloudflaredVersion)
 	logger.Infof("GOOS: %s, GOVersion: %s, GoArch: %s", bi.GoOS, bi.GoVersion, bi.GoArch)
 }

cmd/cloudflared/cliutil/errors.go

@@ -0,0 +1,55 @@
+package cliutil
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+)
+
+type usageError string
+
+func (ue usageError) Error() string {
+	return string(ue)
+}
+
+func UsageError(format string, args ...interface{}) error {
+	if len(args) == 0 {
+		return usageError(format)
+	} else {
+		msg := fmt.Sprintf(format, args...)
+		return usageError(msg)
+	}
+}
+
+// Ensures exit with error code if actionFunc returns an error
+func ErrorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
+	return func(ctx *cli.Context) error {
+		defer logger.SharedWriteManager.Shutdown()
+
+		err := actionFunc(ctx)
+		if err != nil {
+			if _, ok := err.(usageError); ok {
+				msg := fmt.Sprintf("%s\nSee 'cloudflared %s --help'.", err.Error(), ctx.Command.FullName())
+				err = cli.Exit(msg, -1)
+			} else if _, ok := err.(cli.ExitCoder); !ok {
+				err = cli.Exit(err.Error(), 1)
+			}
+		}
+		return err
+	}
+}
+
+// PrintLoggerSetupError returns an error to stdout to notify when a logger can't start
+func PrintLoggerSetupError(msg string, err error) error {
+	l, le := logger.New()
+	if le != nil {
+		log.Printf("%s: %s", msg, err)
+	} else {
+		l.Errorf("%s: %s", msg, err)
+	}
+
+	return errors.Wrap(err, msg)
+}

cmd/cloudflared/config/configuration.go

@@ -1,26 +1,88 @@
 package config
 
 import (
-	"errors"
+	"fmt"
+	"net/url"
 	"os"
 	"path/filepath"
+	"runtime"
+	"time"
 
-	"github.com/cloudflare/cloudflared/validation"
 	homedir "github.com/mitchellh/go-homedir"
-	"gopkg.in/urfave/cli.v2"
-	"gopkg.in/urfave/cli.v2/altsrc"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+	"gopkg.in/yaml.v2"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/validation"
 )
 
 var (
-	// File names from which we attempt to read configuration.
+	// DefaultConfigFiles is the file names from which we attempt to read configuration.
 	DefaultConfigFiles = []string{"config.yml", "config.yaml"}
 
+	// DefaultUnixConfigLocation is the primary location to find a config file
+	DefaultUnixConfigLocation = "/usr/local/etc/cloudflared"
+
+	// DefaultUnixLogLocation is the primary location to find log files
+	DefaultUnixLogLocation = "/var/log/cloudflared"
+
 	// Launchd doesn't set root env variables, so there is default
 	// Windows default config dir was ~/cloudflare-warp in documentation; let's keep it compatible
-	DefaultConfigDirs = []string{"~/.cloudflared", "~/.cloudflare-warp", "~/cloudflare-warp", "/usr/local/etc/cloudflared", "/etc/cloudflared"}
+	defaultUserConfigDirs = []string{"~/.cloudflared", "~/.cloudflare-warp", "~/cloudflare-warp"}
+	defaultNixConfigDirs  = []string{"/etc/cloudflared", DefaultUnixConfigLocation}
+
+	ErrNoConfigFile = fmt.Errorf("Cannot determine default configuration path. No file %v in %v", DefaultConfigFiles, DefaultConfigSearchDirectories())
 )
 
-const DefaultCredentialFile = "cert.pem"
+const (
+	DefaultCredentialFile = "cert.pem"
+
+	// BastionFlag is to enable bastion, or jump host, operation
+	BastionFlag = "bastion"
+)
+
+// DefaultConfigDirectory returns the default directory of the config file
+func DefaultConfigDirectory() string {
+	if runtime.GOOS == "windows" {
+		path := os.Getenv("CFDPATH")
+		if path == "" {
+			path = filepath.Join(os.Getenv("ProgramFiles(x86)"), "cloudflared")
+			if _, err := os.Stat(path); os.IsNotExist(err) { //doesn't exist, so return an empty failure string
+				return ""
+			}
+		}
+		return path
+	}
+	return DefaultUnixConfigLocation
+}
+
+// DefaultLogDirectory returns the default directory for log files
+func DefaultLogDirectory() string {
+	if runtime.GOOS == "windows" {
+		return DefaultConfigDirectory()
+	}
+	return DefaultUnixLogLocation
+}
+
+// DefaultConfigPath returns the default location of a config file
+func DefaultConfigPath() string {
+	dir := DefaultConfigDirectory()
+	if dir == "" {
+		return DefaultConfigFiles[0]
+	}
+	return filepath.Join(dir, DefaultConfigFiles[0])
+}
+
+// DefaultConfigSearchDirectories returns the default folder locations of the config
+func DefaultConfigSearchDirectories() []string {
+	dirs := make([]string, len(defaultUserConfigDirs))
+	copy(dirs, defaultUserConfigDirs)
+	if runtime.GOOS != "windows" {
+		dirs = append(dirs, defaultNixConfigDirs...)
+	}
+	return dirs
+}
 
 // FileExists checks to see if a file exist at the provided path.
 func FileExists(path string) (bool, error) {
@@ -36,19 +98,11 @@ func FileExists(path string) (bool, error) {
 	return true, nil
 }
 
-// FindInputSourceContext pulls the input source from the config flag.
-func FindInputSourceContext(context *cli.Context) (altsrc.InputSourceContext, error) {
-	if context.String("config") != "" {
-		return altsrc.NewYamlSourceFromFile(context.String("config"))
-	}
-	return nil, nil
-}
-
 // FindDefaultConfigPath returns the first path that contains a config file.
-// If none of the combination of DefaultConfigDirs and DefaultConfigFiles
+// If none of the combination of DefaultConfigSearchDirectories() and DefaultConfigFiles
 // contains a config file, return empty string.
 func FindDefaultConfigPath() string {
-	for _, configDir := range DefaultConfigDirs {
+	for _, configDir := range DefaultConfigSearchDirectories() {
 		for _, configFile := range DefaultConfigFiles {
 			dirPath, err := homedir.Expand(configDir)
 			if err != nil {
@@ -63,6 +117,68 @@ func FindDefaultConfigPath() string {
 	return ""
 }
 
+// FindOrCreateConfigPath returns the first path that contains a config file
+// or creates one in the primary default path if it doesn't exist
+func FindOrCreateConfigPath() string {
+	path := FindDefaultConfigPath()
+
+	if path == "" {
+		// create the default directory if it doesn't exist
+		path = DefaultConfigPath()
+		if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
+			return ""
+		}
+
+		// write a new config file out
+		file, err := os.Create(path)
+		if err != nil {
+			return ""
+		}
+		defer file.Close()
+
+		logDir := DefaultLogDirectory()
+		os.MkdirAll(logDir, os.ModePerm) //try and create it. Doesn't matter if it succeed or not, only byproduct will be no logs
+
+		c := Root{
+			LogDirectory: logDir,
+		}
+		if err := yaml.NewEncoder(file).Encode(&c); err != nil {
+			return ""
+		}
+	}
+
+	return path
+}
+
+// FindLogSettings gets the log directory and level from the config file
+func FindLogSettings() (string, string) {
+	configPath := FindOrCreateConfigPath()
+	defaultDirectory := DefaultLogDirectory()
+	defaultLevel := "info"
+
+	file, err := os.Open(configPath)
+	if err != nil {
+		return defaultDirectory, defaultLevel
+	}
+	defer file.Close()
+
+	var config Root
+	if err := yaml.NewDecoder(file).Decode(&config); err != nil {
+		return defaultDirectory, defaultLevel
+	}
+
+	directory := defaultDirectory
+	if config.LogDirectory != "" {
+		directory = config.LogDirectory
+	}
+
+	level := defaultLevel
+	if config.LogLevel != "" {
+		level = config.LogLevel
+	}
+	return directory, level
+}
+
 // ValidateUnixSocket ensures --unix-socket param is used exclusively
 // i.e. it fails if a user specifies both --url and --unix-socket
 func ValidateUnixSocket(c *cli.Context) (string, error) {
@@ -74,14 +190,210 @@ func ValidateUnixSocket(c *cli.Context) (string, error) {
 
 // ValidateUrl will validate url flag correctness. It can be either from --url or argument
 // Notice ValidateUnixSocket, it will enforce --unix-socket is not used with --url or argument
-func ValidateUrl(c *cli.Context) (string, error) {
+func ValidateUrl(c *cli.Context, allowURLFromArgs bool) (*url.URL, error) {
 	var url = c.String("url")
-	if c.NArg() > 0 {
+	if allowURLFromArgs && c.NArg() > 0 {
 		if c.IsSet("url") {
-			return "", errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
+			return nil, errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
 		}
 		url = c.Args().Get(0)
 	}
 	validUrl, err := validation.ValidateUrl(url)
 	return validUrl, err
 }
+
+type UnvalidatedIngressRule struct {
+	Hostname      string
+	Path          string
+	Service       string
+	OriginRequest OriginRequestConfig `yaml:"originRequest"`
+}
+
+// OriginRequestConfig is a set of optional fields that users may set to
+// customize how cloudflared sends requests to origin services. It is used to set
+// up general config that apply to all rules, and also, specific per-rule
+// config.
+// Note: To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
+type OriginRequestConfig struct {
+	// HTTP proxy timeout for establishing a new connection
+	ConnectTimeout *time.Duration `yaml:"connectTimeout"`
+	// HTTP proxy timeout for completing a TLS handshake
+	TLSTimeout *time.Duration `yaml:"tlsTimeout"`
+	// HTTP proxy TCP keepalive duration
+	TCPKeepAlive *time.Duration `yaml:"tcpKeepAlive"`
+	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
+	NoHappyEyeballs *bool `yaml:"noHappyEyeballs"`
+	// HTTP proxy maximum keepalive connection pool size
+	KeepAliveConnections *int `yaml:"keepAliveConnections"`
+	// HTTP proxy timeout for closing an idle connection
+	KeepAliveTimeout *time.Duration `yaml:"keepAliveTimeout"`
+	// Sets the HTTP Host header for the local webserver.
+	HTTPHostHeader *string `yaml:"httpHostHeader"`
+	// Hostname on the origin server certificate.
+	OriginServerName *string `yaml:"originServerName"`
+	// Path to the CA for the certificate of your origin.
+	// This option should be used only if your certificate is not signed by Cloudflare.
+	CAPool *string `yaml:"caPool"`
+	// Disables TLS verification of the certificate presented by your origin.
+	// Will allow any certificate from the origin to be accepted.
+	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
+	NoTLSVerify *bool `yaml:"noTLSVerify"`
+	// Disables chunked transfer encoding.
+	// Useful if you are running a WSGI server.
+	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding"`
+	// Runs as jump host
+	BastionMode *bool `yaml:"bastionMode"`
+	// Listen address for the proxy.
+	ProxyAddress *string `yaml:"proxyAddress"`
+	// Listen port for the proxy.
+	ProxyPort *uint `yaml:"proxyPort"`
+	// Valid options are 'socks' or empty.
+	ProxyType *string `yaml:"proxyType"`
+}
+
+type Configuration struct {
+	TunnelID      string `yaml:"tunnel"`
+	Ingress       []UnvalidatedIngressRule
+	OriginRequest OriginRequestConfig `yaml:"originRequest"`
+	sourceFile    string
+}
+
+type configFileSettings struct {
+	Configuration `yaml:",inline"`
+	// older settings will be aggregated into the generic map, should be read via cli.Context
+	Settings map[string]interface{} `yaml:",inline"`
+}
+
+func (c *Configuration) Source() string {
+	return c.sourceFile
+}
+
+func (c *configFileSettings) Int(name string) (int, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(int); ok {
+			return v, nil
+		}
+		return 0, fmt.Errorf("expected int found %T for %s", raw, name)
+	}
+	return 0, nil
+}
+
+func (c *configFileSettings) Duration(name string) (time.Duration, error) {
+	if raw, ok := c.Settings[name]; ok {
+		switch v := raw.(type) {
+		case time.Duration:
+			return v, nil
+		case string:
+			return time.ParseDuration(v)
+		}
+		return 0, fmt.Errorf("expected duration found %T for %s", raw, name)
+	}
+	return 0, nil
+}
+
+func (c *configFileSettings) Float64(name string) (float64, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(float64); ok {
+			return v, nil
+		}
+		return 0, fmt.Errorf("expected float found %T for %s", raw, name)
+	}
+	return 0, nil
+}
+
+func (c *configFileSettings) String(name string) (string, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(string); ok {
+			return v, nil
+		}
+		return "", fmt.Errorf("expected string found %T for %s", raw, name)
+	}
+	return "", nil
+}
+
+func (c *configFileSettings) StringSlice(name string) ([]string, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if slice, ok := raw.([]interface{}); ok {
+			strSlice := make([]string, len(slice))
+			for i, v := range slice {
+				str, ok := v.(string)
+				if !ok {
+					return nil, fmt.Errorf("expected string, found %T for %v", i, v)
+				}
+				strSlice[i] = str
+			}
+			return strSlice, nil
+		}
+		return nil, fmt.Errorf("expected string slice found %T for %s", raw, name)
+	}
+	return nil, nil
+}
+
+func (c *configFileSettings) IntSlice(name string) ([]int, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if slice, ok := raw.([]interface{}); ok {
+			intSlice := make([]int, len(slice))
+			for i, v := range slice {
+				str, ok := v.(int)
+				if !ok {
+					return nil, fmt.Errorf("expected int, found %T for %v ", v, v)
+				}
+				intSlice[i] = str
+			}
+			return intSlice, nil
+		}
+		if v, ok := raw.([]int); ok {
+			return v, nil
+		}
+		return nil, fmt.Errorf("expected int slice found %T for %s", raw, name)
+	}
+	return nil, nil
+}
+
+func (c *configFileSettings) Generic(name string) (cli.Generic, error) {
+	return nil, errors.New("option type Generic not supported")
+}
+
+func (c *configFileSettings) Bool(name string) (bool, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(bool); ok {
+			return v, nil
+		}
+		return false, fmt.Errorf("expected boolean found %T for %s", raw, name)
+	}
+	return false, nil
+}
+
+var configuration configFileSettings
+
+func GetConfiguration() *Configuration {
+	return &configuration.Configuration
+}
+
+// ReadConfigFile returns InputSourceContext initialized from the configuration file.
+// On repeat calls returns with the same file, returns without reading the file again; however,
+// if value of "config" flag changes, will read the new config file
+func ReadConfigFile(c *cli.Context, log logger.Service) (*configFileSettings, error) {
+	configFile := c.String("config")
+	if configuration.Source() == configFile || configFile == "" {
+		if configuration.Source() == "" {
+			return nil, ErrNoConfigFile
+		}
+		return &configuration, nil
+	}
+
+	log.Debugf("Loading configuration from %s", configFile)
+	file, err := os.Open(configFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			err = ErrNoConfigFile
+		}
+		return nil, err
+	}
+	defer file.Close()
+	if err := yaml.NewDecoder(file).Decode(&configuration); err != nil {
+		return nil, errors.Wrap(err, "error parsing YAML in config file at "+configFile)
+	}
+	configuration.sourceFile = configFile
+	return &configuration, nil
+}

cmd/cloudflared/config/configuration_test.go

@@ -0,0 +1,76 @@
+package config
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v2"
+)
+
+func TestConfigFileSettings(t *testing.T) {
+	var (
+		firstIngress = UnvalidatedIngressRule{
+			Hostname: "tunnel1.example.com",
+			Path:     "/id",
+			Service:  "https://localhost:8000",
+		}
+		secondIngress = UnvalidatedIngressRule{
+			Hostname: "*",
+			Path:     "",
+			Service:  "https://localhost:8001",
+		}
+	)
+	rawYAML := `
+tunnel: config-file-test
+ingress:
+ - hostname: tunnel1.example.com
+   path: /id
+   service: https://localhost:8000
+ - hostname: "*"
+   service: https://localhost:8001
+retries: 5
+grace-period: 30s
+percentage: 3.14
+hostname: example.com
+tag:
+ - test
+ - central-1
+counters:
+ - 123
+ - 456
+`
+	var config configFileSettings
+	err := yaml.Unmarshal([]byte(rawYAML), &config)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "config-file-test", config.TunnelID)
+	assert.Equal(t, firstIngress, config.Ingress[0])
+	assert.Equal(t, secondIngress, config.Ingress[1])
+
+	retries, err := config.Int("retries")
+	assert.NoError(t, err)
+	assert.Equal(t, 5, retries)
+
+	gracePeriod, err := config.Duration("grace-period")
+	assert.NoError(t, err)
+	assert.Equal(t, time.Second*30, gracePeriod)
+
+	percentage, err := config.Float64("percentage")
+	assert.NoError(t, err)
+	assert.Equal(t, 3.14, percentage)
+
+	hostname, err := config.String("hostname")
+	assert.NoError(t, err)
+	assert.Equal(t, "example.com", hostname)
+
+	tags, err := config.StringSlice("tag")
+	assert.NoError(t, err)
+	assert.Equal(t, "test", tags[0])
+	assert.Equal(t, "central-1", tags[1])
+
+	counters, err := config.IntSlice("counters")
+	assert.NoError(t, err)
+	assert.Equal(t, 123, counters[0])
+	assert.Equal(t, 456, counters[1])
+}

cmd/cloudflared/config/manager.go

@@ -0,0 +1,106 @@
+package config
+
+import (
+	"os"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/watcher"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+)
+
+// Notifier sends out config updates
+type Notifier interface {
+	ConfigDidUpdate(Root)
+}
+
+// Manager is the base functions of the config manager
+type Manager interface {
+	Start(Notifier) error
+	Shutdown()
+}
+
+// FileManager watches the yaml config for changes
+// sends updates to the service to reconfigure to match the updated config
+type FileManager struct {
+	watcher    watcher.Notifier
+	notifier   Notifier
+	configPath string
+	logger     logger.Service
+	ReadConfig func(string) (Root, error)
+}
+
+// NewFileManager creates a config manager
+func NewFileManager(watcher watcher.Notifier, configPath string, logger logger.Service) (*FileManager, error) {
+	m := &FileManager{
+		watcher:    watcher,
+		configPath: configPath,
+		logger:     logger,
+		ReadConfig: readConfigFromPath,
+	}
+	err := watcher.Add(configPath)
+	return m, err
+}
+
+// Start starts the runloop to watch for config changes
+func (m *FileManager) Start(notifier Notifier) error {
+	m.notifier = notifier
+
+	// update the notifier with a fresh config on start
+	config, err := m.GetConfig()
+	if err != nil {
+		return err
+	}
+	notifier.ConfigDidUpdate(config)
+
+	m.watcher.Start(m)
+	return nil
+}
+
+// GetConfig reads the yaml file from the disk
+func (m *FileManager) GetConfig() (Root, error) {
+	return m.ReadConfig(m.configPath)
+}
+
+// Shutdown stops the watcher
+func (m *FileManager) Shutdown() {
+	m.watcher.Shutdown()
+}
+
+func readConfigFromPath(configPath string) (Root, error) {
+	if configPath == "" {
+		return Root{}, errors.New("unable to find config file")
+	}
+
+	file, err := os.Open(configPath)
+	if err != nil {
+		return Root{}, err
+	}
+	defer file.Close()
+
+	var config Root
+	if err := yaml.NewDecoder(file).Decode(&config); err != nil {
+		return Root{}, errors.Wrap(err, "error parsing YAML in config file at "+configPath)
+	}
+
+	return config, nil
+}
+
+// File change notifications from the watcher
+
+// WatcherItemDidChange triggers when the yaml config is updated
+// sends the updated config to the service to reload its state
+func (m *FileManager) WatcherItemDidChange(filepath string) {
+	config, err := m.GetConfig()
+	if err != nil {
+		m.logger.Errorf("Failed to read new config: %s", err)
+		return
+	}
+	m.logger.Info("Config file has been updated")
+	m.notifier.ConfigDidUpdate(config)
+}
+
+// WatcherDidError notifies of errors with the file watcher
+func (m *FileManager) WatcherDidError(err error) {
+	m.logger.Errorf("Config watcher encountered an error: %s", err)
+}

cmd/cloudflared/config/manager_test.go

@@ -0,0 +1,88 @@
+package config
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/watcher"
+)
+
+type mockNotifier struct {
+	configs []Root
+}
+
+func (n *mockNotifier) ConfigDidUpdate(c Root) {
+	n.configs = append(n.configs, c)
+}
+
+type mockFileWatcher struct {
+	path     string
+	notifier watcher.Notification
+	ready    chan struct{}
+}
+
+func (w *mockFileWatcher) Start(n watcher.Notification) {
+	w.notifier = n
+	w.ready <- struct{}{}
+}
+
+func (w *mockFileWatcher) Add(string) error {
+	return nil
+}
+
+func (w *mockFileWatcher) Shutdown() {
+
+}
+
+func (w *mockFileWatcher) TriggerChange() {
+	w.notifier.WatcherItemDidChange(w.path)
+}
+
+func TestConfigChanged(t *testing.T) {
+	filePath := "config.yaml"
+	f, err := os.Create(filePath)
+	assert.NoError(t, err)
+	defer func() {
+		f.Close()
+		os.Remove(filePath)
+	}()
+	c := &Root{
+		Forwarders: []Forwarder{
+			{
+				URL:      "test.daltoniam.com",
+				Listener: "127.0.0.1:8080",
+			},
+		},
+	}
+	configRead := func(configPath string) (Root, error) {
+		return *c, nil
+	}
+	wait := make(chan struct{})
+	w := &mockFileWatcher{path: filePath, ready: wait}
+
+	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
+
+	service, err := NewFileManager(w, filePath, logger)
+	service.ReadConfig = configRead
+	assert.NoError(t, err)
+
+	n := &mockNotifier{}
+	go service.Start(n)
+
+	<-wait
+	c.Forwarders = append(c.Forwarders, Forwarder{URL: "add.daltoniam.com", Listener: "127.0.0.1:8081"})
+	w.TriggerChange()
+
+	service.Shutdown()
+
+	assert.Len(t, n.configs, 2, "did not get 2 config updates as expected")
+	assert.Len(t, n.configs[0].Forwarders, 1, "not the amount of forwarders expected")
+	assert.Len(t, n.configs[1].Forwarders, 2, "not the amount of forwarders expected")
+
+	assert.Equal(t, n.configs[0].Forwarders[0].Hash(), c.Forwarders[0].Hash(), "forwarder hashes don't match")
+	assert.Equal(t, n.configs[1].Forwarders[0].Hash(), c.Forwarders[0].Hash(), "forwarder hashes don't match")
+	assert.Equal(t, n.configs[1].Forwarders[1].Hash(), c.Forwarders[1].Hash(), "forwarder hashes don't match")
+}

cmd/cloudflared/config/model.go

@@ -0,0 +1,101 @@
+package config
+
+import (
+	"crypto/md5"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// Forwarder represents a client side listener to forward traffic to the edge
+type Forwarder struct {
+	URL           string `json:"url"`
+	Listener      string `json:"listener"`
+	TokenClientID string `json:"service_token_id" yaml:"serviceTokenID"`
+	TokenSecret   string `json:"secret_token_id" yaml:"serviceTokenSecret"`
+	Destination   string `json:"destination"`
+}
+
+// Tunnel represents a tunnel that should be started
+type Tunnel struct {
+	URL          string `json:"url"`
+	Origin       string `json:"origin"`
+	ProtocolType string `json:"type"`
+}
+
+// DNSResolver represents a client side DNS resolver
+type DNSResolver struct {
+	Enabled    bool     `json:"enabled"`
+	Address    string   `json:"address,omitempty"`
+	Port       uint16   `json:"port,omitempty"`
+	Upstreams  []string `json:"upstreams,omitempty"`
+	Bootstraps []string `json:"bootstraps,omitempty"`
+}
+
+// Root is the base options to configure the service
+type Root struct {
+	LogDirectory string      `json:"log_directory" yaml:"logDirectory,omitempty"`
+	LogLevel     string      `json:"log_level" yaml:"logLevel,omitempty"`
+	Forwarders   []Forwarder `json:"forwarders,omitempty" yaml:"forwarders,omitempty"`
+	Tunnels      []Tunnel    `json:"tunnels,omitempty" yaml:"tunnels,omitempty"`
+	Resolver     DNSResolver `json:"resolver,omitempty" yaml:"resolver,omitempty"`
+}
+
+// Hash returns the computed values to see if the forwarder values change
+func (f *Forwarder) Hash() string {
+	h := md5.New()
+	io.WriteString(h, f.URL)
+	io.WriteString(h, f.Listener)
+	io.WriteString(h, f.TokenClientID)
+	io.WriteString(h, f.TokenSecret)
+	io.WriteString(h, f.Destination)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+// Hash returns the computed values to see if the forwarder values change
+func (r *DNSResolver) Hash() string {
+	h := md5.New()
+	io.WriteString(h, r.Address)
+	io.WriteString(h, strings.Join(r.Bootstraps, ","))
+	io.WriteString(h, strings.Join(r.Upstreams, ","))
+	io.WriteString(h, fmt.Sprintf("%d", r.Port))
+	io.WriteString(h, fmt.Sprintf("%v", r.Enabled))
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+// EnabledOrDefault returns the enabled property
+func (r *DNSResolver) EnabledOrDefault() bool {
+	return r.Enabled
+}
+
+// AddressOrDefault returns the address or returns the default if empty
+func (r *DNSResolver) AddressOrDefault() string {
+	if r.Address != "" {
+		return r.Address
+	}
+	return "localhost"
+}
+
+// PortOrDefault return the port or returns the default if 0
+func (r *DNSResolver) PortOrDefault() uint16 {
+	if r.Port > 0 {
+		return r.Port
+	}
+	return 53
+}
+
+// UpstreamsOrDefault returns the upstreams or returns the default if empty
+func (r *DNSResolver) UpstreamsOrDefault() []string {
+	if len(r.Upstreams) > 0 {
+		return r.Upstreams
+	}
+	return []string{"https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"}
+}
+
+// BootstrapsOrDefault returns the bootstraps or returns the default if empty
+func (r *DNSResolver) BootstrapsOrDefault() []string {
+	if len(r.Bootstraps) > 0 {
+		return r.Bootstraps
+	}
+	return []string{"https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"}
+}

cmd/cloudflared/generic_service.go

@@ -5,7 +5,7 @@ package main
 import (
 	"os"
 
-	cli "gopkg.in/urfave/cli.v2"
+	cli "github.com/urfave/cli/v2"
 )
 
 func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {

cmd/cloudflared/linux_service.go

@@ -7,8 +7,12 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/pkg/errors"
+	cli "github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	cli "gopkg.in/urfave/cli.v2"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
@@ -19,12 +23,18 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 			&cli.Command{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a system service",
-				Action: installLinuxService,
+				Action: cliutil.ErrorHandler(installLinuxService),
+				Flags: []cli.Flag{
+					&cli.BoolFlag{
+						Name:  "legacy",
+						Usage: "Generate service file for non-named tunnels",
+					},
+				},
 			},
 			&cli.Command{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel service",
-				Action: uninstallLinuxService,
+				Action: cliutil.ErrorHandler(uninstallLinuxService),
 			},
 		},
 	})
@@ -37,6 +47,7 @@ const (
 	serviceConfigDir      = "/etc/cloudflared"
 	serviceConfigFile     = "config.yml"
 	serviceCredentialFile = "cert.pem"
+	serviceConfigPath     = serviceConfigDir + "/" + serviceConfigFile
 )
 
 var systemdTemplates = []ServiceTemplate{
@@ -49,7 +60,7 @@ After=network.target
 [Service]
 TimeoutStartSec=0
 Type=notify
-ExecStart={{ .Path }} --config /etc/cloudflared/config.yml --origincert /etc/cloudflared/cert.pem --no-autoupdate
+ExecStart={{ .Path }} --config /etc/cloudflared/config.yml --no-autoupdate{{ range .ExtraArgs }} {{ . }}{{ end }}
 Restart=on-failure
 RestartSec=5s
 
@@ -64,7 +75,7 @@ Description=Update Argo Tunnel
 After=network.target
 
 [Service]
-ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 64 ]; then systemctl restart cloudflared; exit 0; fi; exit $code'
+ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 11 ]; then systemctl restart cloudflared; exit 0; fi; exit $code'
 `,
 	},
 	{
@@ -99,7 +110,7 @@ var sysvTemplate = ServiceTemplate{
 # Description:       Argo Tunnel agent
 ### END INIT INFO
 name=$(basename $(readlink -f $0))
-cmd="{{.Path}} --config /etc/cloudflared/config.yml --origincert /etc/cloudflared/cert.pem --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s"
+cmd="{{.Path}} --config /etc/cloudflared/config.yml --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s{{ range .ExtraArgs }} {{ . }}{{ end }}"
 pid_file="/var/run/$name.pid"
 stdout_log="/var/log/$name.log"
 stderr_log="/var/log/$name.err"
@@ -118,10 +129,6 @@ case "$1" in
             echo "Starting $name"
             $cmd >> "$stdout_log" 2>> "$stderr_log" &
             echo $! > "$pid_file"
-            if ! is_running; then
-                echo "Unable to start, see $stdout_log and $stderr_log"
-                exit 1
-            fi
         fi
     ;;
     stop)
@@ -182,77 +189,126 @@ func isSystemd() bool {
 	return false
 }
 
-func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile string) error {
-	if err := ensureConfigDirExists(serviceConfigDir); err != nil {
-		return err
-	}
+func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile string, logger logger.Service) error {
 	srcCredentialPath := filepath.Join(userConfigDir, userCredentialFile)
 	destCredentialPath := filepath.Join(serviceConfigDir, serviceCredentialFile)
-	if err := copyCredential(srcCredentialPath, destCredentialPath); err != nil {
-		return err
+	if srcCredentialPath != destCredentialPath {
+		if err := copyCredential(srcCredentialPath, destCredentialPath); err != nil {
+			return err
+		}
 	}
+
 	srcConfigPath := filepath.Join(userConfigDir, userConfigFile)
 	destConfigPath := filepath.Join(serviceConfigDir, serviceConfigFile)
-	if err := copyConfig(srcConfigPath, destConfigPath); err != nil {
-		return err
+	if srcConfigPath != destConfigPath {
+		if err := copyConfig(srcConfigPath, destConfigPath); err != nil {
+			return err
+		}
+		logger.Infof("Copied %s to %s", srcConfigPath, destConfigPath)
 	}
+
 	return nil
 }
 
 func installLinuxService(c *cli.Context) error {
+	logger, err := logger.New()
+	if err != nil {
+		return errors.Wrap(err, "error setting up logger")
+	}
+
 	etPath, err := os.Executable()
 	if err != nil {
 		return fmt.Errorf("error determining executable path: %v", err)
 	}
-	templateArgs := ServiceTemplateArgs{Path: etPath}
+	templateArgs := ServiceTemplateArgs{
+		Path: etPath,
+	}
 
-	userConfigDir := filepath.Dir(c.String("config"))
-	userConfigFile := filepath.Base(c.String("config"))
-	userCredentialFile := config.DefaultCredentialFile
-	if err = copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile); err != nil {
-		logger.WithError(err).Infof("Failed to copy user configuration. Before running the service, ensure that %s contains two files, %s and %s",
-			serviceConfigDir, serviceCredentialFile, serviceConfigFile)
+	if err := ensureConfigDirExists(serviceConfigDir); err != nil {
 		return err
 	}
+	if c.Bool("legacy") {
+		userConfigDir := filepath.Dir(c.String("config"))
+		userConfigFile := filepath.Base(c.String("config"))
+		userCredentialFile := config.DefaultCredentialFile
+		if err = copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile, logger); err != nil {
+			logger.Errorf("Failed to copy user configuration: %s. Before running the service, ensure that %s contains two files, %s and %s", err,
+				serviceConfigDir, serviceCredentialFile, serviceConfigFile)
+			return err
+		}
+		templateArgs.ExtraArgs = []string{
+			"--origincert", serviceConfigDir + "/" + serviceCredentialFile,
+		}
+	} else {
+		src, err := config.ReadConfigFile(c, logger)
+		if err != nil {
+			return err
+		}
+
+		// can't use context because this command doesn't define "credentials-file" flag
+		configPresent := func(s string) bool {
+			val, err := src.String(s)
+			return err == nil && val != ""
+		}
+		if src.TunnelID == "" || !configPresent("credentials-file") {
+			return fmt.Errorf(`Configuration file %s must contain entries for the tunnel to run and its associated credentials:
+tunnel: TUNNEL-UUID
+credentials-file: CREDENTIALS-FILE
+`, src.Source())
+		}
+		if src.Source() != serviceConfigPath {
+			if exists, err := config.FileExists(serviceConfigPath); err != nil || exists {
+				return fmt.Errorf("Possible conflicting configuration in %[1]s and %[2]s. Either remove %[2]s or run `cloudflared --config %[2]s service install`", src.Source(), serviceConfigPath)
+			}
+
+			if err := copyFile(src.Source(), serviceConfigPath); err != nil {
+				return fmt.Errorf("failed to copy %s to %s: %w", src.Source(), serviceConfigPath, err)
+			}
+		}
+
+		templateArgs.ExtraArgs = []string{
+			"tunnel", "run",
+		}
+	}
 
 	switch {
 	case isSystemd():
 		logger.Infof("Using Systemd")
-		return installSystemd(&templateArgs)
+		return installSystemd(&templateArgs, logger)
 	default:
-		logger.Infof("Using Sysv")
-		return installSysv(&templateArgs)
+		logger.Infof("Using SysV")
+		return installSysv(&templateArgs, logger)
 	}
 }
 
-func installSystemd(templateArgs *ServiceTemplateArgs) error {
+func installSystemd(templateArgs *ServiceTemplateArgs, logger logger.Service) error {
 	for _, serviceTemplate := range systemdTemplates {
 		err := serviceTemplate.Generate(templateArgs)
 		if err != nil {
-			logger.WithError(err).Infof("error generating service template")
+			logger.Errorf("error generating service template: %s", err)
 			return err
 		}
 	}
 	if err := runCommand("systemctl", "enable", "cloudflared.service"); err != nil {
-		logger.WithError(err).Infof("systemctl enable cloudflared.service error")
+		logger.Errorf("systemctl enable cloudflared.service error: %s", err)
 		return err
 	}
 	if err := runCommand("systemctl", "start", "cloudflared-update.timer"); err != nil {
-		logger.WithError(err).Infof("systemctl start cloudflared-update.timer error")
+		logger.Errorf("systemctl start cloudflared-update.timer error: %s", err)
 		return err
 	}
 	logger.Infof("systemctl daemon-reload")
 	return runCommand("systemctl", "daemon-reload")
 }
 
-func installSysv(templateArgs *ServiceTemplateArgs) error {
+func installSysv(templateArgs *ServiceTemplateArgs, logger logger.Service) error {
 	confPath, err := sysvTemplate.ResolvePath()
 	if err != nil {
-		logger.WithError(err).Infof("error resolving system path")
+		logger.Errorf("error resolving system path: %s", err)
 		return err
 	}
 	if err := sysvTemplate.Generate(templateArgs); err != nil {
-		logger.WithError(err).Infof("error generating system template")
+		logger.Errorf("error generating system template: %s", err)
 		return err
 	}
 	for _, i := range [...]string{"2", "3", "4", "5"} {
@@ -269,28 +325,33 @@ func installSysv(templateArgs *ServiceTemplateArgs) error {
 }
 
 func uninstallLinuxService(c *cli.Context) error {
+	logger, err := logger.New()
+	if err != nil {
+		return errors.Wrap(err, "error setting up logger")
+	}
+
 	switch {
 	case isSystemd():
 		logger.Infof("Using Systemd")
-		return uninstallSystemd()
+		return uninstallSystemd(logger)
 	default:
-		logger.Infof("Using Sysv")
-		return uninstallSysv()
+		logger.Infof("Using SysV")
+		return uninstallSysv(logger)
 	}
 }
 
-func uninstallSystemd() error {
+func uninstallSystemd(logger logger.Service) error {
 	if err := runCommand("systemctl", "disable", "cloudflared.service"); err != nil {
-		logger.WithError(err).Infof("systemctl disable cloudflared.service error")
+		logger.Errorf("systemctl disable cloudflared.service error: %s", err)
 		return err
 	}
 	if err := runCommand("systemctl", "stop", "cloudflared-update.timer"); err != nil {
-		logger.WithError(err).Infof("systemctl stop cloudflared-update.timer error")
+		logger.Errorf("systemctl stop cloudflared-update.timer error: %s", err)
 		return err
 	}
 	for _, serviceTemplate := range systemdTemplates {
 		if err := serviceTemplate.Remove(); err != nil {
-			logger.WithError(err).Infof("error removing service template")
+			logger.Errorf("error removing service template: %s", err)
 			return err
 		}
 	}
@@ -298,9 +359,9 @@ func uninstallSystemd() error {
 	return nil
 }
 
-func uninstallSysv() error {
+func uninstallSysv(logger logger.Service) error {
 	if err := sysvTemplate.Remove(); err != nil {
-		logger.WithError(err).Infof("error removing service template")
+		logger.Errorf("error removing service template: %s", err)
 		return err
 	}
 	for _, i := range [...]string{"2", "3", "4", "5"} {

cmd/cloudflared/macos_service.go

@@ -6,8 +6,10 @@ import (
 	"fmt"
 	"os"
 
-	"gopkg.in/urfave/cli.v2"
+	"github.com/urfave/cli/v2"
 
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/pkg/errors"
 )
 
@@ -23,12 +25,12 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 			{
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as an user launch agent",
-				Action: installLaunchd,
+				Action: cliutil.ErrorHandler(installLaunchd),
 			},
 			{
 				Name:   "uninstall",
 				Usage:  "Uninstall the Argo Tunnel launch agent",
-				Action: uninstallLaunchd,
+				Action: cliutil.ErrorHandler(uninstallLaunchd),
 			},
 		},
 	})
@@ -60,7 +62,7 @@ func newLaunchdTemplate(installPath, stdoutPath, stderrPath string) *ServiceTemp
 			<false/>
 		</dict>
 		<key>ThrottleInterval</key>
-		<integer>20</integer>
+		<integer>5</integer>
 	</dict>
 </plist>`, launchdIdentifier, stdoutPath, stderrPath),
 	}
@@ -105,6 +107,11 @@ func stderrPath() (string, error) {
 }
 
 func installLaunchd(c *cli.Context) error {
+	logger, err := logger.New()
+	if err != nil {
+		return errors.Wrap(err, "error setting up logger")
+	}
+
 	if isRootUser() {
 		logger.Infof("Installing Argo Tunnel client as a system launch daemon. " +
 			"Argo Tunnel client will run at boot")
@@ -116,35 +123,38 @@ func installLaunchd(c *cli.Context) error {
 	}
 	etPath, err := os.Executable()
 	if err != nil {
-		logger.WithError(err).Errorf("Error determining executable path")
+		logger.Errorf("Error determining executable path: %s", err)
 		return fmt.Errorf("Error determining executable path: %v", err)
 	}
 	installPath, err := installPath()
 	if err != nil {
+		logger.Errorf("Error determining install path: %s", err)
 		return errors.Wrap(err, "Error determining install path")
 	}
 	stdoutPath, err := stdoutPath()
 	if err != nil {
+		logger.Errorf("error determining stdout path: %s", err)
 		return errors.Wrap(err, "error determining stdout path")
 	}
 	stderrPath, err := stderrPath()
 	if err != nil {
+		logger.Errorf("error determining stderr path: %s", err)
 		return errors.Wrap(err, "error determining stderr path")
 	}
 	launchdTemplate := newLaunchdTemplate(installPath, stdoutPath, stderrPath)
 	if err != nil {
-		logger.WithError(err).Errorf("error creating launchd template")
+		logger.Errorf("error creating launchd template: %s", err)
 		return errors.Wrap(err, "error creating launchd template")
 	}
 	templateArgs := ServiceTemplateArgs{Path: etPath}
 	err = launchdTemplate.Generate(&templateArgs)
 	if err != nil {
-		logger.WithError(err).Errorf("error generating launchd template")
+		logger.Errorf("error generating launchd template: %s", err)
 		return err
 	}
 	plistPath, err := launchdTemplate.ResolvePath()
 	if err != nil {
-		logger.WithError(err).Infof("error resolving launchd template path")
+		logger.Errorf("error resolving launchd template path: %s", err)
 		return err
 	}
 
@@ -153,6 +163,11 @@ func installLaunchd(c *cli.Context) error {
 }
 
 func uninstallLaunchd(c *cli.Context) error {
+	logger, err := logger.New()
+	if err != nil {
+		return errors.Wrap(err, "error setting up logger")
+	}
+
 	if isRootUser() {
 		logger.Infof("Uninstalling Argo Tunnel as a system launch daemon")
 	} else {
@@ -176,12 +191,12 @@ func uninstallLaunchd(c *cli.Context) error {
 	}
 	plistPath, err := launchdTemplate.ResolvePath()
 	if err != nil {
-		logger.WithError(err).Infof("error resolving launchd template path")
+		logger.Errorf("error resolving launchd template path: %s", err)
 		return err
 	}
 	err = runCommand("launchctl", "unload", plistPath)
 	if err != nil {
-		logger.WithError(err).Infof("error unloading")
+		logger.Errorf("error unloading: %s", err)
 		return err
 	}
 

cmd/cloudflared/main.go

@@ -6,14 +6,19 @@ import (
 	"time"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
-	"github.com/cloudflare/cloudflared/log"
+	log "github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
+	"github.com/cloudflare/cloudflared/overwatch"
+	"github.com/cloudflare/cloudflared/tunneldns"
+	"github.com/cloudflare/cloudflared/watcher"
 
 	raven "github.com/getsentry/raven-go"
 	homedir "github.com/mitchellh/go-homedir"
-	cli "gopkg.in/urfave/cli.v2"
+	cli "github.com/urfave/cli/v2"
 
 	"github.com/pkg/errors"
 )
@@ -25,7 +30,6 @@ const (
 var (
 	Version   = "DEV"
 	BuildTime = "unknown"
-	logger    = log.CreateLogger()
 	// Mostly network errors that we don't want reported back to Sentry, this is done by substring match.
 	ignoredErrors = []string{
 		"connection reset by peer",
@@ -59,7 +63,7 @@ func main() {
 	app := &cli.App{}
 	app.Name = "cloudflared"
 	app.Usage = "Cloudflare's command-line tool and agent"
-	app.ArgsUsage = "origin-url"
+	app.UsageText = "cloudflared [global options] [command] [command options]"
 	app.Copyright = fmt.Sprintf(
 		`(c) %d Cloudflare Inc.
    Your installation of cloudflared software constitutes a symbol of your signature indicating that you accept
@@ -73,21 +77,42 @@ func main() {
 	and configure access control.`
 	app.Flags = flags()
 	app.Action = action(Version, shutdownC, graceShutdownC)
-	app.Before = tunnel.Before
+	app.Before = tunnel.SetFlagsFromConfigFile
 	app.Commands = commands(cli.ShowVersion)
 
 	tunnel.Init(Version, shutdownC, graceShutdownC) // we need this to support the tunnel sub command...
 	access.Init(shutdownC, graceShutdownC)
+	updater.Init(Version)
 	runApp(app, shutdownC, graceShutdownC)
 }
 
 func commands(version func(c *cli.Context)) []*cli.Command {
 	cmds := []*cli.Command{
 		{
-			Name:      "update",
-			Action:    updater.Update,
-			Usage:     "Update the agent if a new version exists",
-			ArgsUsage: " ",
+			Name:   "update",
+			Action: cliutil.ErrorHandler(updater.Update),
+			Usage:  "Update the agent if a new version exists",
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:  "beta",
+					Usage: "specify if you wish to update to the latest beta version",
+				},
+				&cli.BoolFlag{
+					Name:   "force",
+					Usage:  "specify if you wish to force an upgrade to the latest version regardless of the current version",
+					Hidden: true,
+				},
+				&cli.BoolFlag{
+					Name:   "staging",
+					Usage:  "specify if you wish to use the staging url for updating",
+					Hidden: true,
+				},
+				&cli.StringFlag{
+					Name:   "version",
+					Usage:  "specify a version you wish to upgrade or downgrade to",
+					Hidden: false,
+				},
+			},
 			Description: `Looks for a new version on the official download server.
 If a new version exists, updates the agent binary and quits.
 Otherwise, does nothing.
@@ -105,6 +130,7 @@ To determine if an update happened in a script, check for error code 64.`,
 		},
 	}
 	cmds = append(cmds, tunnel.Commands()...)
+	cmds = append(cmds, tunneldns.Command(false))
 	cmds = append(cmds, access.Commands()...)
 	return cmds
 }
@@ -119,23 +145,19 @@ func isEmptyInvocation(c *cli.Context) bool {
 }
 
 func action(version string, shutdownC, graceShutdownC chan struct{}) cli.ActionFunc {
-	return func(c *cli.Context) (err error) {
+	return cliutil.ErrorHandler(func(c *cli.Context) (err error) {
 		if isEmptyInvocation(c) {
-			cli.ShowAppHelpAndExit(c, 1)
+			return handleServiceMode(shutdownC)
 		}
 		tags := make(map[string]string)
 		tags["hostname"] = c.String("hostname")
 		raven.SetTagsContext(tags)
-		raven.CapturePanic(func() { err = tunnel.StartServer(c, version, shutdownC, graceShutdownC) }, nil)
-		exitCode := 0
+		raven.CapturePanic(func() { err = tunnel.TunnelCommand(c) }, nil)
 		if err != nil {
-			handleError(err)
-			exitCode = 1
+			captureError(err)
 		}
-		// we already handle error printing, so we pass an empty string so we
-		// don't have to print again.
-		return cli.Exit("", exitCode)
-	}
+		return err
+	})
 }
 
 func userHomeDir() (string, error) {
@@ -145,14 +167,13 @@ func userHomeDir() (string, error) {
 	// use with sudo.
 	homeDir, err := homedir.Dir()
 	if err != nil {
-		logger.WithError(err).Error("Cannot determine home directory for the user")
 		return "", errors.Wrap(err, "Cannot determine home directory for the user")
 	}
 	return homeDir, nil
 }
 
 // In order to keep the amount of noise sent to Sentry low, typical network errors can be filtered out here by a substring match.
-func handleError(err error) {
+func captureError(err error) {
 	errorMessage := err.Error()
 	for _, ignoredErrorMessage := range ignoredErrors {
 		if strings.Contains(errorMessage, ignoredErrorMessage) {
@@ -161,3 +182,44 @@ func handleError(err error) {
 	}
 	raven.CaptureError(err, nil)
 }
+
+// cloudflared was started without any flags
+func handleServiceMode(shutdownC chan struct{}) error {
+	defer log.SharedWriteManager.Shutdown()
+	logDirectory, logLevel := config.FindLogSettings()
+
+	logger, err := log.New(log.DefaultFile(logDirectory), log.LogLevelString(logLevel))
+	if err != nil {
+		return cliutil.PrintLoggerSetupError("error setting up logger", err)
+	}
+	logger.Infof("logging to directory: %s", logDirectory)
+
+	// start the main run loop that reads from the config file
+	f, err := watcher.NewFile()
+	if err != nil {
+		logger.Errorf("Cannot load config file: %s", err)
+		return err
+	}
+
+	configPath := config.FindOrCreateConfigPath()
+	configManager, err := config.NewFileManager(f, configPath, logger)
+	if err != nil {
+		logger.Errorf("Cannot setup config file for monitoring: %s", err)
+		return err
+	}
+	logger.Infof("monitoring config file at: %s", configPath)
+
+	serviceCallback := func(t string, name string, err error) {
+		if err != nil {
+			logger.Errorf("%s service: %s encountered an error: %s", t, name, err)
+		}
+	}
+	serviceManager := overwatch.NewAppManager(serviceCallback)
+
+	appService := NewAppService(configManager, serviceManager, shutdownC, logger)
+	if err := appService.Run(); err != nil {
+		logger.Errorf("Failed to start app service: %s", err)
+		return err
+	}
+	return nil
+}

cmd/cloudflared/path/path.go

@@ -13,7 +13,7 @@ import (
 
 // GenerateFilePathFromURL will return a filepath for given access application url
 func GenerateFilePathFromURL(url *url.URL, suffix string) (string, error) {
-	configPath, err := homedir.Expand(config.DefaultConfigDirs[0])
+	configPath, err := homedir.Expand(config.DefaultConfigSearchDirectories()[0])
 	if err != nil {
 		return "", err
 	}

cmd/cloudflared/service_template.go

@@ -10,8 +10,9 @@ import (
 	"os/exec"
 	"text/template"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/mitchellh/go-homedir"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 )
 
 type ServiceTemplate struct {
@@ -21,7 +22,8 @@ type ServiceTemplate struct {
 }
 
 type ServiceTemplateArgs struct {
-	Path string
+	Path      string
+	ExtraArgs []string
 }
 
 func (st *ServiceTemplate) ResolvePath() (string, error) {
@@ -73,21 +75,16 @@ func runCommand(command string, args ...string) error {
 	cmd := exec.Command(command, args...)
 	stderr, err := cmd.StderrPipe()
 	if err != nil {
-		logger.WithError(err).Infof("error getting stderr pipe")
 		return fmt.Errorf("error getting stderr pipe: %v", err)
 	}
 	err = cmd.Start()
 	if err != nil {
-		logger.WithError(err).Infof("error starting %s", command)
 		return fmt.Errorf("error starting %s: %v", command, err)
 	}
-	commandErr, _ := ioutil.ReadAll(stderr)
-	if len(commandErr) > 0 {
-		logger.Errorf("%s: %s", command, commandErr)
-	}
+
+	_, _ = ioutil.ReadAll(stderr)
 	err = cmd.Wait()
 	if err != nil {
-		logger.WithError(err).Infof("%s returned error", command)
 		return fmt.Errorf("%s returned with error: %v", command, err)
 	}
 	return nil
@@ -144,12 +141,38 @@ func copyCredential(srcCredentialPath, destCredentialPath string) error {
 	return nil
 }
 
+func copyFile(src, dest string) error {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+
+	destFile, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	ok := false
+	defer func() {
+		destFile.Close()
+		if !ok {
+			_ = os.Remove(dest)
+		}
+	}()
+
+	if _, err := io.Copy(destFile, srcFile); err != nil {
+		return err
+	}
+
+	ok = true
+	return nil
+}
+
 func copyConfig(srcConfigPath, destConfigPath string) error {
 	// Copy or create config
 	destFile, exists, err := openFile(destConfigPath, true)
 	if err != nil {
-		logger.WithError(err).Infof("cannot open %s", destConfigPath)
-		return err
+		return fmt.Errorf("cannot open %s with error: %s", destConfigPath, err)
 	} else if exists {
 		// config already exists, do nothing
 		return nil
@@ -173,7 +196,6 @@ func copyConfig(srcConfigPath, destConfigPath string) error {
 		if err != nil {
 			return fmt.Errorf("unable to copy %s to %s: %v", srcConfigPath, destConfigPath, err)
 		}
-		logger.Infof("Copied %s to %s", srcConfigPath, destConfigPath)
 	}
 
 	return nil

cmd/cloudflared/token/token.go

@@ -2,17 +2,19 @@ package token
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/url"
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/path"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
-	"github.com/cloudflare/cloudflared/log"
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/coreos/go-oidc/jose"
 )
@@ -21,8 +23,6 @@ const (
 	keyName = "token"
 )
 
-var logger = log.CreateLogger()
-
 type lock struct {
 	lockFilePath string
 	backoff      *origin.BackoffHandler
@@ -34,6 +34,21 @@ type signalHandler struct {
 	signals    []os.Signal
 }
 
+type jwtPayload struct {
+	Aud   []string `json:"aud"`
+	Email string   `json:"email"`
+	Exp   int      `json:"exp"`
+	Iat   int      `json:"iat"`
+	Nbf   int      `json:"nbf"`
+	Iss   string   `json:"iss"`
+	Type  string   `json:"type"`
+	Subt  string   `json:"sub"`
+}
+
+func (p jwtPayload) isExpired() bool {
+	return int(time.Now().Unix()) > p.Exp
+}
+
 func (s *signalHandler) register(handler func()) {
 	s.sigChannel = make(chan os.Signal, 1)
 	signal.Notify(s.sigChannel, s.signals...)
@@ -112,8 +127,20 @@ func isTokenLocked(lockFilePath string) bool {
 	return exists && err == nil
 }
 
+// FetchTokenWithRedirect will either load a stored token or generate a new one
+// it appends the full url as the redirect URL to the access cli request if opening the browser
+func FetchTokenWithRedirect(appURL *url.URL, logger logger.Service) (string, error) {
+	return getToken(appURL, false, logger)
+}
+
 // FetchToken will either load a stored token or generate a new one
-func FetchToken(appURL *url.URL) (string, error) {
+// it appends the host of the appURL as the redirect URL to the access cli request if opening the browser
+func FetchToken(appURL *url.URL, logger logger.Service) (string, error) {
+	return getToken(appURL, true, logger)
+}
+
+// getToken will either load a stored token or generate a new one
+func getToken(appURL *url.URL, useHostOnly bool, logger logger.Service) (string, error) {
 	if token, err := GetTokenIfExists(appURL); token != "" && err == nil {
 		return token, nil
 	}
@@ -139,7 +166,7 @@ func FetchToken(appURL *url.URL) (string, error) {
 	// this weird parameter is the resource name (token) and the key/value
 	// we want to send to the transfer service. the key is token and the value
 	// is blank (basically just the id generated in the transfer service)
-	token, err := transfer.Run(appURL, keyName, keyName, "", path, true)
+	token, err := transfer.Run(appURL, keyName, keyName, "", path, true, useHostOnly, logger)
 	if err != nil {
 		return "", err
 	}
@@ -147,7 +174,7 @@ func FetchToken(appURL *url.URL) (string, error) {
 	return string(token), nil
 }
 
-// GetTokenIfExists will return the token from local storage if it exists
+// GetTokenIfExists will return the token from local storage if it exists and not expired
 func GetTokenIfExists(url *url.URL) (string, error) {
 	path, err := path.GenerateFilePathFromURL(url, keyName)
 	if err != nil {
@@ -162,6 +189,17 @@ func GetTokenIfExists(url *url.URL) (string, error) {
 		return "", err
 	}
 
+	var payload jwtPayload
+	err = json.Unmarshal(token.Payload, &payload)
+	if err != nil {
+		return "", err
+	}
+
+	if payload.isExpired() {
+		err := os.Remove(path)
+		return "", err
+	}
+
 	return token.Encode(), nil
 }
 

cmd/cloudflared/transfer/transfer.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/encrypter"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
-	"github.com/cloudflare/cloudflared/log"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
@@ -22,20 +22,18 @@ const (
 	clientTimeout = time.Second * 60
 )
 
-var logger = log.CreateLogger()
-
 // Run does the transfer "dance" with the end result downloading the supported resource.
 // The expanded description is run is encapsulation of shared business logic needed
 // to request a resource (token/cert/etc) from the transfer service (loginhelper).
 // The "dance" we refer to is building a HTTP request, opening that in a browser waiting for
 // the user to complete an action, while it long polls in the background waiting for an
 // action to be completed to download the resource.
-func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncrypt bool) ([]byte, error) {
+func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncrypt bool, useHostOnly bool, logger logger.Service) ([]byte, error) {
 	encrypterClient, err := encrypter.New("cloudflared_priv.pem", "cloudflared_pub.pem")
 	if err != nil {
 		return nil, err
 	}
-	requestURL, err := buildRequestURL(transferURL, key, value+encrypterClient.PublicKey(), shouldEncrypt)
+	requestURL, err := buildRequestURL(transferURL, key, value+encrypterClient.PublicKey(), shouldEncrypt, useHostOnly)
 	if err != nil {
 		return nil, err
 	}
@@ -51,7 +49,7 @@ func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncr
 	var resourceData []byte
 
 	if shouldEncrypt {
-		buf, key, err := transferRequest(baseStoreURL + "transfer/" + encrypterClient.PublicKey())
+		buf, key, err := transferRequest(baseStoreURL+"transfer/"+encrypterClient.PublicKey(), logger)
 		if err != nil {
 			return nil, err
 		}
@@ -67,7 +65,7 @@ func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncr
 
 		resourceData = decrypted
 	} else {
-		buf, _, err := transferRequest(baseStoreURL + encrypterClient.PublicKey())
+		buf, _, err := transferRequest(baseStoreURL+encrypterClient.PublicKey(), logger)
 		if err != nil {
 			return nil, err
 		}
@@ -84,32 +82,34 @@ func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncr
 // BuildRequestURL creates a request suitable for a resource transfer.
 // it will return a constructed url based off the base url and query key/value provided.
 // cli will build a url for cli transfer request.
-func buildRequestURL(baseURL *url.URL, key, value string, cli bool) (string, error) {
+func buildRequestURL(baseURL *url.URL, key, value string, cli, useHostOnly bool) (string, error) {
 	q := baseURL.Query()
 	q.Set(key, value)
 	baseURL.RawQuery = q.Encode()
+	if useHostOnly {
+		baseURL.Path = ""
+	}
 	if !cli {
 		return baseURL.String(), nil
 	}
-
-	q.Set("redirect_url", baseURL.String()) // we add the token as a query param on both the redirect_url
+	q.Set("redirect_url", baseURL.String()) // we add the token as a query param on both the redirect_url and the main url
 	baseURL.RawQuery = q.Encode()           // and this actual baseURL.
 	baseURL.Path = "cdn-cgi/access/cli"
 	return baseURL.String(), nil
 }
 
 // transferRequest downloads the requested resource from the request URL
-func transferRequest(requestURL string) ([]byte, string, error) {
+func transferRequest(requestURL string, logger logger.Service) ([]byte, string, error) {
 	client := &http.Client{Timeout: clientTimeout}
 	const pollAttempts = 10
 	// we do "long polling" on the endpoint to get the resource.
 	for i := 0; i < pollAttempts; i++ {
-		buf, key, err := poll(client, requestURL)
+		buf, key, err := poll(client, requestURL, logger)
 		if err != nil {
 			return nil, "", err
 		} else if len(buf) > 0 {
 			if err := putSuccess(client, requestURL); err != nil {
-				logger.WithError(err).Error("Failed to update resource success")
+				logger.Errorf("Failed to update resource success: %s", err)
 			}
 			return buf, key, nil
 		}
@@ -118,7 +118,7 @@ func transferRequest(requestURL string) ([]byte, string, error) {
 }
 
 // poll the endpoint for the request resource, waiting for the user interaction
-func poll(client *http.Client, requestURL string) ([]byte, string, error) {
+func poll(client *http.Client, requestURL string, logger logger.Service) ([]byte, string, error) {
 	resp, err := client.Get(requestURL)
 	if err != nil {
 		return nil, "", err

cmd/cloudflared/tunnel/configuration.go

@@ -1,20 +1,21 @@
 package tunnel
 
 import (
-	"context"
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
-	"net"
-	"net/http"
 	"os"
 	"path/filepath"
 	"strings"
-	"time"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -23,9 +24,8 @@ import (
 	"github.com/google/uuid"
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
 	"golang.org/x/crypto/ssh/terminal"
-	"gopkg.in/urfave/cli.v2"
 )
 
 var (
@@ -35,10 +35,10 @@ var (
 	argumentsUrl    = developerPortal + "/reference/arguments/"
 )
 
-// returns the first path that contains a cert.pem file. If none of the DefaultConfigDirs
+// returns the first path that contains a cert.pem file. If none of the DefaultConfigSearchDirectories
 // contains a cert.pem file, return empty string
 func findDefaultOriginCertPath() string {
-	for _, defaultConfigDir := range config.DefaultConfigDirs {
+	for _, defaultConfigDir := range config.DefaultConfigSearchDirectories() {
 		originCertPath, _ := homedir.Expand(filepath.Join(defaultConfigDir, config.DefaultCredentialFile))
 		if ok, _ := config.FileExists(originCertPath); ok {
 			return originCertPath
@@ -47,25 +47,16 @@ func findDefaultOriginCertPath() string {
 	return ""
 }
 
-func generateRandomClientID(logger *logrus.Logger) (string, error) {
+func generateRandomClientID(logger logger.Service) (string, error) {
 	u, err := uuid.NewRandom()
 	if err != nil {
-		logger.WithError(err).Error("couldn't create UUID for client ID")
+		logger.Errorf("couldn't create UUID for client ID %s", err)
 		return "", err
 	}
 	return u.String(), nil
 }
 
-func handleDeprecatedOptions(c *cli.Context) error {
-	// Fail if the user provided an old authentication method
-	if c.IsSet("api-key") || c.IsSet("api-email") || c.IsSet("api-ca-key") {
-		logger.Error("You don't need to give us your api-key anymore. Please use the new login method. Just run cloudflared login")
-		return fmt.Errorf("Client provided deprecated options")
-	}
-	return nil
-}
-
-func logClientOptions(c *cli.Context) {
+func logClientOptions(c *cli.Context, logger logger.Service) {
 	flags := make(map[string]interface{})
 	for _, flag := range c.LocalFlagNames() {
 		flags[flag] = c.Generic(flag)
@@ -79,7 +70,7 @@ func logClientOptions(c *cli.Context) {
 	}
 
 	if len(flags) > 0 {
-		logger.WithFields(flags).Info("Flags")
+		logger.Infof("Environment variables %v", flags)
 	}
 
 	envs := make(map[string]string)
@@ -102,27 +93,29 @@ func dnsProxyStandAlone(c *cli.Context) bool {
 	return c.IsSet("proxy-dns") && (!c.IsSet("hostname") && !c.IsSet("tag") && !c.IsSet("hello-world"))
 }
 
-func getOriginCert(c *cli.Context) ([]byte, error) {
-	if c.String("origincert") == "" {
-		logger.Warnf("Cannot determine default origin certificate path. No file %s in %v", config.DefaultCredentialFile, config.DefaultConfigDirs)
+func findOriginCert(c *cli.Context, logger logger.Service) (string, error) {
+	originCertPath := c.String("origincert")
+	if originCertPath == "" {
+		logger.Infof("Cannot determine default origin certificate path. No file %s in %v", config.DefaultCredentialFile, config.DefaultConfigSearchDirectories())
 		if isRunningFromTerminal() {
 			logger.Errorf("You need to specify the origin certificate path with --origincert option, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", argumentsUrl)
-			return nil, fmt.Errorf("Client didn't specify origincert path when running from terminal")
+			return "", fmt.Errorf("Client didn't specify origincert path when running from terminal")
 		} else {
 			logger.Errorf("You need to specify the origin certificate path by specifying the origincert option in the configuration file, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", serviceUrl)
-			return nil, fmt.Errorf("Client didn't specify origincert path")
+			return "", fmt.Errorf("Client didn't specify origincert path")
 		}
 	}
-	// Check that the user has acquired a certificate using the login command
-	originCertPath, err := homedir.Expand(c.String("origincert"))
+	var err error
+	originCertPath, err = homedir.Expand(originCertPath)
 	if err != nil {
-		logger.WithError(err).Errorf("Cannot resolve path %s", c.String("origincert"))
-		return nil, fmt.Errorf("Cannot resolve path %s", c.String("origincert"))
+		logger.Errorf("Cannot resolve path %s: %s", originCertPath, err)
+		return "", fmt.Errorf("Cannot resolve path %s", originCertPath)
 	}
+	// Check that the user has acquired a certificate using the login command
 	ok, err := config.FileExists(originCertPath)
 	if err != nil {
-		logger.Errorf("Cannot check if origin cert exists at path %s", c.String("origincert"))
-		return nil, fmt.Errorf("Cannot check if origin cert exists at path %s", c.String("origincert"))
+		logger.Errorf("Cannot check if origin cert exists at path %s", originCertPath)
+		return "", fmt.Errorf("Cannot check if origin cert exists at path %s", originCertPath)
 	}
 	if !ok {
 		logger.Errorf(`Cannot find a valid certificate for your origin at the path:
@@ -134,159 +127,168 @@ If you don't have a certificate signed by Cloudflare, run the command:
 
 	%s login
 `, originCertPath, os.Args[0])
-		return nil, fmt.Errorf("Cannot find a valid certificate at the path %s", originCertPath)
+		return "", fmt.Errorf("Cannot find a valid certificate at the path %s", originCertPath)
 	}
+
+	return originCertPath, nil
+}
+
+func readOriginCert(originCertPath string, logger logger.Service) ([]byte, error) {
+	logger.Debugf("Reading origin cert from %s", originCertPath)
+
 	// Easier to send the certificate as []byte via RPC than decoding it at this point
 	originCert, err := ioutil.ReadFile(originCertPath)
 	if err != nil {
-		logger.WithError(err).Errorf("Cannot read %s to load origin certificate", originCertPath)
+		logger.Errorf("Cannot read %s to load origin certificate: %s", originCertPath, err)
 		return nil, fmt.Errorf("Cannot read %s to load origin certificate", originCertPath)
 	}
 	return originCert, nil
 }
 
+func getOriginCert(c *cli.Context, logger logger.Service) ([]byte, error) {
+	if originCertPath, err := findOriginCert(c, logger); err != nil {
+		return nil, err
+	} else {
+		return readOriginCert(originCertPath, logger)
+	}
+}
+
 func prepareTunnelConfig(
 	c *cli.Context,
 	buildInfo *buildinfo.BuildInfo,
-	version string, logger,
-	transportLogger *logrus.Logger,
-) (*origin.TunnelConfig, error) {
+	version string,
+	logger logger.Service,
+	transportLogger logger.Service,
+	namedTunnel *connection.NamedTunnelConfig,
+	uiIsEnabled bool,
+) (*origin.TunnelConfig, ingress.Ingress, error) {
+	isNamedTunnel := namedTunnel != nil
+
 	hostname, err := validation.ValidateHostname(c.String("hostname"))
 	if err != nil {
-		logger.WithError(err).Error("Invalid hostname")
-		return nil, errors.Wrap(err, "Invalid hostname")
+		logger.Errorf("Invalid hostname: %s", err)
+		return nil, ingress.Ingress{}, errors.Wrap(err, "Invalid hostname")
 	}
 	isFreeTunnel := hostname == ""
 	clientID := c.String("id")
 	if !c.IsSet("id") {
 		clientID, err = generateRandomClientID(logger)
 		if err != nil {
-			return nil, err
+			return nil, ingress.Ingress{}, err
 		}
 	}
 
 	tags, err := NewTagSliceFromCLI(c.StringSlice("tag"))
 	if err != nil {
-		logger.WithError(err).Error("Tag parse failure")
-		return nil, errors.Wrap(err, "Tag parse failure")
+		logger.Errorf("Tag parse failure: %s", err)
+		return nil, ingress.Ingress{}, errors.Wrap(err, "Tag parse failure")
 	}
 
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
 
-	originURL, err := config.ValidateUrl(c)
-	if err != nil {
-		logger.WithError(err).Error("Error validating origin URL")
-		return nil, errors.Wrap(err, "Error validating origin URL")
-	}
-
 	var originCert []byte
 	if !isFreeTunnel {
-		originCert, err = getOriginCert(c)
+		originCert, err = getOriginCert(c, logger)
 		if err != nil {
-			return nil, errors.Wrap(err, "Error getting origin cert")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "Error getting origin cert")
 		}
 	}
 
-	originCertPool, err := tlsconfig.LoadOriginCA(c, logger)
-	if err != nil {
-		logger.WithError(err).Error("Error loading cert pool")
-		return nil, errors.Wrap(err, "Error loading cert pool")
-	}
-
-	tunnelMetrics := origin.NewTunnelMetrics()
-	httpTransport := &http.Transport{
-		Proxy:                 http.ProxyFromEnvironment,
-		MaxIdleConns:          c.Int("proxy-keepalive-connections"),
-		MaxIdleConnsPerHost:   c.Int("proxy-keepalive-connections"),
-		IdleConnTimeout:       c.Duration("proxy-keepalive-timeout"),
-		TLSHandshakeTimeout:   c.Duration("proxy-tls-timeout"),
-		ExpectContinueTimeout: 1 * time.Second,
-		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: c.IsSet("no-tls-verify")},
-	}
-
-	dialer := &net.Dialer{
-		Timeout:   c.Duration("proxy-connect-timeout"),
-		KeepAlive: c.Duration("proxy-tcp-keepalive"),
-	}
-	if c.Bool("proxy-no-happy-eyeballs") {
-		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
-	}
-	dialContext := dialer.DialContext
-
-	if c.IsSet("unix-socket") {
-		unixSocket, err := config.ValidateUnixSocket(c)
+	var (
+		ingressRules  ingress.Ingress
+		classicTunnel *connection.ClassicTunnelConfig
+	)
+	if isNamedTunnel {
+		clientUUID, err := uuid.NewRandom()
 		if err != nil {
-			logger.WithError(err).Error("Error validating --unix-socket")
-			return nil, errors.Wrap(err, "Error validating --unix-socket")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate clientUUID")
 		}
-
-		logger.Infof("Proxying tunnel requests to unix:%s", unixSocket)
-		httpTransport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
-			// if --unix-socket specified, enforce network type "unix"
-			return dialContext(ctx, "unix", unixSocket)
+		namedTunnel.Client = tunnelpogs.ClientInfo{
+			ClientID: clientUUID[:],
+			Features: []string{origin.FeatureSerializedHeaders},
+			Version:  version,
+			Arch:     fmt.Sprintf("%s_%s", buildInfo.GoOS, buildInfo.GoArch),
+		}
+		ingressRules, err = ingress.ParseIngress(config.GetConfiguration())
+		if err != nil && err != ingress.ErrNoIngressRules {
+			return nil, ingress.Ingress{}, err
+		}
+		if !ingressRules.IsEmpty() && c.IsSet("url") {
+			return nil, ingress.Ingress{}, ingress.ErrURLIncompatibleWithIngress
 		}
 	} else {
-		logger.Infof("Proxying tunnel requests to %s", originURL)
-		httpTransport.DialContext = dialContext
+		classicTunnel = &connection.ClassicTunnelConfig{
+			Hostname:   hostname,
+			OriginCert: originCert,
+			// turn off use of reconnect token and auth refresh when using named tunnels
+			UseReconnectToken: !isNamedTunnel && c.Bool("use-reconnect-token"),
+		}
 	}
 
-	if !c.IsSet("hello-world") && c.IsSet("origin-server-name") {
-		httpTransport.TLSClientConfig.ServerName = c.String("origin-server-name")
+	// Convert single-origin configuration into multi-origin configuration.
+	if ingressRules.IsEmpty() {
+		ingressRules, err = ingress.NewSingleOrigin(c, !isNamedTunnel, logger)
+		if err != nil {
+			return nil, ingress.Ingress{}, err
+		}
 	}
 
-	err = validation.ValidateHTTPService(originURL, hostname, httpTransport)
+	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), namedTunnel, edgediscovery.HTTP2Percentage, origin.ResolveTTL, logger)
 	if err != nil {
-		logger.WithError(err).Error("unable to connect to the origin")
+		return nil, ingress.Ingress{}, err
+	}
+	logger.Infof("Initial protocol %s", protocolSelector.Current())
+
+	edgeTLSConfigs := make(map[connection.Protocol]*tls.Config, len(connection.ProtocolList))
+	for _, p := range connection.ProtocolList {
+		edgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c, p.ServerName())
+		if err != nil {
+			return nil, ingress.Ingress{}, errors.Wrap(err, "unable to create TLS config to connect with edge")
+		}
+		edgeTLSConfigs[p] = edgeTLSConfig
 	}
 
-	toEdgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c)
-	if err != nil {
-		logger.WithError(err).Error("unable to create TLS config to connect with edge")
-		return nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
+	originClient := origin.NewClient(ingressRules, tags, logger)
+	connectionConfig := &connection.Config{
+		OriginClient:    originClient,
+		GracePeriod:     c.Duration("grace-period"),
+		ReplaceExisting: c.Bool("force"),
+	}
+	muxerConfig := &connection.MuxerConfig{
+		HeartbeatInterval:  c.Duration("heartbeat-interval"),
+		MaxHeartbeats:      c.Uint64("heartbeat-count"),
+		CompressionSetting: h2mux.CompressionSetting(c.Uint64("compression-quality")),
+		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
+	}
+
+	var tunnelEventChan chan ui.TunnelEvent
+	if uiIsEnabled {
+		tunnelEventChan = make(chan ui.TunnelEvent, 16)
 	}
 
 	return &origin.TunnelConfig{
-		BuildInfo:            buildInfo,
-		ClientID:             clientID,
-		ClientTlsConfig:      httpTransport.TLSClientConfig,
-		CompressionQuality:   c.Uint64("compression-quality"),
-		EdgeAddrs:            c.StringSlice("edge"),
-		GracePeriod:          c.Duration("grace-period"),
-		HAConnections:        c.Int("ha-connections"),
-		HTTPTransport:        httpTransport,
-		HeartbeatInterval:    c.Duration("heartbeat-interval"),
-		Hostname:             hostname,
-		HTTPHostHeader:       c.String("http-host-header"),
-		IncidentLookup:       origin.NewIncidentLookup(),
-		IsAutoupdated:        c.Bool("is-autoupdated"),
-		IsFreeTunnel:         isFreeTunnel,
-		LBPool:               c.String("lb-pool"),
-		Logger:               logger,
-		MaxHeartbeats:        c.Uint64("heartbeat-count"),
-		Metrics:              tunnelMetrics,
-		MetricsUpdateFreq:    c.Duration("metrics-update-freq"),
-		NoChunkedEncoding:    c.Bool("no-chunked-encoding"),
-		OriginCert:           originCert,
-		OriginUrl:            originURL,
-		ReportedVersion:      version,
-		Retries:              c.Uint("retries"),
-		RunFromTerminal:      isRunningFromTerminal(),
-		Tags:                 tags,
-		TlsConfig:            toEdgeTLSConfig,
-		TransportLogger:      transportLogger,
-		UseDeclarativeTunnel: c.Bool("use-declarative-tunnels"),
-		UseReconnectToken:    c.Bool("use-reconnect-token"),
-		UseQuickReconnects:   c.Bool("use-quick-reconnects"),
-	}, nil
-}
-
-func serviceDiscoverer(c *cli.Context, logger *logrus.Logger) (*edgediscovery.Edge, error) {
-	// If --edge is specfied, resolve edge server addresses
-	if len(c.StringSlice("edge")) > 0 {
-		return edgediscovery.StaticEdge(logger, c.StringSlice("edge"))
-	}
-	// Otherwise lookup edge server addresses through service discovery
-	return edgediscovery.ResolveEdge(logger)
+		ConnectionConfig: connectionConfig,
+		BuildInfo:        buildInfo,
+		ClientID:         clientID,
+		EdgeAddrs:        c.StringSlice("edge"),
+		HAConnections:    c.Int("ha-connections"),
+		IncidentLookup:   origin.NewIncidentLookup(),
+		IsAutoupdated:    c.Bool("is-autoupdated"),
+		IsFreeTunnel:     isFreeTunnel,
+		LBPool:           c.String("lb-pool"),
+		Tags:             tags,
+		Logger:           logger,
+		Observer:         connection.NewObserver(transportLogger, tunnelEventChan),
+		ReportedVersion:  version,
+		Retries:          c.Uint("retries"),
+		RunFromTerminal:  isRunningFromTerminal(),
+		NamedTunnel:      namedTunnel,
+		ClassicTunnel:    classicTunnel,
+		MuxerConfig:      muxerConfig,
+		TunnelEventChan:  tunnelEventChan,
+		ProtocolSelector: protocolSelector,
+		EdgeTLSConfigs:   edgeTLSConfigs,
+	}, ingressRules, nil
 }
 
 func isRunningFromTerminal() bool {

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -0,0 +1,114 @@
+package tunnel
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ingress"
+)
+
+func buildIngressSubcommand() *cli.Command {
+	return &cli.Command{
+		Name:      "ingress",
+		Category:  "Tunnel",
+		Usage:     "Validate and test cloudflared tunnel's ingress configuration",
+		UsageText: "cloudflared tunnel [--config FILEPATH] ingress COMMAND [arguments...]",
+		Hidden:    true,
+		Description: ` Cloudflared lets you route traffic from the internet to multiple different addresses on your
+		origin. Multiple-origin routing is configured by a set of rules. Each rule matches traffic
+		by its hostname or path, and routes it to an address. These rules are configured under the
+		'ingress' key of your config.yaml, for example:
+
+		ingress:
+		  - hostname: www.example.com
+		    service: https://localhost:8000
+		  - hostname: *.example.xyz
+		    path: /[a-zA-Z]+.html
+		    service: https://localhost:8001
+		  - hostname: *
+		    service: https://localhost:8002
+
+		To ensure cloudflared can route all incoming requests, the last rule must be a catch-all
+		rule that matches all traffic. You can validate these rules with the 'ingress validate'
+		command, and test which rule matches a particular URL with 'ingress rule <URL>'.
+
+		Multiple-origin routing is incompatible with the --url flag.`,
+		Subcommands: []*cli.Command{buildValidateIngressCommand(), buildTestURLCommand()},
+	}
+}
+
+func buildValidateIngressCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "validate",
+		Action:      cliutil.ErrorHandler(validateIngressCommand),
+		Usage:       "Validate the ingress configuration ",
+		UsageText:   "cloudflared tunnel [--config FILEPATH] ingress validate",
+		Description: "Validates the configuration file, ensuring your ingress rules are OK.",
+	}
+}
+
+func buildTestURLCommand() *cli.Command {
+	return &cli.Command{
+		Name:      "rule",
+		Action:    cliutil.ErrorHandler(testURLCommand),
+		Usage:     "Check which ingress rule matches a given request URL",
+		UsageText: "cloudflared tunnel [--config FILEPATH] ingress rule URL",
+		ArgsUsage: "URL",
+		Description: "Check which ingress rule matches a given request URL. " +
+			"Ingress rules match a request's hostname and path. Hostname is " +
+			"optional and is either a full hostname like `www.example.com` or a " +
+			"hostname with a `*` for its subdomains, e.g. `*.example.com`. Path " +
+			"is optional and matches a regular expression, like `/[a-zA-Z0-9_]+.html`",
+	}
+}
+
+// validateIngressCommand check the syntax of the ingress rules in the cloudflared config file
+func validateIngressCommand(c *cli.Context) error {
+	conf := config.GetConfiguration()
+	if conf.Source() == "" {
+		fmt.Println("No configuration file was found. Please create one, or use the --config flag to specify its filepath. You can use the help command to learn more about configuration files")
+		return nil
+	}
+	fmt.Println("Validating rules from", conf.Source())
+	if _, err := ingress.ParseIngress(conf); err != nil {
+		return errors.Wrap(err, "Validation failed")
+	}
+	if c.IsSet("url") {
+		return ingress.ErrURLIncompatibleWithIngress
+	}
+	fmt.Println("OK")
+	return nil
+}
+
+// testURLCommand checks which ingress rule matches the given URL.
+func testURLCommand(c *cli.Context) error {
+	requestArg := c.Args().First()
+	if requestArg == "" {
+		return errors.New("cloudflared tunnel rule expects a single argument, the URL to test")
+	}
+
+	requestURL, err := url.Parse(requestArg)
+	if err != nil {
+		return fmt.Errorf("%s is not a valid URL", requestArg)
+	}
+	if requestURL.Hostname() == "" && requestURL.Scheme == "" {
+		return fmt.Errorf("%s doesn't have a hostname, consider adding a scheme", requestArg)
+	}
+
+	conf := config.GetConfiguration()
+	fmt.Println("Using rules from", conf.Source())
+	ing, err := ingress.ParseIngress(conf)
+	if err != nil {
+		return errors.Wrap(err, "Validation failed")
+	}
+
+	_, i := ing.FindMatchingRule(requestURL.Hostname(), requestURL.Path)
+	fmt.Printf("Matched rule #%d\n", i+1)
+	fmt.Println(ing.Rules[i].MultiLineString())
+	return nil
+}

cmd/cloudflared/tunnel/logger.go

@@ -1,75 +0,0 @@
-package tunnel
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/cloudflare/cloudflared/log"
-	"github.com/rifflock/lfshook"
-	"github.com/sirupsen/logrus"
-	"gopkg.in/urfave/cli.v2"
-
-	"github.com/mitchellh/go-homedir"
-	"github.com/pkg/errors"
-)
-
-const debugLevelWarning = "At debug level, request URL, method, protocol, content legnth and header will be logged. " +
-	"Response status, content length and header will also be logged in debug level."
-
-var logger = log.CreateLogger()
-
-func configMainLogger(c *cli.Context) error {
-	logLevel, err := logrus.ParseLevel(c.String("loglevel"))
-	if err != nil {
-		logger.WithError(err).Error("Unknown logging level specified")
-		return errors.Wrap(err, "Unknown logging level specified")
-	}
-	logger.SetLevel(logLevel)
-	if logLevel == logrus.DebugLevel {
-		logger.Warn(debugLevelWarning)
-	}
-	return nil
-}
-
-func configTransportLogger(c *cli.Context) (*logrus.Logger, error) {
-	transportLogLevel, err := logrus.ParseLevel(c.String("transport-loglevel"))
-	if err != nil {
-		logger.WithError(err).Fatal("Unknown transport logging level specified")
-		return nil, errors.Wrap(err, "Unknown transport logging level specified")
-	}
-	transportLogger := logrus.New()
-	transportLogger.Level = transportLogLevel
-	return transportLogger, nil
-}
-
-func initLogFile(c *cli.Context, loggers ...*logrus.Logger) error {
-	filePath, err := homedir.Expand(c.String("logfile"))
-	if err != nil {
-		return errors.Wrap(err, "Cannot resolve logfile path")
-	}
-
-	fileMode := os.O_WRONLY | os.O_APPEND | os.O_CREATE | os.O_TRUNC
-	// do not truncate log file if the client has been autoupdated
-	if c.Bool("is-autoupdated") {
-		fileMode = os.O_WRONLY | os.O_APPEND | os.O_CREATE
-	}
-	f, err := os.OpenFile(filePath, fileMode, 0664)
-	if err != nil {
-		errors.Wrap(err, fmt.Sprintf("Cannot open file %s", filePath))
-	}
-	defer f.Close()
-	pathMap := lfshook.PathMap{
-		logrus.DebugLevel: filePath,
-		logrus.WarnLevel:  filePath,
-		logrus.InfoLevel:  filePath,
-		logrus.ErrorLevel: filePath,
-		logrus.FatalLevel: filePath,
-		logrus.PanicLevel: filePath,
-	}
-
-	for _, l := range loggers {
-		l.Hooks.Add(lfshook.NewHook(pathMap, &logrus.JSONFormatter{}))
-	}
-
-	return nil
-}

cmd/cloudflared/tunnel/login.go

@@ -7,10 +7,14 @@ import (
 	"path/filepath"
 	"syscall"
 
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	cli "github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
-	homedir "github.com/mitchellh/go-homedir"
-	cli "gopkg.in/urfave/cli.v2"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
@@ -18,7 +22,28 @@ const (
 	callbackStoreURL = "https://login.argotunnel.com/"
 )
 
+func buildLoginSubcommand(hidden bool) *cli.Command {
+	return &cli.Command{
+		Name:      "login",
+		Action:    cliutil.ErrorHandler(login),
+		Usage:     "Generate a configuration file with your login details",
+		ArgsUsage: " ",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:   "url",
+				Hidden: true,
+			},
+		},
+		Hidden: hidden,
+	}
+}
+
 func login(c *cli.Context) error {
+	logger, err := logger.New()
+	if err != nil {
+		return errors.Wrap(err, "error setting up logger")
+	}
+
 	path, ok, err := checkForExistingCert()
 	if ok {
 		fmt.Fprintf(os.Stdout, "You have an existing certificate at %s which login would overwrite.\nIf this is intentional, please move or delete that file then run this command again.\n", path)
@@ -33,7 +58,7 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	_, err = transfer.Run(loginURL, "cert", "callback", callbackStoreURL, path, false)
+	_, err = transfer.Run(loginURL, "cert", "callback", callbackStoreURL, path, false, false, logger)
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Failed to write the certificate due to the following error:\n%v\n\nYour browser will download the certificate instead. You will have to manually\ncopy it to the following path:\n\n%s\n", err, path)
 		return err
@@ -44,7 +69,7 @@ func login(c *cli.Context) error {
 }
 
 func checkForExistingCert() (string, bool, error) {
-	configPath, err := homedir.Expand(config.DefaultConfigDirs[0])
+	configPath, err := homedir.Expand(config.DefaultConfigSearchDirectories()[0])
 	if err != nil {
 		return "", false, err
 	}

cmd/cloudflared/tunnel/server.go

@@ -1,30 +1,28 @@
 package tunnel
 
 import (
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunneldns"
 
-	"gopkg.in/urfave/cli.v2"
+	"github.com/urfave/cli/v2"
 
 	"github.com/pkg/errors"
 )
 
-func runDNSProxyServer(c *cli.Context, dnsReadySignal, shutdownC chan struct{}) error {
+func runDNSProxyServer(c *cli.Context, dnsReadySignal, shutdownC chan struct{}, logger logger.Service) error {
 	port := c.Int("proxy-dns-port")
 	if port <= 0 || port > 65535 {
-		logger.Errorf("The 'proxy-dns-port' must be a valid port number in <1, 65535> range.")
 		return errors.New("The 'proxy-dns-port' must be a valid port number in <1, 65535> range.")
 	}
-	listener, err := tunneldns.CreateListener(c.String("proxy-dns-address"), uint16(port), c.StringSlice("proxy-dns-upstream"))
+	listener, err := tunneldns.CreateListener(c.String("proxy-dns-address"), uint16(port), c.StringSlice("proxy-dns-upstream"), c.StringSlice("proxy-dns-bootstrap"), logger)
 	if err != nil {
 		close(dnsReadySignal)
 		listener.Stop()
-		logger.WithError(err).Error("Cannot create the DNS over HTTPS proxy server")
 		return errors.Wrap(err, "Cannot create the DNS over HTTPS proxy server")
 	}
 
 	err = listener.Start(dnsReadySignal)
 	if err != nil {
-		logger.WithError(err).Error("Cannot start the DNS over HTTPS proxy server")
 		return errors.Wrap(err, "Cannot start the DNS over HTTPS proxy server")
 	}
 	<-shutdownC

cmd/cloudflared/tunnel/signal.go

@@ -5,21 +5,25 @@ import (
 	"os/signal"
 	"syscall"
 	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 // waitForSignal notifies all routines to shutdownC immediately by closing the
 // shutdownC when one of the routines in main exits, or when this process receives
 // SIGTERM/SIGINT
-func waitForSignal(errC chan error, shutdownC chan struct{}) error {
+func waitForSignal(errC chan error, shutdownC chan struct{}, logger logger.Service) error {
 	signals := make(chan os.Signal, 10)
 	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
 	defer signal.Stop(signals)
 
 	select {
 	case err := <-errC:
+		logger.Infof("terminating due to error: %v", err)
 		close(shutdownC)
 		return err
-	case <-signals:
+	case s := <-signals:
+		logger.Infof("terminating due to signal %s", s)
 		close(shutdownC)
 	case <-shutdownC:
 	}
@@ -37,6 +41,7 @@ func waitForSignal(errC chan error, shutdownC chan struct{}) error {
 func waitForSignalWithGraceShutdown(errC chan error,
 	shutdownC, graceShutdownC chan struct{},
 	gracePeriod time.Duration,
+	logger logger.Service,
 ) error {
 	signals := make(chan os.Signal, 10)
 	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
@@ -44,14 +49,16 @@ func waitForSignalWithGraceShutdown(errC chan error,
 
 	select {
 	case err := <-errC:
+		logger.Infof("Initiating graceful shutdown due to %v ...", err)
 		close(graceShutdownC)
 		close(shutdownC)
 		return err
-	case <-signals:
+	case s := <-signals:
+		logger.Infof("Initiating graceful shutdown due to signal %s ...", s)
 		close(graceShutdownC)
-		waitForGracePeriod(signals, errC, shutdownC, gracePeriod)
+		waitForGracePeriod(signals, errC, shutdownC, gracePeriod, logger)
 	case <-graceShutdownC:
-		waitForGracePeriod(signals, errC, shutdownC, gracePeriod)
+		waitForGracePeriod(signals, errC, shutdownC, gracePeriod, logger)
 	case <-shutdownC:
 		close(graceShutdownC)
 	}
@@ -63,8 +70,8 @@ func waitForGracePeriod(signals chan os.Signal,
 	errC chan error,
 	shutdownC chan struct{},
 	gracePeriod time.Duration,
+	logger logger.Service,
 ) {
-	logger.Infof("Initiating graceful shutdown...")
 	// Unregister signal handler early, so the client can send a second SIGTERM/SIGINT
 	// to force shutdown cloudflared
 	signal.Stop(signals)

cmd/cloudflared/tunnel/signal_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -27,6 +28,8 @@ func testChannelClosed(t *testing.T, c chan struct{}) {
 }
 
 func TestWaitForSignal(t *testing.T) {
+	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
+
 	// Test handling server error
 	errC := make(chan error)
 	shutdownC := make(chan struct{})
@@ -36,7 +39,7 @@ func TestWaitForSignal(t *testing.T) {
 	}()
 
 	// received error, shutdownC should be closed
-	err := waitForSignal(errC, shutdownC)
+	err := waitForSignal(errC, shutdownC, logger)
 	assert.Equal(t, serverErr, err)
 	testChannelClosed(t, shutdownC)
 
@@ -56,7 +59,7 @@ func TestWaitForSignal(t *testing.T) {
 			syscall.Kill(syscall.Getpid(), sig)
 		}(sig)
 
-		err = waitForSignal(errC, shutdownC)
+		err = waitForSignal(errC, shutdownC, logger)
 		assert.Equal(t, nil, err)
 		assert.Equal(t, shutdownErr, <-errC)
 		testChannelClosed(t, shutdownC)
@@ -73,8 +76,10 @@ func TestWaitForSignalWithGraceShutdown(t *testing.T) {
 		errC <- serverErr
 	}()
 
+	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
+
 	// received error, both shutdownC and graceshutdownC should be closed
-	err := waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick)
+	err := waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
 	assert.Equal(t, serverErr, err)
 	testChannelClosed(t, shutdownC)
 	testChannelClosed(t, graceshutdownC)
@@ -84,7 +89,7 @@ func TestWaitForSignalWithGraceShutdown(t *testing.T) {
 	shutdownC = make(chan struct{})
 	graceshutdownC = make(chan struct{})
 	close(shutdownC)
-	err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick)
+	err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
 	assert.NoError(t, err)
 	testChannelClosed(t, shutdownC)
 	testChannelClosed(t, graceshutdownC)
@@ -94,7 +99,7 @@ func TestWaitForSignalWithGraceShutdown(t *testing.T) {
 	shutdownC = make(chan struct{})
 	graceshutdownC = make(chan struct{})
 	close(graceshutdownC)
-	err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick)
+	err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
 	assert.NoError(t, err)
 	testChannelClosed(t, shutdownC)
 	testChannelClosed(t, graceshutdownC)
@@ -117,7 +122,7 @@ func TestWaitForSignalWithGraceShutdown(t *testing.T) {
 			syscall.Kill(syscall.Getpid(), sig)
 		}(sig)
 
-		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick)
+		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
 		assert.Equal(t, nil, err)
 		assert.Equal(t, graceShutdownErr, <-errC)
 		testChannelClosed(t, shutdownC)
@@ -143,7 +148,7 @@ func TestWaitForSignalWithGraceShutdown(t *testing.T) {
 			syscall.Kill(syscall.Getpid(), sig)
 		}(sig)
 
-		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick)
+		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceshutdownC, tick, logger)
 		assert.Equal(t, nil, err)
 		assert.Equal(t, shutdownErr, <-errC)
 		testChannelClosed(t, shutdownC)

cmd/cloudflared/tunnel/subcommand_context.go

@@ -0,0 +1,374 @@
+package tunnel
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/certutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelstore"
+)
+
+type errInvalidJSONCredential struct {
+	err  error
+	path string
+}
+
+func (e errInvalidJSONCredential) Error() string {
+	return "Invalid JSON when parsing tunnel credentials file"
+}
+
+// subcommandContext carries structs shared between subcommands, to reduce number of arguments needed to
+// pass between subcommands, and make sure they are only initialized once
+type subcommandContext struct {
+	c      *cli.Context
+	logger logger.Service
+
+	// These fields should be accessed using their respective Getter
+	tunnelstoreClient tunnelstore.Client
+	userCredential    *userCredential
+
+	isUIEnabled bool
+}
+
+func newSubcommandContext(c *cli.Context) (*subcommandContext, error) {
+	isUIEnabled := c.IsSet(uiFlag) && c.String("name") != ""
+
+	// If UI is enabled, terminal log output should be disabled -- log should be written into a UI log window instead
+	logger, err := createLogger(c, false, isUIEnabled)
+	if err != nil {
+		return nil, errors.Wrap(err, "error setting up logger")
+	}
+
+	return &subcommandContext{
+		c:           c,
+		logger:      logger,
+		isUIEnabled: isUIEnabled,
+	}, nil
+}
+
+type userCredential struct {
+	cert     *certutil.OriginCert
+	certPath string
+}
+
+func (sc *subcommandContext) client() (tunnelstore.Client, error) {
+	if sc.tunnelstoreClient != nil {
+		return sc.tunnelstoreClient, nil
+	}
+	credential, err := sc.credential()
+	if err != nil {
+		return nil, err
+	}
+	userAgent := fmt.Sprintf("cloudflared/%s", version)
+	client, err := tunnelstore.NewRESTClient(sc.c.String("api-url"), credential.cert.AccountID, credential.cert.ZoneID, credential.cert.ServiceKey, userAgent, sc.logger)
+	if err != nil {
+		return nil, err
+	}
+	sc.tunnelstoreClient = client
+	return client, nil
+}
+
+func (sc *subcommandContext) credential() (*userCredential, error) {
+	if sc.userCredential == nil {
+		originCertPath, err := findOriginCert(sc.c, sc.logger)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error locating origin cert")
+		}
+		blocks, err := readOriginCert(originCertPath, sc.logger)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Can't read origin cert from %s", originCertPath)
+		}
+
+		cert, err := certutil.DecodeOriginCert(blocks)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error decoding origin cert")
+		}
+
+		if cert.AccountID == "" {
+			return nil, errors.Errorf(`Origin certificate needs to be refreshed before creating new tunnels.\nDelete %s and run "cloudflared login" to obtain a new cert.`, originCertPath)
+		}
+
+		sc.userCredential = &userCredential{
+			cert:     cert,
+			certPath: originCertPath,
+		}
+	}
+	return sc.userCredential, nil
+}
+
+func (sc *subcommandContext) readTunnelCredentials(tunnelID uuid.UUID) (*pogs.TunnelAuth, error) {
+	filePath, err := sc.tunnelCredentialsPath(tunnelID)
+	if err != nil {
+		return nil, err
+	}
+	body, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "couldn't read tunnel credentials from %v", filePath)
+	}
+
+	var auth pogs.TunnelAuth
+	if err = json.Unmarshal(body, &auth); err != nil {
+		if strings.HasSuffix(filePath, ".pem") {
+			return nil, fmt.Errorf("The tunnel credentials file should be .json but you gave a .pem. "+
+				"The tunnel credentials file was originally created by `cloudflared tunnel create` and named %s.json."+
+				"You may have accidentally used the filepath to cert.pem, which is generated by `cloudflared tunnel "+
+				"login`.", tunnelID)
+		}
+		return nil, errInvalidJSONCredential{path: filePath, err: err}
+	}
+	return &auth, nil
+}
+
+func (sc *subcommandContext) tunnelCredentialsPath(tunnelID uuid.UUID) (string, error) {
+	if filePath := sc.c.String("credentials-file"); filePath != "" {
+		if validFilePath(filePath) {
+			return filePath, nil
+		}
+		return "", fmt.Errorf("Tunnel credentials file %s doesn't exist or is not a file", filePath)
+	}
+
+	// Fallback to look for tunnel credentials in the origin cert directory
+	if originCertPath, err := findOriginCert(sc.c, sc.logger); err == nil {
+		originCertDir := filepath.Dir(originCertPath)
+		if filePath, err := tunnelFilePath(tunnelID, originCertDir); err == nil {
+			if validFilePath(filePath) {
+				return filePath, nil
+			}
+		}
+	}
+
+	// Last resort look under default config directories
+	for _, configDir := range config.DefaultConfigSearchDirectories() {
+		if filePath, err := tunnelFilePath(tunnelID, configDir); err == nil {
+			if validFilePath(filePath) {
+				return filePath, nil
+			}
+		}
+	}
+	return "", fmt.Errorf("Tunnel credentials file not found")
+}
+
+func (sc *subcommandContext) create(name string) (*tunnelstore.Tunnel, error) {
+	client, err := sc.client()
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't create client to talk to Argo Tunnel backend")
+	}
+
+	tunnelSecret, err := generateTunnelSecret()
+	if err != nil {
+		return nil, errors.Wrap(err, "couldn't generate the secret for your new tunnel")
+	}
+
+	tunnel, err := client.CreateTunnel(name, tunnelSecret)
+	if err != nil {
+		return nil, errors.Wrap(err, "Create Tunnel API call failed")
+	}
+
+	credential, err := sc.credential()
+	if err != nil {
+		return nil, err
+	}
+	if writeFileErr := writeTunnelCredentials(tunnel.ID, credential.cert.AccountID, credential.certPath, tunnelSecret, sc.logger); err != nil {
+		var errorLines []string
+		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write to the tunnel credentials file at %v.json.", tunnel.Name, tunnel.ID, tunnel.ID))
+		errorLines = append(errorLines, fmt.Sprintf("The file-writing error is: %v", writeFileErr))
+		if deleteErr := client.DeleteTunnel(tunnel.ID); deleteErr != nil {
+			errorLines = append(errorLines, fmt.Sprintf("Cloudflared tried to delete the tunnel for you, but encountered an error. You should use `cloudflared tunnel delete %v` to delete the tunnel yourself, because the tunnel can't be run without the tunnelfile.", tunnel.ID))
+			errorLines = append(errorLines, fmt.Sprintf("The delete tunnel error is: %v", deleteErr))
+		} else {
+			errorLines = append(errorLines, fmt.Sprintf("The tunnel was deleted, because the tunnel can't be run without the tunnelfile"))
+		}
+		errorMsg := strings.Join(errorLines, "\n")
+		return nil, errors.New(errorMsg)
+	}
+
+	if outputFormat := sc.c.String(outputFormatFlag.Name); outputFormat != "" {
+		return nil, renderOutput(outputFormat, &tunnel)
+	}
+
+	sc.logger.Infof("Created tunnel %s with id %s", tunnel.Name, tunnel.ID)
+	return tunnel, nil
+}
+
+func (sc *subcommandContext) list(filter *tunnelstore.Filter) ([]*tunnelstore.Tunnel, error) {
+	client, err := sc.client()
+	if err != nil {
+		return nil, err
+	}
+	return client.ListTunnels(filter)
+}
+
+func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
+	forceFlagSet := sc.c.Bool("force")
+
+	client, err := sc.client()
+	if err != nil {
+		return err
+	}
+
+	for _, id := range tunnelIDs {
+		tunnel, err := client.GetTunnel(id)
+		if err != nil {
+			return errors.Wrapf(err, "Can't get tunnel information. Please check tunnel id: %s", tunnel.ID)
+		}
+
+		// Check if tunnel DeletedAt field has already been set
+		if !tunnel.DeletedAt.IsZero() {
+			return fmt.Errorf("Tunnel %s has already been deleted", tunnel.ID)
+		}
+		// Check if tunnel has existing connections and if force flag is set, cleanup connections
+		if len(tunnel.Connections) > 0 {
+			if !forceFlagSet {
+				return fmt.Errorf("You can not delete tunnel %s because it has active connections. To see connections run the 'list' command. If you believe the tunnel is not active, you can use a -f / --force flag with this command.", id)
+			}
+
+			if err := client.CleanupConnections(tunnel.ID); err != nil {
+				return errors.Wrapf(err, "Error cleaning up connections for tunnel %s", tunnel.ID)
+			}
+		}
+
+		if err := client.DeleteTunnel(tunnel.ID); err != nil {
+			return errors.Wrapf(err, "Error deleting tunnel %s", tunnel.ID)
+		}
+
+		tunnelCredentialsPath, err := sc.tunnelCredentialsPath(tunnel.ID)
+		if err != nil {
+			sc.logger.Infof("Cannot locate tunnel credentials to delete, error: %v. Please delete the file manually", err)
+			return nil
+		}
+
+		if err = os.Remove(tunnelCredentialsPath); err != nil {
+			sc.logger.Infof("Cannot delete tunnel credentials, error: %v. Please delete the file manually", err)
+		}
+	}
+	return nil
+}
+
+func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
+	credentials, err := sc.readTunnelCredentials(tunnelID)
+	if err != nil {
+		if e, ok := err.(errInvalidJSONCredential); ok {
+			sc.logger.Errorf("The credentials file at %s contained invalid JSON. This is probably caused by passing the wrong filepath. Reminder: the credentials file is a .json file created via `cloudflared tunnel create`.", e.path)
+			sc.logger.Errorf("Invalid JSON when parsing credentials file: %s", e.err.Error())
+		}
+		return err
+	}
+
+	return StartServer(
+		sc.c,
+		version,
+		shutdownC,
+		graceShutdownC,
+		&connection.NamedTunnelConfig{Auth: *credentials, ID: tunnelID},
+		sc.logger,
+		sc.isUIEnabled,
+	)
+}
+
+func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
+	client, err := sc.client()
+	if err != nil {
+		return err
+	}
+	for _, tunnelID := range tunnelIDs {
+		sc.logger.Infof("Cleanup connection for tunnel %s", tunnelID)
+		if err := client.CleanupConnections(tunnelID); err != nil {
+			sc.logger.Errorf("Error cleaning up connections for tunnel %v, error :%v", tunnelID, err)
+		}
+	}
+	return nil
+}
+
+func (sc *subcommandContext) route(tunnelID uuid.UUID, r tunnelstore.Route) (tunnelstore.RouteResult, error) {
+	client, err := sc.client()
+	if err != nil {
+		return nil, err
+	}
+
+	return client.RouteTunnel(tunnelID, r)
+}
+
+func (sc *subcommandContext) tunnelActive(name string) (*tunnelstore.Tunnel, bool, error) {
+	filter := tunnelstore.NewFilter()
+	filter.NoDeleted()
+	filter.ByName(name)
+	tunnels, err := sc.list(filter)
+	if err != nil {
+		return nil, false, err
+	}
+	if len(tunnels) == 0 {
+		return nil, false, nil
+	}
+	// There should only be 1 active tunnel for a given name
+	return tunnels[0], true, nil
+}
+
+// findID parses the input. If it's a UUID, return the UUID.
+// Otherwise, assume it's a name, and look up the ID of that tunnel.
+func (sc *subcommandContext) findID(input string) (uuid.UUID, error) {
+	if u, err := uuid.Parse(input); err == nil {
+		return u, nil
+	}
+
+	if tunnel, found, err := sc.tunnelActive(input); err != nil {
+		return uuid.Nil, err
+	} else if found {
+		return tunnel.ID, nil
+	}
+
+	return uuid.Nil, fmt.Errorf("%s is neither the ID nor the name of any of your tunnels", input)
+}
+
+// findIDs is just like mapping `findID` over a slice, but it only uses
+// one Tunnelstore API call.
+func (sc *subcommandContext) findIDs(inputs []string) ([]uuid.UUID, error) {
+
+	// First, look up all tunnels the user has
+	filter := tunnelstore.NewFilter()
+	filter.NoDeleted()
+	tunnels, err := sc.list(filter)
+	if err != nil {
+		return nil, err
+	}
+	// Do the pure list-processing in its own function, so that it can be
+	// unit tested easily.
+	return findIDs(tunnels, inputs)
+}
+
+func findIDs(tunnels []*tunnelstore.Tunnel, inputs []string) ([]uuid.UUID, error) {
+	// Put them into a dictionary for faster lookups
+	nameToID := make(map[string]uuid.UUID, len(tunnels))
+	for _, tunnel := range tunnels {
+		nameToID[tunnel.Name] = tunnel.ID
+	}
+
+	// For each input, try to find the tunnel ID.
+	tunnelIDs := make([]uuid.UUID, len(inputs))
+	var badInputs []string
+	for i, input := range inputs {
+		if id, err := uuid.Parse(input); err == nil {
+			tunnelIDs[i] = id
+		} else if id, ok := nameToID[input]; ok {
+			tunnelIDs[i] = id
+		} else {
+			badInputs = append(badInputs, input)
+		}
+	}
+	if len(badInputs) > 0 {
+		msg := "Please specify either the ID or name of a tunnel. The following inputs were neither: %s"
+		return nil, fmt.Errorf(msg, strings.Join(badInputs, ", "))
+	}
+	return tunnelIDs, nil
+}

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -0,0 +1,82 @@
+package tunnel
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/cloudflare/cloudflared/tunnelstore"
+	"github.com/google/uuid"
+)
+
+func Test_findIDs(t *testing.T) {
+	type args struct {
+		tunnels []*tunnelstore.Tunnel
+		inputs  []string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []uuid.UUID
+		wantErr bool
+	}{
+		{
+			name: "input not found",
+			args: args{
+				inputs: []string{"asdf"},
+			},
+			wantErr: true,
+		},
+		{
+			name: "only UUID",
+			args: args{
+				inputs: []string{"a8398a0b-876d-48ed-b609-3fcfd67a4950"},
+			},
+			want: []uuid.UUID{uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950")},
+		},
+		{
+			name: "only name",
+			args: args{
+				tunnels: []*tunnelstore.Tunnel{
+					{
+						ID:   uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950"),
+						Name: "tunnel1",
+					},
+				},
+				inputs: []string{"tunnel1"},
+			},
+			want: []uuid.UUID{uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950")},
+		},
+		{
+			name: "both UUID and name",
+			args: args{
+				tunnels: []*tunnelstore.Tunnel{
+					{
+						ID:   uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950"),
+						Name: "tunnel1",
+					},
+					{
+						ID:   uuid.MustParse("bf028b68-744f-466e-97f8-c46161d80aa5"),
+						Name: "tunnel2",
+					},
+				},
+				inputs: []string{"tunnel1", "bf028b68-744f-466e-97f8-c46161d80aa5"},
+			},
+			want: []uuid.UUID{
+				uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950"),
+				uuid.MustParse("bf028b68-744f-466e-97f8-c46161d80aa5"),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := findIDs(tt.args.tunnels, tt.args.inputs)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("findIDs() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("findIDs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/cloudflared/tunnel/subcommands.go

@@ -0,0 +1,552 @@
+package tunnel
+
+import (
+	"crypto/rand"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"regexp"
+	"sort"
+	"strings"
+	"text/tabwriter"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2/altsrc"
+	"golang.org/x/net/idna"
+	"gopkg.in/yaml.v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelstore"
+)
+
+const (
+	credFileFlagAlias = "cred-file"
+)
+
+var (
+	showDeletedFlag = &cli.BoolFlag{
+		Name:    "show-deleted",
+		Aliases: []string{"d"},
+		Usage:   "Include deleted tunnels in the list",
+	}
+	listNameFlag = &cli.StringFlag{
+		Name:    "name",
+		Aliases: []string{"n"},
+		Usage:   "List tunnels with the given `NAME`",
+	}
+	listExistedAtFlag = &cli.TimestampFlag{
+		Name:        "when",
+		Aliases:     []string{"w"},
+		Usage:       "List tunnels that are active at the given `TIME` in RFC3339 format",
+		Layout:      tunnelstore.TimeLayout,
+		DefaultText: fmt.Sprintf("current time, %s", time.Now().Format(tunnelstore.TimeLayout)),
+	}
+	listIDFlag = &cli.StringFlag{
+		Name:    "id",
+		Aliases: []string{"i"},
+		Usage:   "List tunnel by `ID`",
+	}
+	showRecentlyDisconnected = &cli.BoolFlag{
+		Name:    "show-recently-disconnected",
+		Aliases: []string{"rd"},
+		Usage:   "Include connections that have recently disconnected in the list",
+	}
+	outputFormatFlag = altsrc.NewStringFlag(&cli.StringFlag{
+		Name:    "output",
+		Aliases: []string{"o"},
+		Usage:   "Render output using given `FORMAT`. Valid options are 'json' or 'yaml'",
+	})
+	forceFlag = altsrc.NewBoolFlag(&cli.BoolFlag{
+		Name:    "force",
+		Aliases: []string{"f"},
+		Usage: "By default, if a tunnel is currently being run from a cloudflared, you can't " +
+			"simultaneously rerun it again from a second cloudflared. The --force flag lets you " +
+			"overwrite the previous tunnel. If you want to use a single hostname with multiple " +
+			"tunnels, you can do so with Cloudflare's Load Balancer product.",
+	})
+	credentialsFileFlag = altsrc.NewStringFlag(&cli.StringFlag{
+		Name:    "credentials-file",
+		Aliases: []string{credFileFlagAlias},
+		Usage:   "File path of tunnel credentials",
+		EnvVars: []string{"TUNNEL_CRED_FILE"},
+	})
+	forceDeleteFlag = &cli.BoolFlag{
+		Name:    "force",
+		Aliases: []string{"f"},
+		Usage:   "Allows you to delete a tunnel, even if it has active connections.",
+		EnvVars: []string{"TUNNEL_RUN_FORCE_OVERWRITE"},
+	}
+	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{
+		Name:    "protocol",
+		Value:   "h2mux",
+		Aliases: []string{"p"},
+		Usage:   fmt.Sprintf("Protocol implementation to connect with Cloudflare's edge network. %s", connection.AvailableProtocolFlagMessage),
+		EnvVars: []string{"TUNNEL_TRANSPORT_PROTOCOL"},
+		Hidden:  true,
+	})
+)
+
+func buildCreateCommand() *cli.Command {
+	return &cli.Command{
+		Name:      "create",
+		Action:    cliutil.ErrorHandler(createCommand),
+		Usage:     "Create a new tunnel with given name",
+		UsageText: "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
+		Description: `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
+  Use "cloudflared tunnel route" subcommand to map a DNS name to this tunnel and "cloudflared tunnel run" to start the connection. 
+
+  For example, to create a tunnel named 'my-tunnel' run:
+
+  $ cloudflared tunnel create my-tunnel`,
+		Flags:              []cli.Flag{outputFormatFlag},
+		CustomHelpTemplate: commandHelpTemplate(),
+	}
+}
+
+// generateTunnelSecret as an array of 32 bytes using secure random number generator
+func generateTunnelSecret() ([]byte, error) {
+	randomBytes := make([]byte, 32)
+	_, err := rand.Read(randomBytes)
+	return randomBytes, err
+}
+
+func createCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return errors.Wrap(err, "error setting up logger")
+	}
+
+	if c.NArg() != 1 {
+		return cliutil.UsageError(`"cloudflared tunnel create" requires exactly 1 argument, the name of tunnel to create.`)
+	}
+	name := c.Args().First()
+
+	_, err = sc.create(name)
+	return errors.Wrap(err, "failed to create tunnel")
+}
+
+func tunnelFilePath(tunnelID uuid.UUID, directory string) (string, error) {
+	fileName := fmt.Sprintf("%v.json", tunnelID)
+	filePath := filepath.Clean(fmt.Sprintf("%s/%s", directory, fileName))
+	return homedir.Expand(filePath)
+}
+
+func writeTunnelCredentials(tunnelID uuid.UUID, accountID, originCertPath string, tunnelSecret []byte, logger logger.Service) error {
+	originCertDir := filepath.Dir(originCertPath)
+	filePath, err := tunnelFilePath(tunnelID, originCertDir)
+	if err != nil {
+		return err
+	}
+	body, err := json.Marshal(pogs.TunnelAuth{
+		AccountTag:   accountID,
+		TunnelSecret: tunnelSecret,
+	})
+	if err != nil {
+		return errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
+	}
+	logger.Infof("Writing tunnel credentials to %v. cloudflared chose this file based on where your origin certificate was found.", filePath)
+	logger.Infof("Keep this file secret. To revoke these credentials, delete the tunnel.")
+	return ioutil.WriteFile(filePath, body, 400)
+}
+
+func validFilePath(path string) bool {
+	fileStat, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+	return !fileStat.IsDir()
+}
+
+func buildListCommand() *cli.Command {
+	return &cli.Command{
+		Name:               "list",
+		Action:             cliutil.ErrorHandler(listCommand),
+		Usage:              "List existing tunnels",
+		UsageText:          "cloudflared tunnel [tunnel command options] list [subcommand options]",
+		Description:        "cloudflared tunnel list will display all active tunnels, their created time and associated connections. Use -d flag to include deleted tunnels. See the list of options to filter the list",
+		Flags:              []cli.Flag{outputFormatFlag, showDeletedFlag, listNameFlag, listExistedAtFlag, listIDFlag, showRecentlyDisconnected},
+		CustomHelpTemplate: commandHelpTemplate(),
+	}
+}
+
+func listCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+
+	filter := tunnelstore.NewFilter()
+	if !c.Bool("show-deleted") {
+		filter.NoDeleted()
+	}
+	if name := c.String("name"); name != "" {
+		filter.ByName(name)
+	}
+	if existedAt := c.Timestamp("time"); existedAt != nil {
+		filter.ByExistedAt(*existedAt)
+	}
+	if id := c.String("id"); id != "" {
+		tunnelID, err := uuid.Parse(id)
+		if err != nil {
+			return errors.Wrapf(err, "%s is not a valid tunnel ID", id)
+		}
+		filter.ByTunnelID(tunnelID)
+	}
+
+	tunnels, err := sc.list(filter)
+	if err != nil {
+		return err
+	}
+
+	if outputFormat := c.String(outputFormatFlag.Name); outputFormat != "" {
+		return renderOutput(outputFormat, tunnels)
+	}
+
+	if len(tunnels) > 0 {
+		fmtAndPrintTunnelList(tunnels, c.Bool("show-recently-disconnected"))
+	} else {
+		fmt.Println("You have no tunnels, use 'cloudflared tunnel create' to define a new tunnel")
+	}
+	return nil
+}
+
+func fmtAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconnected bool) {
+	const (
+		minWidth = 0
+		tabWidth = 8
+		padding  = 1
+		padChar  = ' '
+		flags    = 0
+	)
+
+	writer := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
+	defer writer.Flush()
+
+	// Print column headers with tabbed columns
+	fmt.Fprintln(writer, "ID\tNAME\tCREATED\tCONNECTIONS\t")
+
+	// Loop through tunnels, create formatted string for each, and print using tabwriter
+	for _, t := range tunnels {
+		formattedStr := fmt.Sprintf(
+			"%s\t%s\t%s\t%s\t",
+			t.ID,
+			t.Name,
+			t.CreatedAt.Format(time.RFC3339),
+			fmtConnections(t.Connections, showRecentlyDisconnected),
+		)
+		fmt.Fprintln(writer, formattedStr)
+	}
+}
+
+func fmtConnections(connections []tunnelstore.Connection, showRecentlyDisconnected bool) string {
+
+	// Count connections per colo
+	numConnsPerColo := make(map[string]uint, len(connections))
+	for _, connection := range connections {
+		if !connection.IsPendingReconnect || showRecentlyDisconnected {
+			numConnsPerColo[connection.ColoName]++
+		}
+	}
+
+	// Get sorted list of colos
+	sortedColos := []string{}
+	for coloName := range numConnsPerColo {
+		sortedColos = append(sortedColos, coloName)
+	}
+	sort.Strings(sortedColos)
+
+	// Map each colo to its frequency, combine into output string.
+	var output []string
+	for _, coloName := range sortedColos {
+		output = append(output, fmt.Sprintf("%dx%s", numConnsPerColo[coloName], coloName))
+	}
+	return strings.Join(output, ", ")
+}
+
+func buildDeleteCommand() *cli.Command {
+	return &cli.Command{
+		Name:               "delete",
+		Action:             cliutil.ErrorHandler(deleteCommand),
+		Usage:              "Delete existing tunnel by UUID or name",
+		UsageText:          "cloudflared tunnel [tunnel command options] delete [subcommand options] TUNNEL",
+		Description:        "cloudflared tunnel delete will delete tunnels with the given tunnel UUIDs or names. A tunnel cannot be deleted if it has active connections. To delete the tunnel unconditionally, use -f flag.",
+		Flags:              []cli.Flag{credentialsFileFlag, forceDeleteFlag},
+		CustomHelpTemplate: commandHelpTemplate(),
+	}
+}
+
+func deleteCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+
+	if c.NArg() < 1 {
+		return cliutil.UsageError(`"cloudflared tunnel delete" requires at least 1 argument, the ID or name of the tunnel to delete.`)
+	}
+
+	tunnelIDs, err := sc.findIDs(c.Args().Slice())
+	if err != nil {
+		return err
+	}
+
+	return sc.delete(tunnelIDs)
+}
+
+func renderOutput(format string, v interface{}) error {
+	switch format {
+	case "json":
+		encoder := json.NewEncoder(os.Stdout)
+		encoder.SetIndent("", "  ")
+		return encoder.Encode(v)
+	case "yaml":
+		return yaml.NewEncoder(os.Stdout).Encode(v)
+	default:
+		return errors.Errorf("Unknown output format '%s'", format)
+	}
+}
+
+func buildRunCommand() *cli.Command {
+	flags := []cli.Flag{
+		forceFlag,
+		credentialsFileFlag,
+		selectProtocolFlag,
+	}
+	flags = append(flags, configureProxyFlags(false)...)
+	return &cli.Command{
+		Name:      "run",
+		Action:    cliutil.ErrorHandler(runCommand),
+		Before:    SetFlagsFromConfigFile,
+		Usage:     "Proxy a local web server by running the given tunnel",
+		UsageText: "cloudflared tunnel [tunnel command options] run [subcommand options] [TUNNEL]",
+		Description: `Runs the tunnel identified by name or UUUD, creating highly available connections 
+  between your server and the Cloudflare edge. You can provide name or UUID of tunnel to run either as the
+  last command line argument or in the configuration file using "tunnel: TUNNEL".
+
+  This command requires the tunnel credentials file created when "cloudflared tunnel create" was run, 
+  however it does not need access to cert.pem from "cloudflared login" if you identify the tunnel by UUID.
+  If you experience other problems running the tunnel, "cloudflared tunnel cleanup" may help by removing 
+  any old connection records.
+`,
+		Flags:              flags,
+		CustomHelpTemplate: commandHelpTemplate(),
+	}
+}
+
+func runCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+
+	if c.NArg() > 1 {
+		return cliutil.UsageError(`"cloudflared tunnel run" accepts only one argument, the ID or name of the tunnel to run.`)
+	}
+	tunnelRef := c.Args().First()
+	if tunnelRef == "" {
+		// see if tunnel id was in the config file
+		tunnelRef = config.GetConfiguration().TunnelID
+		if tunnelRef == "" {
+			return cliutil.UsageError(`"cloudflared tunnel run" requires the ID or name of the tunnel to run as the last command line argument or in the configuration file.`)
+		}
+	}
+
+	return runNamedTunnel(sc, tunnelRef)
+}
+
+func runNamedTunnel(sc *subcommandContext, tunnelRef string) error {
+	tunnelID, err := sc.findID(tunnelRef)
+	if err != nil {
+		return errors.Wrap(err, "error parsing tunnel ID")
+	}
+
+	sc.logger.Infof("Starting tunnel %s", tunnelID.String())
+
+	return sc.run(tunnelID)
+}
+
+func buildCleanupCommand() *cli.Command {
+	return &cli.Command{
+		Name:               "cleanup",
+		Action:             cliutil.ErrorHandler(cleanupCommand),
+		Usage:              "Cleanup tunnel connections",
+		UsageText:          "cloudflared tunnel [tunnel command options] cleanup [subcommand options] TUNNEL",
+		Description:        "Delete connections for tunnels with the given UUIDs or names.",
+		CustomHelpTemplate: commandHelpTemplate(),
+	}
+}
+
+func cleanupCommand(c *cli.Context) error {
+	if c.NArg() < 1 {
+		return cliutil.UsageError(`"cloudflared tunnel cleanup" requires at least 1 argument, the IDs of the tunnels to cleanup connections.`)
+	}
+
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+
+	tunnelIDs, err := sc.findIDs(c.Args().Slice())
+	if err != nil {
+		return err
+	}
+
+	return sc.cleanupConnections(tunnelIDs)
+}
+
+func buildRouteCommand() *cli.Command {
+	return &cli.Command{
+		Name:      "route",
+		Action:    cliutil.ErrorHandler(routeCommand),
+		Usage:     "Define what hostname or load balancer can route to this tunnel",
+		UsageText: "cloudflared tunnel [tunnel command options] route [subcommand options] dns|lb TUNNEL HOSTNAME [LB-POOL]",
+		Description: `The route defines what hostname or load balancer will proxy requests to this tunnel.
+
+   To route a hostname by creating a CNAME to tunnel's address:
+      cloudflared tunnel route dns <tunnel ID> <hostname>
+   To use this tunnel as a load balancer origin, creating pool and load balancer if necessary:
+      cloudflared tunnel route lb <tunnel ID> <load balancer name> <load balancer pool>`,
+		CustomHelpTemplate: commandHelpTemplate(),
+	}
+}
+
+func dnsRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
+	const (
+		userHostnameIndex = 2
+		expectedNArgs     = 3
+	)
+	if c.NArg() != expectedNArgs {
+		return nil, cliutil.UsageError("Expected %d arguments, got %d", expectedNArgs, c.NArg())
+	}
+	userHostname := c.Args().Get(userHostnameIndex)
+	if userHostname == "" {
+		return nil, cliutil.UsageError("The third argument should be the hostname")
+	} else if !validateHostname(userHostname) {
+		return nil, errors.Errorf("%s is not a valid hostname", userHostname)
+	}
+	return tunnelstore.NewDNSRoute(userHostname), nil
+}
+
+func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
+	const (
+		lbNameIndex   = 2
+		lbPoolIndex   = 3
+		expectedNArgs = 4
+	)
+	if c.NArg() != expectedNArgs {
+		return nil, cliutil.UsageError("Expected %d arguments, got %d", expectedNArgs, c.NArg())
+	}
+	lbName := c.Args().Get(lbNameIndex)
+	if lbName == "" {
+		return nil, cliutil.UsageError("The third argument should be the load balancer name")
+	} else if !validateHostname(lbName) {
+		return nil, errors.Errorf("%s is not a valid load balancer name", lbName)
+	}
+
+	lbPool := c.Args().Get(lbPoolIndex)
+	if lbPool == "" {
+		return nil, cliutil.UsageError("The fourth argument should be the pool name")
+	} else if !validateName(lbPool) {
+		return nil, errors.Errorf("%s is not a valid pool name", lbPool)
+	}
+
+	return tunnelstore.NewLBRoute(lbName, lbPool), nil
+}
+
+var nameRegex = regexp.MustCompile("^[_a-zA-Z0-9][-_.a-zA-Z0-9]*$")
+
+func validateName(s string) bool {
+	return nameRegex.MatchString(s)
+}
+
+func validateHostname(s string) bool {
+	// Slightly stricter than PunyCodeProfile
+	idnaProfile := idna.New(
+		idna.ValidateLabels(true),
+		idna.VerifyDNSLength(true))
+
+	puny, err := idnaProfile.ToASCII(s)
+	return err == nil && validateName(puny)
+}
+
+func routeCommand(c *cli.Context) error {
+	if c.NArg() < 2 {
+		return cliutil.UsageError(`"cloudflared tunnel route" requires the first argument to be the route type(dns or lb), followed by the ID or name of the tunnel`)
+	}
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+
+	const tunnelIDIndex = 1
+
+	routeType := c.Args().First()
+	var route tunnelstore.Route
+	var tunnelID uuid.UUID
+	switch routeType {
+	case "dns":
+		tunnelID, err = sc.findID(c.Args().Get(tunnelIDIndex))
+		if err != nil {
+			return err
+		}
+		route, err = dnsRouteFromArg(c)
+		if err != nil {
+			return err
+		}
+	case "lb":
+		tunnelID, err = sc.findID(c.Args().Get(tunnelIDIndex))
+		if err != nil {
+			return err
+		}
+		route, err = lbRouteFromArg(c)
+		if err != nil {
+			return err
+		}
+	default:
+		return cliutil.UsageError("%s is not a recognized route type. Supported route types are dns and lb", routeType)
+	}
+
+	res, err := sc.route(tunnelID, route)
+	if err != nil {
+		return err
+	}
+
+	sc.logger.Infof(res.SuccessSummary())
+	return nil
+}
+
+func commandHelpTemplate() string {
+	var parentFlagsHelp string
+	for _, f := range configureCloudflaredFlags(false) {
+		parentFlagsHelp += fmt.Sprintf(" %s\n\t", f)
+	}
+	for _, f := range configureLoggingFlags(false) {
+		parentFlagsHelp += fmt.Sprintf(" %s\n\t", f)
+	}
+	const template = `NAME:
+	{{.HelpName}} - {{.Usage}}
+	
+USAGE:
+	{{.UsageText}}
+ 
+DESCRIPTION:
+	{{.Description}}
+ 
+TUNNEL COMMAND OPTIONS:
+	%s
+SUBCOMMAND OPTIONS:
+	{{range .VisibleFlags}}{{.}}
+	{{end}}
+`
+	return fmt.Sprintf(template, parentFlagsHelp)
+}

cmd/cloudflared/tunnel/subcommands_test.go

@@ -0,0 +1,124 @@
+package tunnel
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/cloudflare/cloudflared/tunnelstore"
+	"github.com/mitchellh/go-homedir"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_fmtConnections(t *testing.T) {
+	type args struct {
+		connections []tunnelstore.Connection
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{
+			name: "empty",
+			args: args{
+				connections: []tunnelstore.Connection{},
+			},
+			want: "",
+		},
+		{
+			name: "trivial",
+			args: args{
+				connections: []tunnelstore.Connection{
+					{
+						ColoName: "DFW",
+						ID:       uuid.MustParse("ea550130-57fd-4463-aab1-752822231ddd"),
+					},
+				},
+			},
+			want: "1xDFW",
+		},
+		{
+			name: "with a pending reconnect",
+			args: args{
+				connections: []tunnelstore.Connection{
+					{
+						ColoName:           "DFW",
+						ID:                 uuid.MustParse("ea550130-57fd-4463-aab1-752822231ddd"),
+						IsPendingReconnect: true,
+					},
+				},
+			},
+			want: "",
+		},
+		{
+			name: "many colos",
+			args: args{
+				connections: []tunnelstore.Connection{
+					{
+						ColoName: "YRV",
+						ID:       uuid.MustParse("ea550130-57fd-4463-aab1-752822231ddd"),
+					},
+					{
+						ColoName: "DFW",
+						ID:       uuid.MustParse("c13c0b3b-0fbf-453c-8169-a1990fced6d0"),
+					},
+					{
+						ColoName: "ATL",
+						ID:       uuid.MustParse("70c90639-e386-4e8d-9a4e-7f046d70e63f"),
+					},
+					{
+						ColoName: "DFW",
+						ID:       uuid.MustParse("30ad6251-0305-4635-a670-d3994f474981"),
+					},
+				},
+			},
+			want: "1xATL, 2xDFW, 1xYRV",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := fmtConnections(tt.args.connections, false); got != tt.want {
+				t.Errorf("fmtConnections() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestTunnelfilePath(t *testing.T) {
+	tunnelID, err := uuid.Parse("f48d8918-bc23-4647-9d48-082c5b76de65")
+	assert.NoError(t, err)
+	originCertDir := filepath.Dir("~/.cloudflared/cert.pem")
+	actual, err := tunnelFilePath(tunnelID, originCertDir)
+	assert.NoError(t, err)
+	homeDir, err := homedir.Dir()
+	assert.NoError(t, err)
+	expected := fmt.Sprintf("%s/.cloudflared/%v.json", homeDir, tunnelID)
+	assert.Equal(t, expected, actual)
+}
+
+func TestValidateName(t *testing.T) {
+	tests := []struct {
+		name string
+		want bool
+	}{
+		{name: "", want: false},
+		{name: "-", want: false},
+		{name: ".", want: false},
+		{name: "a b", want: false},
+		{name: "a+b", want: false},
+		{name: "-ab", want: false},
+
+		{name: "ab", want: true},
+		{name: "ab-c", want: true},
+		{name: "abc.def", want: true},
+		{name: "_ab_c.-d-ef", want: true},
+	}
+	for _, tt := range tests {
+		if got := validateName(tt.name); got != tt.want {
+			t.Errorf("validateName() = %v, want %v", got, tt.want)
+		}
+	}
+}

cmd/cloudflared/ui/launch_ui.go

@@ -0,0 +1,244 @@
+package ui
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/logger"
+
+	"github.com/gdamore/tcell"
+	"github.com/rivo/tview"
+)
+
+type connState struct {
+	location string
+	state    status
+}
+
+type status int
+
+const (
+	Disconnected status = iota
+	Connected
+	Reconnecting
+	SetUrl
+	RegisteringTunnel
+)
+
+type TunnelEvent struct {
+	Index     uint8
+	EventType status
+	Location  string
+	Url       string
+}
+
+type uiModel struct {
+	version       string
+	edgeURL       string
+	metricsURL    string
+	localServices []string
+	connections   []connState
+}
+
+type palette struct {
+	url          string
+	connected    string
+	defaultText  string
+	disconnected string
+	reconnecting string
+}
+
+func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haConnections int) *uiModel {
+	localServices := make([]string, len(ing.Rules))
+	for i, rule := range ing.Rules {
+		localServices[i] = rule.Service.String()
+	}
+	return &uiModel{
+		version:       version,
+		edgeURL:       hostname,
+		metricsURL:    metricsURL,
+		localServices: localServices,
+		connections:   make([]connState, haConnections),
+	}
+}
+
+func (data *uiModel) LaunchUI(
+	ctx context.Context,
+	generalLogger, transportLogger logger.Service,
+	logLevels []logger.Level,
+	tunnelEventChan <-chan TunnelEvent,
+) {
+	// Configure the logger to stream logs into the textview
+
+	// Add TextView as a group to write output to
+	logTextView := NewDynamicColorTextView()
+	generalLogger.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+	transportLogger.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+
+	// Construct the UI
+	palette := palette{
+		url:          "lightblue",
+		connected:    "lime",
+		defaultText:  "white",
+		disconnected: "red",
+		reconnecting: "orange",
+	}
+
+	app := tview.NewApplication()
+
+	grid := tview.NewGrid().SetGap(1, 0)
+	frame := tview.NewFrame(grid)
+	header := fmt.Sprintf("cloudflared [::b]%s", data.version)
+
+	frame.AddText(header, true, tview.AlignLeft, tcell.ColorWhite)
+
+	// Create table to store connection info and status
+	connTable := tview.NewTable()
+	// SetColumns takes a value for each column, representing the size of the column
+	// Numbers <= 0 represent proportional widths and positive numbers represent absolute widths
+	grid.SetColumns(20, 0)
+
+	// SetRows takes a value for each row, representing the size of the row
+	grid.SetRows(1, 1, len(data.connections), 1, 0)
+
+	// AddItem takes a primitive tview type, row, column, rowSpan, columnSpan, minGridHeight, minGridWidth, and focus
+	grid.AddItem(tview.NewTextView().SetText("Tunnel:"), 0, 0, 1, 1, 0, 0, false)
+	grid.AddItem(tview.NewTextView().SetText("Status:"), 1, 0, 1, 1, 0, 0, false)
+	grid.AddItem(tview.NewTextView().SetText("Connections:"), 2, 0, 1, 1, 0, 0, false)
+
+	grid.AddItem(tview.NewTextView().SetText("Metrics:"), 3, 0, 1, 1, 0, 0, false)
+
+	tunnelHostText := tview.NewTextView().SetText(data.edgeURL)
+
+	grid.AddItem(tunnelHostText, 0, 1, 1, 1, 0, 0, false)
+	status := fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, strings.Join(data.localServices, ", "))
+	grid.AddItem(NewDynamicColorTextView().SetText(status), 1, 1, 1, 1, 0, 0, false)
+
+	grid.AddItem(connTable, 2, 1, 1, 1, 0, 0, false)
+
+	grid.AddItem(NewDynamicColorTextView().SetText(fmt.Sprintf("Metrics at [%s::b]http://%s/metrics", palette.url, data.metricsURL)), 3, 1, 1, 1, 0, 0, false)
+
+	// Add TextView to stream logs
+	// Logs are displayed in a new grid so a border can be set around them
+	logGrid := tview.NewGrid().SetBorders(true).AddItem(logTextView.SetChangedFunc(handleNewText(app, logTextView)), 0, 0, 5, 2, 0, 0, false)
+	// LogFrame holds the Logs header as well as the grid with the textView for streamed logs
+	logFrame := tview.NewFrame(logGrid).AddText("[::b]Logs:[::-]", true, tview.AlignLeft, tcell.ColorWhite).SetBorders(0, 0, 0, 0, 0, 0)
+	// Footer for log frame
+	logFrame.AddText("[::d]Use Ctrl+C to exit[::-]", false, tview.AlignRight, tcell.ColorWhite)
+	grid.AddItem(logFrame, 4, 0, 5, 2, 0, 0, false)
+
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				app.Stop()
+				return
+			case event := <-tunnelEventChan:
+				switch event.EventType {
+				case Connected:
+					data.setConnTableCell(event, connTable, palette)
+				case Disconnected, Reconnecting:
+					data.changeConnStatus(event, connTable, generalLogger, palette)
+				case SetUrl:
+					tunnelHostText.SetText(event.Url)
+					data.edgeURL = event.Url
+				case RegisteringTunnel:
+					if data.edgeURL == "" {
+						tunnelHostText.SetText("Registering tunnel...")
+					}
+				}
+			}
+			app.Draw()
+		}
+	}()
+
+	go func() {
+		if err := app.SetRoot(frame, true).Run(); err != nil {
+			generalLogger.Errorf("Error launching UI: %s", err)
+		}
+	}()
+}
+
+func NewDynamicColorTextView() *tview.TextView {
+	return tview.NewTextView().SetDynamicColors(true)
+}
+
+// Re-draws application when new logs are streamed to UI
+func handleNewText(app *tview.Application, logTextView *tview.TextView) func() {
+	return func() {
+		app.Draw()
+		// SetFocus to enable scrolling in textview
+		app.SetFocus(logTextView)
+	}
+}
+
+func (data *uiModel) changeConnStatus(event TunnelEvent, table *tview.Table, logger logger.Service, palette palette) {
+	index := int(event.Index)
+	// Get connection location and state
+	connState := data.getConnState(index)
+	// Check if connection is already displayed in UI
+	if connState == nil {
+		logger.Info("Connection is not in the UI table")
+		return
+	}
+
+	locationState := event.Location
+
+	if event.EventType == Disconnected {
+		connState.state = Disconnected
+	} else if event.EventType == Reconnecting {
+		connState.state = Reconnecting
+		locationState = "Reconnecting..."
+	}
+
+	connectionNum := index + 1
+	// Get table cell
+	cell := table.GetCell(index, 0)
+	// Change dot color in front of text as well as location state
+	text := newCellText(palette, connectionNum, locationState, event.EventType)
+	cell.SetText(text)
+}
+
+// Return connection location and row in UI table
+func (data *uiModel) getConnState(connID int) *connState {
+	if connID < len(data.connections) {
+		return &data.connections[connID]
+	}
+
+	return nil
+}
+
+func (data *uiModel) setConnTableCell(event TunnelEvent, table *tview.Table, palette palette) {
+	index := int(event.Index)
+	connectionNum := index + 1
+
+	// Update slice to keep track of connection location and state in UI table
+	data.connections[index].state = Connected
+	data.connections[index].location = event.Location
+
+	// Update text in table cell to show disconnected state
+	text := newCellText(palette, connectionNum, event.Location, event.EventType)
+	cell := tview.NewTableCell(text)
+	table.SetCell(index, 0, cell)
+}
+
+func newCellText(palette palette, connectionNum int, location string, connectedStatus status) string {
+	// HA connection indicator formatted as: "• #<CONNECTION_INDEX>: <COLO>",
+	//  where the left middle dot's color depends on the status of the connection
+	const connFmtString = "[%s]\u2022[%s] #%d: %s"
+
+	var dotColor string
+	switch connectedStatus {
+	case Connected:
+		dotColor = palette.connected
+	case Disconnected:
+		dotColor = palette.disconnected
+	case Reconnecting:
+		dotColor = palette.reconnecting
+	}
+
+	return fmt.Sprintf(connFmtString, dotColor, palette.defaultText, connectionNum, location)
+}

cmd/cloudflared/updater/service.go

@@ -0,0 +1,26 @@
+package updater
+
+// Version is the functions needed to perform an update
+type Version interface {
+	Apply() error
+	String() string
+}
+
+// Service is the functions to get check for new updates
+type Service interface {
+	Check() (Version, error)
+}
+
+const (
+	// OSKeyName is the url parameter key to send to the checkin API for the operating system of the local cloudflared (e.g. windows, darwin, linux)
+	OSKeyName = "os"
+
+	// ArchitectureKeyName is the url parameter key to send to the checkin API for the architecture of the local cloudflared (e.g. amd64, x86)
+	ArchitectureKeyName = "arch"
+
+	// BetaKeyName is the url parameter key to send to the checkin API to signal if the update should be a beta version or not
+	BetaKeyName = "beta"
+
+	// VersionKeyName is the url parameter key to send to the checkin API to specific what version to upgrade or downgrade to
+	VersionKeyName = "version"
+)

cmd/cloudflared/updater/update.go

@@ -2,36 +2,70 @@ package updater
 
 import (
 	"context"
+	"fmt"
 	"os"
+	"path/filepath"
 	"runtime"
 	"time"
 
+	"github.com/urfave/cli/v2"
 	"golang.org/x/crypto/ssh/terminal"
-	"gopkg.in/urfave/cli.v2"
 
-	"github.com/cloudflare/cloudflared/log"
-	"github.com/equinox-io/equinox"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/facebookgo/grace/gracenet"
+	"github.com/pkg/errors"
 )
 
 const (
-	DefaultCheckUpdateFreq   = time.Hour * 24
-	appID                    = "app_idCzgxYerVD"
-	noUpdateInShellMessage   = "cloudflared will not automatically update when run from the shell. To enable auto-updates, run cloudflared as a service: https://developers.cloudflare.com/argo-tunnel/reference/service/"
-	noUpdateOnWindowsMessage = "cloudflared will not automatically update on Windows systems."
+	DefaultCheckUpdateFreq        = time.Hour * 24
+	appID                         = "app_idCzgxYerVD"
+	noUpdateInShellMessage        = "cloudflared will not automatically update when run from the shell. To enable auto-updates, run cloudflared as a service: https://developers.cloudflare.com/argo-tunnel/reference/service/"
+	noUpdateOnWindowsMessage      = "cloudflared will not automatically update on Windows systems."
+	noUpdateManagedPackageMessage = "cloudflared will not automatically update if installed by a package manager."
+	isManagedInstallFile          = ".installedFromPackageManager"
+	UpdateURL                     = "https://update.argotunnel.com"
+	StagingUpdateURL              = "https://staging-update.argotunnel.com"
 )
 
 var (
-	publicKey = []byte(`
------BEGIN ECDSA PUBLIC KEY-----
-MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE4OWZocTVZ8Do/L6ScLdkV+9A0IYMHoOf
-dsCmJ/QZ6aw0w9qkkwEpne1Lmo6+0pGexZzFZOH6w5amShn+RXt7qkSid9iWlzGq
-EKx0BZogHSor9Wy5VztdFaAaVbsJiCbO
------END ECDSA PUBLIC KEY-----
-`)
-	logger = log.CreateLogger()
+	version string
 )
 
+// BinaryUpdated implements ExitCoder interface, the app will exit with status code 11
+// https://pkg.go.dev/github.com/urfave/cli/v2?tab=doc#ExitCoder
+type statusSuccess struct {
+	newVersion string
+}
+
+func (u *statusSuccess) Error() string {
+	return fmt.Sprintf("cloudflared has been updated to version %s", u.newVersion)
+}
+
+func (u *statusSuccess) ExitCode() int {
+	return 11
+}
+
+// UpdateErr implements ExitCoder interface, the app will exit with status code 10
+type statusErr struct {
+	err error
+}
+
+func (e *statusErr) Error() string {
+	return fmt.Sprintf("failed to update cloudflared: %v", e.err)
+}
+
+func (e *statusErr) ExitCode() int {
+	return 10
+}
+
+type updateOptions struct {
+	isBeta    bool
+	isStaging bool
+	isForced  bool
+	version   string
+}
+
 type UpdateOutcome struct {
 	Updated bool
 	Version string
@@ -39,52 +73,93 @@ type UpdateOutcome struct {
 }
 
 func (uo *UpdateOutcome) noUpdate() bool {
-	return uo.Error != nil && uo.Updated == false
+	return uo.Error == nil && uo.Updated == false
 }
 
-func checkForUpdateAndApply() UpdateOutcome {
-	var opts equinox.Options
-	if err := opts.SetPublicKeyPEM(publicKey); err != nil {
-		return UpdateOutcome{Error: err}
-	}
+func Init(v string) {
+	version = v
+}
 
-	resp, err := equinox.Check(appID, opts)
-	switch {
-	case err == equinox.NotAvailableErr:
-		return UpdateOutcome{}
-	case err != nil:
-		return UpdateOutcome{Error: err}
-	}
-
-	err = resp.Apply()
+func checkForUpdateAndApply(options updateOptions) UpdateOutcome {
+	cfdPath, err := os.Executable()
 	if err != nil {
 		return UpdateOutcome{Error: err}
 	}
 
-	return UpdateOutcome{Updated: true, Version: resp.ReleaseVersion}
+	url := UpdateURL
+	if options.isStaging {
+		url = StagingUpdateURL
+	}
+
+	s := NewWorkersService(version, url, cfdPath, Options{IsBeta: options.isBeta,
+		IsForced: options.isForced, RequestedVersion: options.version})
+
+	v, err := s.Check()
+	if err != nil {
+		return UpdateOutcome{Error: err}
+	}
+
+	//already on the latest version
+	if v == nil {
+		return UpdateOutcome{}
+	}
+
+	err = v.Apply()
+	if err != nil {
+		return UpdateOutcome{Error: err}
+	}
+
+	return UpdateOutcome{Updated: true, Version: v.String()}
 }
 
-func Update(_ *cli.Context) error {
-	updateOutcome := loggedUpdate()
+// Update is the handler for the update command from the command line
+func Update(c *cli.Context) error {
+	logger, err := logger.New()
+	if err != nil {
+		return errors.Wrap(err, "error setting up logger")
+	}
+
+	if wasInstalledFromPackageManager() {
+		logger.Error("cloudflared was installed by a package manager. Please update using the same method.")
+		return nil
+	}
+
+	isBeta := c.Bool("beta")
+	if isBeta {
+		logger.Info("cloudflared is set to update to the latest beta version")
+	}
+
+	isStaging := c.Bool("staging")
+	if isStaging {
+		logger.Info("cloudflared is set to update from staging")
+	}
+
+	isForced := c.Bool("force")
+	if isForced {
+		logger.Info("cloudflared is set to upgrade to the latest publish version regardless of the current version")
+	}
+
+	updateOutcome := loggedUpdate(logger, updateOptions{isBeta: isBeta, isStaging: isStaging, isForced: isForced, version: c.String("version")})
 	if updateOutcome.Error != nil {
-		os.Exit(10)
+		return &statusErr{updateOutcome.Error}
 	}
 
 	if updateOutcome.noUpdate() {
 		logger.Infof("cloudflared is up to date (%s)", updateOutcome.Version)
+		return nil
 	}
 
-	return updateOutcome.Error
+	return &statusSuccess{newVersion: updateOutcome.Version}
 }
 
 // Checks for an update and applies it if one is available
-func loggedUpdate() UpdateOutcome {
-	updateOutcome := checkForUpdateAndApply()
+func loggedUpdate(logger logger.Service, options updateOptions) UpdateOutcome {
+	updateOutcome := checkForUpdateAndApply(options)
 	if updateOutcome.Updated {
 		logger.Infof("cloudflared has been updated to version %s", updateOutcome.Version)
 	}
 	if updateOutcome.Error != nil {
-		logger.WithError(updateOutcome.Error).Error("update check failed")
+		logger.Errorf("update check failed: %s", updateOutcome.Error)
 	}
 
 	return updateOutcome
@@ -95,6 +170,7 @@ type AutoUpdater struct {
 	configurable     *configurable
 	listeners        *gracenet.Net
 	updateConfigChan chan *configurable
+	logger           logger.Service
 }
 
 // AutoUpdaterConfigurable is the attributes of AutoUpdater that can be reconfigured during runtime
@@ -103,7 +179,7 @@ type configurable struct {
 	freq    time.Duration
 }
 
-func NewAutoUpdater(freq time.Duration, listeners *gracenet.Net) *AutoUpdater {
+func NewAutoUpdater(freq time.Duration, listeners *gracenet.Net, logger logger.Service) *AutoUpdater {
 	updaterConfigurable := &configurable{
 		enabled: true,
 		freq:    freq,
@@ -116,6 +192,7 @@ func NewAutoUpdater(freq time.Duration, listeners *gracenet.Net) *AutoUpdater {
 		configurable:     updaterConfigurable,
 		listeners:        listeners,
 		updateConfigChan: make(chan *configurable),
+		logger:           logger,
 	}
 }
 
@@ -123,18 +200,21 @@ func (a *AutoUpdater) Run(ctx context.Context) error {
 	ticker := time.NewTicker(a.configurable.freq)
 	for {
 		if a.configurable.enabled {
-			updateOutcome := loggedUpdate()
+			updateOutcome := loggedUpdate(a.logger, updateOptions{})
 			if updateOutcome.Updated {
-				os.Args = append(os.Args, "--is-autoupdated=true")
-				pid, err := a.listeners.StartProcess()
-				if err != nil {
-					logger.WithError(err).Error("Unable to restart server automatically")
-					return err
+				if IsSysV() {
+					// SysV doesn't have a mechanism to keep service alive, we have to restart the process
+					a.logger.Info("Restarting service managed by SysV...")
+					pid, err := a.listeners.StartProcess()
+					if err != nil {
+						a.logger.Errorf("Unable to restart server automatically: %s", err)
+						return &statusErr{err: err}
+					}
+					// stop old process after autoupdate. Otherwise we create a new process
+					// after each update
+					a.logger.Infof("PID of the new process is %d", pid)
 				}
-				// stop old process after autoupdate. Otherwise we create a new process
-				// after each update
-				logger.Infof("PID of the new process is %d", pid)
-				return nil
+				return &statusSuccess{newVersion: updateOutcome.Version}
 			}
 		}
 		select {
@@ -164,19 +244,24 @@ func (a *AutoUpdater) Update(newFreq time.Duration) {
 	a.updateConfigChan <- newConfigurable
 }
 
-func IsAutoupdateEnabled(c *cli.Context) bool {
-	if !SupportAutoUpdate() {
+func IsAutoupdateEnabled(c *cli.Context, l logger.Service) bool {
+	if !SupportAutoUpdate(l) {
 		return false
 	}
 	return !c.Bool("no-autoupdate") && c.Duration("autoupdate-freq") != 0
 }
 
-func SupportAutoUpdate() bool {
+func SupportAutoUpdate(logger logger.Service) bool {
 	if runtime.GOOS == "windows" {
 		logger.Info(noUpdateOnWindowsMessage)
 		return false
 	}
 
+	if wasInstalledFromPackageManager() {
+		logger.Info(noUpdateManagedPackageMessage)
+		return false
+	}
+
 	if isRunningFromTerminal() {
 		logger.Info(noUpdateInShellMessage)
 		return false
@@ -184,6 +269,22 @@ func SupportAutoUpdate() bool {
 	return true
 }
 
+func wasInstalledFromPackageManager() bool {
+	ok, _ := config.FileExists(filepath.Join(config.DefaultUnixConfigLocation, isManagedInstallFile))
+	return ok
+}
+
 func isRunningFromTerminal() bool {
 	return terminal.IsTerminal(int(os.Stdout.Fd()))
 }
+
+func IsSysV() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+
+	if _, err := os.Stat("/run/systemd/system"); err == nil {
+		return false
+	}
+	return true
+}

cmd/cloudflared/updater/update_test.go

@@ -4,13 +4,15 @@ import (
 	"context"
 	"testing"
 
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestDisabledAutoUpdater(t *testing.T) {
 	listeners := &gracenet.Net{}
-	autoupdater := NewAutoUpdater(0, listeners)
+	logger := logger.NewOutputWriter(logger.NewMockWriteManager())
+	autoupdater := NewAutoUpdater(0, listeners, logger)
 	ctx, cancel := context.WithCancel(context.Background())
 	errC := make(chan error)
 	go func() {

cmd/cloudflared/updater/workers_service.go

@@ -0,0 +1,153 @@
+package updater
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"runtime"
+	"strconv"
+	"strings"
+)
+
+// Options are the update options supported by the
+type Options struct {
+	// IsBeta is for beta updates to be installed if available
+	IsBeta bool
+
+	// IsForced is to forcibly download the latest version regardless of the current version
+	IsForced bool
+
+	// RequestedVersion is the specific version to upgrade or downgrade to
+	RequestedVersion string
+}
+
+// VersionResponse is the JSON response from the Workers API endpoint
+type VersionResponse struct {
+	URL          string `json:"url"`
+	Version      string `json:"version"`
+	Checksum     string `json:"checksum"`
+	IsCompressed bool   `json:"compressed"`
+	Error        string `json:"error"`
+}
+
+// WorkersService implements Service.
+// It contains everything needed to check in with the WorkersAPI and download and apply the updates
+type WorkersService struct {
+	currentVersion string
+	url            string
+	targetPath     string
+	opts           Options
+}
+
+// NewWorkersService creates a new updater Service object.
+func NewWorkersService(currentVersion, url, targetPath string, opts Options) Service {
+	return &WorkersService{
+		currentVersion: currentVersion,
+		url:            url,
+		targetPath:     targetPath,
+		opts:           opts,
+	}
+}
+
+// Check does a check in with the Workers API to get a new version update
+func (s *WorkersService) Check() (Version, error) {
+	client := &http.Client{
+		Timeout: clientTimeout,
+	}
+
+	req, err := http.NewRequest(http.MethodGet, s.url, nil)
+	q := req.URL.Query()
+	q.Add(OSKeyName, runtime.GOOS)
+	q.Add(ArchitectureKeyName, runtime.GOARCH)
+
+	if s.opts.IsBeta {
+		q.Add(BetaKeyName, "true")
+	}
+
+	if s.opts.RequestedVersion != "" {
+		q.Add(VersionKeyName, s.opts.RequestedVersion)
+	}
+
+	req.URL.RawQuery = q.Encode()
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var v VersionResponse
+	if err := json.NewDecoder(resp.Body).Decode(&v); err != nil {
+		return nil, err
+	}
+
+	if v.Error != "" {
+		return nil, errors.New(v.Error)
+	}
+
+	if !s.opts.IsForced && !IsNewerVersion(s.currentVersion, v.Version) {
+		return nil, nil
+	}
+
+	return NewWorkersVersion(v.URL, v.Version, v.Checksum, s.targetPath, v.IsCompressed), nil
+}
+
+// IsNewerVersion checks semantic versioning for the latest version
+// cloudflared tagging is more of a date than a semantic version,
+// but the same comparision logic still holds for major.minor.patch
+// e.g. 2020.8.2 is newer than 2020.8.1.
+func IsNewerVersion(current string, check string) bool {
+	if strings.Contains(strings.ToLower(current), "dev") {
+		return false // dev builds shouldn't update
+	}
+
+	cMajor, cMinor, cPatch, err := SemanticParts(current)
+	if err != nil {
+		return false
+	}
+
+	nMajor, nMinor, nPatch, err := SemanticParts(check)
+	if err != nil {
+		return false
+	}
+
+	if nMajor > cMajor {
+		return true
+	}
+
+	if nMajor == cMajor && nMinor > cMinor {
+		return true
+	}
+
+	if nMajor == cMajor && nMinor == cMinor && nPatch > cPatch {
+		return true
+	}
+	return false
+}
+
+// SemanticParts gets the major, minor, and patch version of a semantic version string
+// e.g. 3.1.2 would return 3, 1, 2, nil
+func SemanticParts(version string) (major int, minor int, patch int, err error) {
+	major = 0
+	minor = 0
+	patch = 0
+	parts := strings.Split(version, ".")
+	if len(parts) != 3 {
+		err = errors.New("invalid version")
+		return
+	}
+	major, err = strconv.Atoi(parts[0])
+	if err != nil {
+		return
+	}
+
+	minor, err = strconv.Atoi(parts[1])
+	if err != nil {
+		return
+	}
+
+	patch, err = strconv.Atoi(parts[2])
+	if err != nil {
+		return
+	}
+	return
+}

cmd/cloudflared/updater/workers_service_test.go

@@ -0,0 +1,273 @@
+package updater
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func respondWithJSON(w http.ResponseWriter, v interface{}, status int) {
+	data, _ := json.Marshal(v)
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	w.Write(data)
+}
+
+func respondWithData(w http.ResponseWriter, b []byte, status int) {
+	w.Header().Set("Content-Type", "application/octet-stream")
+	w.WriteHeader(status)
+	w.Write(b)
+}
+
+func updateHandler(w http.ResponseWriter, r *http.Request) {
+	version := "2020.08.05"
+	host := fmt.Sprintf("http://%s", r.Host)
+	url := host + "/download"
+
+	query := r.URL.Query()
+
+	if query.Get(BetaKeyName) == "true" {
+		version = "2020.08.06"
+		url = host + "/beta"
+	}
+
+	requestedVersion := query.Get(VersionKeyName)
+	if requestedVersion != "" {
+		version = requestedVersion
+		url = fmt.Sprintf("%s?version=%s", url, requestedVersion)
+	}
+
+	if query.Get(ArchitectureKeyName) != runtime.GOARCH || query.Get(OSKeyName) != runtime.GOOS {
+		respondWithJSON(w, VersionResponse{Error: "unsupported os and architecture"}, http.StatusBadRequest)
+		return
+	}
+
+	h := sha256.New()
+	fmt.Fprint(h, version)
+	checksum := fmt.Sprintf("%x", h.Sum(nil))
+
+	v := VersionResponse{URL: url, Version: version, Checksum: checksum}
+	respondWithJSON(w, v, http.StatusOK)
+}
+
+func gzipUpdateHandler(w http.ResponseWriter, r *http.Request) {
+	log.Println("got a request!")
+	version := "2020.09.02"
+	h := sha256.New()
+	fmt.Fprint(h, version)
+	checksum := fmt.Sprintf("%x", h.Sum(nil))
+
+	url := fmt.Sprintf("http://%s/gzip-download.tgz", r.Host)
+	v := VersionResponse{URL: url, Version: version, Checksum: checksum}
+	respondWithJSON(w, v, http.StatusOK)
+}
+
+func compressedDownloadHandler(w http.ResponseWriter, r *http.Request) {
+	version := "2020.09.02"
+	buf := new(bytes.Buffer)
+
+	gw := gzip.NewWriter(buf)
+	tw := tar.NewWriter(gw)
+
+	header := &tar.Header{
+		Size: int64(len(version)),
+		Name: "download",
+	}
+	tw.WriteHeader(header)
+	tw.Write([]byte(version))
+
+	tw.Close()
+	gw.Close()
+
+	respondWithData(w, buf.Bytes(), http.StatusOK)
+}
+
+func downloadHandler(w http.ResponseWriter, r *http.Request) {
+	version := "2020.08.05"
+	requestedVersion := r.URL.Query().Get(VersionKeyName)
+	if requestedVersion != "" {
+		version = requestedVersion
+	}
+	respondWithData(w, []byte(version), http.StatusOK)
+}
+
+func betaHandler(w http.ResponseWriter, r *http.Request) {
+	respondWithData(w, []byte("2020.08.06"), http.StatusOK)
+}
+
+func failureHandler(w http.ResponseWriter, r *http.Request) {
+	respondWithJSON(w, VersionResponse{Error: "unsupported os and architecture"}, http.StatusBadRequest)
+}
+
+func createServer() *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/updater", updateHandler)
+	mux.HandleFunc("/download", downloadHandler)
+	mux.HandleFunc("/beta", betaHandler)
+	mux.HandleFunc("/fail", failureHandler)
+	mux.HandleFunc("/compressed", gzipUpdateHandler)
+	mux.HandleFunc("/gzip-download.tgz", compressedDownloadHandler)
+	return httptest.NewServer(mux)
+}
+
+func createTestFile(t *testing.T, path string) {
+	f, err := os.Create("tmpfile")
+	require.NoError(t, err)
+	fmt.Fprint(f, "2020.08.04")
+	f.Close()
+}
+
+func TestUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+	log.Println("server url: ", ts.URL)
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, v.String(), "2020.08.05")
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, string(dat), "2020.08.05")
+}
+
+func TestBetaUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{IsBeta: true})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, v.String(), "2020.08.06")
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, string(dat), "2020.08.06")
+}
+
+func TestFailUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/fail", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.Error(t, err)
+	require.Nil(t, v)
+}
+
+func TestNoUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService("2020.8.5", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Nil(t, v)
+}
+
+func TestForcedUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService("2020.8.5", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{IsForced: true})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, v.String(), "2020.08.05")
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, string(dat), "2020.08.05")
+}
+
+func TestUpdateSpecificVersionService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+	reqVersion := "2020.9.1"
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{RequestedVersion: reqVersion})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, reqVersion, v.String())
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, reqVersion, string(dat))
+}
+
+func TestCompressedUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/compressed", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, "2020.09.02", v.String())
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, "2020.09.02", string(dat))
+}
+
+func TestVersionParsing(t *testing.T) {
+	require.False(t, IsNewerVersion("2020.8.2", "2020.8.2"))
+	require.True(t, IsNewerVersion("2020.8.2", "2020.8.3"))
+	require.True(t, IsNewerVersion("2020.8.2", "2021.1.2"))
+	require.True(t, IsNewerVersion("2020.8.2", "2020.9.1"))
+	require.True(t, IsNewerVersion("2020.8.2", "2020.12.45"))
+	require.False(t, IsNewerVersion("2020.8.2", "2020.6.3"))
+	require.False(t, IsNewerVersion("DEV", "2020.8.5"))
+	require.False(t, IsNewerVersion("2020.8.2", "asdlkfjasdf"))
+	require.True(t, IsNewerVersion("3.0.1", "4.2.1"))
+}

cmd/cloudflared/updater/workers_update.go

@@ -0,0 +1,230 @@
+package updater
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"text/template"
+	"time"
+)
+
+const (
+	clientTimeout = time.Second * 60
+	// stop the service
+	// rename cloudflared.exe to cloudflared.exe.old
+	// rename cloudflared.exe.new to cloudflared.exe
+	// delete cloudflared.exe.old
+	// start the service
+	// delete the batch file
+	windowsUpdateCommandTemplate = `@echo off
+sc stop cloudflared >nul 2>&1
+rename "{{.TargetPath}}" {{.OldName}}
+rename "{{.NewPath}}" {{.BinaryName}}
+del "{{.OldPath}}"
+sc start cloudflared >nul 2>&1
+del {{.BatchName}}`
+	batchFileName = "cfd_update.bat"
+)
+
+// Prepare some data to insert into the template.
+type batchData struct {
+	TargetPath string
+	OldName    string
+	NewPath    string
+	OldPath    string
+	BinaryName string
+	BatchName  string
+}
+
+// WorkersVersion implements the Version interface.
+// It contains everything needed to preform a version upgrade
+type WorkersVersion struct {
+	downloadURL  string
+	checksum     string
+	version      string
+	targetPath   string
+	isCompressed bool
+}
+
+// NewWorkersVersion creates a new Version object. This is normally created by a WorkersService JSON checkin response
+// url is where to download the file
+// version is the version of this update
+// checksum is the expected checksum of the downloaded file
+// target path is where the file should be replace. Normally this the running cloudflared's path
+func NewWorkersVersion(url, version, checksum, targetPath string, isCompressed bool) Version {
+	return &WorkersVersion{downloadURL: url, version: version, checksum: checksum, targetPath: targetPath, isCompressed: isCompressed}
+}
+
+// Apply does the actual verification and update logic.
+// This includes signature and checksum validation,
+// replacing the binary, etc
+func (v *WorkersVersion) Apply() error {
+	newFilePath := fmt.Sprintf("%s.new", v.targetPath)
+	os.Remove(newFilePath) //remove any failed updates before download
+
+	// download the file
+	if err := download(v.downloadURL, newFilePath, v.isCompressed); err != nil {
+		return err
+	}
+
+	// check that the file is what is expected
+	if err := isValidChecksum(v.checksum, newFilePath); err != nil {
+		return err
+	}
+
+	oldFilePath := fmt.Sprintf("%s.old", v.targetPath)
+	// Windows requires more effort to self update, especially when it is running as a service:
+	// you have to stop the service (if running as one) in order to move/rename the binary
+	// but now the binary isn't running though, so an external process
+	// has to move the old binary out and the new one in then start the service
+	// the easiest way to do this is with a batch file (or with a DLL, but that gets ugly for a cross compiled binary like cloudflared)
+	// a batch file isn't ideal, but it is the simplest path forward for the constraints Windows creates
+	if runtime.GOOS == "windows" {
+		if err := writeBatchFile(v.targetPath, newFilePath, oldFilePath); err != nil {
+			return err
+		}
+		rootDir := filepath.Dir(v.targetPath)
+		batchPath := filepath.Join(rootDir, batchFileName)
+		return runWindowsBatch(batchPath)
+	}
+
+	// now move the current file out, move the new file in and delete the old file
+	if err := os.Rename(v.targetPath, oldFilePath); err != nil {
+		return err
+	}
+
+	if err := os.Rename(newFilePath, v.targetPath); err != nil {
+		//attempt rollback
+		os.Rename(oldFilePath, v.targetPath)
+		return err
+	}
+	os.Remove(oldFilePath)
+
+	return nil
+}
+
+// String returns the version number of this update/release (e.g. 2020.08.05)
+func (v *WorkersVersion) String() string {
+	return v.version
+}
+
+// download the file from the link in the json
+func download(url, filepath string, isCompressed bool) error {
+	client := &http.Client{
+		Timeout: clientTimeout,
+	}
+	resp, err := client.Get(url)
+
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	var r io.Reader
+	r = resp.Body
+
+	// compressed macos binary, need to decompress
+	if isCompressed || isCompressedFile(url) {
+		// first the gzip reader
+		gr, err := gzip.NewReader(resp.Body)
+		if err != nil {
+			return err
+		}
+		defer gr.Close()
+
+		// now the tar
+		tr := tar.NewReader(gr)
+
+		// advance the reader pass the header, which will be the single binary file
+		tr.Next()
+
+		r = tr
+	}
+
+	out, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, r)
+	return err
+}
+
+// isCompressedFile is a really simple file extension check to see if this is a macos tar and gzipped
+func isCompressedFile(urlstring string) bool {
+	if strings.HasSuffix(urlstring, ".tgz") {
+		return true
+	}
+
+	u, err := url.Parse(urlstring)
+	if err != nil {
+		return false
+	}
+	return strings.HasSuffix(u.Path, ".tgz")
+}
+
+// checks if the checksum in the json response matches the checksum of the file download
+func isValidChecksum(checksum, filePath string) error {
+	f, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return err
+	}
+
+	hash := fmt.Sprintf("%x", h.Sum(nil))
+
+	if checksum != hash {
+		return errors.New("checksum validation failed")
+	}
+	return nil
+}
+
+// writeBatchFile writes a batch file out to disk
+// see the dicussion on why it has to be done this way
+func writeBatchFile(targetPath string, newPath string, oldPath string) error {
+	os.Remove(batchFileName) //remove any failed updates before download
+	f, err := os.Create(batchFileName)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	cfdName := filepath.Base(targetPath)
+	oldName := filepath.Base(oldPath)
+
+	data := batchData{
+		TargetPath: targetPath,
+		OldName:    oldName,
+		NewPath:    newPath,
+		OldPath:    oldPath,
+		BinaryName: cfdName,
+		BatchName:  batchFileName,
+	}
+
+	t, err := template.New("batch").Parse(windowsUpdateCommandTemplate)
+	if err != nil {
+		return err
+	}
+	return t.Execute(f, data)
+}
+
+// run each OS command for windows
+func runWindowsBatch(batchFile string) error {
+	cmd := exec.Command("cmd", "/c", batchFile)
+	return cmd.Start()
+}

cmd/cloudflared/windows_service.go

@@ -12,7 +12,9 @@ import (
 	"time"
 	"unsafe"
 
-	cli "gopkg.in/urfave/cli.v2"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/pkg/errors"
+	cli "github.com/urfave/cli/v2"
 
 	"golang.org/x/sys/windows"
 	"golang.org/x/sys/windows/svc"
@@ -65,6 +67,12 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 	//     2. get ERROR_FAILED_SERVICE_CONTROLLER_CONNECT
 	// This involves actually trying to start the service.
 
+	logger, err := logger.New()
+	if err != nil {
+		os.Exit(1)
+		return
+	}
+
 	isIntSess, err := svc.IsAnInteractiveSession()
 	if err != nil {
 		logger.Fatalf("failed to determine if we are running in an interactive session: %v", err)
@@ -97,9 +105,15 @@ type windowsService struct {
 
 // called by the package code at the start of the service
 func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeRequest, statusChan chan<- svc.Status) (ssec bool, errno uint32) {
+	logger, err := logger.New()
+	if err != nil {
+		os.Exit(1)
+		return
+	}
+
 	elog, err := eventlog.Open(windowsServiceName)
 	if err != nil {
-		logger.WithError(err).Errorf("Cannot open event log for %s", windowsServiceName)
+		logger.Errorf("Cannot open event log for %s with error: %s", windowsServiceName, err)
 		return
 	}
 	defer elog.Close()
@@ -136,12 +150,11 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 			switch c.Cmd {
 			case svc.Interrogate:
 				statusChan <- c.CurrentStatus
-			case svc.Stop:
+			case svc.Stop, svc.Shutdown:
 				close(s.graceShutdownC)
+				statusChan <- svc.Status{State: svc.Stopped, Accepts: cmdsAccepted}
 				statusChan <- svc.Status{State: svc.StopPending}
-			case svc.Shutdown:
-				close(s.shutdownC)
-				statusChan <- svc.Status{State: svc.StopPending}
+				return
 			default:
 				elog.Error(1, fmt.Sprintf("unexpected control request #%d", c))
 			}
@@ -160,6 +173,11 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 }
 
 func installWindowsService(c *cli.Context) error {
+	logger, err := logger.New()
+	if err != nil {
+		return errors.Wrap(err, "error setting up logger")
+	}
+
 	logger.Infof("Installing Argo Tunnel Windows service")
 	exepath, err := os.Executable()
 	if err != nil {
@@ -168,7 +186,7 @@ func installWindowsService(c *cli.Context) error {
 	}
 	m, err := mgr.Connect()
 	if err != nil {
-		logger.WithError(err).Errorf("Cannot establish a connection to the service control manager")
+		logger.Errorf("Cannot establish a connection to the service control manager: %s", err)
 		return err
 	}
 	defer m.Disconnect()
@@ -189,18 +207,23 @@ func installWindowsService(c *cli.Context) error {
 	err = eventlog.InstallAsEventCreate(windowsServiceName, eventlog.Error|eventlog.Warning|eventlog.Info)
 	if err != nil {
 		s.Delete()
-		logger.WithError(err).Errorf("Cannot install event logger")
+		logger.Errorf("Cannot install event logger: %s", err)
 		return fmt.Errorf("SetupEventLogSource() failed: %s", err)
 	}
 	err = configRecoveryOption(s.Handle)
 	if err != nil {
-		logger.WithError(err).Errorf("Cannot set service recovery actions")
+		logger.Errorf("Cannot set service recovery actions: %s", err)
 		logger.Infof("See %s to manually configure service recovery actions", windowsServiceUrl)
 	}
 	return nil
 }
 
 func uninstallWindowsService(c *cli.Context) error {
+	logger, err := logger.New()
+	if err != nil {
+		return errors.Wrap(err, "error setting up logger")
+	}
+
 	logger.Infof("Uninstalling Argo Tunnel Windows Service")
 	m, err := mgr.Connect()
 	if err != nil {

connection/connection.go

@@ -1,57 +1,62 @@
 package connection
 
 import (
-	"context"
-	"net"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
 	"time"
 
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/google/uuid"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-
-	"github.com/cloudflare/cloudflared/h2mux"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
-	openStreamTimeout = 30 * time.Second
+	lbProbeUserAgentPrefix = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
 )
 
-type Connection struct {
-	id          uuid.UUID
-	muxer       *h2mux.Muxer
-	addr        *net.TCPAddr
-	isLongLived bool
-	longLivedID int
+type Config struct {
+	OriginClient    OriginClient
+	GracePeriod     time.Duration
+	ReplaceExisting bool
 }
 
-func newConnection(muxer *h2mux.Muxer, addr *net.TCPAddr) (*Connection, error) {
-	id, err := uuid.NewRandom()
-	if err != nil {
-		return nil, err
-	}
-	return &Connection{
-		id:    id,
-		muxer: muxer,
-		addr:  addr,
-	}, nil
+type NamedTunnelConfig struct {
+	Auth   pogs.TunnelAuth
+	ID     uuid.UUID
+	Client pogs.ClientInfo
 }
 
-func (c *Connection) Serve(ctx context.Context) error {
-	// Serve doesn't return until h2mux is shutdown
-	return c.muxer.Serve(ctx)
+type ClassicTunnelConfig struct {
+	Hostname   string
+	OriginCert []byte
+	// feature-flag to use new edge reconnect tokens
+	UseReconnectToken bool
 }
 
-// Connect is used to establish connections with cloudflare's edge network
-func (c *Connection) Connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters, logger *logrus.Entry) (tunnelpogs.ConnectResult, error) {
-	tsClient, err := NewRPCClient(ctx, c.muxer, logger.WithField("rpc", "connect"), openStreamTimeout)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot create new RPC connection")
-	}
-	defer tsClient.Close()
-	return tsClient.Connect(ctx, parameters)
+func (c *ClassicTunnelConfig) IsTrialZone() bool {
+	return c.Hostname == ""
 }
 
-func (c *Connection) Shutdown() {
-	c.muxer.Shutdown()
+type OriginClient interface {
+	Proxy(w ResponseWriter, req *http.Request, isWebsocket bool) error
+}
+
+type ResponseWriter interface {
+	WriteRespHeaders(*http.Response) error
+	WriteErrorResponse()
+	io.ReadWriter
+}
+
+type ConnectedFuse interface {
+	Connected()
+	IsConnected() bool
+}
+
+func uint8ToString(input uint8) string {
+	return strconv.FormatUint(uint64(input), 10)
+}
+
+func isServerSentEvent(headers http.Header) bool {
+	return strings.ToLower(headers.Get("content-type")) == "text/event-stream"
 }

connection/connection_test.go

@@ -0,0 +1,113 @@
+package connection
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/gobwas/ws/wsutil"
+)
+
+const (
+	largeFileSize = 2 * 1024 * 1024
+)
+
+var (
+	testConfig = &Config{
+		OriginClient: &mockOriginClient{},
+		GracePeriod:  time.Millisecond * 100,
+	}
+	testLogger, _ = logger.New()
+	testOriginURL = &url.URL{
+		Scheme: "https",
+		Host:   "connectiontest.argotunnel.com",
+	}
+	testTunnelEventChan = make(chan ui.TunnelEvent)
+	testObserver        = &Observer{
+		testLogger,
+		m,
+		testTunnelEventChan,
+	}
+	testLargeResp = make([]byte, largeFileSize)
+)
+
+type testRequest struct {
+	name           string
+	endpoint       string
+	expectedStatus int
+	expectedBody   []byte
+	isProxyError   bool
+}
+
+type mockOriginClient struct {
+}
+
+func (moc *mockOriginClient) Proxy(w ResponseWriter, r *http.Request, isWebsocket bool) error {
+	if isWebsocket {
+		return wsEndpoint(w, r)
+	}
+	switch r.URL.Path {
+	case "/ok":
+		originRespEndpoint(w, http.StatusOK, []byte(http.StatusText(http.StatusOK)))
+	case "/large_file":
+		originRespEndpoint(w, http.StatusOK, testLargeResp)
+	case "/400":
+		originRespEndpoint(w, http.StatusBadRequest, []byte(http.StatusText(http.StatusBadRequest)))
+	case "/500":
+		originRespEndpoint(w, http.StatusInternalServerError, []byte(http.StatusText(http.StatusInternalServerError)))
+	case "/error":
+		return fmt.Errorf("Failed to proxy to origin")
+	default:
+		originRespEndpoint(w, http.StatusNotFound, []byte("page not found"))
+	}
+	return nil
+}
+
+type nowriter struct {
+	io.Reader
+}
+
+func (nowriter) Write(p []byte) (int, error) {
+	return 0, fmt.Errorf("Writer not implemented")
+}
+
+func wsEndpoint(w ResponseWriter, r *http.Request) error {
+	resp := &http.Response{
+		StatusCode: http.StatusSwitchingProtocols,
+	}
+	w.WriteRespHeaders(resp)
+	clientReader := nowriter{r.Body}
+	go func() {
+		for {
+			data, err := wsutil.ReadClientText(clientReader)
+			if err != nil {
+				return
+			}
+			if err := wsutil.WriteServerText(w, data); err != nil {
+				return
+			}
+		}
+	}()
+	<-r.Context().Done()
+	return nil
+}
+
+func originRespEndpoint(w ResponseWriter, status int, data []byte) {
+	resp := &http.Response{
+		StatusCode: status,
+	}
+	w.WriteRespHeaders(resp)
+	w.Write(data)
+}
+
+type mockConnectedFuse struct{}
+
+func (mcf mockConnectedFuse) Connected() {}
+
+func (mcf mockConnectedFuse) IsConnected() bool {
+	return true
+}

connection/errors.go

@@ -0,0 +1,76 @@
+package connection
+
+import (
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	DuplicateConnectionError = "EDUPCONN"
+)
+
+// RegisterTunnel error from client
+type clientRegisterTunnelError struct {
+	cause error
+}
+
+func newRPCError(cause error, counter *prometheus.CounterVec, name rpcName) clientRegisterTunnelError {
+	counter.WithLabelValues(cause.Error(), string(name)).Inc()
+	return clientRegisterTunnelError{cause: cause}
+}
+
+func (e clientRegisterTunnelError) Error() string {
+	return e.cause.Error()
+}
+
+type DupConnRegisterTunnelError struct{}
+
+var errDuplicationConnection = &DupConnRegisterTunnelError{}
+
+func (e DupConnRegisterTunnelError) Error() string {
+	return "already connected to this server, trying another address"
+}
+
+// RegisterTunnel error from server
+type serverRegisterTunnelError struct {
+	cause     error
+	permanent bool
+}
+
+func (e serverRegisterTunnelError) Error() string {
+	return e.cause.Error()
+}
+
+func serverRegistrationErrorFromRPC(err error) *serverRegisterTunnelError {
+	if retryable, ok := err.(*tunnelpogs.RetryableError); ok {
+		return &serverRegisterTunnelError{
+			cause:     retryable.Unwrap(),
+			permanent: false,
+		}
+	}
+	return &serverRegisterTunnelError{
+		cause:     err,
+		permanent: true,
+	}
+}
+
+type muxerShutdownError struct{}
+
+func (e muxerShutdownError) Error() string {
+	return "muxer shutdown"
+}
+
+func isHandshakeErrRecoverable(err error, connIndex uint8, observer *Observer) bool {
+	switch err.(type) {
+	case edgediscovery.DialError:
+		observer.Errorf("Connection %d unable to dial edge: %s", connIndex, err)
+	case h2mux.MuxerHandshakeError:
+		observer.Errorf("Connection %d handshake with edge server failed: %s", connIndex, err)
+	default:
+		observer.Errorf("Connection %d failed: %s", connIndex, err)
+		return false
+	}
+	return true
+}

connection/h2mux.go

@@ -0,0 +1,220 @@
+package connection
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	muxerTimeout      = 5 * time.Second
+	openStreamTimeout = 30 * time.Second
+)
+
+type h2muxConnection struct {
+	config      *Config
+	muxerConfig *MuxerConfig
+	muxer       *h2mux.Muxer
+	// connectionID is only used by metrics, and prometheus requires labels to be string
+	connIndexStr string
+	connIndex    uint8
+
+	observer *Observer
+}
+
+type MuxerConfig struct {
+	HeartbeatInterval  time.Duration
+	MaxHeartbeats      uint64
+	CompressionSetting h2mux.CompressionSetting
+	MetricsUpdateFreq  time.Duration
+}
+
+func (mc *MuxerConfig) H2MuxerConfig(h h2mux.MuxedStreamHandler, logger logger.Service) *h2mux.MuxerConfig {
+	return &h2mux.MuxerConfig{
+		Timeout:            muxerTimeout,
+		Handler:            h,
+		IsClient:           true,
+		HeartbeatInterval:  mc.HeartbeatInterval,
+		MaxHeartbeats:      mc.MaxHeartbeats,
+		Logger:             logger,
+		CompressionQuality: mc.CompressionSetting,
+	}
+}
+
+// NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
+func NewH2muxConnection(ctx context.Context,
+	config *Config,
+	muxerConfig *MuxerConfig,
+	edgeConn net.Conn,
+	connIndex uint8,
+	observer *Observer,
+) (*h2muxConnection, error, bool) {
+	h := &h2muxConnection{
+		config:       config,
+		muxerConfig:  muxerConfig,
+		connIndexStr: uint8ToString(connIndex),
+		connIndex:    connIndex,
+		observer:     observer,
+	}
+
+	// Establish a muxed connection with the edge
+	// Client mux handshake with agent server
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig.H2MuxerConfig(h, observer), h2mux.ActiveStreams)
+	if err != nil {
+		recoverable := isHandshakeErrRecoverable(err, connIndex, observer)
+		return nil, err, recoverable
+	}
+	h.muxer = muxer
+	return h, nil, false
+}
+
+func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelConfig, credentialManager CredentialManager, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() error {
+		stream, err := h.newRPCStream(serveCtx, register)
+		if err != nil {
+			return err
+		}
+		rpcClient := newRegistrationRPCClient(ctx, stream, h.observer)
+		defer rpcClient.Close()
+
+		if err = rpcClient.RegisterConnection(serveCtx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+			return err
+		}
+		connectedFuse.Connected()
+		return nil
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, true)
+		return nil
+	})
+	return errGroup.Wait()
+}
+
+func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel *ClassicTunnelConfig, credentialManager CredentialManager, registrationOptions *tunnelpogs.RegistrationOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() (err error) {
+		defer func() {
+			if err == nil {
+				connectedFuse.Connected()
+			}
+		}()
+		if classicTunnel.UseReconnectToken && connectedFuse.IsConnected() {
+			err := h.reconnectTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
+			if err == nil {
+				return nil
+			}
+			// log errors and proceed to RegisterTunnel
+			h.observer.Errorf("Couldn't reconnect connection %d. Reregistering it instead. Error was: %v", h.connIndex, err)
+		}
+		return h.registerTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, false)
+		return nil
+	})
+	return errGroup.Wait()
+}
+
+func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
+	// All routines should stop when muxer finish serving. When muxer is shutdown
+	// gracefully, it doesn't return an error, so we need to return errMuxerShutdown
+	// here to notify other routines to stop
+	err := h.muxer.Serve(ctx)
+	if err == nil {
+		return muxerShutdownError{}
+	}
+	return err
+}
+
+func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse ConnectedFuse, isNamedTunnel bool) {
+	updateMetricsTickC := time.Tick(h.muxerConfig.MetricsUpdateFreq)
+	for {
+		select {
+		case <-ctx.Done():
+			// UnregisterTunnel blocks until the RPC call returns
+			if connectedFuse.IsConnected() {
+				h.unregister(isNamedTunnel)
+			}
+			h.muxer.Shutdown()
+			return
+		case <-updateMetricsTickC:
+			h.observer.metrics.updateMuxerMetrics(h.connIndexStr, h.muxer.Metrics())
+		}
+	}
+}
+
+func (h *h2muxConnection) newRPCStream(ctx context.Context, rpcName rpcName) (*h2mux.MuxedStream, error) {
+	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer openStreamCancel()
+	stream, err := h.muxer.OpenRPCStream(openStreamCtx)
+	if err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
+
+func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
+	respWriter := &h2muxRespWriter{stream}
+
+	req, reqErr := h.newRequest(stream)
+	if reqErr != nil {
+		respWriter.WriteErrorResponse()
+		return reqErr
+	}
+
+	err := h.config.OriginClient.Proxy(respWriter, req, websocket.IsWebSocketUpgrade(req))
+	if err != nil {
+		respWriter.WriteErrorResponse()
+		return err
+	}
+	return nil
+}
+
+func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request, error) {
+	req, err := http.NewRequest("GET", "http://localhost:8080", h2mux.MuxedStreamReader{MuxedStream: stream})
+	if err != nil {
+		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
+	}
+	err = h2mux.H2RequestHeadersToH1Request(stream.Headers, req)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid request received")
+	}
+	return req, nil
+}
+
+type h2muxRespWriter struct {
+	*h2mux.MuxedStream
+}
+
+func (rp *h2muxRespWriter) WriteRespHeaders(resp *http.Response) error {
+	headers := h2mux.H1ResponseToH2ResponseHeaders(resp)
+	headers = append(headers, h2mux.Header{Name: responseMetaHeaderField, Value: responseMetaHeaderOrigin})
+	return rp.WriteHeaders(headers)
+}
+
+func (rp *h2muxRespWriter) WriteErrorResponse() {
+	rp.WriteHeaders([]h2mux.Header{
+		{Name: ":status", Value: "502"},
+		{Name: responseMetaHeaderField, Value: responseMetaHeaderCfd},
+	})
+	rp.Write([]byte("502 Bad Gateway"))
+}

connection/h2mux_test.go

@@ -0,0 +1,242 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testMuxerConfig = &MuxerConfig{
+		HeartbeatInterval:  time.Second * 5,
+		MaxHeartbeats:      5,
+		CompressionSetting: 0,
+		MetricsUpdateFreq:  time.Second * 5,
+	}
+)
+
+func newH2MuxConnection(ctx context.Context, t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
+	edgeConn, originConn := net.Pipe()
+	edgeMuxChan := make(chan *h2mux.Muxer)
+	go func() {
+		edgeMuxConfig := h2mux.MuxerConfig{
+			Logger: testObserver,
+		}
+		edgeMux, err := h2mux.Handshake(edgeConn, edgeConn, edgeMuxConfig, h2mux.ActiveStreams)
+		require.NoError(t, err)
+		edgeMuxChan <- edgeMux
+	}()
+	var connIndex = uint8(0)
+	h2muxConn, err, _ := NewH2muxConnection(ctx, testConfig, testMuxerConfig, originConn, connIndex, testObserver)
+	require.NoError(t, err)
+	return h2muxConn, <-edgeMuxChan
+}
+
+func TestServeStreamHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "/ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "/large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "/400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "/500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "/error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	for _, test := range tests {
+		headers := []h2mux.Header{
+			{
+				Name:  ":path",
+				Value: test.endpoint,
+			},
+		}
+		stream, err := edgeMux.OpenStream(ctx, headers, nil)
+		require.NoError(t, err)
+		require.True(t, hasHeader(stream, ":status", strconv.Itoa(test.expectedStatus)))
+
+		if test.isProxyError {
+			assert.True(t, hasHeader(stream, responseMetaHeaderField, responseMetaHeaderCfd))
+		} else {
+			assert.True(t, hasHeader(stream, responseMetaHeaderField, responseMetaHeaderOrigin))
+			body := make([]byte, len(test.expectedBody))
+			_, err = stream.Read(body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, body)
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+func TestServeStreamWS(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: "/ws",
+		},
+		{
+			Name:  "connection",
+			Value: "upgrade",
+		},
+		{
+			Name:  "upgrade",
+			Value: "websocket",
+		},
+	}
+
+	readPipe, writePipe := io.Pipe()
+	stream, err := edgeMux.OpenStream(ctx, headers, readPipe)
+	require.NoError(t, err)
+
+	require.True(t, hasHeader(stream, ":status", strconv.Itoa(http.StatusSwitchingProtocols)))
+	assert.True(t, hasHeader(stream, responseMetaHeaderField, responseMetaHeaderOrigin))
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientText(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerText(stream)
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	wg.Wait()
+}
+
+func hasHeader(stream *h2mux.MuxedStream, name, val string) bool {
+	for _, header := range stream.Headers {
+		if header.Name == name && header.Value == val {
+			return true
+		}
+	}
+	return false
+}
+
+func benchmarkServeStreamHTTPSimple(b *testing.B, test testRequest) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, b)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(b, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: test.endpoint,
+		},
+	}
+
+	body := make([]byte, len(test.expectedBody))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		stream, openstreamErr := edgeMux.OpenStream(ctx, headers, nil)
+		_, readBodyErr := stream.Read(body)
+		b.StopTimer()
+
+		require.NoError(b, openstreamErr)
+		assert.True(b, hasHeader(stream, responseMetaHeaderField, responseMetaHeaderOrigin))
+		require.True(b, hasHeader(stream, ":status", strconv.Itoa(http.StatusOK)))
+		require.NoError(b, readBodyErr)
+		require.Equal(b, test.expectedBody, body)
+	}
+
+	cancel()
+	wg.Wait()
+}
+
+func BenchmarkServeStreamHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "/ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}
+
+func BenchmarkServeStreamHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "/large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}

connection/header.go

@@ -0,0 +1,31 @@
+package connection
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+)
+
+const (
+	responseMetaHeaderField = "cf-cloudflared-response-meta"
+)
+
+var (
+	canonicalResponseUserHeadersField = http.CanonicalHeaderKey(h2mux.ResponseUserHeadersField)
+	canonicalResponseMetaHeaderField  = http.CanonicalHeaderKey(responseMetaHeaderField)
+	responseMetaHeaderCfd             = mustInitRespMetaHeader("cloudflared")
+	responseMetaHeaderOrigin          = mustInitRespMetaHeader("origin")
+)
+
+type responseMetaHeader struct {
+	Source string `json:"src"`
+}
+
+func mustInitRespMetaHeader(src string) string {
+	header, err := json.Marshal(responseMetaHeader{Source: src})
+	if err != nil {
+		panic(fmt.Sprintf("Failed to serialize response meta header = %s, err: %v", src, err))
+	}
+	return string(header)
+}

connection/http2.go

@@ -0,0 +1,221 @@
+package connection
+
+import (
+	"context"
+	"errors"
+	"io"
+	"math"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+
+	"golang.org/x/net/http2"
+)
+
+const (
+	internalUpgradeHeader = "Cf-Cloudflared-Proxy-Connection-Upgrade"
+	websocketUpgrade      = "websocket"
+	controlStreamUpgrade  = "control-stream"
+)
+
+var (
+	errNotFlusher = errors.New("ResponseWriter doesn't implement http.Flusher")
+)
+
+type http2Connection struct {
+	conn         net.Conn
+	server       *http2.Server
+	config       *Config
+	namedTunnel  *NamedTunnelConfig
+	connOptions  *tunnelpogs.ConnectionOptions
+	observer     *Observer
+	connIndexStr string
+	connIndex    uint8
+	wg           *sync.WaitGroup
+	// newRPCClientFunc allows us to mock RPCs during testing
+	newRPCClientFunc func(context.Context, io.ReadWriteCloser, logger.Service) NamedTunnelRPCClient
+	connectedFuse    ConnectedFuse
+}
+
+func NewHTTP2Connection(
+	conn net.Conn,
+	config *Config,
+	namedTunnelConfig *NamedTunnelConfig,
+	connOptions *tunnelpogs.ConnectionOptions,
+	observer *Observer,
+	connIndex uint8,
+	connectedFuse ConnectedFuse,
+) *http2Connection {
+	return &http2Connection{
+		conn: conn,
+		server: &http2.Server{
+			MaxConcurrentStreams: math.MaxUint32,
+		},
+		config:           config,
+		namedTunnel:      namedTunnelConfig,
+		connOptions:      connOptions,
+		observer:         observer,
+		connIndexStr:     uint8ToString(connIndex),
+		connIndex:        connIndex,
+		wg:               &sync.WaitGroup{},
+		newRPCClientFunc: newRegistrationRPCClient,
+		connectedFuse:    connectedFuse,
+	}
+}
+
+func (c *http2Connection) Serve(ctx context.Context) {
+	go func() {
+		<-ctx.Done()
+		c.close()
+	}()
+	c.server.ServeConn(c.conn, &http2.ServeConnOpts{
+		Context: ctx,
+		Handler: c,
+	})
+}
+
+func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	c.wg.Add(1)
+	defer c.wg.Done()
+
+	respWriter := &http2RespWriter{
+		r: r.Body,
+		w: w,
+	}
+	flusher, isFlusher := w.(http.Flusher)
+	if !isFlusher {
+		c.observer.Errorf("%T doesn't implement http.Flusher", w)
+		respWriter.WriteErrorResponse()
+		return
+	}
+	respWriter.flusher = flusher
+	var err error
+	if isControlStreamUpgrade(r) {
+		respWriter.shouldFlush = true
+		err = c.serveControlStream(r.Context(), respWriter)
+	} else if isWebsocketUpgrade(r) {
+		respWriter.shouldFlush = true
+		stripWebsocketUpgradeHeader(r)
+		err = c.config.OriginClient.Proxy(respWriter, r, true)
+	} else {
+		err = c.config.OriginClient.Proxy(respWriter, r, false)
+	}
+
+	if err != nil {
+		respWriter.WriteErrorResponse()
+	}
+}
+
+func (c *http2Connection) serveControlStream(ctx context.Context, respWriter *http2RespWriter) error {
+	rpcClient := c.newRPCClientFunc(ctx, respWriter, c.observer)
+	defer rpcClient.Close()
+
+	if err := rpcClient.RegisterConnection(ctx, c.namedTunnel, c.connOptions, c.connIndex, c.observer); err != nil {
+		return err
+	}
+	c.connectedFuse.Connected()
+
+	<-ctx.Done()
+	rpcClient.GracefulShutdown(ctx, c.config.GracePeriod)
+	return nil
+}
+
+func (c *http2Connection) close() {
+	// Wait for all serve HTTP handlers to return
+	c.wg.Wait()
+	c.conn.Close()
+}
+
+type http2RespWriter struct {
+	r           io.Reader
+	w           http.ResponseWriter
+	flusher     http.Flusher
+	shouldFlush bool
+}
+
+func (rp *http2RespWriter) WriteRespHeaders(resp *http.Response) error {
+	dest := rp.w.Header()
+	userHeaders := make(http.Header, len(resp.Header))
+	for header, values := range resp.Header {
+		// Since these are http2 headers, they're required to be lowercase
+		h2name := strings.ToLower(header)
+		for _, v := range values {
+			if h2name == "content-length" {
+				// This header has meaning in HTTP/2 and will be used by the edge,
+				// so it should be sent as an HTTP/2 response header.
+				dest.Add(h2name, v)
+				// Since these are http2 headers, they're required to be lowercase
+			} else if !h2mux.IsControlHeader(h2name) || h2mux.IsWebsocketClientHeader(h2name) {
+				// User headers, on the other hand, must all be serialized so that
+				// HTTP/2 header validation won't be applied to HTTP/1 header values
+				userHeaders.Add(h2name, v)
+			}
+		}
+	}
+
+	// Perform user header serialization and set them in the single header
+	dest.Set(canonicalResponseUserHeadersField, h2mux.SerializeHeaders(userHeaders))
+	rp.setResponseMetaHeader(responseMetaHeaderOrigin)
+	status := resp.StatusCode
+	// HTTP2 removes support for 101 Switching Protocols https://tools.ietf.org/html/rfc7540#section-8.1.1
+	if status == http.StatusSwitchingProtocols {
+		status = http.StatusOK
+	}
+	rp.w.WriteHeader(status)
+	if isServerSentEvent(resp.Header) {
+		rp.shouldFlush = true
+	}
+	if rp.shouldFlush {
+		rp.flusher.Flush()
+	}
+	return nil
+}
+
+func (rp *http2RespWriter) WriteErrorResponse() {
+	rp.setResponseMetaHeader(responseMetaHeaderCfd)
+	rp.w.WriteHeader(http.StatusBadGateway)
+}
+
+func (rp *http2RespWriter) setResponseMetaHeader(value string) {
+	rp.w.Header().Set(canonicalResponseMetaHeaderField, value)
+}
+
+func (rp *http2RespWriter) Read(p []byte) (n int, err error) {
+	return rp.r.Read(p)
+}
+
+func (rp *http2RespWriter) Write(p []byte) (n int, err error) {
+	defer func() {
+		// Implementer of OriginClient should make sure it doesn't write to the connection after Proxy returns
+		// Register a recover routine just in case.
+		if r := recover(); r != nil {
+			println("Recover from http2 response writer panic, error", r)
+		}
+	}()
+	n, err = rp.w.Write(p)
+	if err == nil && rp.shouldFlush {
+		rp.flusher.Flush()
+	}
+	return n, err
+}
+
+func (rp *http2RespWriter) Close() error {
+	return nil
+}
+
+func isControlStreamUpgrade(r *http.Request) bool {
+	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == controlStreamUpgrade
+}
+
+func isWebsocketUpgrade(r *http.Request) bool {
+	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == websocketUpgrade
+}
+
+func stripWebsocketUpgradeHeader(r *http.Request) {
+	r.Header.Del(internalUpgradeHeader)
+}

connection/http2_test.go

@@ -0,0 +1,303 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/http2"
+)
+
+var (
+	testTransport = http2.Transport{}
+)
+
+func newTestHTTP2Connection() (*http2Connection, net.Conn) {
+	edgeConn, originConn := net.Pipe()
+	var connIndex = uint8(0)
+	return NewHTTP2Connection(
+		originConn,
+		testConfig,
+		&NamedTunnelConfig{},
+		&pogs.ConnectionOptions{},
+		testObserver,
+		connIndex,
+		mockConnectedFuse{},
+	), edgeConn
+}
+
+func TestServeHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	for _, test := range tests {
+		endpoint := fmt.Sprintf("http://localhost:8080/%s", test.endpoint)
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+		require.NoError(t, err)
+
+		resp, err := edgeHTTP2Conn.RoundTrip(req)
+		require.NoError(t, err)
+		require.Equal(t, test.expectedStatus, resp.StatusCode)
+		if test.expectedBody != nil {
+			respBody, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, respBody)
+		}
+		if test.isProxyError {
+			require.Equal(t, responseMetaHeaderCfd, resp.Header.Get(responseMetaHeaderField))
+		} else {
+			require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(responseMetaHeaderField))
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+type mockNamedTunnelRPCClient struct {
+	registered   chan struct{}
+	unregistered chan struct{}
+}
+
+func (mc mockNamedTunnelRPCClient) RegisterConnection(
+	c context.Context,
+	config *NamedTunnelConfig,
+	options *tunnelpogs.ConnectionOptions,
+	connIndex uint8,
+	observer *Observer,
+) error {
+	close(mc.registered)
+	return nil
+}
+
+func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+	close(mc.unregistered)
+}
+
+func (mockNamedTunnelRPCClient) Close() {}
+
+type mockRPCClientFactory struct {
+	registered   chan struct{}
+	unregistered chan struct{}
+}
+
+func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, logger.Service) NamedTunnelRPCClient {
+	return mockNamedTunnelRPCClient{
+		registered:   mf.registered,
+		unregistered: mf.unregistered,
+	}
+}
+
+type wsRespWriter struct {
+	*httptest.ResponseRecorder
+	readPipe  *io.PipeReader
+	writePipe *io.PipeWriter
+}
+
+func newWSRespWriter() *wsRespWriter {
+	readPipe, writePipe := io.Pipe()
+	return &wsRespWriter{
+		httptest.NewRecorder(),
+		readPipe,
+		writePipe,
+	}
+}
+
+func (w *wsRespWriter) RespBody() io.ReadWriter {
+	return nowriter{w.readPipe}
+}
+
+func (w *wsRespWriter) Write(data []byte) (n int, err error) {
+	return w.writePipe.Write(data)
+}
+
+func TestServeWS(t *testing.T) {
+	http2Conn, _ := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	respWriter := newWSRespWriter()
+	readPipe, writePipe := io.Pipe()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/ws", readPipe)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, websocketUpgrade)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.ServeHTTP(respWriter, req)
+	}()
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientText(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerText(respWriter.RespBody())
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	resp := respWriter.Result()
+	// http2RespWriter should rewrite status 101 to 200
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(responseMetaHeaderField))
+
+	wg.Wait()
+}
+
+func TestServeControlStream(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+	http2Conn.newRPCClientFunc = rpcClientFactory.newMockRPCClient
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		edgeHTTP2Conn.RoundTrip(req)
+	}()
+
+	<-rpcClientFactory.registered
+	cancel()
+	<-rpcClientFactory.unregistered
+
+	wg.Wait()
+}
+
+func benchmarkServeHTTP(b *testing.B, test testRequest) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	endpoint := fmt.Sprintf("http://localhost:8080/%s", test.endpoint)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+	require.NoError(b, err)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(b, err)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		resp, err := edgeHTTP2Conn.RoundTrip(req)
+		b.StopTimer()
+		require.NoError(b, err)
+		require.Equal(b, test.expectedStatus, resp.StatusCode)
+		if test.expectedBody != nil {
+			respBody, err := ioutil.ReadAll(resp.Body)
+			require.NoError(b, err)
+			require.Equal(b, test.expectedBody, respBody)
+		}
+		resp.Body.Close()
+	}
+
+	cancel()
+	wg.Wait()
+}
+func BenchmarkServeHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeHTTP(b, test)
+}
+
+func BenchmarkServeHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeHTTP(b, test)
+}

connection/json.go

@@ -0,0 +1,7 @@
+package connection
+
+import (
+	jsoniter "github.com/json-iterator/go"
+)
+
+var json = jsoniter.ConfigFastest

connection/manager.go

@@ -1,302 +0,0 @@
-package connection
-
-import (
-	"context"
-	"crypto/tls"
-	"fmt"
-	"sync"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
-
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
-	"github.com/cloudflare/cloudflared/edgediscovery"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/streamhandler"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-const (
-	quickStartLink       = "https://developers.cloudflare.com/argo-tunnel/quickstart/"
-	faqLink              = "https://developers.cloudflare.com/argo-tunnel/faq/"
-	defaultRetryAfter    = time.Second * 5
-	packageNamespace     = "connection"
-	edgeManagerSubsystem = "edgemanager"
-)
-
-// EdgeManager manages connections with the edge
-type EdgeManager struct {
-	// streamHandler handles stream opened by the edge
-	streamHandler *streamhandler.StreamHandler
-	// TLSConfig is the TLS configuration to connect with edge
-	tlsConfig *tls.Config
-	// cloudflaredConfig is the cloudflared configuration that is determined when the process first starts
-	cloudflaredConfig *CloudflaredConfig
-	// serviceDiscoverer returns the next edge addr to connect to
-	serviceDiscoverer *edgediscovery.Edge
-	// state is attributes of ConnectionManager that can change during runtime.
-	state *edgeManagerState
-
-	logger *logrus.Entry
-
-	metrics *metrics
-}
-
-type metrics struct {
-	// activeStreams is a gauge shared by all muxers of this process to expose the total number of active streams
-	activeStreams prometheus.Gauge
-}
-
-func newMetrics(namespace, subsystem string) *metrics {
-	return &metrics{
-		activeStreams: h2mux.NewActiveStreamsMetrics(namespace, subsystem),
-	}
-}
-
-// EdgeManagerConfigurable is the configurable attributes of a EdgeConnectionManager
-type EdgeManagerConfigurable struct {
-	TunnelHostnames []h2mux.TunnelHostname
-	*tunnelpogs.EdgeConnectionConfig
-}
-
-type CloudflaredConfig struct {
-	CloudflaredID uuid.UUID
-	Tags          []tunnelpogs.Tag
-	BuildInfo     *buildinfo.BuildInfo
-	IntentLabel   string
-}
-
-func NewEdgeManager(
-	streamHandler *streamhandler.StreamHandler,
-	edgeConnMgrConfigurable *EdgeManagerConfigurable,
-	userCredential []byte,
-	tlsConfig *tls.Config,
-	serviceDiscoverer *edgediscovery.Edge,
-	cloudflaredConfig *CloudflaredConfig,
-	logger *logrus.Logger,
-) *EdgeManager {
-	return &EdgeManager{
-		streamHandler:     streamHandler,
-		tlsConfig:         tlsConfig,
-		cloudflaredConfig: cloudflaredConfig,
-		serviceDiscoverer: serviceDiscoverer,
-		state:             newEdgeConnectionManagerState(edgeConnMgrConfigurable, userCredential),
-		logger:            logger.WithField("subsystem", "connectionManager"),
-		metrics:           newMetrics(packageNamespace, edgeManagerSubsystem),
-	}
-}
-
-func (em *EdgeManager) Run(ctx context.Context) error {
-	defer em.shutdown()
-
-	// Currently, declarative tunnels don't have any concept of a stable connection
-	// Each edge connection is transient and when it dies, it is replaced by a different one,
-	// not restarted.
-	// So in the future we should really change this so that n connections are stored individually
-	connIndex := 0
-	for {
-		select {
-		case <-ctx.Done():
-			return errors.Wrap(ctx.Err(), "EdgeConnectionManager terminated")
-		default:
-			time.Sleep(1 * time.Second)
-		}
-		// Create/delete connection one at a time, so we don't need to adjust for connections that are being created/deleted
-		// in shouldCreateConnection or shouldReduceConnection calculation
-		if em.state.shouldCreateConnection(em.serviceDiscoverer.AvailableAddrs()) {
-			if connErr := em.newConnection(ctx, connIndex); connErr != nil {
-				if !connErr.ShouldRetry {
-					em.logger.WithError(connErr).Error(em.noRetryMessage())
-					return connErr
-				}
-				em.logger.WithError(connErr).Error("cannot create new connection")
-			} else {
-				connIndex++
-			}
-		} else if em.state.shouldReduceConnection() {
-			if err := em.closeConnection(ctx); err != nil {
-				em.logger.WithError(err).Error("cannot close connection")
-			}
-		}
-	}
-}
-
-func (em *EdgeManager) UpdateConfigurable(newConfigurable *EdgeManagerConfigurable) {
-	em.logger.Infof("New edge connection manager configuration %+v", newConfigurable)
-	em.state.updateConfigurable(newConfigurable)
-}
-
-func (em *EdgeManager) newConnection(ctx context.Context, index int) *tunnelpogs.ConnectError {
-	edgeTCPAddr, err := em.serviceDiscoverer.GetAddr(index)
-	if err != nil {
-		return retryConnection(fmt.Sprintf("edge address discovery error: %v", err))
-	}
-	configurable := em.state.getConfigurable()
-	edgeConn, err := DialEdge(ctx, configurable.Timeout, em.tlsConfig, edgeTCPAddr)
-	if err != nil {
-		return retryConnection(fmt.Sprintf("dial edge error: %v", err))
-	}
-	// Establish a muxed connection with the edge
-	// Client mux handshake with agent server
-	muxer, err := h2mux.Handshake(edgeConn, edgeConn, h2mux.MuxerConfig{
-		Timeout:           configurable.Timeout,
-		Handler:           em.streamHandler,
-		IsClient:          true,
-		HeartbeatInterval: configurable.HeartbeatInterval,
-		MaxHeartbeats:     configurable.MaxFailedHeartbeats,
-		Logger:            em.logger.WithField("subsystem", "muxer"),
-	}, em.metrics.activeStreams)
-	if err != nil {
-		retryConnection(fmt.Sprintf("couldn't perform handshake with edge: %v", err))
-	}
-
-	h2muxConn, err := newConnection(muxer, edgeTCPAddr)
-	if err != nil {
-		return retryConnection(fmt.Sprintf("couldn't create h2mux connection: %v", err))
-	}
-
-	go em.serveConn(ctx, h2muxConn)
-
-	connResult, err := h2muxConn.Connect(ctx, &tunnelpogs.ConnectParameters{
-		CloudflaredID:       em.cloudflaredConfig.CloudflaredID,
-		CloudflaredVersion:  em.cloudflaredConfig.BuildInfo.CloudflaredVersion,
-		NumPreviousAttempts: 0,
-		OriginCert:          em.state.getUserCredential(),
-		IntentLabel:         em.cloudflaredConfig.IntentLabel,
-		Tags:                em.cloudflaredConfig.Tags,
-	}, em.logger)
-	if err != nil {
-		h2muxConn.Shutdown()
-		return retryConnection(fmt.Sprintf("couldn't connect to edge: %v", err))
-	}
-
-	if connErr := connResult.ConnectError(); connErr != nil {
-		return connErr
-	}
-
-	em.state.newConnection(h2muxConn)
-	em.logger.Infof("connected to %s", connResult.ConnectedTo())
-
-	if connResult.ClientConfig() != nil {
-		em.streamHandler.UseConfiguration(ctx, connResult.ClientConfig())
-	}
-	return nil
-}
-
-func (em *EdgeManager) closeConnection(ctx context.Context) error {
-	conn := em.state.getFirstConnection()
-	if conn == nil {
-		return fmt.Errorf("no connection to close")
-	}
-	conn.Shutdown()
-	// teardown will be handled by EdgeManager.serveConn in another goroutine
-	return nil
-}
-
-func (em *EdgeManager) serveConn(ctx context.Context, conn *Connection) {
-	err := conn.Serve(ctx)
-	em.logger.WithError(err).Warn("Connection closed")
-	em.state.closeConnection(conn)
-	em.serviceDiscoverer.GiveBack(conn.addr)
-}
-
-func (em *EdgeManager) noRetryMessage() string {
-	messageTemplate := "cloudflared could not register an Argo Tunnel on your account. Please confirm the following before trying again:" +
-		"1. You have Argo Smart Routing enabled in your account, See Enable Argo section of %s." +
-		"2. Your credential at %s is still valid. See %s."
-	return fmt.Sprintf(messageTemplate, quickStartLink, em.state.getConfigurable().UserCredentialPath, faqLink)
-}
-
-func (em *EdgeManager) shutdown() {
-	em.state.shutdown()
-}
-
-type edgeManagerState struct {
-	sync.RWMutex
-	configurable   *EdgeManagerConfigurable
-	userCredential []byte
-	conns          map[uuid.UUID]*Connection
-}
-
-func newEdgeConnectionManagerState(configurable *EdgeManagerConfigurable, userCredential []byte) *edgeManagerState {
-	return &edgeManagerState{
-		configurable:   configurable,
-		userCredential: userCredential,
-		conns:          make(map[uuid.UUID]*Connection),
-	}
-}
-
-func (ems *edgeManagerState) shouldCreateConnection(availableEdgeAddrs int) bool {
-	ems.RLock()
-	defer ems.RUnlock()
-	expectedHAConns := int(ems.configurable.NumHAConnections)
-	if availableEdgeAddrs < expectedHAConns {
-		expectedHAConns = availableEdgeAddrs
-	}
-	return len(ems.conns) < expectedHAConns
-}
-
-func (ems *edgeManagerState) shouldReduceConnection() bool {
-	ems.RLock()
-	defer ems.RUnlock()
-	return uint8(len(ems.conns)) > ems.configurable.NumHAConnections
-}
-
-func (ems *edgeManagerState) newConnection(conn *Connection) {
-	ems.Lock()
-	defer ems.Unlock()
-	ems.conns[conn.id] = conn
-}
-
-func (ems *edgeManagerState) closeConnection(conn *Connection) {
-	ems.Lock()
-	defer ems.Unlock()
-	delete(ems.conns, conn.id)
-}
-
-func (ems *edgeManagerState) getFirstConnection() *Connection {
-	ems.RLock()
-	defer ems.RUnlock()
-
-	for _, conn := range ems.conns {
-		return conn
-	}
-	return nil
-}
-
-func (ems *edgeManagerState) shutdown() {
-	ems.Lock()
-	defer ems.Unlock()
-	for _, conn := range ems.conns {
-		conn.Shutdown()
-	}
-}
-
-func (ems *edgeManagerState) getConfigurable() *EdgeManagerConfigurable {
-	ems.Lock()
-	defer ems.Unlock()
-	return ems.configurable
-}
-
-func (ems *edgeManagerState) updateConfigurable(newConfigurable *EdgeManagerConfigurable) {
-	ems.Lock()
-	defer ems.Unlock()
-	ems.configurable = newConfigurable
-}
-
-func (ems *edgeManagerState) getUserCredential() []byte {
-	ems.RLock()
-	defer ems.RUnlock()
-	return ems.userCredential
-}
-
-func retryConnection(cause string) *tunnelpogs.ConnectError {
-	return &tunnelpogs.ConnectError{
-		Cause:       cause,
-		RetryAfter:  defaultRetryAfter,
-		ShouldRetry: true,
-	}
-}

connection/manager_test.go

@@ -1,77 +0,0 @@
-package connection
-
-import (
-	"net"
-	"testing"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/assert"
-
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
-	"github.com/cloudflare/cloudflared/edgediscovery"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/streamhandler"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-var (
-	configurable = &EdgeManagerConfigurable{
-		[]h2mux.TunnelHostname{
-			"http.example.com",
-			"ws.example.com",
-			"hello.example.com",
-		},
-		&pogs.EdgeConnectionConfig{
-			NumHAConnections:    1,
-			HeartbeatInterval:   1 * time.Second,
-			Timeout:             5 * time.Second,
-			MaxFailedHeartbeats: 3,
-			UserCredentialPath:  "/etc/cloudflared/cert.pem",
-		},
-	}
-	cloudflaredConfig = &CloudflaredConfig{
-		CloudflaredID: uuid.New(),
-		Tags: []pogs.Tag{
-			{Name: "pool", Value: "east-6"},
-		},
-		BuildInfo: &buildinfo.BuildInfo{
-			GoOS:               "linux",
-			GoVersion:          "1.12",
-			GoArch:             "amd64",
-			CloudflaredVersion: "2019.6.0",
-		},
-	}
-)
-
-func mockEdgeManager() *EdgeManager {
-	newConfigChan := make(chan<- *pogs.ClientConfig)
-	useConfigResultChan := make(<-chan *pogs.UseConfigurationResult)
-	logger := logrus.New()
-	edge := edgediscovery.MockEdge(logger, []*net.TCPAddr{})
-	return NewEdgeManager(
-		streamhandler.NewStreamHandler(newConfigChan, useConfigResultChan, logger),
-		configurable,
-		[]byte{},
-		nil,
-		edge,
-		cloudflaredConfig,
-		logger,
-	)
-}
-
-func TestUpdateConfigurable(t *testing.T) {
-	m := mockEdgeManager()
-	newConfigurable := &EdgeManagerConfigurable{
-		[]h2mux.TunnelHostname{
-			"second.example.com",
-		},
-		&pogs.EdgeConnectionConfig{
-			NumHAConnections: 2,
-		},
-	}
-	m.UpdateConfigurable(newConfigurable)
-
-	assert.Equal(t, newConfigurable, m.state.getConfigurable())
-}

connection/metrics.go

@@ -0,0 +1,405 @@
+package connection
+
+import (
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	MetricsNamespace = "cloudflared"
+	TunnelSubsystem  = "tunnel"
+	muxerSubsystem   = "muxer"
+)
+
+type muxerMetrics struct {
+	rtt              *prometheus.GaugeVec
+	rttMin           *prometheus.GaugeVec
+	rttMax           *prometheus.GaugeVec
+	receiveWindowAve *prometheus.GaugeVec
+	sendWindowAve    *prometheus.GaugeVec
+	receiveWindowMin *prometheus.GaugeVec
+	receiveWindowMax *prometheus.GaugeVec
+	sendWindowMin    *prometheus.GaugeVec
+	sendWindowMax    *prometheus.GaugeVec
+	inBoundRateCurr  *prometheus.GaugeVec
+	inBoundRateMin   *prometheus.GaugeVec
+	inBoundRateMax   *prometheus.GaugeVec
+	outBoundRateCurr *prometheus.GaugeVec
+	outBoundRateMin  *prometheus.GaugeVec
+	outBoundRateMax  *prometheus.GaugeVec
+	compBytesBefore  *prometheus.GaugeVec
+	compBytesAfter   *prometheus.GaugeVec
+	compRateAve      *prometheus.GaugeVec
+}
+
+type tunnelMetrics struct {
+	timerRetries    prometheus.Gauge
+	serverLocations *prometheus.GaugeVec
+	// locationLock is a mutex for oldServerLocations
+	locationLock sync.Mutex
+	// oldServerLocations stores the last server the tunnel was connected to
+	oldServerLocations map[string]string
+
+	regSuccess *prometheus.CounterVec
+	regFail    *prometheus.CounterVec
+	rpcFail    *prometheus.CounterVec
+
+	muxerMetrics        *muxerMetrics
+	tunnelsHA           tunnelsForHA
+	userHostnamesCounts *prometheus.CounterVec
+}
+
+func newMuxerMetrics() *muxerMetrics {
+	rtt := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt",
+			Help:      "Round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rtt)
+
+	rttMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_min",
+			Help:      "Shortest round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rttMin)
+
+	rttMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_max",
+			Help:      "Longest round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rttMax)
+
+	receiveWindowAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_ave",
+			Help:      "Average receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowAve)
+
+	sendWindowAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_ave",
+			Help:      "Average send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowAve)
+
+	receiveWindowMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_min",
+			Help:      "Smallest receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowMin)
+
+	receiveWindowMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_max",
+			Help:      "Largest receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowMax)
+
+	sendWindowMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_min",
+			Help:      "Smallest send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowMin)
+
+	sendWindowMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_max",
+			Help:      "Largest send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowMax)
+
+	inBoundRateCurr := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_curr",
+			Help:      "Current inbounding bytes per second, 0 if there is no incoming connection",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateCurr)
+
+	inBoundRateMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero inbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateMin)
+
+	inBoundRateMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_max",
+			Help:      "Maximum inbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateMax)
+
+	outBoundRateCurr := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_curr",
+			Help:      "Current outbounding bytes per second, 0 if there is no outgoing traffic",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateCurr)
+
+	outBoundRateMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero outbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateMin)
+
+	outBoundRateMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_max",
+			Help:      "Maximum outbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateMax)
+
+	compBytesBefore := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_before",
+			Help:      "Bytes sent via cross-stream compression, pre compression",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compBytesBefore)
+
+	compBytesAfter := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_after",
+			Help:      "Bytes sent via cross-stream compression, post compression",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compBytesAfter)
+
+	compRateAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_rate_ave",
+			Help:      "Average outbound cross-stream compression ratio",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compRateAve)
+
+	return &muxerMetrics{
+		rtt:              rtt,
+		rttMin:           rttMin,
+		rttMax:           rttMax,
+		receiveWindowAve: receiveWindowAve,
+		sendWindowAve:    sendWindowAve,
+		receiveWindowMin: receiveWindowMin,
+		receiveWindowMax: receiveWindowMax,
+		sendWindowMin:    sendWindowMin,
+		sendWindowMax:    sendWindowMax,
+		inBoundRateCurr:  inBoundRateCurr,
+		inBoundRateMin:   inBoundRateMin,
+		inBoundRateMax:   inBoundRateMax,
+		outBoundRateCurr: outBoundRateCurr,
+		outBoundRateMin:  outBoundRateMin,
+		outBoundRateMax:  outBoundRateMax,
+		compBytesBefore:  compBytesBefore,
+		compBytesAfter:   compBytesAfter,
+		compRateAve:      compRateAve,
+	}
+}
+
+func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
+	m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
+	m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
+	m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
+	m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
+	m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
+	m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
+	m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
+	m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
+	m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
+	m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
+	m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
+	m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
+	m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
+	m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
+	m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
+	m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
+	m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
+	m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
+}
+
+func convertRTTMilliSec(t time.Duration) float64 {
+	return float64(t / time.Millisecond)
+}
+
+// Metrics that can be collected without asking the edge
+func newTunnelMetrics() *tunnelMetrics {
+	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "max_concurrent_requests_per_tunnel",
+			Help:      "Largest number of concurrent requests proxied through each tunnel so far",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
+
+	timerRetries := prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "timer_retries",
+			Help:      "Unacknowledged heart beats count",
+		})
+	prometheus.MustRegister(timerRetries)
+
+	serverLocations := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "server_locations",
+			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
+		},
+		[]string{"connection_id", "location"},
+	)
+	prometheus.MustRegister(serverLocations)
+
+	rpcFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_rpc_fail",
+			Help:      "Count of RPC connection errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(rpcFail)
+
+	registerFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_register_fail",
+			Help:      "Count of tunnel registration errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(registerFail)
+
+	userHostnamesCounts := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "user_hostnames_counts",
+			Help:      "Which user hostnames cloudflared is serving",
+		},
+		[]string{"userHostname"},
+	)
+	prometheus.MustRegister(userHostnamesCounts)
+
+	registerSuccess := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_register_success",
+			Help:      "Count of successful tunnel registrations",
+		},
+		[]string{"rpcName"},
+	)
+	prometheus.MustRegister(registerSuccess)
+
+	return &tunnelMetrics{
+		timerRetries:        timerRetries,
+		serverLocations:     serverLocations,
+		oldServerLocations:  make(map[string]string),
+		muxerMetrics:        newMuxerMetrics(),
+		tunnelsHA:           NewTunnelsForHA(),
+		regSuccess:          registerSuccess,
+		regFail:             registerFail,
+		rpcFail:             rpcFail,
+		userHostnamesCounts: userHostnamesCounts,
+	}
+}
+
+func (t *tunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
+	t.muxerMetrics.update(connectionID, metrics)
+}
+
+func (t *tunnelMetrics) registerServerLocation(connectionID, loc string) {
+	t.locationLock.Lock()
+	defer t.locationLock.Unlock()
+	if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
+		return
+	} else if ok {
+		t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
+	}
+	t.serverLocations.WithLabelValues(connectionID, loc).Inc()
+	t.oldServerLocations[connectionID] = loc
+}

connection/observer.go

@@ -0,0 +1,99 @@
+package connection
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+type Observer struct {
+	logger.Service
+	metrics         *tunnelMetrics
+	tunnelEventChan chan<- ui.TunnelEvent
+}
+
+func NewObserver(logger logger.Service, tunnelEventChan chan<- ui.TunnelEvent) *Observer {
+	return &Observer{
+		logger,
+		newTunnelMetrics(),
+		tunnelEventChan,
+	}
+}
+
+func (o *Observer) logServerInfo(connIndex uint8, location, msg string) {
+	// If launch-ui flag is set, send connect msg
+	if o.tunnelEventChan != nil {
+		o.tunnelEventChan <- ui.TunnelEvent{Index: connIndex, EventType: ui.Connected, Location: location}
+	}
+	o.Infof(msg)
+	o.metrics.registerServerLocation(uint8ToString(connIndex), location)
+}
+
+func (o *Observer) logTrialHostname(registration *tunnelpogs.TunnelRegistration) error {
+	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
+	if o.tunnelEventChan == nil {
+		if registrationURL, err := url.Parse(registration.Url); err == nil {
+			for _, line := range asciiBox(trialZoneMsg(registrationURL.String()), 2) {
+				o.Info(line)
+			}
+		} else {
+			o.Error("Failed to connect tunnel, please try again.")
+			return fmt.Errorf("empty URL in response from Cloudflare edge")
+		}
+	}
+	return nil
+}
+
+// Print out the given lines in a nice ASCII box.
+func asciiBox(lines []string, padding int) (box []string) {
+	maxLen := maxLen(lines)
+	spacer := strings.Repeat(" ", padding)
+
+	border := "+" + strings.Repeat("-", maxLen+(padding*2)) + "+"
+
+	box = append(box, border)
+	for _, line := range lines {
+		box = append(box, "|"+spacer+line+strings.Repeat(" ", maxLen-len(line))+spacer+"|")
+	}
+	box = append(box, border)
+	return
+}
+
+func maxLen(lines []string) int {
+	max := 0
+	for _, line := range lines {
+		if len(line) > max {
+			max = len(line)
+		}
+	}
+	return max
+}
+
+func trialZoneMsg(url string) []string {
+	return []string{
+		"Your free tunnel has started! Visit it:",
+		"  " + url,
+	}
+}
+
+func (o *Observer) sendRegisteringEvent() {
+	if o.tunnelEventChan != nil {
+		o.tunnelEventChan <- ui.TunnelEvent{EventType: ui.RegisteringTunnel}
+	}
+}
+
+func (o *Observer) sendConnectedEvent(connIndex uint8, location string) {
+	if o.tunnelEventChan != nil {
+		o.tunnelEventChan <- ui.TunnelEvent{Index: connIndex, EventType: ui.Connected, Location: location}
+	}
+}
+
+func (o *Observer) sendURL(url string) {
+	if o.tunnelEventChan != nil {
+		o.tunnelEventChan <- ui.TunnelEvent{EventType: ui.SetUrl, Url: url}
+	}
+}

connection/observer_test.go

@@ -0,0 +1,45 @@
+package connection
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// can only be called once
+var m = newTunnelMetrics()
+
+func TestRegisterServerLocation(t *testing.T) {
+	tunnels := 20
+	var wg sync.WaitGroup
+	wg.Add(tunnels)
+	for i := 0; i < tunnels; i++ {
+		go func(i int) {
+			id := strconv.Itoa(i)
+			m.registerServerLocation(id, "LHR")
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	for i := 0; i < tunnels; i++ {
+		id := strconv.Itoa(i)
+		assert.Equal(t, "LHR", m.oldServerLocations[id])
+	}
+
+	wg.Add(tunnels)
+	for i := 0; i < tunnels; i++ {
+		go func(i int) {
+			id := strconv.Itoa(i)
+			m.registerServerLocation(id, "AUS")
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	for i := 0; i < tunnels; i++ {
+		id := strconv.Itoa(i)
+		assert.Equal(t, "AUS", m.oldServerLocations[id])
+	}
+
+}

connection/protocol.go

@@ -0,0 +1,179 @@
+package connection
+
+import (
+	"fmt"
+	"hash/fnv"
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+)
+
+const (
+	AvailableProtocolFlagMessage = "Available protocols: http2 - Go's implementation, h2mux - Cloudflare's implementation of HTTP/2, and auto - automatically select between http2 and h2mux"
+	// edgeH2muxTLSServerName is the server name to establish h2mux connection with edge
+	edgeH2muxTLSServerName = "cftunnel.com"
+	// edgeH2TLSServerName is the server name to establish http2 connection with edge
+	edgeH2TLSServerName = "h2.cftunnel.com"
+	// threshold to switch back to h2mux when the user intentionally pick --protocol http2
+	explicitHTTP2FallbackThreshold = -1
+	autoSelectFlag                 = "auto"
+)
+
+var (
+	ProtocolList = []Protocol{H2mux, HTTP2}
+)
+
+type Protocol int64
+
+const (
+	H2mux Protocol = iota
+	HTTP2
+)
+
+func (p Protocol) ServerName() string {
+	switch p {
+	case H2mux:
+		return edgeH2muxTLSServerName
+	case HTTP2:
+		return edgeH2TLSServerName
+	default:
+		return ""
+	}
+}
+
+// Fallback returns the fallback protocol and whether the protocol has a fallback
+func (p Protocol) fallback() (Protocol, bool) {
+	switch p {
+	case H2mux:
+		return 0, false
+	case HTTP2:
+		return H2mux, true
+	default:
+		return 0, false
+	}
+}
+
+func (p Protocol) String() string {
+	switch p {
+	case H2mux:
+		return "h2mux"
+	case HTTP2:
+		return "http2"
+	default:
+		return fmt.Sprintf("unknown protocol")
+	}
+}
+
+type ProtocolSelector interface {
+	Current() Protocol
+	Fallback() (Protocol, bool)
+}
+
+type staticProtocolSelector struct {
+	current Protocol
+}
+
+func (s *staticProtocolSelector) Current() Protocol {
+	return s.current
+}
+
+func (s *staticProtocolSelector) Fallback() (Protocol, bool) {
+	return 0, false
+}
+
+type autoProtocolSelector struct {
+	lock           sync.RWMutex
+	current        Protocol
+	switchThrehold int32
+	fetchFunc      PercentageFetcher
+	refreshAfter   time.Time
+	ttl            time.Duration
+	logger         logger.Service
+}
+
+func newAutoProtocolSelector(
+	current Protocol,
+	switchThrehold int32,
+	fetchFunc PercentageFetcher,
+	ttl time.Duration,
+	logger logger.Service,
+) *autoProtocolSelector {
+	return &autoProtocolSelector{
+		current:        current,
+		switchThrehold: switchThrehold,
+		fetchFunc:      fetchFunc,
+		refreshAfter:   time.Now().Add(ttl),
+		ttl:            ttl,
+		logger:         logger,
+	}
+}
+
+func (s *autoProtocolSelector) Current() Protocol {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	if time.Now().Before(s.refreshAfter) {
+		return s.current
+	}
+
+	percentage, err := s.fetchFunc()
+	if err != nil {
+		s.logger.Errorf("Failed to refresh protocol, err: %v", err)
+		return s.current
+	}
+
+	if s.switchThrehold < percentage {
+		s.current = HTTP2
+	} else {
+		s.current = H2mux
+	}
+	s.refreshAfter = time.Now().Add(s.ttl)
+	return s.current
+}
+
+func (s *autoProtocolSelector) Fallback() (Protocol, bool) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	return s.current.fallback()
+}
+
+type PercentageFetcher func() (int32, error)
+
+func NewProtocolSelector(protocolFlag string, namedTunnel *NamedTunnelConfig, fetchFunc PercentageFetcher, ttl time.Duration, logger logger.Service) (ProtocolSelector, error) {
+	if namedTunnel == nil {
+		return &staticProtocolSelector{
+			current: H2mux,
+		}, nil
+	}
+	if protocolFlag == H2mux.String() {
+		return &staticProtocolSelector{
+			current: H2mux,
+		}, nil
+	}
+
+	http2Percentage, err := fetchFunc()
+	if err != nil {
+		return nil, err
+	}
+	if protocolFlag == HTTP2.String() {
+		if http2Percentage < 0 {
+			return newAutoProtocolSelector(H2mux, explicitHTTP2FallbackThreshold, fetchFunc, ttl, logger), nil
+		}
+		return newAutoProtocolSelector(HTTP2, explicitHTTP2FallbackThreshold, fetchFunc, ttl, logger), nil
+	}
+
+	if protocolFlag != autoSelectFlag {
+		return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
+	}
+	threshold := switchThreshold(namedTunnel.Auth.AccountTag)
+	if threshold < http2Percentage {
+		return newAutoProtocolSelector(HTTP2, threshold, fetchFunc, ttl, logger), nil
+	}
+	return newAutoProtocolSelector(H2mux, threshold, fetchFunc, ttl, logger), nil
+}
+
+func switchThreshold(accountTag string) int32 {
+	h := fnv.New32a()
+	h.Write([]byte(accountTag))
+	return int32(h.Sum32() % 100)
+}

connection/protocol_test.go

@@ -0,0 +1,220 @@
+package connection
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testNoTTL = 0
+)
+
+var (
+	testNamedTunnelConfig = &NamedTunnelConfig{
+		Auth: pogs.TunnelAuth{
+			AccountTag: "testAccountTag",
+		},
+	}
+)
+
+func mockFetcher(percentage int32) PercentageFetcher {
+	return func() (int32, error) {
+		return percentage, nil
+	}
+}
+
+func mockFetcherWithError() PercentageFetcher {
+	return func() (int32, error) {
+		return 0, fmt.Errorf("failed to fetch precentage")
+	}
+}
+
+type dynamicMockFetcher struct {
+	percentage int32
+	err        error
+}
+
+func (dmf *dynamicMockFetcher) fetch() PercentageFetcher {
+	return func() (int32, error) {
+		if dmf.err != nil {
+			return 0, dmf.err
+		}
+		return dmf.percentage, nil
+	}
+}
+
+func TestNewProtocolSelector(t *testing.T) {
+	tests := []struct {
+		name              string
+		protocol          string
+		expectedProtocol  Protocol
+		hasFallback       bool
+		expectedFallback  Protocol
+		namedTunnelConfig *NamedTunnelConfig
+		fetchFunc         PercentageFetcher
+		wantErr           bool
+	}{
+		{
+			name:              "classic tunnel",
+			protocol:          "h2mux",
+			expectedProtocol:  H2mux,
+			namedTunnelConfig: nil,
+		},
+		{
+			name:              "named tunnel over h2mux",
+			protocol:          "h2mux",
+			expectedProtocol:  H2mux,
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel over http2",
+			protocol:          "http2",
+			expectedProtocol:  HTTP2,
+			hasFallback:       true,
+			expectedFallback:  H2mux,
+			fetchFunc:         mockFetcher(0),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel http2 disabled",
+			protocol:          "http2",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(-1),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto all http2 disabled",
+			protocol:          "auto",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(-1),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto to h2mux",
+			protocol:          "auto",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(0),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto to http2",
+			protocol:          "auto",
+			expectedProtocol:  HTTP2,
+			hasFallback:       true,
+			expectedFallback:  H2mux,
+			fetchFunc:         mockFetcher(100),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			// None named tunnel can only use h2mux, so specifying an unknown protocol is not an error
+			name:             "classic tunnel unknown protocol",
+			protocol:         "unknown",
+			expectedProtocol: H2mux,
+		},
+		{
+			name:              "named tunnel unknown protocol",
+			protocol:          "unknown",
+			fetchFunc:         mockFetcher(100),
+			namedTunnelConfig: testNamedTunnelConfig,
+			wantErr:           true,
+		},
+		{
+			name:              "named tunnel fetch error",
+			protocol:          "unknown",
+			fetchFunc:         mockFetcherWithError(),
+			namedTunnelConfig: testNamedTunnelConfig,
+			wantErr:           true,
+		},
+	}
+	logger, _ := logger.New()
+	for _, test := range tests {
+		selector, err := NewProtocolSelector(test.protocol, test.namedTunnelConfig, test.fetchFunc, testNoTTL, logger)
+		if test.wantErr {
+			assert.Error(t, err, fmt.Sprintf("test %s failed", test.name))
+		} else {
+			assert.NoError(t, err, fmt.Sprintf("test %s failed", test.name))
+			assert.Equal(t, test.expectedProtocol, selector.Current(), fmt.Sprintf("test %s failed", test.name))
+			fallback, ok := selector.Fallback()
+			assert.Equal(t, test.hasFallback, ok, fmt.Sprintf("test %s failed", test.name))
+			if test.hasFallback {
+				assert.Equal(t, test.expectedFallback, fallback, fmt.Sprintf("test %s failed", test.name))
+			}
+		}
+	}
+}
+
+func TestAutoProtocolSelectorRefresh(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{}
+	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.err = fmt.Errorf("failed to fetch")
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	fetcher.err = nil
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+}
+
+func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{}
+	selector, err := NewProtocolSelector("http2", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.err = fmt.Errorf("failed to fetch")
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	fetcher.err = nil
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	assert.Equal(t, H2mux, selector.Current())
+}
+
+func TestProtocolSelectorRefreshTTL(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{percentage: 100}
+	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), time.Hour, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+}

connection/rpc.go

@@ -3,47 +3,293 @@ package connection
 import (
 	"context"
 	"fmt"
+	"io"
 	"time"
 
-	"github.com/sirupsen/logrus"
-	rpc "zombiezen.com/go/capnproto2/rpc"
-
-	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"zombiezen.com/go/capnproto2/rpc"
 )
 
-// NewRPCClient creates and returns a new RPC client, which will communicate
-// using a stream on the given muxer
-func NewRPCClient(
+type tunnelServerClient struct {
+	client    tunnelpogs.TunnelServer_PogsClient
+	transport rpc.Transport
+}
+
+// NewTunnelRPCClient creates and returns a new RPC client, which will communicate using a stream on the given muxer.
+// This method is exported for supervisor to call Authenticate RPC
+func NewTunnelServerClient(
 	ctx context.Context,
-	muxer *h2mux.Muxer,
-	logger *logrus.Entry,
-	openStreamTimeout time.Duration,
-) (client tunnelpogs.TunnelServer_PogsClient, err error) {
-	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
-	defer openStreamCancel()
-	stream, err := muxer.OpenRPCStream(openStreamCtx)
+	stream io.ReadWriteCloser,
+	logger logger.Service,
+) *tunnelServerClient {
+	transport := tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream))
+	conn := rpc.NewConn(
+		transport,
+		tunnelrpc.ConnLog(logger),
+	)
+	registrationClient := tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
+	return &tunnelServerClient{
+		client:    tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn},
+		transport: transport,
+	}
+}
+
+func (tsc *tunnelServerClient) Authenticate(ctx context.Context, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) (tunnelpogs.AuthOutcome, error) {
+	authResp, err := tsc.client.Authenticate(ctx, classicTunnel.OriginCert, classicTunnel.Hostname, registrationOptions)
+	if err != nil {
+		return nil, err
+	}
+	return authResp.Outcome(), nil
+}
+
+func (tsc *tunnelServerClient) Close() {
+	// Closing the client will also close the connection
+	tsc.client.Close()
+	tsc.transport.Close()
+}
+
+type NamedTunnelRPCClient interface {
+	RegisterConnection(
+		c context.Context,
+		config *NamedTunnelConfig,
+		options *tunnelpogs.ConnectionOptions,
+		connIndex uint8,
+		observer *Observer,
+	) error
+	GracefulShutdown(ctx context.Context, gracePeriod time.Duration)
+	Close()
+}
+
+type registrationServerClient struct {
+	client    tunnelpogs.RegistrationServer_PogsClient
+	transport rpc.Transport
+}
+
+func newRegistrationRPCClient(
+	ctx context.Context,
+	stream io.ReadWriteCloser,
+	logger logger.Service,
+) NamedTunnelRPCClient {
+	transport := tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream))
+	conn := rpc.NewConn(
+		transport,
+		tunnelrpc.ConnLog(logger),
+	)
+	return &registrationServerClient{
+		client:    tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn},
+		transport: transport,
+	}
+}
+
+func (rsc *registrationServerClient) RegisterConnection(
+	ctx context.Context,
+	config *NamedTunnelConfig,
+	options *tunnelpogs.ConnectionOptions,
+	connIndex uint8,
+	observer *Observer,
+) error {
+	conn, err := rsc.client.RegisterConnection(
+		ctx,
+		config.Auth,
+		config.ID,
+		connIndex,
+		options,
+	)
+	if err != nil {
+		if err.Error() == DuplicateConnectionError {
+			observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
+			return errDuplicationConnection
+		}
+		observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
+		return serverRegistrationErrorFromRPC(err)
+	}
+
+	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
+
+	observer.logServerInfo(connIndex, conn.Location, fmt.Sprintf("Connection %d registered with %s using ID %s", connIndex, conn.Location, conn.UUID))
+	observer.sendConnectedEvent(connIndex, conn.Location)
+
+	return nil
+}
+
+func (rsc *registrationServerClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+	ctx, cancel := context.WithTimeout(ctx, gracePeriod)
+	defer cancel()
+	rsc.client.UnregisterConnection(ctx)
+}
+
+func (rsc *registrationServerClient) Close() {
+	// Closing the client will also close the connection
+	rsc.client.Close()
+	// Closing the transport also closes the stream
+	rsc.transport.Close()
+}
+
+type rpcName string
+
+const (
+	register     rpcName = "register"
+	reconnect    rpcName = "reconnect"
+	unregister   rpcName = "unregister"
+	authenticate rpcName = " authenticate"
+)
+
+func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+	h.observer.sendRegisteringEvent()
+
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := NewTunnelServerClient(ctx, stream, h.observer)
+	defer rpcClient.Close()
+
+	h.logServerInfo(ctx, rpcClient)
+	registration := rpcClient.client.RegisterTunnel(
+		ctx,
+		classicTunnel.OriginCert,
+		classicTunnel.Hostname,
+		registrationOptions,
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// RegisterTunnel RPC failure
+		return h.processRegisterTunnelError(registrationErr, register)
+	}
+
+	// Send free tunnel URL to UI
+	h.observer.sendURL(registration.Url)
+	credentialSetter.SetEventDigest(h.connIndex, registration.EventDigest)
+	return h.processRegistrationSuccess(registration, register, credentialSetter, classicTunnel)
+}
+
+type CredentialManager interface {
+	ReconnectToken() ([]byte, error)
+	EventDigest(connID uint8) ([]byte, error)
+	SetEventDigest(connID uint8, digest []byte)
+	ConnDigest(connID uint8) ([]byte, error)
+	SetConnDigest(connID uint8, digest []byte)
+}
+
+func (h *h2muxConnection) processRegistrationSuccess(
+	registration *tunnelpogs.TunnelRegistration,
+	name rpcName,
+	credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig,
+) error {
+	for _, logLine := range registration.LogLines {
+		h.observer.Info(logLine)
+	}
+
+	if registration.TunnelID != "" {
+		h.observer.metrics.tunnelsHA.AddTunnelID(h.connIndex, registration.TunnelID)
+		h.observer.Infof("Each HA connection's tunnel IDs: %v", h.observer.metrics.tunnelsHA.String())
+	}
+
+	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
+	if classicTunnel.IsTrialZone() {
+		err := h.observer.logTrialHostname(registration)
+		if err != nil {
+			return err
+		}
+	}
+
+	credentialManager.SetConnDigest(h.connIndex, registration.ConnDigest)
+	h.observer.metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
+
+	h.observer.Infof("Route propagating, it may take up to 1 minute for your new route to become functional")
+	h.observer.metrics.regSuccess.WithLabelValues(string(name)).Inc()
+	return nil
+}
+
+func (h *h2muxConnection) processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, name rpcName) error {
+	if err.Error() == DuplicateConnectionError {
+		h.observer.metrics.regFail.WithLabelValues("dup_edge_conn", string(name)).Inc()
+		return errDuplicationConnection
+	}
+	h.observer.metrics.regFail.WithLabelValues("server_error", string(name)).Inc()
+	return serverRegisterTunnelError{
+		cause:     err,
+		permanent: err.IsPermanent(),
+	}
+}
+
+func (h *h2muxConnection) reconnectTunnel(ctx context.Context, credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+	token, err := credentialManager.ReconnectToken()
+	if err != nil {
+		return err
+	}
+	eventDigest, err := credentialManager.EventDigest(h.connIndex)
+	if err != nil {
+		return err
+	}
+	connDigest, err := credentialManager.ConnDigest(h.connIndex)
+	if err != nil {
+		return err
+	}
+
+	h.observer.Debug("initiating RPC stream to reconnect")
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := NewTunnelServerClient(ctx, stream, h.observer)
+	defer rpcClient.Close()
+
+	h.logServerInfo(ctx, rpcClient)
+	registration := rpcClient.client.ReconnectTunnel(
+		ctx,
+		token,
+		eventDigest,
+		connDigest,
+		classicTunnel.Hostname,
+		registrationOptions,
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// ReconnectTunnel RPC failure
+		return h.processRegisterTunnelError(registrationErr, reconnect)
+	}
+	return h.processRegistrationSuccess(registration, reconnect, credentialManager, classicTunnel)
+}
+
+func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelServerClient) error {
+	// Request server info without blocking tunnel registration; must use capnp library directly.
+	serverInfoPromise := tunnelrpc.TunnelServer{Client: rpcClient.client.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+		return nil
+	})
+	serverInfoMessage, err := serverInfoPromise.Result().Struct()
+	if err != nil {
+		h.observer.Errorf("Failed to retrieve server information: %s", err)
+		return err
+	}
+	serverInfo, err := tunnelpogs.UnmarshalServerInfo(serverInfoMessage)
+	if err != nil {
+		h.observer.Errorf("Failed to retrieve server information: %s", err)
+		return err
+	}
+	h.observer.logServerInfo(h.connIndex, serverInfo.LocationName, fmt.Sprintf("Connnection %d connected to %s", h.connIndex, serverInfo.LocationName))
+	return nil
+}
+
+func (h *h2muxConnection) unregister(isNamedTunnel bool) {
+	unregisterCtx, cancel := context.WithTimeout(context.Background(), h.config.GracePeriod)
+	defer cancel()
+
+	stream, err := h.newRPCStream(unregisterCtx, register)
 	if err != nil {
 		return
 	}
 
-	if !isRPCStreamResponse(stream.Headers) {
-		stream.Close()
-		err = fmt.Errorf("rpc: bad response headers: %v", stream.Headers)
-		return
+	if isNamedTunnel {
+		rpcClient := newRegistrationRPCClient(unregisterCtx, stream, h.observer)
+		defer rpcClient.Close()
+
+		rpcClient.GracefulShutdown(unregisterCtx, h.config.GracePeriod)
+	} else {
+		rpcClient := NewTunnelServerClient(unregisterCtx, stream, h.observer)
+		defer rpcClient.Close()
+
+		// gracePeriod is encoded in int64 using capnproto
+		rpcClient.client.UnregisterTunnel(unregisterCtx, h.config.GracePeriod.Nanoseconds())
 	}
-
-	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
-		tunnelrpc.ConnLog(logger),
-	)
-	client = tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	return client, nil
-}
-
-func isRPCStreamResponse(headers []h2mux.Header) bool {
-	return len(headers) == 1 &&
-		headers[0].Name == ":status" &&
-		headers[0].Value == "200"
 }

connection/tunnelsforha.go

@@ -0,0 +1,50 @@
+package connection
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// tunnelsForHA maps this cloudflared instance's HA connections to the tunnel IDs they serve.
+type tunnelsForHA struct {
+	sync.Mutex
+	metrics *prometheus.GaugeVec
+	entries map[uint8]string
+}
+
+// NewTunnelsForHA initializes the Prometheus metrics etc for a tunnelsForHA.
+func NewTunnelsForHA() tunnelsForHA {
+	metrics := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "tunnel_ids",
+			Help: "The ID of all tunnels (and their corresponding HA connection ID) running in this instance of cloudflared.",
+		},
+		[]string{"tunnel_id", "ha_conn_id"},
+	)
+	prometheus.MustRegister(metrics)
+
+	return tunnelsForHA{
+		metrics: metrics,
+		entries: make(map[uint8]string),
+	}
+}
+
+// Track a new tunnel ID, removing the disconnected tunnel (if any) and update metrics.
+func (t *tunnelsForHA) AddTunnelID(haConn uint8, tunnelID string) {
+	t.Lock()
+	defer t.Unlock()
+	haStr := fmt.Sprintf("%v", haConn)
+	if oldTunnelID, ok := t.entries[haConn]; ok {
+		t.metrics.WithLabelValues(oldTunnelID, haStr).Dec()
+	}
+	t.entries[haConn] = tunnelID
+	t.metrics.WithLabelValues(tunnelID, haStr).Inc()
+}
+
+func (t *tunnelsForHA) String() string {
+	t.Lock()
+	defer t.Unlock()
+	return fmt.Sprintf("%v", t.entries)
+}

dbconnect/cmd.go

@@ -6,8 +6,8 @@ import (
 	"net"
 	"strconv"
 
-	"gopkg.in/urfave/cli.v2"
-	"gopkg.in/urfave/cli.v2/altsrc"
+	"github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2/altsrc"
 )
 
 // Cmd is the entrypoint command for dbconnect.
@@ -15,11 +15,19 @@ import (
 // The tunnel package is responsible for appending this to tunnel.Commands().
 func Cmd() *cli.Command {
 	return &cli.Command{
-		Category:  "Database Connect (ALPHA)",
+		Category:  "Database Connect (ALPHA) - Deprecated",
 		Name:      "db-connect",
-		Usage:     "Access your SQL database from Cloudflare Workers or the browser",
+		Usage:     "deprecated: Access your SQL database from Cloudflare Workers or the browser",
 		ArgsUsage: " ",
 		Description: `
+		This feature has been deprecated. 
+		Please see: 
+
+		cloudflared access tcp --help 
+
+		for setting up database connections to the cloudflare edge.
+		
+
 		Creates a connection between your database and the Cloudflare edge.
 		Now you can execute SQL commands anywhere you can send HTTPS requests.
 

dbconnect/cmd_test.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
-	"gopkg.in/urfave/cli.v2"
+	"github.com/urfave/cli/v2"
 )
 
 func TestCmd(t *testing.T) {

dbconnect/proxy.go

@@ -11,17 +11,17 @@ import (
 	"time"
 
 	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/validation"
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 )
 
 // Proxy is an HTTP server that proxies requests to a Client.
 type Proxy struct {
 	client          Client
 	accessValidator *validation.Access
-	logger          *logrus.Logger
+	logger          logger.Service
 }
 
 // NewInsecureProxy creates a Proxy that talks to a Client at an origin.
@@ -43,7 +43,12 @@ func NewInsecureProxy(ctx context.Context, origin string) (*Proxy, error) {
 		return nil, errors.Wrap(err, "could not connect to the database")
 	}
 
-	return &Proxy{client, nil, logrus.New()}, nil
+	logger, err := logger.New()
+	if err != nil {
+		return nil, errors.Wrap(err, "error setting up logger")
+	}
+
+	return &Proxy{client, nil, logger}, nil
 }
 
 // NewSecureProxy creates a Proxy that talks to a Client at an origin.
@@ -90,7 +95,8 @@ func (proxy *Proxy) IsAllowed(r *http.Request, verbose ...bool) bool {
 	// Warn administrators that invalid JWTs are being rejected. This is indicative
 	// of either a misconfiguration of the CLI or a massive failure of upstream systems.
 	if len(verbose) > 0 {
-		proxy.httpLog(r, err).Error("Failed JWT authentication")
+		cfRay := proxy.getRayHeader(r)
+		proxy.logger.Infof("dbproxy: Failed JWT authentication: cf-ray: %s %s", cfRay, err)
 	}
 
 	return false
@@ -234,13 +240,14 @@ func (proxy *Proxy) httpRespondErr(w http.ResponseWriter, r *http.Request, defau
 
 	proxy.httpRespond(w, r, status, err.Error())
 	if len(err.Error()) > 0 {
-		proxy.httpLog(r, err).Warn("Database proxy error")
+		cfRay := proxy.getRayHeader(r)
+		proxy.logger.Infof("dbproxy: Database proxy error: cf-ray: %s %s", cfRay, err)
 	}
 }
 
-// httpLog returns a logrus.Entry that is formatted to output a request Cf-ray.
-func (proxy *Proxy) httpLog(r *http.Request, err error) *logrus.Entry {
-	return proxy.logger.WithContext(r.Context()).WithField("CF-RAY", r.Header.Get("Cf-ray")).WithError(err)
+// getRayHeader returns the request's Cf-ray header.
+func (proxy *Proxy) getRayHeader(r *http.Request) string {
+	return r.Header.Get("Cf-ray")
 }
 
 // httpError extracts common errors and returns an status code and friendly error.

dev.Dockerfile

@@ -1,4 +1,9 @@
-FROM golang:1.12 as builder
+FROM golang:1.15.3 as builder
+ENV GO111MODULE=on \
+    CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 RUN apt-get update
 COPY . .
+# compile cloudflared
+RUN make cloudflared
+RUN cp /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/

edgediscovery/allregions/discovery.go

@@ -7,8 +7,8 @@ import (
 	"net"
 	"time"
 
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -58,15 +58,15 @@ var friendlyDNSErrorLines = []string{
 }
 
 // EdgeDiscovery implements HA service discovery lookup.
-func edgeDiscovery(logger *logrus.Entry) ([][]*net.TCPAddr, error) {
+func edgeDiscovery(logger logger.Service) ([][]*net.TCPAddr, error) {
 	_, addrs, err := netLookupSRV(srvService, srvProto, srvName)
 	if err != nil {
 		_, fallbackAddrs, fallbackErr := fallbackLookupSRV(srvService, srvProto, srvName)
 		if fallbackErr != nil || len(fallbackAddrs) == 0 {
 			// use the original DNS error `err` in messages, not `fallbackErr`
-			logger.Errorln("Error looking up Cloudflare edge IPs: the DNS query failed:", err)
+			logger.Errorf("Error looking up Cloudflare edge IPs: the DNS query failed: %s", err)
 			for _, s := range friendlyDNSErrorLines {
-				logger.Errorln(s)
+				logger.Error(s)
 			}
 			return nil, errors.Wrapf(err, "Could not lookup srv records on _%v._%v.%v", srvService, srvProto, srvName)
 		}
@@ -121,15 +121,15 @@ func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
 }
 
 // ResolveAddrs resolves TCP address given a list of addresses. Address can be a hostname, however, it will return at most one
-// of the hostname's IP addresses
-func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) {
-	var tcpAddrs []*net.TCPAddr
+// of the hostname's IP addresses.
+func ResolveAddrs(addrs []string, logger logger.Service) (resolved []*net.TCPAddr) {
 	for _, addr := range addrs {
 		tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
 		if err != nil {
-			return nil, err
+			logger.Errorf("Failed to resolve %s, err: %v", addr, err)
+		} else {
+			resolved = append(resolved, tcpAddr)
 		}
-		tcpAddrs = append(tcpAddrs, tcpAddr)
 	}
-	return tcpAddrs, nil
+	return
 }

edgediscovery/allregions/discovery_test.go

@@ -3,7 +3,7 @@ package allregions
 import (
 	"testing"
 
-	"github.com/sirupsen/logrus"
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -19,7 +19,8 @@ func TestEdgeDiscovery(t *testing.T) {
 		}
 	}
 
-	addrLists, err := edgeDiscovery(logrus.New().WithFields(logrus.Fields{}))
+	l := logger.NewOutputWriter(logger.NewMockWriteManager())
+	addrLists, err := edgeDiscovery(l)
 	assert.NoError(t, err)
 	actualAddrSet := map[string]bool{}
 	for _, addrs := range addrLists {

edgediscovery/allregions/region.go

@@ -56,6 +56,10 @@ func (r Region) GetUnusedIP(excluding *net.TCPAddr) *net.TCPAddr {
 
 // Use the address, assigning it to a proxy connection.
 func (r Region) Use(addr *net.TCPAddr, connID int) {
+	if addr == nil {
+		//logrus.Errorf("Attempted to use nil address for connection %d", connID)
+		return
+	}
 	r.connFor[addr] = InUse(connID)
 }
 

edgediscovery/allregions/regions.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"net"
 
-	"github.com/sirupsen/logrus"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 // Regions stores Cloudflare edge network IPs, partitioned into two regions.
@@ -19,7 +19,7 @@ type Regions struct {
 // ------------------------------------
 
 // ResolveEdge resolves the Cloudflare edge, returning all regions discovered.
-func ResolveEdge(logger *logrus.Entry) (*Regions, error) {
+func ResolveEdge(logger logger.Service) (*Regions, error) {
 	addrLists, err := edgeDiscovery(logger)
 	if err != nil {
 		return nil, err
@@ -35,12 +35,12 @@ func ResolveEdge(logger *logrus.Entry) (*Regions, error) {
 
 // StaticEdge creates a list of edge addresses from the list of hostnames.
 // Mainly used for testing connectivity.
-func StaticEdge(hostnames []string) (*Regions, error) {
-	addrs, err := ResolveAddrs(hostnames)
-	if err != nil {
-		return nil, err
+func StaticEdge(hostnames []string, logger logger.Service) (*Regions, error) {
+	resolved := ResolveAddrs(hostnames, logger)
+	if len(resolved) == 0 {
+		return nil, fmt.Errorf("failed to resolve any edge address")
 	}
-	return NewNoResolve(addrs), nil
+	return NewNoResolve(resolved), nil
 }
 
 // NewNoResolve doesn't resolve the edge. Instead it just uses the given addresses.
@@ -86,21 +86,28 @@ func (rs *Regions) AddrUsedBy(connID int) *net.TCPAddr {
 // GetUnusedAddr gets an unused addr from the edge, excluding the given addr. Prefer to use addresses
 // evenly across both regions.
 func (rs *Regions) GetUnusedAddr(excluding *net.TCPAddr, connID int) *net.TCPAddr {
-	var addr *net.TCPAddr
 	if rs.region1.AvailableAddrs() > rs.region2.AvailableAddrs() {
-		addr = rs.region1.GetUnusedIP(excluding)
-		rs.region1.Use(addr, connID)
-	} else {
-		addr = rs.region2.GetUnusedIP(excluding)
-		rs.region2.Use(addr, connID)
+		return getAddrs(excluding, connID, &rs.region1, &rs.region2)
 	}
 
-	if addr == nil {
-		return nil
+	return getAddrs(excluding, connID, &rs.region2, &rs.region1)
+}
+
+// getAddrs tries to grab address form `first` region, then `second` region
+// this is an unrolled loop over 2 element array
+func getAddrs(excluding *net.TCPAddr, connID int, first *Region, second *Region) *net.TCPAddr {
+	addr := first.GetUnusedIP(excluding)
+	if addr != nil {
+		first.Use(addr, connID)
+		return addr
+	}
+	addr = second.GetUnusedIP(excluding)
+	if addr != nil {
+		second.Use(addr, connID)
+		return addr
 	}
 
-	// Mark the address as used and return it
-	return addr
+	return nil
 }
 
 // AvailableAddrs returns how many edge addresses aren't used.

edgediscovery/allregions/regions_test.go

@@ -58,6 +58,7 @@ func TestRegions_Giveback_Region1(t *testing.T) {
 	rs.GiveBack(&addr0)
 	assert.Equal(t, &addr0, rs.GetUnusedAddr(nil, 3))
 }
+
 func TestRegions_Giveback_Region2(t *testing.T) {
 	rs := makeRegions()
 	rs.region1.Use(&addr0, 0)

edgediscovery/dial.go

@@ -1,4 +1,4 @@
-package connection
+package edgediscovery
 
 import (
 	"context"
@@ -9,7 +9,7 @@ import (
 	"github.com/pkg/errors"
 )
 
-// DialEdge makes a TLS connection to a Cloudflare edge node
+// DialEdgeWithH2Mux makes a TLS connection to a Cloudflare edge node
 func DialEdge(
 	ctx context.Context,
 	timeout time.Duration,
@@ -25,6 +25,7 @@ func DialEdge(
 	if err != nil {
 		return nil, newDialError(err, "DialContext error")
 	}
+
 	tlsEdgeConn := tls.Client(edgeConn, tlsConfig)
 	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
 

edgediscovery/edgediscovery.go

@@ -6,8 +6,7 @@ import (
 	"sync"
 
 	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
-
-	"github.com/sirupsen/logrus"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
@@ -20,7 +19,7 @@ var errNoAddressesLeft = fmt.Errorf("There are no free edge addresses left")
 type Edge struct {
 	regions *allregions.Regions
 	sync.Mutex
-	logger *logrus.Entry
+	logger logger.Service
 }
 
 // ------------------------------------
@@ -29,37 +28,34 @@ type Edge struct {
 
 // ResolveEdge runs the initial discovery of the Cloudflare edge, finding Addrs that can be allocated
 // to connections.
-func ResolveEdge(l *logrus.Logger) (*Edge, error) {
-	logger := l.WithField("subsystem", subsystem)
-	regions, err := allregions.ResolveEdge(logger)
+func ResolveEdge(l logger.Service) (*Edge, error) {
+	regions, err := allregions.ResolveEdge(l)
 	if err != nil {
 		return new(Edge), err
 	}
 	return &Edge{
-		logger:  logger,
+		logger:  l,
 		regions: regions,
 	}, nil
 }
 
 // StaticEdge creates a list of edge addresses from the list of hostnames. Mainly used for testing connectivity.
-func StaticEdge(l *logrus.Logger, hostnames []string) (*Edge, error) {
-	logger := l.WithField("subsystem", subsystem)
-	regions, err := allregions.StaticEdge(hostnames)
+func StaticEdge(l logger.Service, hostnames []string) (*Edge, error) {
+	regions, err := allregions.StaticEdge(hostnames, l)
 	if err != nil {
 		return new(Edge), err
 	}
 	return &Edge{
-		logger:  logger,
+		logger:  l,
 		regions: regions,
 	}, nil
 }
 
 // MockEdge creates a Cloudflare Edge from arbitrary TCP addresses. Used for testing.
-func MockEdge(l *logrus.Logger, addrs []*net.TCPAddr) *Edge {
-	logger := l.WithField("subsystem", subsystem)
+func MockEdge(l logger.Service, addrs []*net.TCPAddr) *Edge {
 	regions := allregions.NewNoResolve(addrs)
 	return &Edge{
-		logger:  logger,
+		logger:  l,
 		regions: regions,
 	}
 }
@@ -83,24 +79,20 @@ func (ed *Edge) GetAddrForRPC() (*net.TCPAddr, error) {
 func (ed *Edge) GetAddr(connID int) (*net.TCPAddr, error) {
 	ed.Lock()
 	defer ed.Unlock()
-	logger := ed.logger.WithFields(logrus.Fields{
-		"connID":   connID,
-		"function": "GetAddr",
-	})
 
 	// If this connection has already used an edge addr, return it.
 	if addr := ed.regions.AddrUsedBy(connID); addr != nil {
-		logger.Debug("Returning same address back to proxy connection")
+		ed.logger.Debugf("edgediscovery - GetAddr: Returning same address back to proxy connection: connID: %d", connID)
 		return addr, nil
 	}
 
 	// Otherwise, give it an unused one
 	addr := ed.regions.GetUnusedAddr(nil, connID)
 	if addr == nil {
-		logger.Debug("No addresses left to give proxy connection")
+		ed.logger.Debugf("edgediscovery - GetAddr: No addresses left to give proxy connection: connID: %d", connID)
 		return nil, errNoAddressesLeft
 	}
-	logger.Debug("Giving connection its new address")
+	ed.logger.Debugf("edgediscovery - GetAddr: Giving connection its new address %s: connID: %d", addr, connID)
 	return addr, nil
 }
 
@@ -108,10 +100,6 @@ func (ed *Edge) GetAddr(connID int) (*net.TCPAddr, error) {
 func (ed *Edge) GetDifferentAddr(connID int) (*net.TCPAddr, error) {
 	ed.Lock()
 	defer ed.Unlock()
-	logger := ed.logger.WithFields(logrus.Fields{
-		"connID":   connID,
-		"function": "GetDifferentAddr",
-	})
 
 	oldAddr := ed.regions.AddrUsedBy(connID)
 	if oldAddr != nil {
@@ -119,10 +107,11 @@ func (ed *Edge) GetDifferentAddr(connID int) (*net.TCPAddr, error) {
 	}
 	addr := ed.regions.GetUnusedAddr(oldAddr, connID)
 	if addr == nil {
-		logger.Debug("No addresses left to give proxy connection")
+		ed.logger.Debugf("edgediscovery - GetDifferentAddr: No addresses left to give proxy connection: connID: %d", connID)
+		// note: if oldAddr were not nil, it will become available on the next iteration
 		return nil, errNoAddressesLeft
 	}
-	logger.Debug("Giving connection its new address")
+	ed.logger.Debugf("edgediscovery - GetDifferentAddr: Giving connection its new address %s: connID: %d", addr, connID)
 	return addr, nil
 }
 
@@ -138,6 +127,6 @@ func (ed *Edge) AvailableAddrs() int {
 func (ed *Edge) GiveBack(addr *net.TCPAddr) bool {
 	ed.Lock()
 	defer ed.Unlock()
-	ed.logger.WithField("function", "GiveBack").Debug("Address now unused")
+	ed.logger.Debug("edgediscovery - GiveBack: Address now unused")
 	return ed.regions.GiveBack(addr)
 }

edgediscovery/edgediscovery_test.go

@@ -4,7 +4,7 @@ import (
 	"net"
 	"testing"
 
-	"github.com/sirupsen/logrus"
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -32,7 +32,7 @@ var (
 )
 
 func TestGiveBack(t *testing.T) {
-	l := logrus.New()
+	l := logger.NewOutputWriter(logger.NewMockWriteManager())
 	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
 
 	// Give this connection an address
@@ -49,7 +49,7 @@ func TestGiveBack(t *testing.T) {
 }
 
 func TestRPCAndProxyShareSingleEdgeIP(t *testing.T) {
-	l := logrus.New()
+	l := logger.NewOutputWriter(logger.NewMockWriteManager())
 
 	// Make an edge with a single IP
 	edge := MockEdge(l, []*net.TCPAddr{&addr0})
@@ -66,7 +66,7 @@ func TestRPCAndProxyShareSingleEdgeIP(t *testing.T) {
 }
 
 func TestGetAddrForRPC(t *testing.T) {
-	l := logrus.New()
+	l := logger.NewOutputWriter(logger.NewMockWriteManager())
 	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
 
 	// Get a connection
@@ -83,8 +83,32 @@ func TestGetAddrForRPC(t *testing.T) {
 	assert.Equal(t, 4, edge.AvailableAddrs())
 }
 
+func TestOnePerRegion(t *testing.T) {
+	l := logger.NewOutputWriter(logger.NewMockWriteManager())
+
+	// Make an edge with only one address
+	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1})
+
+	// Use the only address
+	const connID = 0
+	a1, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, a1)
+
+	// if the first address is bad, get the second one
+	a2, err := edge.GetDifferentAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, a2)
+	assert.NotEqual(t, a1, a2)
+
+	// now that second one is bad, get the first one again
+	a3, err := edge.GetDifferentAddr(connID)
+	assert.NoError(t, err)
+	assert.Equal(t, a1, a3)
+}
+
 func TestOnlyOneAddrLeft(t *testing.T) {
-	l := logrus.New()
+	l := logger.NewOutputWriter(logger.NewMockWriteManager())
 
 	// Make an edge with only one address
 	edge := MockEdge(l, []*net.TCPAddr{&addr0})
@@ -98,10 +122,15 @@ func TestOnlyOneAddrLeft(t *testing.T) {
 	// If that edge address is "bad", there's no alternative address.
 	_, err = edge.GetDifferentAddr(connID)
 	assert.Error(t, err)
+
+	// previously bad address should become available again on next iteration.
+	addr, err = edge.GetDifferentAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
 }
 
 func TestNoAddrsLeft(t *testing.T) {
-	l := logrus.New()
+	l := logger.NewOutputWriter(logger.NewMockWriteManager())
 
 	// Make an edge with no addresses
 	edge := MockEdge(l, []*net.TCPAddr{})
@@ -113,7 +142,7 @@ func TestNoAddrsLeft(t *testing.T) {
 }
 
 func TestGetAddr(t *testing.T) {
-	l := logrus.New()
+	l := logger.NewOutputWriter(logger.NewMockWriteManager())
 	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
 
 	// Give this connection an address
@@ -129,7 +158,7 @@ func TestGetAddr(t *testing.T) {
 }
 
 func TestGetDifferentAddr(t *testing.T) {
-	l := logrus.New()
+	l := logger.NewOutputWriter(logger.NewMockWriteManager())
 	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
 
 	// Give this connection an address

edgediscovery/mocks_for_test.go

@@ -1,4 +1,4 @@
-package connection
+package edgediscovery
 
 import (
 	"fmt"

edgediscovery/protocol.go

@@ -0,0 +1,45 @@
+package edgediscovery
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+)
+
+const (
+	protocolRecord = "protocol.argotunnel.com"
+)
+
+var (
+	errNoProtocolRecord = fmt.Errorf("No TXT record found for %s to determine connection protocol", protocolRecord)
+)
+
+func HTTP2Percentage() (int32, error) {
+	records, err := net.LookupTXT(protocolRecord)
+	if err != nil {
+		return 0, err
+	}
+	if len(records) == 0 {
+		return 0, errNoProtocolRecord
+	}
+	return parseHTTP2Precentage(records[0])
+}
+
+// The record looks like http2=percentage
+func parseHTTP2Precentage(record string) (int32, error) {
+	const key = "http2"
+	slices := strings.Split(record, "=")
+	if len(slices) != 2 {
+		return 0, fmt.Errorf("Malformed TXT record %s, expect http2=percentage", record)
+	}
+	if slices[0] != key {
+		return 0, fmt.Errorf("Incorrect key %s, expect %s", slices[0], key)
+	}
+	percentage, err := strconv.ParseInt(slices[1], 10, 32)
+	if err != nil {
+		return 0, err
+	}
+	return int32(percentage), nil
+
+}

edgediscovery/protocol_test.go

@@ -0,0 +1,80 @@
+package edgediscovery
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHTTP2Percentage(t *testing.T) {
+	_, err := HTTP2Percentage()
+	assert.NoError(t, err)
+}
+
+func TestParseHTTP2Precentage(t *testing.T) {
+	tests := []struct {
+		record     string
+		percentage int32
+		wantErr    bool
+	}{
+		{
+			record:     "http2=-1",
+			percentage: -1,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=0",
+			percentage: 0,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=50",
+			percentage: 50,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=100",
+			percentage: 100,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=1000",
+			percentage: 1000,
+			wantErr:    false,
+		},
+		{
+			record:  "http2=10.5",
+			wantErr: true,
+		},
+		{
+			record:  "http2=10 h2mux=90",
+			wantErr: true,
+		},
+		{
+			record:  "http2=ten",
+			wantErr: true,
+		},
+
+		{
+			record:  "h2mux=100",
+			wantErr: true,
+		},
+		{
+			record:  "http2",
+			wantErr: true,
+		},
+		{
+			record:  "http2=",
+			wantErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		p, err := parseHTTP2Precentage(test.record)
+		if test.wantErr {
+			assert.Error(t, err)
+		} else {
+			assert.Equal(t, test.percentage, p)
+		}
+	}
+}

github_message.py

@@ -0,0 +1,118 @@
+#!/usr/bin/python3
+"""
+Creates Github Releases Notes with content hashes
+"""
+
+import argparse
+import logging
+import os
+import hashlib
+import glob
+
+from github import Github, GithubException, UnknownObjectException
+
+FORMAT = "%(levelname)s - %(asctime)s: %(message)s"
+logging.basicConfig(format=FORMAT)
+
+CLOUDFLARED_REPO = os.environ.get("GITHUB_REPO", "cloudflare/cloudflared")
+GITHUB_CONFLICT_CODE = "already_exists"
+
+def get_sha256(filename):
+    """ get the sha256 of a file """
+    sha256_hash = hashlib.sha256()
+    with open(filename,"rb") as f:
+        for byte_block in iter(lambda: f.read(4096),b""):
+            sha256_hash.update(byte_block)
+        return sha256_hash.hexdigest()
+
+
+def update_or_add_message(msg, name, sha):
+    """ 
+    updates or builds the github version message for each new asset's sha256. 
+    Searches the existing message string to update or create. 
+    """
+    new_text = '{0}: {1}\n'.format(name, sha)
+    start = msg.find(name)
+    if (start != -1):
+        end = msg.find("\n", start)
+        if (end != -1):
+            return msg.replace(msg[start:end+1], new_text)
+    back = msg.rfind("```")
+    if (back != -1):
+        return '{0}{1}```'.format(msg[:back], new_text)
+    return '{0} \n### SHA256 Checksums:\n```\n{1}```'.format(msg, new_text)
+
+def get_release(repo, version):
+    """ Get a Github Release matching the version tag. """
+    try:
+        release = repo.get_release(version)
+        logging.info("Release %s found", version)
+        return release
+    except UnknownObjectException:
+        logging.info("Release %s not found", version)
+
+
+def parse_args():
+    """ Parse and validate args """
+    parser = argparse.ArgumentParser(
+        description="Creates Github Releases and uploads assets."
+    )
+    parser.add_argument(
+        "--api-key", default=os.environ.get("API_KEY"), help="Github API key"
+    )
+    parser.add_argument(
+        "--release-version",
+        metavar="version",
+        default=os.environ.get("VERSION"),
+        help="Release version",
+    )
+    parser.add_argument(
+        "--dry-run", action="store_true", help="Do not modify the release message"
+    )
+
+    args = parser.parse_args()
+    is_valid = True
+    if not args.release_version:
+        logging.error("Missing release version")
+        is_valid = False
+
+    if not args.api_key:
+        logging.error("Missing API key")
+        is_valid = False
+
+    if is_valid:
+        return args
+
+    parser.print_usage()
+    exit(1)
+
+
+def main():
+    """ Attempts to update the Github Release message with the github asset's checksums """
+    try:
+        args = parse_args()
+        client = Github(args.api_key)
+        repo = client.get_repo(CLOUDFLARED_REPO)
+        release = get_release(repo, args.release_version)
+
+        msg = release.body
+
+        for filepath in glob.glob("artifacts/*"):
+            pkg_hash = get_sha256(filepath)
+            # add the sha256 of the new artifact to the release message body
+            name = os.path.basename(filepath)
+            msg = update_or_add_message(msg, name, pkg_hash)
+
+        if args.dry_run:
+            logging.info("Skipping asset upload because of dry-run")
+            return
+
+        # update the release body text
+        release.update_release(args.release_version, msg)
+
+    except Exception as e:
+        logging.exception(e)
+        exit(1)
+
+
+main()

github_release.py

@@ -0,0 +1,213 @@
+#!/usr/bin/python3
+"""
+Creates Github Releases and uploads assets
+"""
+
+import argparse
+import logging
+import os
+import shutil
+import hashlib
+import requests
+import tarfile
+
+from github import Github, GithubException, UnknownObjectException
+
+FORMAT = "%(levelname)s - %(asctime)s: %(message)s"
+logging.basicConfig(format=FORMAT)
+
+CLOUDFLARED_REPO = os.environ.get("GITHUB_REPO", "cloudflare/cloudflared")
+GITHUB_CONFLICT_CODE = "already_exists"
+BASE_KV_URL = 'https://api.cloudflare.com/client/v4/accounts/'
+UPDATER_PREFIX = 'update'
+
+def get_sha256(filename):
+    """ get the sha256 of a file """
+    sha256_hash = hashlib.sha256()
+    with open(filename,"rb") as f:
+        for byte_block in iter(lambda: f.read(4096),b""):
+            sha256_hash.update(byte_block)
+        return sha256_hash.hexdigest()
+
+def send_hash(pkg_hash, name, version, account, namespace, api_token):
+    """ send the checksum of a file to workers kv """
+    key = '{0}_{1}_{2}'.format(UPDATER_PREFIX, version, name)
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": "Bearer " + api_token,
+    }
+    response = requests.put(
+            BASE_KV_URL + account + "/storage/kv/namespaces/" + namespace + "/values/" + key,
+            headers=headers,
+            data=pkg_hash
+    )
+
+    if response.status_code != 200:
+        jsonResponse = response.json()
+        errors = jsonResponse["errors"]
+        if len(errors) > 0:
+            raise Exception("failed to upload checksum: {0}", errors[0])
+
+
+
+def assert_tag_exists(repo, version):
+    """ Raise exception if repo does not contain a tag matching version """
+    tags = repo.get_tags()
+    if not tags or tags[0].name != version:
+        raise Exception("Tag {} not found".format(version))
+
+
+def get_or_create_release(repo, version, dry_run=False):
+    """
+    Get a Github Release matching the version tag or create a new one.
+    If a conflict occurs on creation, attempt to fetch the Release on last time
+    """
+    try:
+        release = repo.get_release(version)
+        logging.info("Release %s found", version)
+        return release
+    except UnknownObjectException:
+        logging.info("Release %s not found", version)
+
+    # We dont want to create a new release tag if one doesnt already exist
+    assert_tag_exists(repo, version)
+
+    if dry_run:
+        logging.info("Skipping Release creation because of dry-run")
+        return
+
+    try:
+        logging.info("Creating release %s", version)
+        return repo.create_git_release(version, version, "")
+    except GithubException as e:
+        errors = e.data.get("errors", [])
+        if e.status == 422 and any(
+            [err.get("code") == GITHUB_CONFLICT_CODE for err in errors]
+        ):
+            logging.warning(
+                "Conflict: Release was likely just made by a different build: %s",
+                e.data,
+            )
+            return repo.get_release(version)
+        raise e
+
+
+def parse_args():
+    """ Parse and validate args """
+    parser = argparse.ArgumentParser(
+        description="Creates Github Releases and uploads assets."
+    )
+    parser.add_argument(
+        "--api-key", default=os.environ.get("API_KEY"), help="Github API key"
+    )
+    parser.add_argument(
+        "--release-version",
+        metavar="version",
+        default=os.environ.get("VERSION"),
+        help="Release version",
+    )
+    parser.add_argument(
+        "--path", default=os.environ.get("ASSET_PATH"), help="Asset path"
+    )
+    parser.add_argument(
+        "--name", default=os.environ.get("ASSET_NAME"), help="Asset Name"
+    )
+    parser.add_argument(
+        "--namespace-id", default=os.environ.get("KV_NAMESPACE"), help="workersKV namespace id"
+    )
+    parser.add_argument(
+        "--kv-account-id", default=os.environ.get("KV_ACCOUNT"), help="workersKV account id"
+    )
+    parser.add_argument(
+        "--kv-api-token", default=os.environ.get("KV_API_TOKEN"), help="workersKV API Token"
+    )
+    parser.add_argument(
+        "--dry-run", action="store_true", help="Do not create release or upload asset"
+    )
+
+    args = parser.parse_args()
+    is_valid = True
+    if not args.release_version:
+        logging.error("Missing release version")
+        is_valid = False
+
+    if not args.path:
+        logging.error("Missing asset path")
+        is_valid = False
+
+    if not args.name:
+        logging.error("Missing asset name")
+        is_valid = False
+
+    if not args.api_key:
+        logging.error("Missing API key")
+        is_valid = False
+    
+    if not args.namespace_id:
+        logging.error("Missing KV namespace id")
+        is_valid = False
+
+    if not args.kv_account_id:
+        logging.error("Missing KV account id")
+        is_valid = False
+
+    if not args.kv_api_token:
+        logging.error("Missing KV API token")
+        is_valid = False
+
+    if is_valid:
+        return args
+
+    parser.print_usage()
+    exit(1)
+
+
+def main():
+    """ Attempts to upload Asset to Github Release. Creates Release if it doesnt exist """
+    try:
+        args = parse_args()
+        client = Github(args.api_key)
+        repo = client.get_repo(CLOUDFLARED_REPO)
+        release = get_or_create_release(repo, args.release_version, args.dry_run)
+
+        if args.dry_run:
+            logging.info("Skipping asset upload because of dry-run")
+            return
+
+        release.upload_asset(args.path, name=args.name)
+
+        # check and extract if the file is a tar and gzipped file (as is the case with the macos builds)
+        binary_path = args.path
+        if binary_path.endswith("tgz"):
+            try:
+                shutil.rmtree('cfd') 
+            except OSError as e:
+                pass
+            zipfile = tarfile.open(binary_path, "r:gz")
+            zipfile.extractall('cfd') # specify which folder to extract to
+            zipfile.close()
+
+            binary_path = os.path.join(os.getcwd(), 'cfd', 'cloudflared') 
+
+        # send the sha256 (the checksum) to workers kv
+        pkg_hash = get_sha256(binary_path)
+        send_hash(pkg_hash, args.name, args.release_version, args.kv_account_id, args.namespace_id, args.kv_api_token)
+
+        # create the artifacts directory if it doesn't exist 
+        artifact_path = os.path.join(os.getcwd(), 'artifacts') 
+        if not os.path.isdir(artifact_path):
+            os.mkdir(artifact_path)
+
+        # copy the binary to the path
+        copy_path = os.path.join(artifact_path, args.name)
+        try:
+            shutil.copy(args.path, copy_path)
+        except shutil.SameFileError:
+            pass # the macOS release copy fails with being the same file (already in the artifacts directory). Catching to ignore.
+
+    except Exception as e:
+        logging.exception(e)
+        exit(1)
+
+
+main()