Release 2023.2.0

.github/workflows/check.yaml

@@ -4,15 +4,15 @@ jobs:
   check:
     strategy:
       matrix:
-        go-version: [1.21.x]
+        go-version: [1.19.x]
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
     - name: Install Go
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v2
       with:
         go-version: ${{ matrix.go-version }}
     - name: Checkout code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
     - name: Test
       run: make test

.teamcity/build-macos.sh

@@ -143,7 +143,7 @@ if [[ ! -z "$CODE_SIGN_NAME" ]]; then
   codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME}
   
  # notarize the binary
- # TODO: TUN-5789
+ # TODO: https://jira.cfdata.org/browse/TUN-5789
 fi
 
 # creating build directory
@@ -169,7 +169,7 @@ if [[ ! -z "$PKG_SIGN_NAME" ]]; then
       ${PKGNAME}
 
       # notarize the package
-      # TODO: TUN-5789
+      # TODO: https://jira.cfdata.org/browse/TUN-5789
 else
     pkgbuild --identifier com.cloudflare.${PRODUCT} \
       --version ${VERSION} \

.teamcity/install-cloudflare-go.sh

@@ -1,8 +0,0 @@
-# !/usr/bin/env bash
-
-cd /tmp
-git clone -q https://github.com/cloudflare/go
-cd go/src
-# https://github.com/cloudflare/go/tree/34129e47042e214121b6bbff0ded4712debed18e is version go1.21.5-devel-cf
-git checkout -q 34129e47042e214121b6bbff0ded4712debed18e
-./make.bash

.teamcity/mac/install-cloudflare-go.sh

@@ -1,10 +0,0 @@
-rm -rf /tmp/go
-export GOCACHE=/tmp/gocache
-rm -rf $GOCACHE
-
-./.teamcity/install-cloudflare-go.sh
-
-export PATH="/tmp/go/bin:$PATH"
-go version
-which go
-go env

.teamcity/update-homebrew-core.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if ! VERSION="$(git describe --tags --exact-match 2>/dev/null)" ; then
+    echo "Skipping public release for an untagged commit."
+    echo "##teamcity[buildStatus status='SUCCESS' text='Skipped due to lack of tag']"
+    exit 0
+fi
+
+if [[ "${HOMEBREW_GITHUB_API_TOKEN:-}" == "" ]] ; then
+    echo "Missing GITHUB_API_TOKEN"
+    exit 1
+fi
+
+# "install" Homebrew
+git clone https://github.com/Homebrew/brew tmp/homebrew
+eval "$(tmp/homebrew/bin/brew shellenv)"
+brew update --force --quiet
+chmod -R go-w "$(brew --prefix)/share/zsh"
+
+git config --global user.name "cloudflare-warp-bot"
+git config --global user.email "warp-bot@cloudflare.com"
+
+# bump formula pr
+brew bump-formula-pr cloudflared --version="$VERSION" --no-browse --no-audit

.teamcity/update-homebrew.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -euo pipefail
+
+FILENAME="${PWD}/artifacts/cloudflared-darwin-amd64.tgz"
+
+if ! VERSION="$(git describe --tags --exact-match 2>/dev/null)" ; then
+    echo "Skipping public release for an untagged commit."
+    echo "##teamcity[buildStatus status='SUCCESS' text='Skipped due to lack of tag']"
+    exit 0
+fi
+
+if [[ ! -f "$FILENAME" ]] ; then
+    echo "Missing $FILENAME"
+    exit 1
+fi
+
+if [[ "${GITHUB_PRIVATE_KEY:-}" == "" ]] ; then
+    echo "Missing GITHUB_PRIVATE_KEY"
+    exit 1
+fi
+
+# upload to s3 bucket for use by Homebrew formula
+s3cmd \
+    --acl-public --signature-v2 --access_key="$AWS_ACCESS_KEY_ID" --secret_key="$AWS_SECRET_ACCESS_KEY" --host-bucket="%(bucket)s.s3.cfdata.org" \
+    put "$FILENAME" "s3://cftunnel-docs/dl/cloudflared-$VERSION-darwin-amd64.tgz"
+s3cmd \
+    --acl-public --signature-v2 --access_key="$AWS_ACCESS_KEY_ID" --secret_key="$AWS_SECRET_ACCESS_KEY" --host-bucket="%(bucket)s.s3.cfdata.org" \
+    cp "s3://cftunnel-docs/dl/cloudflared-$VERSION-darwin-amd64.tgz" "s3://cftunnel-docs/dl/cloudflared-stable-darwin-amd64.tgz"
+SHA256=$(sha256sum "$FILENAME" | cut -b1-64)
+
+# set up git (note that UserKnownHostsFile is an absolute path so we can cd wherever)
+mkdir -p tmp
+ssh-keyscan -t rsa github.com > tmp/github.txt
+echo "$GITHUB_PRIVATE_KEY" > tmp/private.key
+chmod 0400 tmp/private.key
+export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=$PWD/tmp/github.txt -i $PWD/tmp/private.key -o IdentitiesOnly=yes"
+
+# clone Homebrew repo into tmp/homebrew-cloudflare
+git clone git@github.com:cloudflare/homebrew-cloudflare.git tmp/homebrew-cloudflare    
+cd tmp/homebrew-cloudflare
+git checkout -f master
+git reset --hard origin/master
+
+# modify cloudflared.rb
+URL="https://packages.argotunnel.com/dl/cloudflared-$VERSION-darwin-amd64.tgz"
+tee cloudflared.rb <<EOF
+class Cloudflared < Formula
+  desc 'Cloudflare Tunnel'
+  homepage 'https://developers.cloudflare.com/cloudflare-one/connections/connect-apps'
+  url '$URL'
+  sha256 '$SHA256'
+  version '$VERSION'
+  def install
+    bin.install 'cloudflared'
+  end
+end
+EOF
+
+# push cloudflared.rb
+git add cloudflared.rb
+git diff
+git config user.name "cloudflare-warp-bot"
+git config user.email "warp-bot@cloudflare.com"
+git commit -m "Release Cloudflare Tunnel $VERSION"
+
+git push -v origin master

.teamcity/windows/builds.ps1

@@ -1,28 +0,0 @@
-Set-StrictMode -Version Latest
-$ErrorActionPreference = "Stop"
-$ProgressPreference = "SilentlyContinue"
-
-# Relative path to working directory
-$CloudflaredDirectory = "go\src\github.com\cloudflare\cloudflared"
-
-cd $CloudflaredDirectory
-
-Write-Output "Building for amd64"
-$env:TARGET_OS = "windows"
-$env:CGO_ENABLED = 1
-$env:TARGET_ARCH = "amd64"
-$env:Path = "$Env:Temp\go\bin;$($env:Path)"
-
-go env
-go version
-
-& make cloudflared
-if ($LASTEXITCODE -ne 0) { throw "Failed to build cloudflared for amd64" }
-copy .\cloudflared.exe .\cloudflared-windows-amd64.exe
-
-Write-Output "Building for 386"
-$env:CGO_ENABLED = 0
-$env:TARGET_ARCH = "386"
-make cloudflared
-if ($LASTEXITCODE -ne 0) { throw "Failed to build cloudflared for 386" }
-copy .\cloudflared.exe .\cloudflared-windows-386.exe

.teamcity/windows/component-test.ps1

@@ -1,82 +0,0 @@
-Set-StrictMode -Version Latest
-$ErrorActionPreference = "Stop"
-$ProgressPreference = "SilentlyContinue"
-
-$WorkingDirectory = Get-Location
-$CloudflaredDirectory = "$WorkingDirectory\go\src\github.com\cloudflare\cloudflared"
-
-Write-Output "Installing python..."
-
-$PythonVersion = "3.10.11"
-$PythonZipFile = "$env:Temp\python-$PythonVersion-embed-amd64.zip"
-$PipInstallFile = "$env:Temp\get-pip.py"
-$PythonZipUrl = "https://www.python.org/ftp/python/$PythonVersion/python-$PythonVersion-embed-amd64.zip"
-$PythonPath = "$WorkingDirectory\Python"
-$PythonBinPath = "$PythonPath\python.exe"
-
-# Download Python zip file
-Invoke-WebRequest -Uri $PythonZipUrl -OutFile $PythonZipFile
-
-# Download Python pip file
-Invoke-WebRequest -Uri "https://bootstrap.pypa.io/get-pip.py" -OutFile $PipInstallFile
-
-# Extract Python files
-Expand-Archive $PythonZipFile -DestinationPath $PythonPath -Force
-
-# Add Python to PATH
-$env:Path = "$PythonPath\Scripts;$PythonPath;$($env:Path)"
-
-Write-Output "Installed to $PythonPath"
-
-# Install pip
-& $PythonBinPath $PipInstallFile
-
-# Add package paths in pythonXX._pth to unblock python -m pip
-$PythonImportPathFile = "$PythonPath\python310._pth"
-$ComponentTestsDir = "$CloudflaredDirectory\component-tests\"
-@($ComponentTestsDir, "Lib\site-packages", $(Get-Content $PythonImportPathFile)) | Set-Content $PythonImportPathFile
-
-# Test Python installation
-& $PythonBinPath --version
-& $PythonBinPath -m pip --version
-
-go env
-go version
-
-$env:TARGET_OS = "windows"
-$env:CGO_ENABLED = 1
-$env:TARGET_ARCH = "amd64"
-$env:Path = "$Env:Temp\go\bin;$($env:Path)"
-
-& $PythonBinPath --version
-& $PythonBinPath -m pip --version
-
-cd $CloudflaredDirectory
-
-go env
-go version
-
-Write-Output "Building cloudflared"
-
-& make cloudflared
-if ($LASTEXITCODE -ne 0) { throw "Failed to build cloudflared" }
-
-echo $LASTEXITCODE
-
-Write-Output "Running unit tests"
-
-# Not testing with race detector because of https://github.com/golang/go/issues/61058
-# We already test it on other platforms
-& go test -failfast -mod=vendor ./...
-if ($LASTEXITCODE -ne 0) { throw "Failed unit tests" }
-
-Write-Output "Running component tests"
-
-& $PythonBinPath -m pip install --upgrade -r component-tests/requirements.txt
-& $PythonBinPath component-tests/setup.py --type create
-& $PythonBinPath -m pytest component-tests -o log_cli=true --log-cli-level=INFO
-if ($LASTEXITCODE -ne 0) {
-    & $PythonBinPath component-tests/setup.py --type cleanup
-    throw "Failed component tests"
-}
-& $PythonBinPath component-tests/setup.py --type cleanup

.teamcity/windows/install-cloudflare-go.ps1

@@ -1,16 +0,0 @@
-Set-StrictMode -Version Latest
-$ErrorActionPreference = "Stop"
-$ProgressPreference = "SilentlyContinue"
-
-Write-Output "Downloading cloudflare go..."
-
-Set-Location "$Env:Temp"
-
-git clone -q https://github.com/cloudflare/go
-Write-Output "Building go..."
-cd go/src
-# https://github.com/cloudflare/go/tree/34129e47042e214121b6bbff0ded4712debed18e is version go1.21.5-devel-cf
-git checkout -q 34129e47042e214121b6bbff0ded4712debed18e
-& ./make.bat
-
-Write-Output "Installed"

.teamcity/windows/install-go-msi.ps1

@@ -1,20 +0,0 @@
-$ErrorActionPreference = "Stop"
-$ProgressPreference = "SilentlyContinue"
-$GoMsiVersion = "go1.21.5.windows-amd64.msi"
- 
-Write-Output "Downloading go installer..."
- 
-Set-Location "$Env:Temp"
- 
-(New-Object System.Net.WebClient).DownloadFile(
-    "https://go.dev/dl/$GoMsiVersion",
-    "$Env:Temp\$GoMsiVersion"
-)
- 
-Write-Output "Installing go..."
-Install-Package "$Env:Temp\$GoMsiVersion" -Force
- 
-# Go installer updates global $PATH
-go env
- 
-Write-Output "Installed"

CHANGES.md

@@ -1,42 +1,6 @@
-## 2023.9.0
-### Notices
-- The `warp-routing` `enabled: boolean` flag is no longer supported in the configuration file. Warp Routing traffic (eg TCP, UDP, ICMP) traffic is proxied to cloudflared if routes to the target tunnel are configured. This change does not affect remotely managed tunnels, but for locally managed tunnels, users that might be relying on this feature flag to block traffic should instead guarantee that tunnel has no Private Routes configured for the tunnel.
-## 2023.7.0
-### New Features
-- You can now enable additional diagnostics over the management.argotunnel.com service for your active cloudflared connectors via a new runtime flag `--management-diagnostics` (or env `TUNNEL_MANAGEMENT_DIAGNOSTICS`). This feature is provided as opt-in and requires the flag to enable. Endpoints such as /metrics provides your prometheus metrics endpoint another mechanism to be reached. Additionally /debug/pprof/(goroutine|heap) are also introduced to allow for remotely retrieving active pprof information from a running cloudflared connector.
-
-## 2023.4.1
-### New Features
-- You can now stream your logs from your remote cloudflared to your local terminal with `cloudflared tail <TUNNEL-ID>`. This new feature requires the remote cloudflared to be version 2023.4.1 or higher.
-
-## 2023.3.2
-### Notices
-- Due to the nature of QuickTunnels (https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/do-more-with-tunnels/trycloudflare/) and its intended usage for testing and experiment of Cloudflare Tunnels, starting from 2023.3.2, QuickTunnels only make a single connection to the edge. If users want to use Tunnels in a production environment, they should move to Named Tunnels instead. (https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/tunnel-guide/remote/#set-up-a-tunnel-remotely-dashboard-setup)
-
-## 2023.3.1
-### Breaking Change
-- Running a tunnel without ingress rules defined in configuration file nor from the CLI flags will no longer provide a default ingress rule to localhost:8080 and instead will return HTTP response code 503 for all incoming HTTP requests.
-
-### Security Fixes
-- Windows 32 bit machines MSI now defaults to Program Files to install cloudflared. (See CVE-2023-1314). The cloudflared client itself is unaffected. This just changes how the installer works on 32 bit windows machines.
-
-### Bug Fixes
-- Fixed a bug that would cause running tunnel on Bastion mode and without ingress rules to crash.
-
-## 2023.2.2
-### Notices
-- Legacy tunnels were officially deprecated on December 1, 2022. Starting with this version, cloudflared no longer supports connecting legacy tunnels.
-- h2mux tunnel connection protocol is no longer supported. Any tunnels still configured to use this protocol will alert and use http2 tunnel protocol instead. We recommend using quic protocol for all tunnels going forward.
-
-## 2023.2.1
-### Bug fixes
-- Fixed a bug in TCP connection proxy that could result in the connection being closed before all data was written.
-- cloudflared now correctly aborts body write if connection to origin service fails after response headers were sent already.
-- Fixed a bug introduced in the previous release where debug endpoints were removed.
-
 ## 2022.12.0
 ### Improvements
-- cloudflared now attempts to try other edge addresses before falling back to a lower protocol.
+- cloudflared now attempts to try other edge addresses before falling back to a lower protoocol.
 - cloudflared tunnel no longer spins up a quick tunnel. The call has to be explicit and provide a --url flag.
 - cloudflared will now randomly pick the first or second region to connect to instead of always connecting to region2 first.
 

Dockerfile

@@ -1,21 +1,19 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.21.5 as builder
+FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \
     TARGET_GOARCH=${TARGET_GOARCH}
-
+    
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
 # copy our sources into the builder image
 COPY . .
 
-RUN .teamcity/install-cloudflare-go.sh
-
 # compile cloudflared
-RUN PATH="/tmp/go/bin:$PATH" make cloudflared
+RUN make cloudflared
 
 # use a distroless base image with glibc
 FROM gcr.io/distroless/base-debian11:nonroot

Dockerfile.amd64

@@ -1,5 +1,5 @@
 # use a builder image for building cloudflare
-FROM golang:1.21.5 as builder
+FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 
     
@@ -8,10 +8,8 @@ WORKDIR /go/src/github.com/cloudflare/cloudflared/
 # copy our sources into the builder image
 COPY . .
 
-RUN .teamcity/install-cloudflare-go.sh
-
 # compile cloudflared
-RUN GOOS=linux GOARCH=amd64 PATH="/tmp/go/bin:$PATH" make cloudflared
+RUN GOOS=linux GOARCH=amd64 make cloudflared
 
 # use a distroless base image with glibc
 FROM gcr.io/distroless/base-debian11:nonroot

Dockerfile.arm64

@@ -1,5 +1,5 @@
 # use a builder image for building cloudflare
-FROM golang:1.21.5 as builder
+FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 
     
@@ -8,10 +8,8 @@ WORKDIR /go/src/github.com/cloudflare/cloudflared/
 # copy our sources into the builder image
 COPY . .
 
-RUN .teamcity/install-cloudflare-go.sh
-
 # compile cloudflared
-RUN GOOS=linux GOARCH=arm64 PATH="/tmp/go/bin:$PATH" make cloudflared
+RUN GOOS=linux GOARCH=arm64 make cloudflared
 
 # use a distroless base image with glibc
 FROM gcr.io/distroless/base-debian11:nonroot-arm64

Makefile

@@ -1,6 +1,3 @@
-# The targets cannot be run in parallel
-.NOTPARALLEL:
-
 VERSION       := $(shell git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
 MSI_VERSION   := $(shell git tag -l --sort=v:refname | grep "w" | tail -1 | cut -c2-)
 #MSI_VERSION expects the format of the tag to be: (wX.X.X). Starts with the w character to not break cfsetup.
@@ -52,8 +49,6 @@ PACKAGE_DIR    := $(CURDIR)/packaging
 PREFIX         := /usr
 INSTALL_BINDIR := $(PREFIX)/bin/
 INSTALL_MANDIR := $(PREFIX)/share/man/man1/
-CF_GO_PATH     := /tmp/go
-PATH           := $(CF_GO_PATH)/bin:$(PATH)
 
 LOCAL_ARCH ?= $(shell uname -m)
 ifneq ($(GOARCH),)
@@ -87,8 +82,6 @@ else ifeq ($(LOCAL_OS),windows)
     TARGET_OS ?= windows
 else ifeq ($(LOCAL_OS),freebsd)
     TARGET_OS ?= freebsd
-else ifeq ($(LOCAL_OS),openbsd)
-    TARGET_OS ?= openbsd
 else
     $(error This system's OS $(LOCAL_OS) isn't supported)
 endif
@@ -131,7 +124,7 @@ ifeq ($(FIPS), true)
 	$(info Building cloudflared with go-fips)
 	cp -f fips/fips.go.linux-amd64 cmd/cloudflared/fips.go
 endif
-	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) $(ARM_COMMAND) go build -mod=vendor $(GO_BUILD_TAGS) $(LDFLAGS) $(IMPORT_PATH)/cmd/cloudflared
+	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) $(ARM_COMMAND) go build -v -mod=vendor $(GO_BUILD_TAGS) $(LDFLAGS) $(IMPORT_PATH)/cmd/cloudflared
 ifeq ($(FIPS), true)
 	rm -f cmd/cloudflared/fips.go
 	./check-fips.sh cloudflared
@@ -145,7 +138,6 @@ container:
 generate-docker-version:
 	echo latest $(VERSION) > versions
 
-
 .PHONY: test
 test: vet
 ifndef CI
@@ -153,35 +145,17 @@ ifndef CI
 else
 	@mkdir -p .cover
 	go test -v -mod=vendor -race $(LDFLAGS) -coverprofile=".cover/c.out" ./...
+	go tool cover -html ".cover/c.out" -o .cover/all.html
 endif
 
-.PHONY: cover
-cover:
-	@echo ""
-	@echo "=====> Total test coverage: <====="
-	@echo ""
-	# Print the overall coverage here for quick access.
-	$Q go tool cover -func ".cover/c.out" | grep "total:" | awk '{print $$3}'
-	# Generate the HTML report that can be viewed from the browser in CI.
-	$Q go tool cover -html ".cover/c.out" -o .cover/all.html
-
 .PHONY: test-ssh-server
 test-ssh-server:
 	docker-compose -f ssh_server_tests/docker-compose.yml up
 
-.PHONY: install-go
-install-go:
-	rm -rf ${CF_GO_PATH}
-	./.teamcity/install-cloudflare-go.sh
-
-.PHONY: cleanup-go
-cleanup-go:
-	rm -rf ${CF_GO_PATH}
-
 cloudflared.1: cloudflared_man_template
 	sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' cloudflared_man_template > cloudflared.1
 
-install: install-go cloudflared cloudflared.1 cleanup-go
+install: cloudflared cloudflared.1
 	mkdir -p $(DESTDIR)$(INSTALL_BINDIR) $(DESTDIR)$(INSTALL_MANDIR)
 	install -m755 cloudflared $(DESTDIR)$(INSTALL_BINDIR)/cloudflared
 	install -m644 cloudflared.1 $(DESTDIR)$(INSTALL_MANDIR)/cloudflared.1
@@ -223,6 +197,67 @@ cloudflared-darwin-amd64.tgz: cloudflared
 	tar czf cloudflared-darwin-amd64.tgz cloudflared
 	rm cloudflared
 
+.PHONY: cloudflared-junos
+cloudflared-junos: cloudflared jetez-certificate.pem jetez-key.pem
+	jetez --source . \
+		  -j jet.yaml \
+		  --key jetez-key.pem \
+		  --cert jetez-certificate.pem \
+		  --version $(VERSION)
+	rm jetez-*.pem
+
+jetez-certificate.pem:
+ifndef JETEZ_CERT
+	$(error JETEZ_CERT not defined)
+endif
+	@echo "Writing JetEZ certificate"
+	@echo "$$JETEZ_CERT" > jetez-certificate.pem
+
+jetez-key.pem:
+ifndef JETEZ_KEY
+	$(error JETEZ_KEY not defined)
+endif
+	@echo "Writing JetEZ key"
+	@echo "$$JETEZ_KEY" > jetez-key.pem
+
+.PHONY: publish-cloudflared-junos
+publish-cloudflared-junos: cloudflared-junos cloudflared-x86-64.latest.s3
+ifndef S3_ENDPOINT
+	$(error S3_HOST not defined)
+endif
+ifndef S3_URI
+	$(error S3_URI not defined)
+endif
+ifndef S3_ACCESS_KEY
+	$(error S3_ACCESS_KEY not defined)
+endif
+ifndef S3_SECRET_KEY
+	$(error S3_SECRET_KEY not defined)
+endif
+	sha256sum cloudflared-x86-64-$(VERSION).tgz | awk '{printf $$1}' > cloudflared-x86-64-$(VERSION).tgz.shasum
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+		put cloudflared-x86-64-$(VERSION).tgz $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+		put cloudflared-x86-64-$(VERSION).tgz.shasum $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz.shasum
+	dpkg --compare-versions "$(VERSION)" gt "$(shell cat cloudflared-x86-64.latest.s3)" && \
+		echo -n "$(VERSION)" > cloudflared-x86-64.latest && \
+		s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+			put cloudflared-x86-64.latest $(S3_URI)/cloudflared-x86-64.latest || \
+		echo "Latest version not updated"
+
+cloudflared-x86-64.latest.s3:
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force \
+		get $(S3_URI)/cloudflared-x86-64.latest cloudflared-x86-64.latest.s3
+
+.PHONY: homebrew-upload
+homebrew-upload: cloudflared-darwin-amd64.tgz
+	aws s3 --endpoint-url $(S3_ENDPOINT) cp --acl public-read $$^ $(S3_URI)/cloudflared-$$(VERSION)-$1.tgz
+	aws s3 --endpoint-url $(S3_ENDPOINT) cp --acl public-read $(S3_URI)/cloudflared-$$(VERSION)-$1.tgz  $(S3_URI)/cloudflared-stable-$1.tgz
+
+.PHONY: homebrew-release
+homebrew-release: homebrew-upload
+	./publish-homebrew-formula.sh cloudflared-darwin-amd64.tgz $(VERSION) homebrew-cloudflare
+
 .PHONY: github-release
 github-release: cloudflared
 	python3 github_release.py --path $(EXECUTABLE_PATH) --release-version $(VERSION)
@@ -265,7 +300,7 @@ quic-deps:
 
 .PHONY: vet
 vet:
-	go vet -mod=vendor github.com/cloudflare/cloudflared/...
+	go vet -v -mod=vendor github.com/cloudflare/cloudflared/...
 
 .PHONY: fmt
 fmt:

README.md

@@ -31,7 +31,7 @@ Downloads are available as standalone binaries, a Docker image, and Debian, RPM,
 * Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#linux)
 * A Docker image of `cloudflared` is [available on DockerHub](https://hub.docker.com/r/cloudflare/cloudflared)
 * You can install on Windows machines with the [steps here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#windows)
-* To build from source, first you need to download the go toolchain by running `./.teamcity/install-cloudflare-go.sh` and follow the output. Then you can run `make cloudflared`
+* Build from source with the [instructions here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#build-from-source)
 
 User documentation for Cloudflare Tunnel can be found at https://developers.cloudflare.com/cloudflare-one/connections/connect-apps
 
@@ -53,6 +53,9 @@ Want to test Cloudflare Tunnel before adding a website to Cloudflare? You can do
 
 ## Deprecated versions
 
-Cloudflare currently supports versions of cloudflared that are **within one year** of the most recent release. Breaking changes unrelated to feature availability may be introduced that will impact versions released more than one year ago. You can read more about upgrading cloudflared in our [developer documentation](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/downloads/#updating-cloudflared).
+Cloudflare currently supports versions of `cloudflared` 2020.5.1 and later. Breaking changes unrelated to feature availability may be introduced that will impact versions released prior to 2020.5.1. You can read more about upgrading `cloudflared` in our [developer documentation](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#updating-cloudflared).
 
-For example, as of January 2023 Cloudflare will support cloudflared version 2023.1.1 to cloudflared 2022.1.1.
+| Version(s) | Deprecation status |
+|---|---|
+| 2020.5.1 and later | Supported |
+| Versions prior to 2020.5.1 | No longer supported |

RELEASE_NOTES

@@ -1,196 +1,13 @@
-2024.2.0
-- 2024-02-07 TUN-8224: Count and collect metrics on stream connect successes/errors
+2023.2.0
+- 2023-01-31 TUN-7147: Revert wrong removal of debug endpoint from metrics port
+- 2023-01-30 TUN-7146: Avoid data race in closing origin connection too early
+- 2023-01-22 TUN-7097: Fix bug checking proxy-dns config on tunnel cmd execution
+- 2023-01-20 ZTC-446: Allow to force delete a vnet
+- 2023-01-17 TUN-7077: Specific name in cloudflare tunnel route lb command
+- 2023-01-16 TUN-7073: Fix propagating of bad stream request from origin to downstream
+- 2023-01-11 TUN-7065: Remove classic tunnel creation
+- 2023-01-10 TUN-7066: Bump coredns to v1.10.0
 
-2024.1.5
-- 2024-01-22 TUN-8176: Support ARM platforms that don't have an FPU or have it enabled in kernel
-- 2024-01-15 TUN-8158: Bring back commit e6537418859afcac29e56a39daa08bcabc09e048 and fixes infinite loop on linux when the socket is closed
-
-2024.1.4
-- 2024-01-19 Revert "TUN-8158: Add logging to confirm when ICMP reply is returned to the edge"
-
-2024.1.3
-- 2024-01-15 TUN-8161: Fix broken ARM build for armv6
-- 2024-01-15 TUN-8158: Add logging to confirm when ICMP reply is returned to the edge
-
-2024.1.2
-- 2024-01-11 TUN-8147: Disable ECN usage due to bugs in detecting if supported
-- 2024-01-11 TUN-8146: Fix export path for install-go command
-- 2024-01-11 TUN-8146: Fix Makefile targets should not be run in parallel and install-go script was missing shebang
-- 2024-01-10 TUN-8140: Remove homebrew scripts
-
-2024.1.1
-- 2024-01-10 TUN-8134: Revert installed prefix to /usr
-- 2024-01-09 TUN-8130: Fix path to install go for mac build
-- 2024-01-09 TUN-8129: Use the same build command between branch and release builds
-- 2024-01-09 TUN-8130: Install go tool chain in /tmp on build agents
-- 2024-01-09 TUN-8134: Install cloudflare go as part of make install
-- 2024-01-08 TUN-8118: Disable FIPS module to build with go-boring without CGO_ENABLED
-
-2024.1.0
-- 2024-01-01 TUN-7934: Update quic-go to a version that queues datagrams for better throughput and drops large datagram
-- 2023-12-20 TUN-8072: Need to set GOCACHE in mac go installation script
-- 2023-12-17 TUN-8072: Add script to download cloudflare go for Mac build agents
-- 2023-12-15 Fix nil pointer dereference segfault when passing "null" config json to cloudflared tunnel ingress validate (#1070)
-- 2023-12-15 configuration.go: fix developerPortal link (#960)
-- 2023-12-14 tunnelrpc/pogs: fix dropped test errors (#1106)
-- 2023-12-14 cmd/cloudflared/updater: fix dropped error (#1055)
-- 2023-12-14 use os.Executable to discover the path to cloudflared (#1040)
-- 2023-12-14 Remove extraneous `period` from Path Environment Variable (#1009)
-- 2023-12-14 Use CLI context when running tunnel (#597)
-- 2023-12-14 TUN-8066: Define scripts to build on Windows agents
-- 2023-12-11 TUN-8052: Update go to 1.21.5
-- 2023-12-07 TUN-7970: Default to enable post quantum encryption for quic transport
-- 2023-12-04 TUN-8006: Update quic-go to latest upstream
-- 2023-11-15 VULN-44842 Add a flag that allows users to not send the Access JWT to stdout
-- 2023-11-13 TUN-7965: Remove legacy incident status page check
-- 2023-11-13 AUTH-5682 Org token flow in Access logins should pass CF_AppSession cookie
-
-2023.10.0
-- 2023-10-06 TUN-7864: Document cloudflared versions support
-- 2023-10-03 CUSTESC-33731: Make rule match test report rule in 0-index base
-- 2023-09-22 TUN-7824: Fix usage of systemctl status to detect which services are installed
-- 2023-09-20 TUN-7813: Improve tunnel delete command to use cascade delete
-- 2023-09-20 TUN-7787: cloudflared only list ip routes targeted for cfd_tunnel
-- 2023-09-15 TUN-7787: Refactor cloudflared to use new route endpoints based on route IDs
-- 2023-09-08 TUN-7776: Remove warp-routing flag from cloudflared
-- 2023-09-05 TUN-7756: Clarify that QUIC is mandatory to support ICMP proxying
-
-2023.8.2
-- 2023-08-25 TUN-7700: Implement feature selector to determine if connections will prefer post quantum cryptography
-- 2023-08-22 TUN-7707: Use X25519Kyber768Draft00 curve when post-quantum feature is enabled
-
-2023.8.1
-- 2023-08-23 TUN-7718: Update R2 Token to no longer encode secret
-
-2023.8.0
-- 2023-07-26 TUN-7584: Bump go 1.20.6
-
-2023.7.3
-- 2023-07-25 TUN-7628: Correct Host parsing for Access
-- 2023-07-24 TUN-7624: Fix flaky TestBackoffGracePeriod test in cloudflared
-
-2023.7.2
-- 2023-07-19 TUN-7599: Onboard cloudflared to Software Dashboard
-- 2023-07-19 TUN-7587: Remove junos builds
-- 2023-07-18 TUN-7597: Add flag to disable auto-update services to be installed
-- 2023-07-17 TUN-7594: Add nightly arm64 cloudflared internal deb publishes
-- 2023-07-14 TUN-7586: Upgrade go-jose/go-jose/v3 and core-os/go-oidc/v3
-- 2023-07-14 TUN-7589: Remove legacy golang.org/x/crypto/ssh/terminal package usage
-- 2023-07-14 TUN-7590: Remove usages of ioutil
-- 2023-07-14 TUN-7585: Remove h2mux compression
-- 2023-07-14 TUN-7588: Update package coreos/go-systemd
-
-2023.7.1
-- 2023-07-13 TUN-7582: Correct changelog wording for --management-diagnostics
-- 2023-07-12 TUN-7575: Add option to disable PTMU discovery over QUIC
-
-2023.7.0
-- 2023-07-06 TUN-7558: Flush on Writes for StreamBasedOriginProxy
-- 2023-07-05 TUN-7553: Add flag to enable management diagnostic services
-- 2023-07-05 TUN-7564: Support cf-trace-id for cloudflared access
-- 2023-07-05 TUN-7477: Decrement UDP sessions on shutdown
-- 2023-07-03 TUN-7545: Add support for full bidirectionally streaming with close signal propagation
-- 2023-06-30 TUN-7549: Add metrics route to management service
-- 2023-06-30 TUN-7551: Complete removal of raven-go to sentry-go
-- 2023-06-30 TUN-7550: Add pprof endpoint to management service
-- 2023-06-29 TUN-7543: Add --debug-stream flag to cloudflared access ssh
-- 2023-06-26 TUN-6011: Remove docker networks from ICMP Proxy test
-- 2023-06-20 AUTH-5328 Pass cloudflared_token_check param when running cloudflared access login
-
-2023.6.1
-- 2023-06-19 TUN-7480: Added a timeout for unregisterUDP.
-- 2023-06-16 TUN-7477: Add UDP/TCP session metrics
-- 2023-06-14 TUN-7468: Increase the limit of incoming streams
-
-2023.6.0
-- 2023-06-15 TUN-7471: Fixes cloudflared not closing the quic stream on unregister UDP session
-- 2023-06-09 TUN-7463: Add default ingress rule if no ingress rules are provided when updating the configuration
-- 2023-05-31 TUN-7447: Add a cover build to report code coverage
-
-2023.5.1
-- 2023-05-16 TUN-7424: Add CORS headers to host_details responses
-- 2023-05-11 TUN-7421: Add *.cloudflare.com to permitted Origins for management WebSocket requests
-- 2023-05-05 TUN-7404: Default configuration version set to -1
-- 2023-05-05 TUN-7227: Migrate to devincarr/quic-go
-
-2023.5.0
-- 2023-04-27 TUN-7398: Add support for quic safe stream to set deadline
-- 2023-04-26 TUN-7394: Retry StartFirstTunnel on quic.ApplicationErrors
-- 2023-04-26 TUN-7392: Ignore release checksum upload if asset already uploaded
-- 2023-04-25 TUN-7392: Ignore duplicate artifact uploads for github release
-- 2023-04-25 TUN-7393: Add json output for cloudflared tail
-- 2023-04-24 TUN-7390: Remove Debian stretch builds
-
-2023.4.2
-- 2023-04-24 TUN-7133: Add sampling support for streaming logs
-- 2023-04-21 TUN-7141: Add component tests for streaming logs
-- 2023-04-21 TUN-7373: Streaming logs override for same actor
-- 2023-04-20 TUN-7383: Bump requirements.txt
-- 2023-04-19 TUN-7361: Add a label to override hostname
-- 2023-04-19 TUN-7378: Remove RPC debug logs
-- 2023-04-18 TUN-7360: Add Get Host Details handler in management service
-- 2023-04-17 AUTH-3122 Verify that Access tokens are still valid in curl command
-- 2023-04-17 TUN-7129: Categorize TCP logs for streaming logs
-- 2023-04-17 TUN-7130: Categorize UDP logs for streaming logs
-- 2023-04-10 AUTH-4887 Add aud parameter to token transfer url
-
-2023.4.1
-- 2023-04-13 TUN-7368: Report destination address for TCP requests in logs
-- 2023-04-12 TUN-7134: Acquire token for cloudflared tail
-- 2023-04-12 TUN-7131: Add cloudflared log event to connection messages and enable streaming logs
-- 2023-04-11 TUN-7132 TUN-7136: Add filter support for streaming logs
-- 2023-04-06 TUN-7354: Don't warn for empty ingress rules when using --token
-- 2023-04-06 TUN-7128: Categorize logs from public hostname locations
-- 2023-04-06 TUN-7351: Add streaming logs session ping and timeout
-- 2023-04-06 TUN-7335: Fix cloudflared update not working in windows
-
-2023.4.0
-- 2023-04-07 TUN-7356: Bump golang.org/x/net package to 0.7.0
-- 2023-04-07 TUN-7357: Bump to go 1.19.6
-- 2023-04-06 TUN-7127: Disconnect logger level requirement for management
-- 2023-04-05 TUN-7332: Remove legacy tunnel force flag
-- 2023-04-05 TUN-7135: Add cloudflared tail
-- 2023-04-04 Add suport for OpenBSD (#916)
-- 2023-04-04 Fix typo (#918)
-- 2023-04-04 TUN-7125: Add management streaming logs WebSocket protocol
-- 2023-03-30 TUN-9999: Remove classic tunnel component tests
-- 2023-03-30 TUN-7126: Add Management logger io.Writer
-- 2023-03-29 TUN-7324: Add http.Hijacker to connection.ResponseWriter
-- 2023-03-29 TUN-7333: Default features checkable at runtime across all packages
-- 2023-03-21 TUN-7124: Add intercept ingress rule for management requests
-
-2023.3.1
-- 2023-03-13 TUN-7271: Return 503 status code when no ingress rules configured
-- 2023-03-10 TUN-7272: Fix cloudflared returning non supported status service which breaks configuration migration
-- 2023-03-09 TUN-7259: Add warning for missing ingress rules
-- 2023-03-09 TUN-7268: Default to Program Files as location for win32
-- 2023-03-07 TUN-7252: Remove h2mux connection
-- 2023-03-07 TUN-7253: Adopt http.ResponseWriter for connection.ResponseWriter
-- 2023-03-06 TUN-7245: Add bastion flag to origin service check
-- 2023-03-06 EDGESTORE-108: Remove deprecated s3v2 signature
-- 2023-03-02 TUN-7226: Fixed a missed rename
-
-2023.3.0
-- 2023-03-01 GH-352: Add Tunnel CLI option "edge-bind-address" (#870)
-- 2023-03-01 Fixed WIX template to allow MSI upgrades (#838)
-- 2023-02-28 TUN-7213: Decode Base64 encoded key before writing it
-- 2023-02-28 check.yaml: update actions to v3 (#876)
-- 2023-02-27 TUN-7213: Debug homebrew-cloudflare  build
-- 2023-02-15 RTG-2476 Add qtls override for Go 1.20
-
-2023.2.2
-- 2023-02-22 TUN-7197: Add connIndex tag to debug messages of incoming requests
-- 2023-02-08 TUN-7167: Respect protocol overrides with --token
-- 2023-02-06 TUN-7065: Remove classic tunnel creation
-- 2023-02-06 TUN-6938: Force h2mux protocol to http2 for named tunnels
-- 2023-02-06 TUN-6938: Provide QUIC as first in protocol list
-- 2023-02-03 TUN-7158: Correct TCP tracing propagation
-- 2023-02-01 TUN-7151: Update changes file with latest release notices
-
-2023.2.1
-- 2023-02-01 TUN-7065: Revert Ingress Rule check for named tunnel configurations
-- 2023-02-01 Revert "TUN-7065: Revert Ingress Rule check for named tunnel configurations"
-- 2023-02-01 Revert "TUN-7065: Remove classic tunnel creation"
 2023.1.0
 - 2023-01-10 TUN-7064: RPM digests are now sha256 instead of md5sum
 - 2023-01-04 RTG-2418 Update qtls

build-packages-fips.sh

@@ -1,4 +1,3 @@
-#!/bin/bash
 VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
 echo $VERSION
 

build-packages.sh

@@ -1,9 +1,7 @@
-#!/bin/bash
 VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
 echo $VERSION
 
-# Disable FIPS module in go-boring
-export GOEXPERIMENT=noboringcrypto
+# Avoid depending on C code since we don't need it.
 export CGO_ENABLED=0
 
 # This controls the directory the built artifacts go into
@@ -15,12 +13,6 @@ export TARGET_OS=linux
 for arch in ${linuxArchs[@]}; do
     unset TARGET_ARM
     export TARGET_ARCH=$arch
-
-    ## Support for arm platforms without hardware FPU enabled
-    if [[ $arch == arm ]] ; then
-        export TARGET_ARCH=arm
-        export TARGET_ARM=5
-    fi
     
     ## Support for armhf builds 
     if [[ $arch == armhf ]] ; then

catalog-info.yaml

@@ -1,16 +0,0 @@
-apiVersion: backstage.io/v1alpha1
-kind: Component
-metadata:
-  name: cloudflared
-  description: Client for Cloudflare Tunnels
-  annotations:
-    backstage.io/source-location: url:https://bitbucket.cfdata.org/projects/TUN/repos/cloudflared/browse
-    cloudflare.com/software-excellence-opt-in: "true"
-    cloudflare.com/jira-project-key: "TUN"
-    cloudflare.com/jira-project-component: "Cloudflare Tunnel"
-  tags:
-    - internal
-spec:
-  type: "service"
-  lifecycle: "Active"
-  owner: "teams/tunnel-teams-routing"

certutil/certutil.go

@@ -0,0 +1,58 @@
+package certutil
+
+import (
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+)
+
+type namedTunnelToken struct {
+	ZoneID    string `json:"zoneID"`
+	AccountID string `json:"accountID"`
+	APIToken  string `json:"apiToken"`
+}
+
+type OriginCert struct {
+	ZoneID    string
+	APIToken  string
+	AccountID string
+}
+
+func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
+	if len(blocks) == 0 {
+		return nil, fmt.Errorf("Cannot decode empty certificate")
+	}
+	originCert := OriginCert{}
+	block, rest := pem.Decode(blocks)
+	for {
+		if block == nil {
+			break
+		}
+		switch block.Type {
+		case "PRIVATE KEY", "CERTIFICATE":
+			// this is for legacy purposes.
+			break
+		case "ARGO TUNNEL TOKEN":
+			if originCert.ZoneID != "" || originCert.APIToken != "" {
+				return nil, fmt.Errorf("Found multiple tokens in the certificate")
+			}
+			// The token is a string,
+			// Try the newer JSON format
+			ntt := namedTunnelToken{}
+			if err := json.Unmarshal(block.Bytes, &ntt); err == nil {
+				originCert.ZoneID = ntt.ZoneID
+				originCert.APIToken = ntt.APIToken
+				originCert.AccountID = ntt.AccountID
+			}
+		default:
+			return nil, fmt.Errorf("Unknown block %s in the certificate", block.Type)
+		}
+		block, rest = pem.Decode(rest)
+	}
+
+	if originCert.ZoneID == "" || originCert.APIToken == "" {
+		return nil, fmt.Errorf("Missing token in the certificate")
+	}
+
+	return &originCert, nil
+}

certutil/certutil_test.go

@@ -0,0 +1,51 @@
+package certutil
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadOriginCert(t *testing.T) {
+	cert, err := DecodeOriginCert([]byte{})
+	assert.Equal(t, fmt.Errorf("Cannot decode empty certificate"), err)
+	assert.Nil(t, cert)
+
+	blocks, err := ioutil.ReadFile("test-cert-unknown-block.pem")
+	assert.Nil(t, err)
+	cert, err = DecodeOriginCert(blocks)
+	assert.Equal(t, fmt.Errorf("Unknown block RSA PRIVATE KEY in the certificate"), err)
+	assert.Nil(t, cert)
+}
+
+func TestJSONArgoTunnelTokenEmpty(t *testing.T) {
+	cert, err := DecodeOriginCert([]byte{})
+	blocks, err := ioutil.ReadFile("test-cert-no-token.pem")
+	assert.Nil(t, err)
+	cert, err = DecodeOriginCert(blocks)
+	assert.Equal(t, fmt.Errorf("Missing token in the certificate"), err)
+	assert.Nil(t, cert)
+}
+
+func TestJSONArgoTunnelToken(t *testing.T) {
+	// The given cert's Argo Tunnel Token was generated by base64 encoding this JSON:
+	// {
+	// "zoneID": "7b0a4d77dfb881c1a3b7d61ea9443e19",
+	// "apiToken": "test-service-key",
+	// "accountID": "abcdabcdabcdabcd1234567890abcdef"
+	// }
+	CloudflareTunnelTokenTest(t, "test-cloudflare-tunnel-cert-json.pem")
+}
+
+func CloudflareTunnelTokenTest(t *testing.T, path string) {
+	blocks, err := ioutil.ReadFile(path)
+	assert.Nil(t, err)
+	cert, err := DecodeOriginCert(blocks)
+	assert.Nil(t, err)
+	assert.NotNil(t, cert)
+	assert.Equal(t, "7b0a4d77dfb881c1a3b7d61ea9443e19", cert.ZoneID)
+	key := "test-service-key"
+	assert.Equal(t, key, cert.APIToken)
+}

cfapi/client.go

@@ -8,8 +8,7 @@ type TunnelClient interface {
 	CreateTunnel(name string, tunnelSecret []byte) (*TunnelWithToken, error)
 	GetTunnel(tunnelID uuid.UUID) (*Tunnel, error)
 	GetTunnelToken(tunnelID uuid.UUID) (string, error)
-	GetManagementToken(tunnelID uuid.UUID) (string, error)
-	DeleteTunnel(tunnelID uuid.UUID, cascade bool) error
+	DeleteTunnel(tunnelID uuid.UUID) error
 	ListTunnels(filter *TunnelFilter) ([]*Tunnel, error)
 	ListActiveClients(tunnelID uuid.UUID) ([]*ActiveClient, error)
 	CleanupConnections(tunnelID uuid.UUID, params *CleanupParams) error
@@ -22,7 +21,7 @@ type HostnameClient interface {
 type IPRouteClient interface {
 	ListRoutes(filter *IpRouteFilter) ([]*DetailedRoute, error)
 	AddRoute(newRoute NewRoute) (Route, error)
-	DeleteRoute(id uuid.UUID) error
+	DeleteRoute(params DeleteRouteParams) error
 	GetByIP(params GetRouteByIpParams) (DetailedRoute, error)
 }
 

cfapi/ip_route.go

@@ -75,12 +75,10 @@ type NewRoute struct {
 // MarshalJSON handles fields with non-JSON types (e.g. net.IPNet).
 func (r NewRoute) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&struct {
-		Network  string     `json:"network"`
 		TunnelID uuid.UUID  `json:"tunnel_id"`
 		Comment  string     `json:"comment"`
 		VNetID   *uuid.UUID `json:"virtual_network_id,omitempty"`
 	}{
-		Network:  r.Network.String(),
 		TunnelID: r.TunnelID,
 		Comment:  r.Comment,
 		VNetID:   r.VNetID,
@@ -89,7 +87,6 @@ func (r NewRoute) MarshalJSON() ([]byte, error) {
 
 // DetailedRoute is just a Route with some extra fields, e.g. TunnelName.
 type DetailedRoute struct {
-	ID       uuid.UUID `json:"id"`
 	Network  CIDR      `json:"network"`
 	TunnelID uuid.UUID `json:"tunnel_id"`
 	// Optional field. When unset, it means the DetailedRoute belongs to the default virtual network.
@@ -118,8 +115,7 @@ func (r DetailedRoute) TableString() string {
 	}
 
 	return fmt.Sprintf(
-		"%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
-		r.ID,
+		"%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
 		r.Network.String(),
 		vnetColumn,
 		r.Comment,
@@ -130,6 +126,12 @@ func (r DetailedRoute) TableString() string {
 	)
 }
 
+type DeleteRouteParams struct {
+	Network net.IPNet
+	// Optional field. If unset, backend will assume the default vnet for the account.
+	VNetID *uuid.UUID
+}
+
 type GetRouteByIpParams struct {
 	Ip net.IP
 	// Optional field. If unset, backend will assume the default vnet for the account.
@@ -156,7 +158,7 @@ func (r *RESTClient) ListRoutes(filter *IpRouteFilter) ([]*DetailedRoute, error)
 // AddRoute calls the Tunnelstore POST endpoint for a given route.
 func (r *RESTClient) AddRoute(newRoute NewRoute) (Route, error) {
 	endpoint := r.baseEndpoints.accountRoutes
-	endpoint.Path = path.Join(endpoint.Path)
+	endpoint.Path = path.Join(endpoint.Path, "network", url.PathEscape(newRoute.Network.String()))
 	resp, err := r.sendRequest("POST", endpoint, newRoute)
 	if err != nil {
 		return Route{}, errors.Wrap(err, "REST request failed")
@@ -171,9 +173,10 @@ func (r *RESTClient) AddRoute(newRoute NewRoute) (Route, error) {
 }
 
 // DeleteRoute calls the Tunnelstore DELETE endpoint for a given route.
-func (r *RESTClient) DeleteRoute(id uuid.UUID) error {
+func (r *RESTClient) DeleteRoute(params DeleteRouteParams) error {
 	endpoint := r.baseEndpoints.accountRoutes
-	endpoint.Path = path.Join(endpoint.Path, url.PathEscape(id.String()))
+	endpoint.Path = path.Join(endpoint.Path, "network", url.PathEscape(params.Network.String()))
+	setVnetParam(&endpoint, params.VNetID)
 
 	resp, err := r.sendRequest("DELETE", endpoint, nil)
 	if err != nil {

cfapi/ip_route_filter.go

@@ -58,29 +58,31 @@ type IpRouteFilter struct {
 
 // NewIpRouteFilterFromCLI parses CLI flags to discover which filters should get applied.
 func NewIpRouteFilterFromCLI(c *cli.Context) (*IpRouteFilter, error) {
-	f := NewIPRouteFilter()
+	f := &IpRouteFilter{
+		queryParams: url.Values{},
+	}
 
 	// Set deletion filter
 	if flag := filterIpRouteDeleted.Name; c.IsSet(flag) && c.Bool(flag) {
-		f.Deleted()
+		f.deleted()
 	} else {
-		f.NotDeleted()
+		f.notDeleted()
 	}
 
 	if subset, err := cidrFromFlag(c, filterSubsetIpRoute); err != nil {
 		return nil, err
 	} else if subset != nil {
-		f.NetworkIsSupersetOf(*subset)
+		f.networkIsSupersetOf(*subset)
 	}
 
 	if superset, err := cidrFromFlag(c, filterSupersetIpRoute); err != nil {
 		return nil, err
 	} else if superset != nil {
-		f.NetworkIsSupersetOf(*superset)
+		f.networkIsSupersetOf(*superset)
 	}
 
 	if comment := c.String(filterIpRouteComment.Name); comment != "" {
-		f.CommentIs(comment)
+		f.commentIs(comment)
 	}
 
 	if tunnelID := c.String(filterIpRouteTunnelID.Name); tunnelID != "" {
@@ -88,7 +90,7 @@ func NewIpRouteFilterFromCLI(c *cli.Context) (*IpRouteFilter, error) {
 		if err != nil {
 			return nil, errors.Wrapf(err, "Couldn't parse UUID from %s", filterIpRouteTunnelID.Name)
 		}
-		f.TunnelID(u)
+		f.tunnelID(u)
 	}
 
 	if vnetId := c.String(filterIpRouteByVnet.Name); vnetId != "" {
@@ -96,7 +98,7 @@ func NewIpRouteFilterFromCLI(c *cli.Context) (*IpRouteFilter, error) {
 		if err != nil {
 			return nil, errors.Wrapf(err, "Couldn't parse UUID from %s", filterIpRouteByVnet.Name)
 		}
-		f.VNetID(u)
+		f.vnetID(u)
 	}
 
 	if maxFetch := c.Int("max-fetch-size"); maxFetch > 0 {
@@ -122,44 +124,35 @@ func cidrFromFlag(c *cli.Context, flag cli.StringFlag) (*net.IPNet, error) {
 	return subset, nil
 }
 
-func NewIPRouteFilter() *IpRouteFilter {
-	values := &IpRouteFilter{queryParams: url.Values{}}
-
-	// always list cfd_tunnel routes only
-	values.queryParams.Set("tun_types", "cfd_tunnel")
-
-	return values
-}
-
-func (f *IpRouteFilter) CommentIs(comment string) {
+func (f *IpRouteFilter) commentIs(comment string) {
 	f.queryParams.Set("comment", comment)
 }
 
-func (f *IpRouteFilter) NotDeleted() {
+func (f *IpRouteFilter) notDeleted() {
 	f.queryParams.Set("is_deleted", "false")
 }
 
-func (f *IpRouteFilter) Deleted() {
+func (f *IpRouteFilter) deleted() {
 	f.queryParams.Set("is_deleted", "true")
 }
 
-func (f *IpRouteFilter) NetworkIsSubsetOf(superset net.IPNet) {
+func (f *IpRouteFilter) networkIsSubsetOf(superset net.IPNet) {
 	f.queryParams.Set("network_subset", superset.String())
 }
 
-func (f *IpRouteFilter) NetworkIsSupersetOf(subset net.IPNet) {
+func (f *IpRouteFilter) networkIsSupersetOf(subset net.IPNet) {
 	f.queryParams.Set("network_superset", subset.String())
 }
 
-func (f *IpRouteFilter) ExistedAt(existedAt time.Time) {
+func (f *IpRouteFilter) existedAt(existedAt time.Time) {
 	f.queryParams.Set("existed_at", existedAt.Format(time.RFC3339))
 }
 
-func (f *IpRouteFilter) TunnelID(id uuid.UUID) {
+func (f *IpRouteFilter) tunnelID(id uuid.UUID) {
 	f.queryParams.Set("tunnel_id", id.String())
 }
 
-func (f *IpRouteFilter) VNetID(id uuid.UUID) {
+func (f *IpRouteFilter) vnetID(id uuid.UUID) {
 	f.queryParams.Set("virtual_network_id", id.String())
 }
 

cfapi/ip_route_test.go

@@ -69,7 +69,6 @@ func TestDetailedRouteJsonRoundtrip(t *testing.T) {
 	}{
 		{
 			`{
-				"id":"91ebc578-cc99-4641-9937-0fb630505fa0",
 				"network":"10.1.2.40/29",
 				"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
 				"comment":"test",
@@ -81,7 +80,6 @@ func TestDetailedRouteJsonRoundtrip(t *testing.T) {
 		},
 		{
 			`{
-				"id":"91ebc578-cc99-4641-9937-0fb630505fa0",
 				"network":"10.1.2.40/29",
 				"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
 				"virtual_network_id":"38c95083-8191-4110-8339-3f438d44fdb9",
@@ -169,10 +167,9 @@ func TestRouteTableString(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, network)
 	r := DetailedRoute{
-		ID:      uuid.Nil,
 		Network: CIDR(*network),
 	}
 	row := r.TableString()
 	fmt.Println(row)
-	require.True(t, strings.HasPrefix(row, "00000000-0000-0000-0000-000000000000\t1.2.3.4/32"))
+	require.True(t, strings.HasPrefix(row, "1.2.3.4/32"))
 }

cfapi/tunnel.go

@@ -50,10 +50,6 @@ type newTunnel struct {
 	TunnelSecret []byte `json:"tunnel_secret"`
 }
 
-type managementRequest struct {
-	Resources []string `json:"resources"`
-}
-
 type CleanupParams struct {
 	queryParams url.Values
 }
@@ -137,36 +133,9 @@ func (r *RESTClient) GetTunnelToken(tunnelID uuid.UUID) (token string, err error
 	return "", r.statusCodeToError("get tunnel token", resp)
 }
 
-func (r *RESTClient) GetManagementToken(tunnelID uuid.UUID) (token string, err error) {
-	endpoint := r.baseEndpoints.accountLevel
-	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/management", tunnelID))
-
-	body := &managementRequest{
-		Resources: []string{"logs"},
-	}
-
-	resp, err := r.sendRequest("POST", endpoint, body)
-	if err != nil {
-		return "", errors.Wrap(err, "REST request failed")
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode == http.StatusOK {
-		err = parseResponse(resp.Body, &token)
-		return token, err
-	}
-
-	return "", r.statusCodeToError("get tunnel token", resp)
-}
-
-func (r *RESTClient) DeleteTunnel(tunnelID uuid.UUID, cascade bool) error {
+func (r *RESTClient) DeleteTunnel(tunnelID uuid.UUID) error {
 	endpoint := r.baseEndpoints.accountLevel
 	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v", tunnelID))
-	// Cascade will delete all tunnel dependencies (connections, routes, etc.) that
-	// are linked to the deleted tunnel.
-	if cascade {
-		endpoint.RawQuery = "cascade=true"
-	}
 	resp, err := r.sendRequest("DELETE", endpoint, nil)
 	if err != nil {
 		return errors.Wrap(err, "REST request failed")

cfapi/tunnel_test.go

@@ -3,7 +3,7 @@ package cfapi
 import (
 	"bytes"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"net"
 	"reflect"
 	"strings"
@@ -49,7 +49,7 @@ func Test_parseListTunnels(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			body := io.NopCloser(bytes.NewReader([]byte(tt.args.body)))
+			body := ioutil.NopCloser(bytes.NewReader([]byte(tt.args.body)))
 			got, err := parseListTunnels(body)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("parseListTunnels() error = %v, wantErr %v", err, tt.wantErr)

cfsetup.yaml

@@ -1,44 +1,34 @@
-pinned_go: &pinned_go go-boring=1.21.5-1
+pinned_go: &pinned_go go=1.19.3-1
+pinned_go_fips: &pinned_go_fips go-boring=1.19.3-1
 
 build_dir: &build_dir /cfsetup_build
 default-flavor: bullseye
-buster: &buster
+stretch: &stretch
   build:
     build_dir: *build_dir
     builddeps: &build_deps
       - *pinned_go
       - build-essential
       - gotest-to-teamcity
-      - fakeroot
-      - rubygem-fpm
-      - rpm
-      - libffi-dev
-      - reprepro
-      - createrepo
     pre-cache: &build_pre_cache
       - export GOCACHE=/cfsetup_build/.cache/go-build
       - go install golang.org/x/tools/cmd/goimports@latest
     post-cache:
-      # TODO: TUN-8126 this is temporary to make sure packages can be built before release
-      - ./build-packages.sh
-      # Build binary for component test
-      - GOOS=linux GOARCH=amd64 make cloudflared
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - make cloudflared
   build-fips:
     build_dir: *build_dir
-    builddeps: *build_deps
+    builddeps: &build_deps_fips
+      - *pinned_go_fips
+      - build-essential
+      - gotest-to-teamcity
     pre-cache: *build_pre_cache
     post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
       - export FIPS=true
-      # TODO: TUN-8126 this is temporary to make sure packages can be built before release
-      - ./build-packages-fips.sh
-      # Build binary for component test
-      - GOOS=linux GOARCH=amd64 make cloudflared
-  cover: 
-    build_dir: *build_dir
-    builddeps: *build_deps
-    pre-cache: *build_pre_cache
-    post-cache: 
-     - make cover
+      - make cloudflared
   # except FIPS (handled in github-fips-release-pkgs) and macos (handled in github-release-macos-amd64)
   github-release-pkgs:
     build_dir: *build_dir
@@ -76,7 +66,7 @@ buster: &buster
   github-fips-release-pkgs:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
       - fakeroot
       - rubygem-fpm
@@ -115,7 +105,7 @@ buster: &buster
   build-fips-internal-deb:
     build_dir: *build_dir
     builddeps: &build_fips_deb_deps
-      - *pinned_go
+      - *pinned_go_fips
       - build-essential
       - fakeroot
       - rubygem-fpm
@@ -125,7 +115,7 @@ buster: &buster
       - export FIPS=true
       - export ORIGINAL_NAME=true
       - make cloudflared-deb
-  build-internal-deb-nightly-amd64:
+  build-fips-internal-deb-nightly:
     build_dir: *build_dir
     builddeps: *build_fips_deb_deps
     post-cache:
@@ -135,16 +125,6 @@ buster: &buster
       - export FIPS=true
       - export ORIGINAL_NAME=true
       - make cloudflared-deb
-  build-internal-deb-nightly-arm64:
-    build_dir: *build_dir
-    builddeps: *build_fips_deb_deps
-    post-cache:
-      - export GOOS=linux
-      - export GOARCH=arm64
-      - export NIGHTLY=true
-      #- export FIPS=true # TUN-7595
-      - export ORIGINAL_NAME=true
-      - make cloudflared-deb
   build-deb-arm64:
     build_dir: *build_dir
     builddeps: *build_deb_deps
@@ -199,7 +179,7 @@ buster: &buster
       - make test | gotest-to-teamcity
   test-fips:
     build_dir: *build_dir
-    builddeps: *build_deps
+    builddeps: *build_deps_fips
     pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
@@ -231,7 +211,7 @@ buster: &buster
   component-test-fips:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go
+      - *pinned_go_fips
       - python3.7
       - python3-pip
       - python3-setuptools
@@ -242,12 +222,52 @@ buster: &buster
       - component-tests/requirements.txt
     pre-cache: *component_test_pre_cache
     post-cache: *component_test_post_cache
+  update-homebrew:
+    builddeps:
+      - openssh-client
+      - s3cmd
+      - jq
+      - build-essential
+      - procps
+    post-cache:
+      - .teamcity/update-homebrew.sh
+      - .teamcity/update-homebrew-core.sh
   github-message-release:
     build_dir: *build_dir
     builddeps: *build_pygithub
     pre-cache: *install_pygithub
     post-cache:
       - make github-message
+  build-junos:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - python3
+      - genisoimage
+    pre-cache:
+      - ln -s /usr/bin/genisoimage /usr/bin/mkisofs
+    post-cache:
+      - export CGO_ENABLED=0
+      - export GOOS=freebsd
+      - export GOARCH=amd64
+      - make cloudflared
+  publish-junos:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - python3
+      - genisoimage
+      - jetez
+      - s4cmd
+    pre-cache:
+      - ln -s /usr/bin/genisoimage /usr/bin/mkisofs
+    post-cache:
+      - export GOOS=freebsd
+      - export GOARCH=amd64
+      - make publish-cloudflared-junos
 
-bullseye: *buster
-bookworm: *buster
+buster: *stretch
+bullseye: *stretch
+bookworm: *stretch

cloudflared.wxs

@@ -1,64 +1,62 @@
 <?xml version="1.0"?>
 
-<?if $(var.Platform)="x64" ?>
-    <?define Program_Files="ProgramFiles64Folder"?>
-<?else ?>
+<?if $(var.Platform)="x86"?>
     <?define Program_Files="ProgramFilesFolder"?>
-<?endif ?>
+<?else?>
+    <?define Program_Files="ProgramFiles64Folder"?>
+<?endif?>
 <?ifndef var.Version?>
     <?error Undefined Version variable?>
-<?endif ?>
+<?endif?>
 <?ifndef var.Path?>
     <?error Undefined Path variable?>
-<?endif ?>
+<?endif?>
 
 <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
-    <Product Id="*"
+    <Product Id="35e5e858-9372-4449-bf73-1cd6f7267128" 
         UpgradeCode="23f90fdd-9328-47ea-ab52-5380855a4b12"
         Name="cloudflared"
         Version="$(var.Version)"
         Manufacturer="cloudflare"
         Language="1033">
 
-        <Package InstallerVersion="200" Compressed="yes" Comments="Windows Installer Package" InstallScope="perMachine" />
+      <Package InstallerVersion="200" Compressed="yes" Comments="Windows Installer Package" InstallScope="perMachine"/>
 
-        <Media Id="1" Cabinet="product.cab" EmbedCab="yes" />
+      <Media Id="1" Cabinet="product.cab" EmbedCab="yes"/>
 
-        <MajorUpgrade DowngradeErrorMessage="A later version of [ProductName] is already installed. Setup will now exit." />
+      <Upgrade Id="23f90fdd-9328-47ea-ab52-5380855a4b12">
+         <UpgradeVersion Minimum="$(var.Version)" OnlyDetect="yes" Property="NEWERVERSIONDETECTED"/>
+         <UpgradeVersion Minimum="2020.8.0" Maximum="$(var.Version)" IncludeMinimum="yes" IncludeMaximum="no"
+                         Property="OLDERVERSIONBEINGUPGRADED"/>
+      </Upgrade>
+      <Condition Message="A newer version of this software is already installed.">NOT NEWERVERSIONDETECTED</Condition>
 
-        <Upgrade Id="23f90fdd-9328-47ea-ab52-5380855a4b12">
-            <UpgradeVersion Minimum="$(var.Version)" OnlyDetect="yes" Property="NEWERVERSIONDETECTED" />
-            <UpgradeVersion Minimum="2020.8.0" Maximum="$(var.Version)" IncludeMinimum="yes" IncludeMaximum="no"
-                Property="OLDERVERSIONBEINGUPGRADED" />
-        </Upgrade>
-        <Condition Message="A newer version of this software is already installed.">NOT NEWERVERSIONDETECTED</Condition>
-
-        <Directory Id="TARGETDIR" Name="SourceDir">
-            <!--This specifies where the cloudflared.exe is moved to in the windows Operation System-->
-            <Directory Id="$(var.Program_Files)">
-                <Directory Id="INSTALLDIR" Name="cloudflared">
-                    <Component Id="ApplicationFiles" Guid="35e5e858-9372-4449-bf73-1cd6f7267128">
-                        <File Id="ApplicationFile0" Source="$(var.Path)" />
-                    </Component>
-                </Directory>
+    <Directory Id="TARGETDIR" Name="SourceDir">
+        <!--This specifies where the cloudflared.exe is moved to in the windows Operation System-->
+        <Directory Id="$(var.Program_Files)">
+            <Directory Id="INSTALLDIR" Name="cloudflared">
+                <Component Id="ApplicationFiles" Guid="35e5e858-9372-4449-bf73-1cd6f7267128">
+                    <File Id="ApplicationFile0" Source="$(var.Path)"/>
+                </Component>
             </Directory>
-            <Component Id="ENVS" Guid="6bb74449-d10d-4f4a-933e-6fc9fa006eae">
-                <!--Set the cloudflared bin location to the Path Environment Variable-->
-                <Environment Id="ENV0"
-                    Name="PATH"
-                    Value="[INSTALLDIR]"
-                    Permanent="no"
-                    Part="last"
-                    Action="create"
-                    System="yes" />
-            </Component>
         </Directory>
+        <Component Id="ENVS" Guid="6bb74449-d10d-4f4a-933e-6fc9fa006eae">
+        <!--Set the cloudflared bin location to the Path Environment Variable-->
+         <Environment Id="ENV0"
+           Name="PATH"
+           Value="[INSTALLDIR]."
+           Permanent="no"
+           Part="last"
+           Action="create"
+           System="yes" />
+        </Component>
+    </Directory>
 
 
-        <Feature Id='Complete' Level='1'>
-            <ComponentRef Id="ENVS" />
-            <ComponentRef Id='ApplicationFiles' />
-        </Feature>
+    <Feature Id='Complete' Level='1'>
+      <ComponentRef Id="ENVS"/>
+      <ComponentRef Id='ApplicationFiles' />
+    </Feature>
 
     </Product>
 </Wix>

cmd/cloudflared/access/carrier.go

@@ -3,7 +3,6 @@ package access
 import (
 	"crypto/tls"
 	"fmt"
-	"io"
 	"net/http"
 	"strings"
 
@@ -14,7 +13,6 @@ import (
 	"github.com/cloudflare/cloudflared/carrier"
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/stream"
 	"github.com/cloudflare/cloudflared/validation"
 )
 
@@ -40,7 +38,6 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, log *z
 	if forwarder.TokenSecret != "" {
 		headers.Set(cfAccessClientSecretHeader, forwarder.TokenSecret)
 	}
-	headers.Set("User-Agent", userAgent)
 
 	carrier.SetBastionDest(headers, forwarder.Destination)
 
@@ -61,37 +58,31 @@ func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, log *z
 // useful for proxying other protocols (like ssh) over websockets
 // (which you can put Access in front of)
 func ssh(c *cli.Context) error {
-	// If not running as a forwarder, disable terminal logs as it collides with the stdin/stdout of the parent process
-	outputTerminal := logger.DisableTerminalLog
-	if c.IsSet(sshURLFlag) {
-		outputTerminal = logger.EnableTerminalLog
-	}
-	log := logger.CreateSSHLoggerFromContext(c, outputTerminal)
+	log := logger.CreateSSHLoggerFromContext(c, logger.EnableTerminalLog)
 
 	// get the hostname from the cmdline and error out if its not provided
 	rawHostName := c.String(sshHostnameFlag)
-	url, err := parseURL(rawHostName)
-	if err != nil {
-		log.Err(err).Send()
+	hostname, err := validation.ValidateHostname(rawHostName)
+	if err != nil || rawHostName == "" {
 		return cli.ShowCommandHelp(c, "ssh")
 	}
+	originURL := ensureURLScheme(hostname)
 
 	// get the headers from the cmdline and add them
-	headers := parseRequestHeaders(c.StringSlice(sshHeaderFlag))
+	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
 		headers.Set(cfAccessClientIDHeader, c.String(sshTokenIDFlag))
 	}
 	if c.IsSet(sshTokenSecretFlag) {
 		headers.Set(cfAccessClientSecretHeader, c.String(sshTokenSecretFlag))
 	}
-	headers.Set("User-Agent", userAgent)
 
 	carrier.SetBastionDest(headers, c.String(sshDestinationFlag))
 
 	options := &carrier.StartOptions{
-		OriginURL: url.String(),
+		OriginURL: originURL,
 		Headers:   headers,
-		Host:      url.Host,
+		Host:      hostname,
 	}
 
 	if connectTo := c.String(sshConnectTo); connectTo != "" {
@@ -130,17 +121,16 @@ func ssh(c *cli.Context) error {
 		return err
 	}
 
-	var s io.ReadWriter
-	s = &carrier.StdinoutStream{}
-	if c.IsSet(sshDebugStream) {
-		maxMessages := c.Uint64(sshDebugStream)
-		if maxMessages == 0 {
-			// default to 10 if provided but unset
-			maxMessages = 10
-		}
-		logger := log.With().Str("host", url.Host).Logger()
-		s = stream.NewDebugStream(s, &logger, maxMessages)
-	}
-	carrier.StartClient(wsConn, s, options)
-	return nil
+	return carrier.StartClient(wsConn, &carrier.StdinoutStream{}, options)
+}
+
+func buildRequestHeaders(values []string) http.Header {
+	headers := make(http.Header)
+	for _, valuePair := range values {
+		split := strings.Split(valuePair, ":")
+		if len(split) > 1 {
+			headers.Add(strings.TrimSpace(split[0]), strings.TrimSpace(split[1]))
+		}
+	}
+	return headers
 }

cmd/cloudflared/access/carrier_test.go

@@ -0,0 +1,18 @@
+package access
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuildRequestHeaders(t *testing.T) {
+	headers := make(http.Header)
+	headers.Add("client", "value")
+	headers.Add("secret", "safe-value")
+
+	values := buildRequestHeaders([]string{"client: value", "secret: safe-value", "trash"})
+	assert.Equal(t, headers.Get("client"), values.Get("client"))
+	assert.Equal(t, headers.Get("secret"), values.Get("secret"))
+}

cmd/cloudflared/access/cmd.go

@@ -26,7 +26,6 @@ import (
 )
 
 const (
-	loginQuietFlag     = "quiet"
 	sshHostnameFlag    = "hostname"
 	sshDestinationFlag = "destination"
 	sshURLFlag         = "url"
@@ -35,7 +34,6 @@ const (
 	sshTokenSecretFlag = "service-token-secret"
 	sshGenCertFlag     = "short-lived-cert"
 	sshConnectTo       = "connect-to"
-	sshDebugStream     = "debug-stream"
 	sshConfigTemplate  = `
 Add to your {{.Home}}/.ssh/config:
 
@@ -91,13 +89,6 @@ func Commands() []*cli.Command {
 					Once authenticated with your identity provider, the login command will generate a JSON Web Token (JWT)
 					scoped to your identity, the application you intend to reach, and valid for a session duration set by your
 					administrator. cloudflared stores the token in local storage.`,
-					Flags: []cli.Flag{
-						&cli.BoolFlag{
-							Name:    loginQuietFlag,
-							Aliases: []string{"q"},
-							Usage:   "do not print the jwt to the command line",
-						},
-					},
 				},
 				{
 					Name:   "curl",
@@ -160,12 +151,9 @@ func Commands() []*cli.Command {
 							EnvVars: []string{"TUNNEL_SERVICE_TOKEN_SECRET"},
 						},
 						&cli.StringFlag{
-							Name:  logger.LogFileFlag,
-							Usage: "Save application log to this file for reporting issues.",
-						},
-						&cli.StringFlag{
-							Name:  logger.LogSSHDirectoryFlag,
-							Usage: "Save application log to this directory for reporting issues.",
+							Name:    logger.LogSSHDirectoryFlag,
+							Aliases: []string{"logfile"}, //added to match the tunnel side
+							Usage:   "Save application log to this directory for reporting issues.",
 						},
 						&cli.StringFlag{
 							Name:    logger.LogSSHLevelFlag,
@@ -177,11 +165,6 @@ func Commands() []*cli.Command {
 							Hidden: true,
 							Usage:  "Connect to alternate location for testing, value is host, host:port, or sni:port:host",
 						},
-						&cli.Uint64Flag{
-							Name:   sshDebugStream,
-							Hidden: true,
-							Usage:  "Writes up-to the max provided stream payloads to the logger as debug statements.",
-						},
 					},
 				},
 				{
@@ -230,7 +213,8 @@ func login(c *cli.Context) error {
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	args := c.Args()
-	appURL, err := parseURL(args.First())
+	rawURL := ensureURLScheme(args.First())
+	appURL, err := url.Parse(rawURL)
 	if args.Len() < 1 || err != nil {
 		log.Error().Msg("Please provide the url of the Access application")
 		return err
@@ -254,15 +238,21 @@ func login(c *cli.Context) error {
 		fmt.Fprintln(os.Stderr, "token for provided application was empty.")
 		return errors.New("empty application token")
 	}
-
-	if c.Bool(loginQuietFlag) {
-		return nil
-	}
 	fmt.Fprintf(os.Stdout, "Successfully fetched your token:\n\n%s\n\n", cfdToken)
 
 	return nil
 }
 
+// ensureURLScheme prepends a URL with https:// if it doesn't have a scheme. http:// URLs will not be converted.
+func ensureURLScheme(url string) string {
+	url = strings.Replace(strings.ToLower(url), "http://", "https://", 1)
+	if !strings.HasPrefix(url, "https://") {
+		url = fmt.Sprintf("https://%s", url)
+
+	}
+	return url
+}
+
 // curl provides a wrapper around curl, passing Access JWT along in request
 func curl(c *cli.Context) error {
 	err := sentry.Init(sentry.ClientOptions{
@@ -290,13 +280,6 @@ func curl(c *cli.Context) error {
 	if err != nil {
 		return err
 	}
-
-	// Verify that the existing token is still good; if not fetch a new one
-	if err := verifyTokenAtEdge(appURL, appInfo, c, log); err != nil {
-		log.Err(err).Msg("Could not verify token")
-		return err
-	}
-
 	tok, err := token.GetAppTokenIfExists(appInfo)
 	if err != nil || tok == "" {
 		if allowRequest {
@@ -346,7 +329,7 @@ func generateToken(c *cli.Context) error {
 	if err != nil {
 		return err
 	}
-	appURL, err := parseURL(c.String("app"))
+	appURL, err := url.Parse(ensureURLScheme(c.String("app")))
 	if err != nil || c.NumFlags() < 1 {
 		fmt.Fprintln(os.Stderr, "Please provide a url.")
 		return err
@@ -399,7 +382,7 @@ func sshGen(c *cli.Context) error {
 		return cli.ShowCommandHelp(c, "ssh-gen")
 	}
 
-	originURL, err := parseURL(hostname)
+	originURL, err := url.Parse(ensureURLScheme(hostname))
 	if err != nil {
 		return err
 	}
@@ -478,11 +461,6 @@ func processURL(s string) (*url.URL, error) {
 
 // cloudflaredPath pulls the full path of cloudflared on disk
 func cloudflaredPath() string {
-	path, err := os.Executable()
-	if err == nil && isFileThere(path) {
-		return path
-	}
-
 	for _, p := range strings.Split(os.Getenv("PATH"), ":") {
 		path := fmt.Sprintf("%s/%s", p, "cloudflared")
 		if isFileThere(path) {
@@ -505,7 +483,7 @@ func isFileThere(candidate string) bool {
 // Then makes a request to to the origin with the token to ensure it is valid.
 // Returns nil if token is valid.
 func verifyTokenAtEdge(appUrl *url.URL, appInfo *token.AppInfo, c *cli.Context, log *zerolog.Logger) error {
-	headers := parseRequestHeaders(c.StringSlice(sshHeaderFlag))
+	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
 	if c.IsSet(sshTokenIDFlag) {
 		headers.Add(cfAccessClientIDHeader, c.String(sshTokenIDFlag))
 	}
@@ -540,11 +518,6 @@ func isTokenValid(options *carrier.StartOptions, log *zerolog.Logger) (bool, err
 		return false, errors.Wrap(err, "Could not create access request")
 	}
 	req.Header.Set("User-Agent", userAgent)
-
-	query := req.URL.Query()
-	query.Set("cloudflared_token_check", "true")
-	req.URL.RawQuery = query.Encode()
-
 	// Do not follow redirects
 	client := &http.Client{
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {

cmd/cloudflared/access/cmd_test.go

@@ -0,0 +1,25 @@
+package access
+
+import "testing"
+
+func Test_ensureURLScheme(t *testing.T) {
+	type args struct {
+		url string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{"no scheme", args{"localhost:123"}, "https://localhost:123"},
+		{"http scheme", args{"http://test"}, "https://test"},
+		{"https scheme", args{"https://test"}, "https://test"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := ensureURLScheme(tt.args.url); got != tt.want {
+				t.Errorf("ensureURLScheme() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/cloudflared/access/validation.go

@@ -1,55 +0,0 @@
-package access
-
-import (
-	"errors"
-	"fmt"
-	"net/http"
-	"net/url"
-	"strings"
-
-	"golang.org/x/net/http/httpguts"
-)
-
-// parseRequestHeaders will take user-provided header values as strings "Content-Type: application/json" and create
-// a http.Header object.
-func parseRequestHeaders(values []string) http.Header {
-	headers := make(http.Header)
-	for _, valuePair := range values {
-		header, value, found := strings.Cut(valuePair, ":")
-		if found {
-			headers.Add(strings.TrimSpace(header), strings.TrimSpace(value))
-		}
-	}
-	return headers
-}
-
-// parseHostname will attempt to convert a user provided URL string into a string with some light error checking on
-// certain expectations from the URL.
-// Will convert all HTTP URLs to HTTPS
-func parseURL(input string) (*url.URL, error) {
-	if input == "" {
-		return nil, errors.New("no input provided")
-	}
-	if !strings.HasPrefix(input, "https://") && !strings.HasPrefix(input, "http://") {
-		input = fmt.Sprintf("https://%s", input)
-	}
-	url, err := url.ParseRequestURI(input)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse as URL: %w", err)
-	}
-	if url.Scheme != "https" {
-		url.Scheme = "https"
-	}
-	if url.Host == "" {
-		return nil, errors.New("failed to parse Host")
-	}
-	host, err := httpguts.PunycodeHostPort(url.Host)
-	if err != nil || host == "" {
-		return nil, err
-	}
-	if !httpguts.ValidHostHeader(host) {
-		return nil, errors.New("invalid Host provided")
-	}
-	url.Host = host
-	return url, nil
-}

cmd/cloudflared/access/validation_test.go

@@ -1,80 +0,0 @@
-package access
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestParseRequestHeaders(t *testing.T) {
-	values := parseRequestHeaders([]string{"client: value", "secret: safe-value", "trash", "cf-trace-id: 000:000:0:1:asd"})
-	assert.Len(t, values, 3)
-	assert.Equal(t, "value", values.Get("client"))
-	assert.Equal(t, "safe-value", values.Get("secret"))
-	assert.Equal(t, "000:000:0:1:asd", values.Get("cf-trace-id"))
-}
-
-func TestParseURL(t *testing.T) {
-	schemes := []string{
-		"http://",
-		"https://",
-		"",
-	}
-	hosts := []struct {
-		input    string
-		expected string
-	}{
-		{"localhost", "localhost"},
-		{"127.0.0.1", "127.0.0.1"},
-		{"127.0.0.1:9090", "127.0.0.1:9090"},
-		{"::1", "::1"},
-		{"::1:8080", "::1:8080"},
-		{"[::1]", "[::1]"},
-		{"[::1]:8080", "[::1]:8080"},
-		{":8080", ":8080"},
-		{"example.com", "example.com"},
-		{"hello.example.com", "hello.example.com"},
-		{"bücher.example.com", "xn--bcher-kva.example.com"},
-	}
-	paths := []string{
-		"",
-		"/test",
-		"/example.com?qwe=123",
-	}
-	for i, scheme := range schemes {
-		for j, host := range hosts {
-			for k, path := range paths {
-				t.Run(fmt.Sprintf("%d_%d_%d", i, j, k), func(t *testing.T) {
-					input := fmt.Sprintf("%s%s%s", scheme, host.input, path)
-					expected := fmt.Sprintf("%s%s%s", "https://", host.expected, path)
-					url, err := parseURL(input)
-					assert.NoError(t, err, "input: %s\texpected: %s", input, expected)
-					assert.Equal(t, expected, url.String())
-					assert.Equal(t, host.expected, url.Host)
-					assert.Equal(t, "https", url.Scheme)
-				})
-			}
-		}
-	}
-
-	t.Run("no input", func(t *testing.T) {
-		_, err := parseURL("")
-		assert.ErrorContains(t, err, "no input provided")
-	})
-
-	t.Run("missing host", func(t *testing.T) {
-		_, err := parseURL("https:///host")
-		assert.ErrorContains(t, err, "failed to parse Host")
-	})
-
-	t.Run("invalid path only", func(t *testing.T) {
-		_, err := parseURL("/host")
-		assert.ErrorContains(t, err, "failed to parse Host")
-	})
-
-	t.Run("invalid parse URL", func(t *testing.T) {
-		_, err := parseURL("https://host\\host")
-		assert.ErrorContains(t, err, "failed to parse as URL")
-	})
-}

cmd/cloudflared/cliutil/build_info.go

@@ -47,7 +47,3 @@ func (bi *BuildInfo) GetBuildTypeMsg() string {
 	}
 	return fmt.Sprintf(" with %s", bi.BuildType)
 }
-
-func (bi *BuildInfo) UserAgent() string {
-	return fmt.Sprintf("cloudflared/%s", bi.CloudflaredVersion)
-}

cmd/cloudflared/cliutil/logger.go

@@ -1,51 +0,0 @@
-package cliutil
-
-import (
-	"github.com/urfave/cli/v2"
-	"github.com/urfave/cli/v2/altsrc"
-
-	"github.com/cloudflare/cloudflared/logger"
-)
-
-var (
-	debugLevelWarning = "At debug level cloudflared will log request URL, method, protocol, content length, as well as, all request and response headers. " +
-		"This can expose sensitive information in your logs."
-)
-
-func ConfigureLoggingFlags(shouldHide bool) []cli.Flag {
-	return []cli.Flag{
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    logger.LogLevelFlag,
-			Value:   "info",
-			Usage:   "Application logging level {debug, info, warn, error, fatal}. " + debugLevelWarning,
-			EnvVars: []string{"TUNNEL_LOGLEVEL"},
-			Hidden:  shouldHide,
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    logger.LogTransportLevelFlag,
-			Aliases: []string{"proto-loglevel"}, // This flag used to be called proto-loglevel
-			Value:   "info",
-			Usage:   "Transport logging level(previously called protocol logging level) {debug, info, warn, error, fatal}",
-			EnvVars: []string{"TUNNEL_PROTO_LOGLEVEL", "TUNNEL_TRANSPORT_LOGLEVEL"},
-			Hidden:  shouldHide,
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    logger.LogFileFlag,
-			Usage:   "Save application log to this file for reporting issues.",
-			EnvVars: []string{"TUNNEL_LOGFILE"},
-			Hidden:  shouldHide,
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    logger.LogDirectoryFlag,
-			Usage:   "Save application log to this directory for reporting issues.",
-			EnvVars: []string{"TUNNEL_LOGDIRECTORY"},
-			Hidden:  shouldHide,
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "trace-output",
-			Usage:   "Name of trace output file, generated when cloudflared stops.",
-			EnvVars: []string{"TUNNEL_TRACE_OUTPUT"},
-			Hidden:  shouldHide,
-		}),
-	}
-}

cmd/cloudflared/generic_service.go

@@ -1,4 +1,5 @@
 //go:build !windows && !darwin && !linux
+// +build !windows,!darwin,!linux
 
 package main
 

cmd/cloudflared/linux_service.go

@@ -1,4 +1,5 @@
 //go:build linux
+// +build linux
 
 package main
 
@@ -24,9 +25,6 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 				Name:   "install",
 				Usage:  "Install cloudflared as a system service",
 				Action: cliutil.ConfiguredAction(installLinuxService),
-				Flags: []cli.Flag{
-					noUpdateServiceFlag,
-				},
 			},
 			{
 				Name:   "uninstall",
@@ -41,17 +39,15 @@ func runApp(app *cli.App, graceShutdownC chan struct{}) {
 // The directory and files that are used by the service.
 // These are hard-coded in the templates below.
 const (
-	serviceConfigDir         = "/etc/cloudflared"
-	serviceConfigFile        = "config.yml"
-	serviceCredentialFile    = "cert.pem"
-	serviceConfigPath        = serviceConfigDir + "/" + serviceConfigFile
-	cloudflaredService       = "cloudflared.service"
-	cloudflaredUpdateService = "cloudflared-update.service"
-	cloudflaredUpdateTimer   = "cloudflared-update.timer"
+	serviceConfigDir      = "/etc/cloudflared"
+	serviceConfigFile     = "config.yml"
+	serviceCredentialFile = "cert.pem"
+	serviceConfigPath     = serviceConfigDir + "/" + serviceConfigFile
+	cloudflaredService    = "cloudflared.service"
 )
 
-var systemdAllTemplates = map[string]ServiceTemplate{
-	cloudflaredService: {
+var systemdTemplates = []ServiceTemplate{
+	{
 		Path: fmt.Sprintf("/etc/systemd/system/%s", cloudflaredService),
 		Content: `[Unit]
 Description=cloudflared
@@ -68,8 +64,8 @@ RestartSec=5s
 WantedBy=multi-user.target
 `,
 	},
-	cloudflaredUpdateService: {
-		Path: fmt.Sprintf("/etc/systemd/system/%s", cloudflaredUpdateService),
+	{
+		Path: "/etc/systemd/system/cloudflared-update.service",
 		Content: `[Unit]
 Description=Update cloudflared
 After=network.target
@@ -78,8 +74,8 @@ After=network.target
 ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 11 ]; then systemctl restart cloudflared; exit 0; fi; exit $code'
 `,
 	},
-	cloudflaredUpdateTimer: {
-		Path: fmt.Sprintf("/etc/systemd/system/%s", cloudflaredUpdateTimer),
+	{
+		Path: "/etc/systemd/system/cloudflared-update.timer",
 		Content: `[Unit]
 Description=Update cloudflared
 
@@ -110,7 +106,7 @@ var sysvTemplate = ServiceTemplate{
 # Description:       cloudflared agent
 ### END INIT INFO
 name=$(basename $(readlink -f $0))
-cmd="{{.Path}} --pidfile /var/run/$name.pid {{ range .ExtraArgs }} {{ . }}{{ end }}"
+cmd="{{.Path}} --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s{{ range .ExtraArgs }} {{ . }}{{ end }}"
 pid_file="/var/run/$name.pid"
 stdout_log="/var/log/$name.log"
 stderr_log="/var/log/$name.err"
@@ -182,14 +178,6 @@ exit 0
 `,
 }
 
-var (
-	noUpdateServiceFlag = &cli.BoolFlag{
-		Name:  "no-update-service",
-		Usage: "Disable auto-update of the cloudflared linux service, which restarts the server to upgrade for new versions.",
-		Value: false,
-	}
-)
-
 func isSystemd() bool {
 	if _, err := os.Stat("/run/systemd/system"); err == nil {
 		return true
@@ -208,9 +196,6 @@ func installLinuxService(c *cli.Context) error {
 		Path: etPath,
 	}
 
-	// Check if the "no update flag" is set
-	autoUpdate := !c.IsSet(noUpdateServiceFlag.Name)
-
 	var extraArgsFunc func(c *cli.Context, log *zerolog.Logger) ([]string, error)
 	if c.NArg() == 0 {
 		extraArgsFunc = buildArgsForConfig
@@ -228,10 +213,10 @@ func installLinuxService(c *cli.Context) error {
 	switch {
 	case isSystemd():
 		log.Info().Msgf("Using Systemd")
-		err = installSystemd(&templateArgs, autoUpdate, log)
+		err = installSystemd(&templateArgs, log)
 	default:
 		log.Info().Msgf("Using SysV")
-		err = installSysv(&templateArgs, autoUpdate, log)
+		err = installSysv(&templateArgs, log)
 	}
 
 	if err == nil {
@@ -276,20 +261,7 @@ credentials-file: CREDENTIALS-FILE
 	}, nil
 }
 
-func installSystemd(templateArgs *ServiceTemplateArgs, autoUpdate bool, log *zerolog.Logger) error {
-	var systemdTemplates []ServiceTemplate
-	if autoUpdate {
-		systemdTemplates = []ServiceTemplate{
-			systemdAllTemplates[cloudflaredService],
-			systemdAllTemplates[cloudflaredUpdateService],
-			systemdAllTemplates[cloudflaredUpdateTimer],
-		}
-	} else {
-		systemdTemplates = []ServiceTemplate{
-			systemdAllTemplates[cloudflaredService],
-		}
-	}
-
+func installSystemd(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
 	for _, serviceTemplate := range systemdTemplates {
 		err := serviceTemplate.Generate(templateArgs)
 		if err != nil {
@@ -301,14 +273,10 @@ func installSystemd(templateArgs *ServiceTemplateArgs, autoUpdate bool, log *zer
 		log.Err(err).Msgf("systemctl enable %s error", cloudflaredService)
 		return err
 	}
-
-	if autoUpdate {
-		if err := runCommand("systemctl", "start", cloudflaredUpdateTimer); err != nil {
-			log.Err(err).Msgf("systemctl start %s error", cloudflaredUpdateTimer)
-			return err
-		}
+	if err := runCommand("systemctl", "start", "cloudflared-update.timer"); err != nil {
+		log.Err(err).Msg("systemctl start cloudflared-update.timer error")
+		return err
 	}
-
 	if err := runCommand("systemctl", "daemon-reload"); err != nil {
 		log.Err(err).Msg("systemctl daemon-reload error")
 		return err
@@ -316,19 +284,12 @@ func installSystemd(templateArgs *ServiceTemplateArgs, autoUpdate bool, log *zer
 	return runCommand("systemctl", "start", cloudflaredService)
 }
 
-func installSysv(templateArgs *ServiceTemplateArgs, autoUpdate bool, log *zerolog.Logger) error {
+func installSysv(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
 	confPath, err := sysvTemplate.ResolvePath()
 	if err != nil {
 		log.Err(err).Msg("error resolving system path")
 		return err
 	}
-
-	if autoUpdate {
-		templateArgs.ExtraArgs = append([]string{"--autoupdate-freq 24h0m0s"}, templateArgs.ExtraArgs...)
-	} else {
-		templateArgs.ExtraArgs = append([]string{"--no-autoupdate"}, templateArgs.ExtraArgs...)
-	}
-
 	if err := sysvTemplate.Generate(templateArgs); err != nil {
 		log.Err(err).Msg("error generating system template")
 		return err
@@ -366,35 +327,19 @@ func uninstallLinuxService(c *cli.Context) error {
 }
 
 func uninstallSystemd(log *zerolog.Logger) error {
-	// Get only the installed services
-	installedServices := make(map[string]ServiceTemplate)
-	for serviceName, serviceTemplate := range systemdAllTemplates {
-		if err := runCommand("systemctl", "list-units", "--all", "|", "grep", serviceName); err == nil {
-			installedServices[serviceName] = serviceTemplate
-		} else {
-			log.Info().Msgf("Service '%s' not installed, skipping its uninstall", serviceName)
-		}
+	if err := runCommand("systemctl", "disable", cloudflaredService); err != nil {
+		log.Err(err).Msgf("systemctl disable %s error", cloudflaredService)
+		return err
 	}
-
-	if _, exists := installedServices[cloudflaredService]; exists {
-		if err := runCommand("systemctl", "disable", cloudflaredService); err != nil {
-			log.Err(err).Msgf("systemctl disable %s error", cloudflaredService)
-			return err
-		}
-		if err := runCommand("systemctl", "stop", cloudflaredService); err != nil {
-			log.Err(err).Msgf("systemctl stop %s error", cloudflaredService)
-			return err
-		}
+	if err := runCommand("systemctl", "stop", cloudflaredService); err != nil {
+		log.Err(err).Msgf("systemctl stop %s error", cloudflaredService)
+		return err
 	}
-
-	if _, exists := installedServices[cloudflaredUpdateTimer]; exists {
-		if err := runCommand("systemctl", "stop", cloudflaredUpdateTimer); err != nil {
-			log.Err(err).Msgf("systemctl stop %s error", cloudflaredUpdateTimer)
-			return err
-		}
+	if err := runCommand("systemctl", "stop", "cloudflared-update.timer"); err != nil {
+		log.Err(err).Msg("systemctl stop cloudflared-update.timer error")
+		return err
 	}
-
-	for _, serviceTemplate := range installedServices {
+	for _, serviceTemplate := range systemdTemplates {
 		if err := serviceTemplate.Remove(); err != nil {
 			log.Err(err).Msg("error removing service template")
 			return err

cmd/cloudflared/macos_service.go

@@ -1,4 +1,5 @@
 //go:build darwin
+// +build darwin
 
 package main
 

cmd/cloudflared/main.go

@@ -3,7 +3,6 @@ package main
 import (
 	"fmt"
 	"math/rand"
-	"os"
 	"strings"
 	"time"
 
@@ -16,7 +15,6 @@ import (
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/tail"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/config"
@@ -50,9 +48,6 @@ var (
 )
 
 func main() {
-	// FIXME: TUN-8148: Disable QUIC_GO ECN due to bugs in proper detection if supported
-	os.Setenv("QUIC_GO_DISABLE_ECN", "1")
-
 	rand.Seed(time.Now().UnixNano())
 	metrics.RegisterBuildInfo(BuildType, BuildTime, Version)
 	maxprocs.Set()
@@ -94,7 +89,6 @@ func main() {
 	updater.Init(Version)
 	tracing.Init(Version)
 	token.Init(Version)
-	tail.Init(bInfo)
 	runApp(app, graceShutdownC)
 }
 
@@ -144,7 +138,6 @@ To determine if an update happened in a script, check for error code 11.`,
 	cmds = append(cmds, tunnel.Commands()...)
 	cmds = append(cmds, proxydns.Command(false))
 	cmds = append(cmds, access.Commands()...)
-	cmds = append(cmds, tail.Command())
 	return cmds
 }
 

cmd/cloudflared/service_template.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"path"
@@ -63,7 +64,7 @@ func (st *ServiceTemplate) Generate(args *ServiceTemplateArgs) error {
 		return fmt.Errorf("error creating %s: %v", plistFolder, err)
 	}
 
-	err = os.WriteFile(resolvedPath, buffer.Bytes(), fileMode)
+	err = ioutil.WriteFile(resolvedPath, buffer.Bytes(), fileMode)
 	if err != nil {
 		return fmt.Errorf("error writing %s: %v", resolvedPath, err)
 	}
@@ -102,7 +103,7 @@ func runCommand(command string, args ...string) error {
 		return fmt.Errorf("error starting %s: %v", command, err)
 	}
 
-	output, _ := io.ReadAll(stderr)
+	output, _ := ioutil.ReadAll(stderr)
 	err = cmd.Wait()
 	if err != nil {
 		return fmt.Errorf("%s %v returned with error code %v due to: %v", command, args, err, string(output))

cmd/cloudflared/tail/cmd.go

@@ -1,428 +0,0 @@
-package tail
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"net/http"
-	"net/url"
-	"os"
-	"os/signal"
-	"syscall"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/mattn/go-colorable"
-	"github.com/rs/zerolog"
-	"github.com/urfave/cli/v2"
-	"nhooyr.io/websocket"
-
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
-	"github.com/cloudflare/cloudflared/credentials"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/management"
-)
-
-var (
-	buildInfo *cliutil.BuildInfo
-)
-
-func Init(bi *cliutil.BuildInfo) {
-	buildInfo = bi
-}
-
-func Command() *cli.Command {
-	subcommands := []*cli.Command{
-		buildTailManagementTokenSubcommand(),
-	}
-
-	return buildTailCommand(subcommands)
-}
-
-func buildTailManagementTokenSubcommand() *cli.Command {
-	return &cli.Command{
-		Name:        "token",
-		Action:      cliutil.ConfiguredAction(managementTokenCommand),
-		Usage:       "Get management access jwt",
-		UsageText:   "cloudflared tail token TUNNEL_ID",
-		Description: `Get management access jwt for a tunnel`,
-		Hidden:      true,
-	}
-}
-
-func managementTokenCommand(c *cli.Context) error {
-	log := createLogger(c)
-	token, err := getManagementToken(c, log)
-	if err != nil {
-		return err
-	}
-	var tokenResponse = struct {
-		Token string `json:"token"`
-	}{Token: token}
-
-	return json.NewEncoder(os.Stdout).Encode(tokenResponse)
-}
-
-func buildTailCommand(subcommands []*cli.Command) *cli.Command {
-	return &cli.Command{
-		Name:      "tail",
-		Action:    Run,
-		Usage:     "Stream logs from a remote cloudflared",
-		UsageText: "cloudflared tail [tail command options] [TUNNEL-ID]",
-		Flags: []cli.Flag{
-			&cli.StringFlag{
-				Name:    "connector-id",
-				Usage:   "Access a specific cloudflared instance by connector id (for when a tunnel has multiple cloudflared's)",
-				Value:   "",
-				EnvVars: []string{"TUNNEL_MANAGEMENT_CONNECTOR"},
-			},
-			&cli.StringSliceFlag{
-				Name:    "event",
-				Usage:   "Filter by specific Events (cloudflared, http, tcp, udp) otherwise, defaults to send all events",
-				EnvVars: []string{"TUNNEL_MANAGEMENT_FILTER_EVENTS"},
-			},
-			&cli.StringFlag{
-				Name:    "level",
-				Usage:   "Filter by specific log levels (debug, info, warn, error). Filters by debug log level by default.",
-				EnvVars: []string{"TUNNEL_MANAGEMENT_FILTER_LEVEL"},
-				Value:   "debug",
-			},
-			&cli.Float64Flag{
-				Name:    "sample",
-				Usage:   "Sample log events by percentage (0.0 .. 1.0). No sampling by default.",
-				EnvVars: []string{"TUNNEL_MANAGEMENT_FILTER_SAMPLE"},
-				Value:   1.0,
-			},
-			&cli.StringFlag{
-				Name:    "token",
-				Usage:   "Access token for a specific tunnel",
-				Value:   "",
-				EnvVars: []string{"TUNNEL_MANAGEMENT_TOKEN"},
-			},
-			&cli.StringFlag{
-				Name:    "output",
-				Usage:   "Output format for the logs (default, json)",
-				Value:   "default",
-				EnvVars: []string{"TUNNEL_MANAGEMENT_OUTPUT"},
-			},
-			&cli.StringFlag{
-				Name:    "management-hostname",
-				Usage:   "Management hostname to signify incoming management requests",
-				EnvVars: []string{"TUNNEL_MANAGEMENT_HOSTNAME"},
-				Hidden:  true,
-				Value:   "management.argotunnel.com",
-			},
-			&cli.StringFlag{
-				Name:   "trace",
-				Usage:  "Set a cf-trace-id for the request",
-				Hidden: true,
-				Value:  "",
-			},
-			&cli.StringFlag{
-				Name:    logger.LogLevelFlag,
-				Value:   "info",
-				Usage:   "Application logging level {debug, info, warn, error, fatal}",
-				EnvVars: []string{"TUNNEL_LOGLEVEL"},
-			},
-			&cli.StringFlag{
-				Name:    credentials.OriginCertFlag,
-				Usage:   "Path to the certificate generated for your origin when you run cloudflared login.",
-				EnvVars: []string{"TUNNEL_ORIGIN_CERT"},
-				Value:   credentials.FindDefaultOriginCertPath(),
-			},
-		},
-		Subcommands: subcommands,
-	}
-}
-
-// Middleware validation error struct for returning to the eyeball
-type managementError struct {
-	Code    int    `json:"code,omitempty"`
-	Message string `json:"message,omitempty"`
-}
-
-// Middleware validation error HTTP response JSON for returning to the eyeball
-type managementErrorResponse struct {
-	Success bool              `json:"success,omitempty"`
-	Errors  []managementError `json:"errors,omitempty"`
-}
-
-func handleValidationError(resp *http.Response, log *zerolog.Logger) {
-	if resp.StatusCode == 530 {
-		log.Error().Msgf("no cloudflared connector available or reachable via management request (a recent version of cloudflared is required to use streaming logs)")
-	}
-	var managementErr managementErrorResponse
-	err := json.NewDecoder(resp.Body).Decode(&managementErr)
-	if err != nil {
-		log.Error().Msgf("unable to start management log streaming session: http response code returned %d", resp.StatusCode)
-		return
-	}
-	if managementErr.Success || len(managementErr.Errors) == 0 {
-		log.Error().Msgf("management tunnel validation returned success with invalid HTTP response code to convert to a WebSocket request")
-		return
-	}
-	for _, e := range managementErr.Errors {
-		log.Error().Msgf("management request failed validation: (%d) %s", e.Code, e.Message)
-	}
-}
-
-// logger will be created to emit only against the os.Stderr as to not obstruct with normal output from
-// management requests
-func createLogger(c *cli.Context) *zerolog.Logger {
-	level, levelErr := zerolog.ParseLevel(c.String(logger.LogLevelFlag))
-	if levelErr != nil {
-		level = zerolog.InfoLevel
-	}
-	log := zerolog.New(zerolog.ConsoleWriter{
-		Out:        colorable.NewColorable(os.Stderr),
-		TimeFormat: time.RFC3339,
-	}).With().Timestamp().Logger().Level(level)
-	return &log
-}
-
-// parseFilters will attempt to parse provided filters to send to with the EventStartStreaming
-func parseFilters(c *cli.Context) (*management.StreamingFilters, error) {
-	var level *management.LogLevel
-	var events []management.LogEventType
-	var sample float64
-
-	argLevel := c.String("level")
-	argEvents := c.StringSlice("event")
-	argSample := c.Float64("sample")
-
-	if argLevel != "" {
-		l, ok := management.ParseLogLevel(argLevel)
-		if !ok {
-			return nil, fmt.Errorf("invalid --level filter provided, please use one of the following Log Levels: debug, info, warn, error")
-		}
-		level = &l
-	}
-
-	for _, v := range argEvents {
-		t, ok := management.ParseLogEventType(v)
-		if !ok {
-			return nil, fmt.Errorf("invalid --event filter provided, please use one of the following EventTypes: cloudflared, http, tcp, udp")
-		}
-		events = append(events, t)
-	}
-
-	if argSample <= 0.0 || argSample > 1.0 {
-		return nil, fmt.Errorf("invalid --sample value provided, please make sure it is in the range (0.0 .. 1.0)")
-	}
-	sample = argSample
-
-	if level == nil && len(events) == 0 && argSample != 1.0 {
-		// When no filters are provided, do not return a StreamingFilters struct
-		return nil, nil
-	}
-
-	return &management.StreamingFilters{
-		Level:    level,
-		Events:   events,
-		Sampling: sample,
-	}, nil
-}
-
-// getManagementToken will make a call to the Cloudflare API to acquire a management token for the requested tunnel.
-func getManagementToken(c *cli.Context, log *zerolog.Logger) (string, error) {
-	userCreds, err := credentials.Read(c.String(credentials.OriginCertFlag), log)
-	if err != nil {
-		return "", err
-	}
-
-	client, err := userCreds.Client(c.String("api-url"), buildInfo.UserAgent(), log)
-	if err != nil {
-		return "", err
-	}
-
-	tunnelIDString := c.Args().First()
-	if tunnelIDString == "" {
-		return "", errors.New("no tunnel ID provided")
-	}
-	tunnelID, err := uuid.Parse(tunnelIDString)
-	if err != nil {
-		return "", errors.New("unable to parse provided tunnel id as a valid UUID")
-	}
-
-	token, err := client.GetManagementToken(tunnelID)
-	if err != nil {
-		return "", err
-	}
-
-	return token, nil
-}
-
-// buildURL will build the management url to contain the required query parameters to authenticate the request.
-func buildURL(c *cli.Context, log *zerolog.Logger) (url.URL, error) {
-	var err error
-	managementHostname := c.String("management-hostname")
-	token := c.String("token")
-	if token == "" {
-		token, err = getManagementToken(c, log)
-		if err != nil {
-			return url.URL{}, fmt.Errorf("unable to acquire management token for requested tunnel id: %w", err)
-		}
-	}
-	query := url.Values{}
-	query.Add("access_token", token)
-	connector := c.String("connector-id")
-	if connector != "" {
-		connectorID, err := uuid.Parse(connector)
-		if err != nil {
-			return url.URL{}, fmt.Errorf("unabled to parse 'connector-id' flag into a valid UUID: %w", err)
-		}
-		query.Add("connector_id", connectorID.String())
-	}
-	return url.URL{Scheme: "wss", Host: managementHostname, Path: "/logs", RawQuery: query.Encode()}, nil
-}
-
-func printLine(log *management.Log, logger *zerolog.Logger) {
-	fields, err := json.Marshal(log.Fields)
-	if err != nil {
-		fields = []byte("unable to parse fields")
-		logger.Debug().Msgf("unable to parse fields from event %+v", log)
-	}
-	fmt.Printf("%s %s %s %s %s\n", log.Time, log.Level, log.Event, log.Message, fields)
-}
-
-func printJSON(log *management.Log, logger *zerolog.Logger) {
-	output, err := json.Marshal(log)
-	if err != nil {
-		logger.Debug().Msgf("unable to parse event to json %+v", log)
-	} else {
-		fmt.Println(string(output))
-	}
-}
-
-// Run implements a foreground runner
-func Run(c *cli.Context) error {
-	log := createLogger(c)
-
-	signals := make(chan os.Signal, 10)
-	signal.Notify(signals, syscall.SIGTERM, syscall.SIGINT)
-	defer signal.Stop(signals)
-
-	output := "default"
-	switch c.String("output") {
-	case "default", "":
-		output = "default"
-	case "json":
-		output = "json"
-	default:
-		log.Err(errors.New("invalid --output value provided, please make sure it is one of: default, json")).Send()
-	}
-
-	filters, err := parseFilters(c)
-	if err != nil {
-		log.Error().Err(err).Msgf("invalid filters provided")
-		return nil
-	}
-
-	u, err := buildURL(c, log)
-	if err != nil {
-		log.Err(err).Msg("unable to construct management request URL")
-		return nil
-	}
-
-	header := make(http.Header)
-	header.Add("User-Agent", buildInfo.UserAgent())
-	trace := c.String("trace")
-	if trace != "" {
-		header["cf-trace-id"] = []string{trace}
-	}
-	ctx := c.Context
-	conn, resp, err := websocket.Dial(ctx, u.String(), &websocket.DialOptions{
-		HTTPHeader: header,
-	})
-	if err != nil {
-		if resp != nil && resp.StatusCode != http.StatusSwitchingProtocols {
-			handleValidationError(resp, log)
-			return nil
-		}
-		log.Error().Err(err).Msgf("unable to start management log streaming session")
-		return nil
-	}
-	defer conn.Close(websocket.StatusInternalError, "management connection was closed abruptly")
-
-	// Once connection is established, send start_streaming event to begin receiving logs
-	err = management.WriteEvent(conn, ctx, &management.EventStartStreaming{
-		ClientEvent: management.ClientEvent{Type: management.StartStreaming},
-		Filters:     filters,
-	})
-	if err != nil {
-		log.Error().Err(err).Msg("unable to request logs from management tunnel")
-		return nil
-	}
-	log.Debug().
-		Str("tunnel-id", c.Args().First()).
-		Str("connector-id", c.String("connector-id")).
-		Interface("filters", filters).
-		Msg("connected")
-
-	readerDone := make(chan struct{})
-
-	go func() {
-		defer close(readerDone)
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			default:
-				event, err := management.ReadServerEvent(conn, ctx)
-				if err != nil {
-					if closeErr := management.AsClosed(err); closeErr != nil {
-						// If the client (or the server) already closed the connection, don't continue to
-						// attempt to read from the client.
-						if closeErr.Code == websocket.StatusNormalClosure {
-							return
-						}
-						// Only log abnormal closures
-						log.Error().Msgf("received remote closure: (%d) %s", closeErr.Code, closeErr.Reason)
-						return
-					}
-					log.Err(err).Msg("unable to read event from server")
-					return
-				}
-				switch event.Type {
-				case management.Logs:
-					logs, ok := management.IntoServerEvent(event, management.Logs)
-					if !ok {
-						log.Error().Msgf("invalid logs event")
-						continue
-					}
-					// Output all the logs received to stdout
-					for _, l := range logs.Logs {
-						if output == "json" {
-							printJSON(l, log)
-						} else {
-							printLine(l, log)
-						}
-					}
-				case management.UnknownServerEventType:
-					fallthrough
-				default:
-					log.Debug().Msgf("unexpected log event type: %s", event.Type)
-				}
-			}
-		}
-	}()
-
-	for {
-		select {
-		case <-ctx.Done():
-			return nil
-		case <-readerDone:
-			return nil
-		case <-signals:
-			log.Debug().Msg("closing management connection")
-			// Cleanly close the connection by sending a close message and then
-			// waiting (with timeout) for the server to close the connection.
-			conn.Close(websocket.StatusNormalClosure, "")
-			select {
-			case <-readerDone:
-			case <-time.After(time.Second):
-			}
-			return nil
-		}
-	}
-}

cmd/cloudflared/tunnel/cmd.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"runtime/trace"
@@ -11,7 +12,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/coreos/go-systemd/v22/daemon"
+	"github.com/coreos/go-systemd/daemon"
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/getsentry/sentry-go"
 	"github.com/google/uuid"
@@ -27,27 +28,19 @@ import (
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/credentials"
-	"github.com/cloudflare/cloudflared/edgediscovery"
-	"github.com/cloudflare/cloudflared/features"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/management"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/orchestration"
 	"github.com/cloudflare/cloudflared/signal"
 	"github.com/cloudflare/cloudflared/supervisor"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunneldns"
-	"github.com/cloudflare/cloudflared/validation"
 )
 
 const (
 	sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b:3e8827f6f9f740738eb11138f7bebb68@sentry.io/189878"
 
-	// ha-Connections specifies how many connections to make to the edge
-	haConnectionsFlag = "ha-connections"
-
 	// sshPortFlag is the port on localhost the cloudflared ssh server will run on
 	sshPortFlag = "local-ssh-port"
 
@@ -78,32 +71,26 @@ const (
 	// hostKeyPath is the path of the dir to save SSH host keys too
 	hostKeyPath = "host-key-path"
 
-	// udpUnregisterSessionTimeout is how long we wait before we stop trying to unregister a UDP session from the edge
-	udpUnregisterSessionTimeoutFlag = "udp-unregister-session-timeout"
-
-	// quicDisablePathMTUDiscovery sets if QUIC should not perform PTMU discovery and use a smaller (safe) packet size.
-	// Packets will then be at most 1252 (IPv4) / 1232 (IPv6) bytes in size.
-	// Note that this may result in packet drops for UDP proxying, since we expect being able to send at least 1280 bytes of inner packets.
-	quicDisablePathMTUDiscovery = "quic-disable-pmtu-discovery"
-
 	// uiFlag is to enable launching cloudflared in interactive UI mode
 	uiFlag = "ui"
 
+	debugLevelWarning = "At debug level cloudflared will log request URL, method, protocol, content length, as well as, all request and response headers. " +
+		"This can expose sensitive information in your logs."
+
 	LogFieldCommand             = "command"
 	LogFieldExpandedPath        = "expandedPath"
 	LogFieldPIDPathname         = "pidPathname"
 	LogFieldTmpTraceFilename    = "tmpTraceFilename"
 	LogFieldTraceOutputFilepath = "traceOutputFilepath"
 
-	tunnelCmdErrorMessage = `You did not specify any valid additional argument to the cloudflared tunnel command.
+	tunnelCmdErrorMessage = `You did not specify any valid additional argument to the cloudflared tunnel command. 
 
-If you are trying to run a Quick Tunnel then you need to explicitly pass the --url flag.
-Eg. cloudflared tunnel --url localhost:8080/.
+If you are trying to run a Quick Tunnel then you need to explicitly pass the --url flag. 
+Eg. cloudflared tunnel --url localhost:8080/. 
 
-Please note that Quick Tunnels are meant to be ephemeral and should only be used for testing purposes.
+Please note that Quick Tunnels are meant to be ephemeral and should only be used for testing purposes. 
 For production usage, we recommend creating Named Tunnels. (https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/tunnel-guide/)
 `
-	connectorLabelFlag = "label"
 )
 
 var (
@@ -190,46 +177,26 @@ func TunnelCommand(c *cli.Context) error {
 		return err
 	}
 
-	// Run a adhoc named tunnel
-	// Allows for the creation, routing (optional), and startup of a tunnel in one command
-	// --name required
-	// --url or --hello-world required
-	// --hostname optional
-	if name := c.String("name"); name != "" {
-		hostname, err := validation.ValidateHostname(c.String("hostname"))
-		if err != nil {
-			return errors.Wrap(err, "Invalid hostname provided")
-		}
-		url := c.String("url")
-		if url == hostname && url != "" && hostname != "" {
-			return fmt.Errorf("hostname and url shouldn't match. See --help for more information")
-		}
-
+	if name := c.String("name"); name != "" { // Start a named tunnel
 		return runAdhocNamedTunnel(sc, name, c.String(CredFileFlag))
 	}
 
-	// Run a quick tunnel
-	// A unauthenticated named tunnel hosted on <random>.<quick-tunnels-service>.com
-	// We don't support running proxy-dns and a quick tunnel at the same time as the same process
-	shouldRunQuickTunnel := c.IsSet("url") || c.IsSet(ingress.HelloWorldFlag)
-	if !c.IsSet("proxy-dns") && c.String("quick-service") != "" && shouldRunQuickTunnel {
+	// Unauthenticated named tunnel on <random>.<quick-tunnels-service>.com
+	shouldRunQuickTunnel := c.IsSet("url") || c.IsSet("hello-world")
+	if !dnsProxyStandAlone(c, nil) && c.String("quick-service") != "" && shouldRunQuickTunnel {
 		return RunQuickTunnel(sc)
 	}
 
-	// If user provides a config, check to see if they meant to use `tunnel run` instead
 	if ref := config.GetConfiguration().TunnelID; ref != "" {
 		return fmt.Errorf("Use `cloudflared tunnel run` to start tunnel %s", ref)
 	}
 
-	// Classic tunnel usage is no longer supported
+	// classic tunnel usage is no longer supported
 	if c.String("hostname") != "" {
 		return deprecatedClassicTunnelErr
 	}
 
 	if c.IsSet("proxy-dns") {
-		if shouldRunQuickTunnel {
-			return fmt.Errorf("running a quick tunnel with `proxy-dns` is not supported")
-		}
 		// NamedTunnelProperties are nil since proxy dns server does not need it.
 		// This is supported for legacy reasons: dns proxy server is not a tunnel and ideally should
 		// not run as part of cloudflared tunnel.
@@ -304,7 +271,7 @@ func StartServer(
 	}
 
 	if c.IsSet("trace-output") {
-		tmpTraceFile, err := os.CreateTemp("", "trace")
+		tmpTraceFile, err := ioutil.TempFile("", "trace")
 		if err != nil {
 			log.Err(err).Msg("Failed to create new temporary file to save trace output")
 		}
@@ -340,7 +307,7 @@ func StartServer(
 	logClientOptions(c, log)
 
 	// this context drives the server, when it's cancelled tunnel and all other components (origins, dns, etc...) should stop
-	ctx, cancel := context.WithCancel(c.Context)
+	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
 	go waitForSignal(graceShutdownC, log)
@@ -372,7 +339,7 @@ func StartServer(
 		errC <- autoupdater.Run(ctx)
 	}()
 
-	// Serve DNS proxy stand-alone if no tunnel type (quick, adhoc, named) is going to run
+	// Serve DNS proxy stand-alone if no hostname or tag or app is going to run
 	if dnsProxyStandAlone(c, namedTunnel) {
 		connectedSignal.Notify()
 		// no grace period, handle SIGINT/SIGTERM immediately
@@ -392,7 +359,7 @@ func StartServer(
 		observer.SendURL(quickTunnelURL)
 	}
 
-	tunnelConfig, orchestratorConfig, err := prepareTunnelConfig(ctx, c, info, log, logTransport, observer, namedTunnel)
+	tunnelConfig, orchestratorConfig, err := prepareTunnelConfig(c, info, log, logTransport, observer, namedTunnel)
 	if err != nil {
 		log.Err(err).Msg("Couldn't start tunnel")
 		return err
@@ -406,27 +373,7 @@ func StartServer(
 		}
 	}
 
-	internalRules := []ingress.Rule{}
-	if features.Contains(features.FeatureManagementLogs) {
-		serviceIP := c.String("service-op-ip")
-		if edgeAddrs, err := edgediscovery.ResolveEdge(log, tunnelConfig.Region, tunnelConfig.EdgeIPVersion); err == nil {
-			if serviceAddr, err := edgeAddrs.GetAddrForRPC(); err == nil {
-				serviceIP = serviceAddr.TCP.String()
-			}
-		}
-
-		mgmt := management.New(
-			c.String("management-hostname"),
-			c.Bool("management-diagnostics"),
-			serviceIP,
-			clientID,
-			c.String(connectorLabelFlag),
-			logger.ManagementLogger.Log,
-			logger.ManagementLogger,
-		)
-		internalRules = []ingress.Rule{ingress.NewManagementRule(mgmt)}
-	}
-	orchestrator, err := orchestration.NewOrchestrator(ctx, orchestratorConfig, tunnelConfig.Tags, internalRules, tunnelConfig.Log)
+	orchestrator, err := orchestration.NewOrchestrator(ctx, orchestratorConfig, tunnelConfig.Tags, tunnelConfig.Log)
 	if err != nil {
 		return err
 	}
@@ -450,7 +397,7 @@ func StartServer(
 		errC <- metrics.ServeMetrics(metricsListener, ctx, metricsConfig, log)
 	}()
 
-	reconnectCh := make(chan supervisor.ReconnectSignal, c.Int(haConnectionsFlag))
+	reconnectCh := make(chan supervisor.ReconnectSignal, c.Int("ha-connections"))
 	if c.IsSet("stdin-control") {
 		log.Info().Msg("Enabling control through stdin")
 		go stdinControl(reconnectCh, log)
@@ -563,7 +510,7 @@ func addPortIfMissing(uri *url.URL, port int) string {
 func tunnelFlags(shouldHide bool) []cli.Flag {
 	flags := configureCloudflaredFlags(shouldHide)
 	flags = append(flags, configureProxyFlags(shouldHide)...)
-	flags = append(flags, cliutil.ConfigureLoggingFlags(shouldHide)...)
+	flags = append(flags, configureLoggingFlags(shouldHide)...)
 	flags = append(flags, configureProxyDNSFlags(shouldHide)...)
 	flags = append(flags, []cli.Flag{
 		credentialsFileFlag,
@@ -586,17 +533,11 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    "edge-ip-version",
-			Usage:   "Cloudflare Edge IP address version to connect with. {4, 6, auto}",
+			Usage:   "Cloudflare Edge ip address version to connect with. {4, 6, auto}",
 			EnvVars: []string{"TUNNEL_EDGE_IP_VERSION"},
 			Value:   "4",
 			Hidden:  false,
 		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "edge-bind-address",
-			Usage:   "Bind to IP address for outgoing connections to Cloudflare Edge.",
-			EnvVars: []string{"TUNNEL_EDGE_BIND_ADDRESS"},
-			Hidden:  false,
-		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    tlsconfig.CaCertFlag,
 			Usage:   "Certificate Authority authenticating connections with Cloudflare's edge network.",
@@ -687,27 +628,10 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewIntFlag(&cli.IntFlag{
-			Name:   haConnectionsFlag,
+			Name:   "ha-connections",
 			Value:  4,
 			Hidden: true,
 		}),
-		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   udpUnregisterSessionTimeoutFlag,
-			Value:  5 * time.Second,
-			Hidden: true,
-		}),
-		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    quicDisablePathMTUDiscovery,
-			EnvVars: []string{"TUNNEL_DISABLE_QUIC_PMTU"},
-			Usage:   "Use this option to disable PTMU discovery for QUIC connections. This will result in lower packet sizes. Not however, that this may cause instability for UDP proxying.",
-			Value:   false,
-			Hidden:  true,
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:  connectorLabelFlag,
-			Usage: "Use this option to give a meaningful label to a specific connector. When a tunnel starts up, a connector id unique to the tunnel is generated. This is a uuid. To make it easier to identify a connector, we will use the hostname of the machine the tunnel is running on along with the connector ID. This option exists if one wants to have more control over what their individual connectors are called.",
-			Value: "",
-		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:    "grace-period",
 			Usage:   "When cloudflared receives SIGINT/SIGTERM it will stop accepting new requests, wait for in-progress requests to terminate, then shutdown. Waiting for in-progress requests will timeout after this grace period, or when a second SIGTERM/SIGINT is received.",
@@ -776,12 +700,6 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_POST_QUANTUM"},
 			Hidden:  FipsEnabled,
 		}),
-		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "management-diagnostics",
-			Usage:   "Enables the in-depth diagnostic routes to be made available over the management service (/debug/pprof, /metrics, etc.)",
-			EnvVars: []string{"TUNNEL_MANAGEMENT_DIAGNOSTICS"},
-			Value:   false,
-		}),
 		selectProtocolFlag,
 		overwriteDNSFlag,
 	}...)
@@ -799,10 +717,10 @@ func configureCloudflaredFlags(shouldHide bool) []cli.Flag {
 			Hidden: shouldHide,
 		},
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    credentials.OriginCertFlag,
+			Name:    "origincert",
 			Usage:   "Path to the certificate generated for your origin when you run cloudflared login.",
 			EnvVars: []string{"TUNNEL_ORIGIN_CERT"},
-			Value:   credentials.FindDefaultOriginCertPath(),
+			Value:   findDefaultOriginCertPath(),
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
@@ -844,7 +762,7 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    ingress.HelloWorldFlag,
+			Name:    "hello-world",
 			Value:   false,
 			Usage:   "Run Hello World Server",
 			EnvVars: []string{"TUNNEL_HELLO_WORLD"},
@@ -947,20 +865,6 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			Hidden:  shouldHide,
 			Value:   false,
 		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "management-hostname",
-			Usage:   "Management hostname to signify incoming management requests",
-			EnvVars: []string{"TUNNEL_MANAGEMENT_HOSTNAME"},
-			Hidden:  true,
-			Value:   "management.argotunnel.com",
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "service-op-ip",
-			Usage:   "Fallback IP for service operations run by the management service.",
-			EnvVars: []string{"TUNNEL_SERVICE_OP_IP"},
-			Hidden:  true,
-			Value:   "198.41.200.113:80",
-		}),
 	}
 	return append(flags, sshFlags(shouldHide)...)
 }
@@ -1069,6 +973,44 @@ func sshFlags(shouldHide bool) []cli.Flag {
 	}
 }
 
+func configureLoggingFlags(shouldHide bool) []cli.Flag {
+	return []cli.Flag{
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    logger.LogLevelFlag,
+			Value:   "info",
+			Usage:   "Application logging level {debug, info, warn, error, fatal}. " + debugLevelWarning,
+			EnvVars: []string{"TUNNEL_LOGLEVEL"},
+			Hidden:  shouldHide,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    logger.LogTransportLevelFlag,
+			Aliases: []string{"proto-loglevel"}, // This flag used to be called proto-loglevel
+			Value:   "info",
+			Usage:   "Transport logging level(previously called protocol logging level) {debug, info, warn, error, fatal}",
+			EnvVars: []string{"TUNNEL_PROTO_LOGLEVEL", "TUNNEL_TRANSPORT_LOGLEVEL"},
+			Hidden:  shouldHide,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    logger.LogFileFlag,
+			Usage:   "Save application log to this file for reporting issues.",
+			EnvVars: []string{"TUNNEL_LOGFILE"},
+			Hidden:  shouldHide,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    logger.LogDirectoryFlag,
+			Usage:   "Save application log to this directory for reporting issues.",
+			EnvVars: []string{"TUNNEL_LOGDIRECTORY"},
+			Hidden:  shouldHide,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    "trace-output",
+			Usage:   "Name of trace output file, generated when cloudflared stops.",
+			EnvVars: []string{"TUNNEL_TRACE_OUTPUT"},
+			Hidden:  shouldHide,
+		}),
+	}
+}
+
 func configureProxyDNSFlags(shouldHide bool) []cli.Flag {
 	return []cli.Flag{
 		altsrc.NewBoolFlag(&cli.BoolFlag{

cmd/cloudflared/tunnel/config_test.go

@@ -4,12 +4,10 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
-
-	"github.com/cloudflare/cloudflared/features"
 )
 
 func TestDedup(t *testing.T) {
 	expected := []string{"a", "b"}
-	actual := features.Dedup([]string{"a", "b", "a"})
+	actual := dedup([]string{"a", "b", "a"})
 	require.ElementsMatch(t, expected, actual)
 }

cmd/cloudflared/tunnel/configuration.go

@@ -1,28 +1,32 @@
 package tunnel
 
 import (
-	"context"
 	"crypto/tls"
 	"fmt"
+	"io/ioutil"
+	mathRand "math/rand"
 	"net"
 	"net/netip"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/google/uuid"
+	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
 	"github.com/urfave/cli/v2/altsrc"
-	"golang.org/x/term"
+	"golang.org/x/crypto/ssh/terminal"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
+
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
-	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
-	"github.com/cloudflare/cloudflared/features"
+	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/orchestration"
 	"github.com/cloudflare/cloudflared/supervisor"
@@ -30,21 +34,32 @@ import (
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
-const (
-	secretValue       = "*****"
-	icmpFunnelTimeout = time.Second * 10
-)
+const LogFieldOriginCertPath = "originCertPath"
+const secretValue = "*****"
 
 var (
-	developerPortal = "https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup"
-	serviceUrl      = developerPortal + "/tunnel-guide/local/as-a-service/"
-	argumentsUrl    = developerPortal + "/tunnel-guide/local/local-management/arguments/"
+	developerPortal = "https://developers.cloudflare.com/argo-tunnel"
+	serviceUrl      = developerPortal + "/reference/service/"
+	argumentsUrl    = developerPortal + "/reference/arguments/"
 
-	secretFlags = [2]*altsrc.StringFlag{credentialsContentsFlag, tunnelTokenFlag}
+	secretFlags     = [2]*altsrc.StringFlag{credentialsContentsFlag, tunnelTokenFlag}
+	defaultFeatures = []string{supervisor.FeatureAllowRemoteConfig, supervisor.FeatureSerializedHeaders, supervisor.FeatureDatagramV2, supervisor.FeatureQUICSupportEOF}
 
-	configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq", "edge-ip-version", "edge-bind-address"}
+	configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq", "edge-ip-version"}
 )
 
+// returns the first path that contains a cert.pem file. If none of the DefaultConfigSearchDirectories
+// contains a cert.pem file, return empty string
+func findDefaultOriginCertPath() string {
+	for _, defaultConfigDir := range config.DefaultConfigSearchDirectories() {
+		originCertPath, _ := homedir.Expand(filepath.Join(defaultConfigDir, config.DefaultCredentialFile))
+		if ok, _ := config.FileExists(originCertPath); ok {
+			return originCertPath
+		}
+	}
+	return ""
+}
+
 func generateRandomClientID(log *zerolog.Logger) (string, error) {
 	u, err := uuid.NewRandom()
 	if err != nil {
@@ -109,14 +124,66 @@ func isSecretEnvVar(key string) bool {
 }
 
 func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelProperties) bool {
-	return c.IsSet("proxy-dns") &&
-		!(c.IsSet("name") || // adhoc-named tunnel
-			c.IsSet(ingress.HelloWorldFlag) || // quick or named tunnel
-			namedTunnel != nil) // named tunnel
+	return c.IsSet("proxy-dns") && (!c.IsSet("tag") && !c.IsSet("hello-world") && namedTunnel == nil)
+}
+
+func findOriginCert(originCertPath string, log *zerolog.Logger) (string, error) {
+	if originCertPath == "" {
+		log.Info().Msgf("Cannot determine default origin certificate path. No file %s in %v", config.DefaultCredentialFile, config.DefaultConfigSearchDirectories())
+		if isRunningFromTerminal() {
+			log.Error().Msgf("You need to specify the origin certificate path with --origincert option, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", argumentsUrl)
+			return "", fmt.Errorf("client didn't specify origincert path when running from terminal")
+		} else {
+			log.Error().Msgf("You need to specify the origin certificate path by specifying the origincert option in the configuration file, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", serviceUrl)
+			return "", fmt.Errorf("client didn't specify origincert path")
+		}
+	}
+	var err error
+	originCertPath, err = homedir.Expand(originCertPath)
+	if err != nil {
+		log.Err(err).Msgf("Cannot resolve origin certificate path")
+		return "", fmt.Errorf("cannot resolve path %s", originCertPath)
+	}
+	// Check that the user has acquired a certificate using the login command
+	ok, err := config.FileExists(originCertPath)
+	if err != nil {
+		log.Error().Err(err).Msgf("Cannot check if origin cert exists at path %s", originCertPath)
+		return "", fmt.Errorf("cannot check if origin cert exists at path %s", originCertPath)
+	}
+	if !ok {
+		log.Error().Msgf(`Cannot find a valid certificate for your origin at the path:
+
+    %s
+
+If the path above is wrong, specify the path with the -origincert option.
+If you don't have a certificate signed by Cloudflare, run the command:
+
+	%s login
+`, originCertPath, os.Args[0])
+		return "", fmt.Errorf("cannot find a valid certificate at the path %s", originCertPath)
+	}
+
+	return originCertPath, nil
+}
+
+func readOriginCert(originCertPath string) ([]byte, error) {
+	// Easier to send the certificate as []byte via RPC than decoding it at this point
+	originCert, err := ioutil.ReadFile(originCertPath)
+	if err != nil {
+		return nil, fmt.Errorf("cannot read %s to load origin certificate", originCertPath)
+	}
+	return originCert, nil
+}
+
+func getOriginCert(originCertPath string, log *zerolog.Logger) ([]byte, error) {
+	if originCertPath, err := findOriginCert(originCertPath, log); err != nil {
+		return nil, err
+	} else {
+		return readOriginCert(originCertPath)
+	}
 }
 
 func prepareTunnelConfig(
-	ctx context.Context,
 	c *cli.Context,
 	info *cliutil.BuildInfo,
 	log, logTransport *zerolog.Logger,
@@ -136,49 +203,66 @@ func prepareTunnelConfig(
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID.String()})
 
 	transportProtocol := c.String("protocol")
-
-	clientFeatures := features.Dedup(append(c.StringSlice("features"), features.DefaultFeatures...))
-
-	staticFeatures := features.StaticFeatures{}
-	if c.Bool("post-quantum") {
+	needPQ := c.Bool("post-quantum")
+	if needPQ {
 		if FipsEnabled {
 			return nil, nil, fmt.Errorf("post-quantum not supported in FIPS mode")
 		}
-		pqMode := features.PostQuantumStrict
-		staticFeatures.PostQuantumMode = &pqMode
-	}
-	featureSelector, err := features.NewFeatureSelector(ctx, namedTunnel.Credentials.AccountTag, staticFeatures, log)
-	if err != nil {
-		return nil, nil, errors.Wrap(err, "Failed to create feature selector")
-	}
-	pqMode := featureSelector.PostQuantumMode()
-	if pqMode == features.PostQuantumStrict {
 		// Error if the user tries to force a non-quic transport protocol
 		if transportProtocol != connection.AutoSelectFlag && transportProtocol != connection.QUIC.String() {
 			return nil, nil, fmt.Errorf("post-quantum is only supported with the quic transport")
 		}
 		transportProtocol = connection.QUIC.String()
-		clientFeatures = append(clientFeatures, features.FeaturePostQuantum)
-
-		log.Info().Msgf(
-			"Using hybrid post-quantum key agreement %s",
-			supervisor.PQKexName,
-		)
 	}
 
+	protocolFetcher := edgediscovery.ProtocolPercentage
+
+	features := append(c.StringSlice("features"), defaultFeatures...)
+	if needPQ {
+		features = append(features, supervisor.FeaturePostQuantum)
+	}
+	if c.IsSet(TunnelTokenFlag) {
+		if transportProtocol == connection.AutoSelectFlag {
+			protocolFetcher = func() (edgediscovery.ProtocolPercents, error) {
+				// If the Tunnel is remotely managed and no protocol is set, we prefer QUIC, but still allow fall-back.
+				preferQuic := []edgediscovery.ProtocolPercent{
+					{
+						Protocol:   connection.QUIC.String(),
+						Percentage: 100,
+					},
+					{
+						Protocol:   connection.HTTP2.String(),
+						Percentage: 100,
+					},
+				}
+				return preferQuic, nil
+			}
+		}
+		log.Info().Msg("Will be fetching remotely managed configuration from Cloudflare API. Defaulting to protocol: quic")
+	}
 	namedTunnel.Client = tunnelpogs.ClientInfo{
 		ClientID: clientID[:],
-		Features: clientFeatures,
+		Features: dedup(features),
 		Version:  info.Version(),
 		Arch:     info.OSArch(),
 	}
 	cfg := config.GetConfiguration()
-	ingressRules, err := ingress.ParseIngressFromConfigAndCLI(cfg, c, log)
-	if err != nil {
+	ingressRules, err := ingress.ParseIngress(cfg)
+	if err != nil && err != ingress.ErrNoIngressRules {
 		return nil, nil, err
 	}
+	// Only for quick tunnels will we attempt to parse the --url flag for a tunnel ingress rule
+	if ingressRules.IsEmpty() && c.IsSet("url") && namedTunnel.QuickTunnelUrl != "" {
+		ingressRules, err = ingress.NewSingleOrigin(c, true)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+	if ingressRules.IsEmpty() {
+		return nil, nil, ingress.ErrNoIngressRules
+	}
 
-	protocolSelector, err := connection.NewProtocolSelector(transportProtocol, namedTunnel.Credentials.AccountTag, c.IsSet(TunnelTokenFlag), c.Bool("post-quantum"), edgediscovery.ProtocolPercentage, connection.ResolveTTL, log)
+	protocolSelector, err := connection.NewProtocolSelector(transportProtocol, cfg.WarpRouting.Enabled, namedTunnel, protocolFetcher, supervisor.ResolveTTL, log, c.Bool("post-quantum"))
 	if err != nil {
 		return nil, nil, err
 	}
@@ -204,21 +288,26 @@ func prepareTunnelConfig(
 	if err != nil {
 		return nil, nil, err
 	}
+	muxerConfig := &connection.MuxerConfig{
+		HeartbeatInterval: c.Duration("heartbeat-interval"),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		MaxHeartbeats: uint64(c.Int("heartbeat-count")),
+		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
+		CompressionSetting: h2mux.CompressionSetting(uint64(c.Int("compression-quality"))),
+		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
+	}
 	edgeIPVersion, err := parseConfigIPVersion(c.String("edge-ip-version"))
 	if err != nil {
 		return nil, nil, err
 	}
-	edgeBindAddr, err := parseConfigBindAddress(c.String("edge-bind-address"))
-	if err != nil {
-		return nil, nil, err
-	}
-	if err := testIPBindable(edgeBindAddr); err != nil {
-		return nil, nil, fmt.Errorf("invalid edge-bind-address %s: %v", edgeBindAddr, err)
-	}
-	edgeIPVersion, err = adjustIPVersionByBindAddress(edgeIPVersion, edgeBindAddr)
-	if err != nil {
-		// This is not a fatal error, we just overrode edgeIPVersion
-		log.Warn().Str("edgeIPVersion", edgeIPVersion.String()).Err(err).Msg("Overriding edge-ip-version")
+
+	var pqKexIdx int
+	if needPQ {
+		pqKexIdx = mathRand.Intn(len(supervisor.PQKexes))
+		log.Info().Msgf(
+			"Using experimental hybrid post-quantum key agreement %s",
+			supervisor.PQKexNames[supervisor.PQKexes[pqKexIdx]],
+		)
 	}
 
 	tunnelConfig := &supervisor.TunnelConfig{
@@ -229,8 +318,8 @@ func prepareTunnelConfig(
 		EdgeAddrs:       c.StringSlice("edge"),
 		Region:          c.String("region"),
 		EdgeIPVersion:   edgeIPVersion,
-		EdgeBindAddr:    edgeBindAddr,
-		HAConnections:   c.Int(haConnectionsFlag),
+		HAConnections:   c.Int("ha-connections"),
+		IncidentLookup:  supervisor.NewIncidentLookup(),
 		IsAutoupdated:   c.Bool("is-autoupdated"),
 		LBPool:          c.String("lb-pool"),
 		Tags:            tags,
@@ -239,15 +328,15 @@ func prepareTunnelConfig(
 		Observer:        observer,
 		ReportedVersion: info.Version(),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
-		Retries:                     uint(c.Int("retries")),
-		RunFromTerminal:             isRunningFromTerminal(),
-		NamedTunnel:                 namedTunnel,
-		ProtocolSelector:            protocolSelector,
-		EdgeTLSConfigs:              edgeTLSConfigs,
-		FeatureSelector:             featureSelector,
-		MaxEdgeAddrRetries:          uint8(c.Int("max-edge-addr-retries")),
-		UDPUnregisterSessionTimeout: c.Duration(udpUnregisterSessionTimeoutFlag),
-		DisableQUICPathMTUDiscovery: c.Bool(quicDisablePathMTUDiscovery),
+		Retries:            uint(c.Int("retries")),
+		RunFromTerminal:    isRunningFromTerminal(),
+		NamedTunnel:        namedTunnel,
+		MuxerConfig:        muxerConfig,
+		ProtocolSelector:   protocolSelector,
+		EdgeTLSConfigs:     edgeTLSConfigs,
+		NeedPQ:             needPQ,
+		PQKexIdx:           pqKexIdx,
+		MaxEdgeAddrRetries: uint8(c.Int("max-edge-addr-retries")),
 	}
 	packetConfig, err := newPacketConfig(c, log)
 	if err != nil {
@@ -284,7 +373,26 @@ func gracePeriod(c *cli.Context) (time.Duration, error) {
 }
 
 func isRunningFromTerminal() bool {
-	return term.IsTerminal(int(os.Stdout.Fd()))
+	return terminal.IsTerminal(int(os.Stdout.Fd()))
+}
+
+// Remove any duplicates from the slice
+func dedup(slice []string) []string {
+
+	// Convert the slice into a set
+	set := make(map[string]bool, 0)
+	for _, str := range slice {
+		set[str] = true
+	}
+
+	// Convert the set back into a slice
+	keys := make([]string, len(set))
+	i := 0
+	for str := range set {
+		keys[i] = str
+		i++
+	}
+	return keys
 }
 
 // ParseConfigIPVersion returns the IP version from possible expected values from config
@@ -302,51 +410,6 @@ func parseConfigIPVersion(version string) (v allregions.ConfigIPVersion, err err
 	return
 }
 
-func parseConfigBindAddress(ipstr string) (net.IP, error) {
-	// Unspecified - it's fine
-	if ipstr == "" {
-		return nil, nil
-	}
-	ip := net.ParseIP(ipstr)
-	if ip == nil {
-		return nil, fmt.Errorf("invalid value for edge-bind-address: %s", ipstr)
-	}
-	return ip, nil
-}
-
-func testIPBindable(ip net.IP) error {
-	// "Unspecified" = let OS choose, so always bindable
-	if ip == nil {
-		return nil
-	}
-
-	addr := &net.UDPAddr{IP: ip, Port: 0}
-	listener, err := net.ListenUDP("udp", addr)
-	if err != nil {
-		return err
-	}
-	listener.Close()
-	return nil
-}
-
-func adjustIPVersionByBindAddress(ipVersion allregions.ConfigIPVersion, ip net.IP) (allregions.ConfigIPVersion, error) {
-	if ip == nil {
-		return ipVersion, nil
-	}
-	// https://pkg.go.dev/net#IP.To4: "If ip is not an IPv4 address, To4 returns nil."
-	if ip.To4() != nil {
-		if ipVersion == allregions.IPv6Only {
-			return allregions.IPv4Only, fmt.Errorf("IPv4 bind address is specified, but edge-ip-version is IPv6")
-		}
-		return allregions.IPv4Only, nil
-	} else {
-		if ipVersion == allregions.IPv4Only {
-			return allregions.IPv6Only, fmt.Errorf("IPv6 bind address is specified, but edge-ip-version is IPv4")
-		}
-		return allregions.IPv6Only, nil
-	}
-}
-
 func newPacketConfig(c *cli.Context, logger *zerolog.Logger) (*ingress.GlobalRouterConfig, error) {
 	ipv4Src, err := determineICMPv4Src(c.String("icmpv4-src"), logger)
 	if err != nil {
@@ -364,7 +427,7 @@ func newPacketConfig(c *cli.Context, logger *zerolog.Logger) (*ingress.GlobalRou
 		logger.Info().Msgf("ICMP proxy will use %s as source for IPv6", ipv6Src)
 	}
 
-	icmpRouter, err := ingress.NewICMPRouter(ipv4Src, ipv6Src, zone, logger, icmpFunnelTimeout)
+	icmpRouter, err := ingress.NewICMPRouter(ipv4Src, ipv6Src, zone, logger)
 	if err != nil {
 		return nil, err
 	}

cmd/cloudflared/tunnel/configuration_test.go

@@ -1,4 +1,5 @@
 //go:build ignore
+// +build ignore
 
 // TODO: Remove the above build tag and include this test when we start compiling with Golang 1.10.0+
 
@@ -8,7 +9,6 @@ import (
 	"crypto/x509"
 	"crypto/x509/pkix"
 	"encoding/asn1"
-	"net"
 	"os"
 	"testing"
 
@@ -214,23 +214,3 @@ func getCertPoolSubjects(certPool *x509.CertPool) ([]*pkix.Name, error) {
 func isUnrecoverableError(err error) bool {
 	return err != nil && err.Error() != "crypto/x509: system root pool is not available on Windows"
 }
-
-func TestTestIPBindable(t *testing.T) {
-	assert.Nil(t, testIPBindable(nil))
-
-	// Public services - if one of these IPs is on the machine, the test environment is too weird
-	assert.NotNil(t, testIPBindable(net.ParseIP("8.8.8.8")))
-	assert.NotNil(t, testIPBindable(net.ParseIP("1.1.1.1")))
-
-	addrs, err := net.InterfaceAddrs()
-	if err != nil {
-		t.Fatal(err)
-	}
-	for i, addr := range addrs {
-		if i >= 3 {
-			break
-		}
-		ip := addr.(*net.IPNet).IP
-		assert.Nil(t, testIPBindable(ip))
-	}
-}

cmd/cloudflared/tunnel/credential_finder.go

@@ -5,7 +5,6 @@ import (
 	"path/filepath"
 
 	"github.com/cloudflare/cloudflared/config"
-	"github.com/cloudflare/cloudflared/credentials"
 
 	"github.com/google/uuid"
 	"github.com/rs/zerolog"
@@ -57,13 +56,13 @@ func newSearchByID(id uuid.UUID, c *cli.Context, log *zerolog.Logger, fs fileSys
 }
 
 func (s searchByID) Path() (string, error) {
-	originCertPath := s.c.String(credentials.OriginCertFlag)
+	originCertPath := s.c.String("origincert")
 	originCertLog := s.log.With().
-		Str("originCertPath", originCertPath).
+		Str(LogFieldOriginCertPath, originCertPath).
 		Logger()
 
 	// Fallback to look for tunnel credentials in the origin cert directory
-	if originCertPath, err := credentials.FindOriginCert(originCertPath, &originCertLog); err == nil {
+	if originCertPath, err := findOriginCert(originCertPath, &originCertLog); err == nil {
 		originCertDir := filepath.Dir(originCertPath)
 		if filePath, err := tunnelFilePath(s.id, originCertDir); err == nil {
 			if s.fs.validFilePath(filePath) {

cmd/cloudflared/tunnel/filesystem.go

@@ -1,6 +1,7 @@
 package tunnel
 
 import (
+	"io/ioutil"
 	"os"
 )
 
@@ -22,5 +23,5 @@ func (fs realFileSystem) validFilePath(path string) bool {
 }
 
 func (fs realFileSystem) readFile(filePath string) ([]byte, error) {
-	return os.ReadFile(filePath)
+	return ioutil.ReadFile(filePath)
 }

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -139,7 +139,7 @@ func testURLCommand(c *cli.Context) error {
 	}
 
 	_, i := ing.FindMatchingRule(requestURL.Hostname(), requestURL.Path)
-	fmt.Printf("Matched rule #%d\n", i)
+	fmt.Printf("Matched rule #%d\n", i+1)
 	fmt.Println(ing.Rules[i].MultiLineString())
 	return nil
 }

cmd/cloudflared/tunnel/login.go

@@ -2,6 +2,7 @@ package tunnel
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -13,7 +14,6 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/config"
-	"github.com/cloudflare/cloudflared/credentials"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/token"
 )
@@ -52,7 +52,6 @@ func login(c *cli.Context) error {
 
 	resourceData, err := token.RunTransfer(
 		loginURL,
-		"",
 		"cert",
 		"callback",
 		callbackStoreURL,
@@ -65,7 +64,7 @@ func login(c *cli.Context) error {
 		return err
 	}
 
-	if err := os.WriteFile(path, resourceData, 0600); err != nil {
+	if err := ioutil.WriteFile(path, resourceData, 0600); err != nil {
 		return errors.Wrap(err, fmt.Sprintf("error writing cert to %s", path))
 	}
 
@@ -86,7 +85,7 @@ func checkForExistingCert() (string, bool, error) {
 	if err != nil {
 		return "", false, err
 	}
-	path := filepath.Join(configPath, credentials.DefaultCredentialFile)
+	path := filepath.Join(configPath, config.DefaultCredentialFile)
 	fileInfo, err := os.Stat(path)
 	if err == nil && fileInfo.Size() > 0 {
 		return path, true, nil

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -73,9 +73,6 @@ func RunQuickTunnel(sc *subcommandContext) error {
 		sc.c.Set("protocol", "quic")
 	}
 
-	// Override the number of connections used. Quick tunnels shouldn't be used for production usage,
-	// so, use a single connection instead.
-	sc.c.Set(haConnectionsFlag, "1")
 	return StartServer(
 		sc.c,
 		buildInfo,

cmd/cloudflared/tunnel/signal_test.go

@@ -1,4 +1,5 @@
 //go:build !windows
+// +build !windows
 
 package tunnel
 

cmd/cloudflared/tunnel/subcommand_context.go

@@ -13,9 +13,9 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
 
+	"github.com/cloudflare/cloudflared/certutil"
 	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/credentials"
 	"github.com/cloudflare/cloudflared/logger"
 )
 
@@ -37,7 +37,7 @@ type subcommandContext struct {
 
 	// These fields should be accessed using their respective Getter
 	tunnelstoreClient cfapi.Client
-	userCredential    *credentials.User
+	userCredential    *userCredential
 }
 
 func newSubcommandContext(c *cli.Context) (*subcommandContext, error) {
@@ -56,28 +56,65 @@ func (sc *subcommandContext) credentialFinder(tunnelID uuid.UUID) CredFinder {
 	return newSearchByID(tunnelID, sc.c, sc.log, sc.fs)
 }
 
+type userCredential struct {
+	cert     *certutil.OriginCert
+	certPath string
+}
+
 func (sc *subcommandContext) client() (cfapi.Client, error) {
 	if sc.tunnelstoreClient != nil {
 		return sc.tunnelstoreClient, nil
 	}
-	cred, err := sc.credential()
+	credential, err := sc.credential()
 	if err != nil {
 		return nil, err
 	}
-	sc.tunnelstoreClient, err = cred.Client(sc.c.String("api-url"), buildInfo.UserAgent(), sc.log)
+	userAgent := fmt.Sprintf("cloudflared/%s", buildInfo.Version())
+	client, err := cfapi.NewRESTClient(
+		sc.c.String("api-url"),
+		credential.cert.AccountID,
+		credential.cert.ZoneID,
+		credential.cert.APIToken,
+		userAgent,
+		sc.log,
+	)
+
 	if err != nil {
 		return nil, err
 	}
-	return sc.tunnelstoreClient, nil
+	sc.tunnelstoreClient = client
+	return client, nil
 }
 
-func (sc *subcommandContext) credential() (*credentials.User, error) {
+func (sc *subcommandContext) credential() (*userCredential, error) {
 	if sc.userCredential == nil {
-		uc, err := credentials.Read(sc.c.String(credentials.OriginCertFlag), sc.log)
+		originCertPath := sc.c.String("origincert")
+		originCertLog := sc.log.With().
+			Str(LogFieldOriginCertPath, originCertPath).
+			Logger()
+
+		originCertPath, err := findOriginCert(originCertPath, &originCertLog)
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrap(err, "Error locating origin cert")
+		}
+		blocks, err := readOriginCert(originCertPath)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Can't read origin cert from %s", originCertPath)
+		}
+
+		cert, err := certutil.DecodeOriginCert(blocks)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error decoding origin cert")
+		}
+
+		if cert.AccountID == "" {
+			return nil, errors.Errorf(`Origin certificate needs to be refreshed before creating new tunnels.\nDelete %s and run "cloudflared login" to obtain a new cert.`, originCertPath)
+		}
+
+		sc.userCredential = &userCredential{
+			cert:     cert,
+			certPath: originCertPath,
 		}
-		sc.userCredential = uc
 	}
 	return sc.userCredential, nil
 }
@@ -138,13 +175,13 @@ func (sc *subcommandContext) create(name string, credentialsFilePath string, sec
 		return nil, err
 	}
 	tunnelCredentials := connection.Credentials{
-		AccountTag:   credential.AccountID(),
+		AccountTag:   credential.cert.AccountID,
 		TunnelSecret: tunnelSecret,
 		TunnelID:     tunnel.ID,
 	}
 	usedCertPath := false
 	if credentialsFilePath == "" {
-		originCertDir := filepath.Dir(credential.CertPath())
+		originCertDir := filepath.Dir(credential.certPath)
 		credentialsFilePath, err = tunnelFilePath(tunnelCredentials.TunnelID, originCertDir)
 		if err != nil {
 			return nil, err
@@ -156,7 +193,7 @@ func (sc *subcommandContext) create(name string, credentialsFilePath string, sec
 		var errorLines []string
 		errorLines = append(errorLines, fmt.Sprintf("Your tunnel '%v' was created with ID %v. However, cloudflared couldn't write tunnel credentials to %s.", tunnel.Name, tunnel.ID, credentialsFilePath))
 		errorLines = append(errorLines, fmt.Sprintf("The file-writing error is: %v", writeFileErr))
-		if deleteErr := client.DeleteTunnel(tunnel.ID, true); deleteErr != nil {
+		if deleteErr := client.DeleteTunnel(tunnel.ID); deleteErr != nil {
 			errorLines = append(errorLines, fmt.Sprintf("Cloudflared tried to delete the tunnel for you, but encountered an error. You should use `cloudflared tunnel delete %v` to delete the tunnel yourself, because the tunnel can't be run without the tunnelfile.", tunnel.ID))
 			errorLines = append(errorLines, fmt.Sprintf("The delete tunnel error is: %v", deleteErr))
 		} else {
@@ -206,8 +243,13 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 		if !tunnel.DeletedAt.IsZero() {
 			return fmt.Errorf("Tunnel %s has already been deleted", tunnel.ID)
 		}
+		if forceFlagSet {
+			if err := client.CleanupConnections(tunnel.ID, cfapi.NewCleanupParams()); err != nil {
+				return errors.Wrapf(err, "Error cleaning up connections for tunnel %s", tunnel.ID)
+			}
+		}
 
-		if err := client.DeleteTunnel(tunnel.ID, forceFlagSet); err != nil {
+		if err := client.DeleteTunnel(tunnel.ID); err != nil {
 			return errors.Wrapf(err, "Error deleting tunnel %s", tunnel.ID)
 		}
 

cmd/cloudflared/tunnel/subcommand_context_teamnet.go

@@ -1,9 +1,6 @@
 package tunnel
 
 import (
-	"net"
-
-	"github.com/google/uuid"
 	"github.com/pkg/errors"
 
 	"github.com/cloudflare/cloudflared/cfapi"
@@ -27,12 +24,12 @@ func (sc *subcommandContext) addRoute(newRoute cfapi.NewRoute) (cfapi.Route, err
 	return client.AddRoute(newRoute)
 }
 
-func (sc *subcommandContext) deleteRoute(id uuid.UUID) error {
+func (sc *subcommandContext) deleteRoute(params cfapi.DeleteRouteParams) error {
 	client, err := sc.client()
 	if err != nil {
 		return errors.Wrap(err, noClientMsg)
 	}
-	return client.DeleteRoute(id)
+	return client.DeleteRoute(params)
 }
 
 func (sc *subcommandContext) getRouteByIP(params cfapi.GetRouteByIpParams) (cfapi.DetailedRoute, error) {
@@ -42,25 +39,3 @@ func (sc *subcommandContext) getRouteByIP(params cfapi.GetRouteByIpParams) (cfap
 	}
 	return client.GetByIP(params)
 }
-
-func (sc *subcommandContext) getRouteId(network net.IPNet, vnetId *uuid.UUID) (uuid.UUID, error) {
-	filters := cfapi.NewIPRouteFilter()
-	filters.NotDeleted()
-	filters.NetworkIsSubsetOf(network)
-	filters.NetworkIsSupersetOf(network)
-
-	if vnetId != nil {
-		filters.VNetID(*vnetId)
-	}
-
-	result, err := sc.listRoutes(filters)
-	if err != nil {
-		return uuid.Nil, err
-	}
-
-	if len(result) != 1 {
-		return uuid.Nil, errors.New("unable to find route for provided network and vnet")
-	}
-
-	return result[0].ID, nil
-}

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -16,7 +16,6 @@ import (
 
 	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/credentials"
 )
 
 type mockFileSystem struct {
@@ -38,7 +37,7 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 		log               *zerolog.Logger
 		fs                fileSystem
 		tunnelstoreClient cfapi.Client
-		userCredential    *credentials.User
+		userCredential    *userCredential
 	}
 	type args struct {
 		tunnelID uuid.UUID
@@ -219,7 +218,7 @@ func (d *deleteMockTunnelStore) GetTunnelToken(tunnelID uuid.UUID) (string, erro
 	return "token", nil
 }
 
-func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID, cascade bool) error {
+func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
 	tunnel, ok := d.mockTunnels[tunnelID]
 	if !ok {
 		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
@@ -250,7 +249,7 @@ func Test_subcommandContext_Delete(t *testing.T) {
 		isUIEnabled       bool
 		fs                fileSystem
 		tunnelstoreClient *deleteMockTunnelStore
-		userCredential    *credentials.User
+		userCredential    *userCredential
 	}
 	type args struct {
 		tunnelIDs []uuid.UUID

cmd/cloudflared/tunnel/subcommands.go

@@ -5,6 +5,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -94,6 +95,14 @@ var (
 		Usage:   "Inverts the sort order of the tunnel list.",
 		EnvVars: []string{"TUNNEL_LIST_INVERT_SORT"},
 	}
+	forceFlag = altsrc.NewBoolFlag(&cli.BoolFlag{
+		Name:    "force",
+		Aliases: []string{"f"},
+		Usage: "By default, if a tunnel is currently being run from a cloudflared, you can't " +
+			"simultaneously rerun it again from a second cloudflared. The --force flag lets you " +
+			"overwrite the previous tunnel. If you want to use a single hostname with multiple " +
+			"tunnels, you can do so with Cloudflare's Load Balancer product.",
+	})
 	featuresFlag = altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
 		Name:    "features",
 		Aliases: []string{"F"},
@@ -119,8 +128,8 @@ var (
 	forceDeleteFlag = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
-		Usage: "Deletes a tunnel even if tunnel is connected and it has dependencies associated to it. (eg. IP routes)." +
-			" It is not possible to delete tunnels that have connections or non-deleted dependencies, without this flag.",
+		Usage: "Cleans up any stale connections before the tunnel is deleted. cloudflared will not " +
+			"delete a tunnel with connections without this flag.",
 		EnvVars: []string{"TUNNEL_RUN_FORCE_OVERWRITE"},
 	}
 	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{
@@ -240,7 +249,7 @@ func writeTunnelCredentials(filePath string, credentials *connection.Credentials
 	if err != nil {
 		return errors.Wrap(err, "Unable to marshal tunnel credentials to JSON")
 	}
-	return os.WriteFile(filePath, body, 0400)
+	return ioutil.WriteFile(filePath, body, 400)
 }
 
 func buildListCommand() *cli.Command {
@@ -607,6 +616,7 @@ func renderOutput(format string, v interface{}) error {
 
 func buildRunCommand() *cli.Command {
 	flags := []cli.Flag{
+		forceFlag,
 		credentialsFileFlag,
 		credentialsContentsFlag,
 		postQuantumFlag,
@@ -622,7 +632,7 @@ func buildRunCommand() *cli.Command {
 		Action:    cliutil.ConfiguredAction(runCommand),
 		Usage:     "Proxy a local web server by running the given tunnel",
 		UsageText: "cloudflared tunnel [tunnel command options] run [subcommand options] [TUNNEL]",
-		Description: `Runs the tunnel identified by name or UUID, creating highly available connections
+		Description: `Runs the tunnel identified by name or UUUD, creating highly available connections
   between your server and the Cloudflare edge. You can provide name or UUID of tunnel to run either as the
   last command line argument or in the configuration file using "tunnel: TUNNEL".
 
@@ -931,7 +941,7 @@ func commandHelpTemplate() string {
 	for _, f := range configureCloudflaredFlags(false) {
 		parentFlagsHelp += fmt.Sprintf(" %s\n\t", f)
 	}
-	for _, f := range cliutil.ConfigureLoggingFlags(false) {
+	for _, f := range configureLoggingFlags(false) {
 		parentFlagsHelp += fmt.Sprintf(" %s\n\t", f)
 	}
 	const template = `NAME:

cmd/cloudflared/tunnel/teamnet_subcommands.go

@@ -21,8 +21,6 @@ var (
 		Aliases: []string{"vn"},
 		Usage:   "The ID or name of the virtual network to which the route is associated to.",
 	}
-
-	routeAddError = errors.New("You must supply exactly one argument, the ID or CIDR of the route you want to delete")
 )
 
 func buildRouteIPSubcommand() *cli.Command {
@@ -70,9 +68,11 @@ which virtual network's routing table you want to add the route to with:
 				Name:      "delete",
 				Action:    cliutil.ConfiguredAction(deleteRouteCommand),
 				Usage:     "Delete a row from your organization's private routing table",
-				UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [flags] [Route ID or CIDR]",
-				Description: `Deletes the row for the given route ID from your routing table. That portion of your network
-will no longer be reachable.`,
+				UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [flags] [CIDR]",
+				Description: `Deletes the row for a given CIDR from your routing table. That portion of your network
+will no longer be reachable by the WARP clients. Note that if you use virtual
+networks, then you have to tell which virtual network whose routing table you
+have a row deleted from.`,
 				Flags: []cli.Flag{vnetFlag},
 			},
 			{
@@ -187,36 +187,33 @@ func deleteRouteCommand(c *cli.Context) error {
 	}
 
 	if c.NArg() != 1 {
-		return routeAddError
+		return errors.New("You must supply exactly one argument, the network whose route you want to delete (in CIDR form e.g. 1.2.3.4/32)")
 	}
 
-	var routeId uuid.UUID
-	routeId, err = uuid.Parse(c.Args().First())
+	_, network, err := net.ParseCIDR(c.Args().First())
 	if err != nil {
-		_, network, err := net.ParseCIDR(c.Args().First())
-		if err != nil || network == nil {
-			return routeAddError
-		}
+		return errors.Wrap(err, "Invalid network CIDR")
+	}
+	if network == nil {
+		return errors.New("Invalid network CIDR")
+	}
 
-		var vnetId *uuid.UUID
-		if c.IsSet(vnetFlag.Name) {
-			id, err := getVnetId(sc, c.String(vnetFlag.Name))
-			if err != nil {
-				return err
-			}
-			vnetId = &id
-		}
+	params := cfapi.DeleteRouteParams{
+		Network: *network,
+	}
 
-		routeId, err = sc.getRouteId(*network, vnetId)
+	if c.IsSet(vnetFlag.Name) {
+		vnetId, err := getVnetId(sc, c.String(vnetFlag.Name))
 		if err != nil {
 			return err
 		}
+		params.VNetID = &vnetId
 	}
 
-	if err := sc.deleteRoute(routeId); err != nil {
+	if err := sc.deleteRoute(params); err != nil {
 		return errors.Wrap(err, "API error")
 	}
-	fmt.Printf("Successfully deleted route with ID %s\n", routeId)
+	fmt.Printf("Successfully deleted route for %s\n", network)
 	return nil
 }
 
@@ -272,7 +269,7 @@ func formatAndPrintRouteList(routes []*cfapi.DetailedRoute) {
 	defer writer.Flush()
 
 	// Print column headers with tabbed columns
-	_, _ = fmt.Fprintln(writer, "ID\tNETWORK\tVIRTUAL NET ID\tCOMMENT\tTUNNEL ID\tTUNNEL NAME\tCREATED\tDELETED\t")
+	_, _ = fmt.Fprintln(writer, "NETWORK\tVIRTUAL NET ID\tCOMMENT\tTUNNEL ID\tTUNNEL NAME\tCREATED\tDELETED\t")
 
 	// Loop through routes, create formatted string for each, and print using tabwriter
 	for _, route := range routes {

cmd/cloudflared/updater/update.go

@@ -6,13 +6,12 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
-	"strings"
 	"time"
 
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
-	"golang.org/x/term"
+	"golang.org/x/crypto/ssh/terminal"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
@@ -96,24 +95,11 @@ func CheckForUpdate(options updateOptions) (CheckResult, error) {
 		url = StagingUpdateURL
 	}
 
-	if runtime.GOOS == "windows" {
-		cfdPath = encodeWindowsPath(cfdPath)
-	}
-
 	s := NewWorkersService(version, url, cfdPath, Options{IsBeta: options.isBeta,
 		IsForced: options.isForced, RequestedVersion: options.intendedVersion})
 
 	return s.Check()
 }
-func encodeWindowsPath(path string) string {
-	// We do this because Windows allows spaces in directories such as
-	// Program Files but does not allow these directories to be spaced in batch files.
-	targetPath := strings.Replace(path, "Program Files (x86)", "PROGRA~2", -1)
-	// This is to do the same in 32 bit systems. We do this second so that the first
-	// replace is for x86 dirs.
-	targetPath = strings.Replace(targetPath, "Program Files", "PROGRA~1", -1)
-	return targetPath
-}
 
 func applyUpdate(options updateOptions, update CheckResult) UpdateOutcome {
 	if update.Version() == "" || options.updateDisabled {
@@ -304,7 +290,7 @@ func wasInstalledFromPackageManager() bool {
 }
 
 func isRunningFromTerminal() bool {
-	return term.IsTerminal(int(os.Stdout.Fd()))
+	return terminal.IsTerminal(int(os.Stdout.Fd()))
 }
 
 func IsSysV() bool {

cmd/cloudflared/updater/workers_service.go

@@ -56,9 +56,6 @@ func (s *WorkersService) Check() (CheckResult, error) {
 	}
 
 	req, err := http.NewRequest(http.MethodGet, s.url, nil)
-	if err != nil {
-		return nil, err
-	}
 	q := req.URL.Query()
 	q.Add(OSKeyName, runtime.GOOS)
 	q.Add(ArchitectureKeyName, runtime.GOARCH)

cmd/cloudflared/updater/workers_service_test.go

@@ -1,4 +1,5 @@
 //go:build !windows
+// +build !windows
 
 package updater
 
@@ -10,6 +11,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"net/http/httptest"
@@ -222,7 +224,7 @@ func TestUpdateService(t *testing.T) {
 	require.Equal(t, v.Version(), mostRecentVersion)
 
 	require.NoError(t, v.Apply())
-	dat, err := os.ReadFile(testFilePath)
+	dat, err := ioutil.ReadFile(testFilePath)
 	require.NoError(t, err)
 
 	require.Equal(t, string(dat), mostRecentVersion)
@@ -241,7 +243,7 @@ func TestBetaUpdateService(t *testing.T) {
 	require.Equal(t, v.Version(), mostRecentBetaVersion)
 
 	require.NoError(t, v.Apply())
-	dat, err := os.ReadFile(testFilePath)
+	dat, err := ioutil.ReadFile(testFilePath)
 	require.NoError(t, err)
 
 	require.Equal(t, string(dat), mostRecentBetaVersion)
@@ -287,7 +289,7 @@ func TestForcedUpdateService(t *testing.T) {
 	require.Equal(t, v.Version(), mostRecentVersion)
 
 	require.NoError(t, v.Apply())
-	dat, err := os.ReadFile(testFilePath)
+	dat, err := ioutil.ReadFile(testFilePath)
 	require.NoError(t, err)
 
 	require.Equal(t, string(dat), mostRecentVersion)
@@ -307,7 +309,7 @@ func TestUpdateSpecificVersionService(t *testing.T) {
 	require.Equal(t, reqVersion, v.Version())
 
 	require.NoError(t, v.Apply())
-	dat, err := os.ReadFile(testFilePath)
+	dat, err := ioutil.ReadFile(testFilePath)
 	require.NoError(t, err)
 
 	require.Equal(t, reqVersion, string(dat))
@@ -326,7 +328,7 @@ func TestCompressedUpdateService(t *testing.T) {
 	require.Equal(t, "2020.09.02", v.Version())
 
 	require.NoError(t, v.Apply())
-	dat, err := os.ReadFile(testFilePath)
+	dat, err := ioutil.ReadFile(testFilePath)
 	require.NoError(t, err)
 
 	require.Equal(t, "2020.09.02", string(dat))

cmd/cloudflared/updater/workers_update.go

@@ -25,13 +25,14 @@ const (
 	// rename cloudflared.exe.new to cloudflared.exe
 	// delete cloudflared.exe.old
 	// start the service
-	// exit with code 0 if we've reached this point indicating success.
-	windowsUpdateCommandTemplate = `sc stop cloudflared >nul 2>&1
+	// delete the batch file
+	windowsUpdateCommandTemplate = `@echo off
+sc stop cloudflared >nul 2>&1
 rename "{{.TargetPath}}" {{.OldName}}
 rename "{{.NewPath}}" {{.BinaryName}}
 del "{{.OldPath}}"
 sc start cloudflared >nul 2>&1
-exit /b 0`
+del {{.BatchName}}`
 	batchFileName = "cfd_update.bat"
 )
 
@@ -213,9 +214,8 @@ func isValidChecksum(checksum, filePath string) error {
 // writeBatchFile writes a batch file out to disk
 // see the dicussion on why it has to be done this way
 func writeBatchFile(targetPath string, newPath string, oldPath string) error {
-	batchFilePath := filepath.Join(filepath.Dir(targetPath), batchFileName)
-	os.Remove(batchFilePath) //remove any failed updates before download
-	f, err := os.Create(batchFilePath)
+	os.Remove(batchFileName) //remove any failed updates before download
+	f, err := os.Create(batchFileName)
 	if err != nil {
 		return err
 	}
@@ -241,16 +241,6 @@ func writeBatchFile(targetPath string, newPath string, oldPath string) error {
 
 // run each OS command for windows
 func runWindowsBatch(batchFile string) error {
-	defer os.Remove(batchFile)
-	cmd := exec.Command("cmd", "/C", batchFile)
-	_, err := cmd.Output()
-	// Remove the batch file we created. Don't let this interfere with the error
-	// we report.
-	if err != nil {
-		if exitError, ok := err.(*exec.ExitError); ok {
-			return fmt.Errorf("Error during update : %s;", string(exitError.Stderr))
-		}
-
-	}
-	return err
+	cmd := exec.Command("cmd", "/c", batchFile)
+	return cmd.Start()
 }

cmd/cloudflared/windows_service.go

@@ -1,4 +1,5 @@
 //go:build windows
+// +build windows
 
 package main
 

component-tests/README.md

@@ -10,6 +10,7 @@
 cloudflared_binary: "cloudflared"
 tunnel: "3d539f97-cd3a-4d8e-c33b-65e9099c7a8d"
 credentials_file: "/Users/tunnel/.cloudflared/3d539f97-cd3a-4d8e-c33b-65e9099c7a8d.json"
+classic_hostname: "classic-tunnel-component-tests.example.com"
 origincert: "/Users/tunnel/.cloudflared/cert.pem"
 ingress:
 - hostname: named-tunnel-component-tests.example.com

component-tests/cli.py

@@ -2,9 +2,7 @@ import json
 import subprocess
 from time import sleep
 
-from constants import MANAGEMENT_HOST_NAME
 from setup import get_config_from_file
-from util import get_tunnel_connector_id
 
 SINGLE_CASE_TIMEOUT = 600
 
@@ -30,36 +28,6 @@ class CloudflaredCli:
         listed = self._run_command(cmd_args, "list")
         return json.loads(listed.stdout)
 
-    def get_management_token(self, config, config_path):
-        basecmd = [config.cloudflared_binary]
-        if config_path is not None:
-            basecmd += ["--config", str(config_path)]
-        origincert = get_config_from_file()["origincert"]
-        if origincert:
-            basecmd += ["--origincert", origincert]
-
-        cmd_args = ["tail", "token", config.get_tunnel_id()]
-        cmd = basecmd + cmd_args
-        result = run_subprocess(cmd, "token", self.logger, check=True, capture_output=True, timeout=15)
-        return json.loads(result.stdout.decode("utf-8").strip())["token"]
-    
-    def get_management_url(self, path, config, config_path):
-        access_jwt = self.get_management_token(config, config_path)
-        connector_id = get_tunnel_connector_id()
-        return f"https://{MANAGEMENT_HOST_NAME}/{path}?connector_id={connector_id}&access_token={access_jwt}"
-    
-    def get_management_wsurl(self, path, config, config_path):
-        access_jwt = self.get_management_token(config, config_path)
-        connector_id = get_tunnel_connector_id()
-        return f"wss://{MANAGEMENT_HOST_NAME}/{path}?connector_id={connector_id}&access_token={access_jwt}"
-
-    def get_connector_id(self, config): 
-        op = self.get_tunnel_info(config.get_tunnel_id())
-        connectors = []
-        for conn in op["conns"]:
-            connectors.append(conn["id"])
-        return connectors
-
     def get_tunnel_info(self, tunnel_id):
         info = self._run_command(["info", "--output", "json", tunnel_id], "info")
         return json.loads(info.stdout)

component-tests/config.py

@@ -30,7 +30,6 @@ class NamedTunnelBaseConfig(BaseConfig):
     tunnel: str = None
     credentials_file: str = None
     ingress: list = None
-    hostname: str = None
 
     def __post_init__(self):
         if self.tunnel is None:
@@ -42,10 +41,8 @@ class NamedTunnelBaseConfig(BaseConfig):
 
     def merge_config(self, additional):
         config = super(NamedTunnelBaseConfig, self).merge_config(additional)
-        if 'tunnel' not in config:
-            config['tunnel'] = self.tunnel
-        if 'credentials-file' not in config:
-            config['credentials-file'] = self.credentials_file
+        config['tunnel'] = self.tunnel
+        config['credentials-file'] = self.credentials_file
         # In some cases we want to override default ingress, such as in config tests
         if 'ingress' not in config:
             config['ingress'] = self.ingress
@@ -64,7 +61,7 @@ class NamedTunnelConfig(NamedTunnelBaseConfig):
                            self.merge_config(additional_config))
 
     def get_url(self):
-        return "https://" + self.hostname
+        return "https://" + self.ingress[0]['hostname']
 
     def base_config(self):
         config = self.full_config.copy()
@@ -87,9 +84,28 @@ class NamedTunnelConfig(NamedTunnelBaseConfig):
     def get_credentials_json(self):
         with open(self.credentials_file) as json_file:
             return json.load(json_file)
-        
+
+
 @dataclass(frozen=True)
-class QuickTunnelConfig(BaseConfig):
+class ClassicTunnelBaseConfig(BaseConfig):
+    hostname: str = None
+    origincert: str = None
+
+    def __post_init__(self):
+        if self.hostname is None:
+            raise TypeError("Field tunnel is not set")
+        if self.origincert is None:
+            raise TypeError("Field credentials_file is not set")
+
+    def merge_config(self, additional):
+        config = super(ClassicTunnelBaseConfig, self).merge_config(additional)
+        config['hostname'] = self.hostname
+        config['origincert'] = self.origincert
+        return config
+
+
+@dataclass(frozen=True)
+class ClassicTunnelConfig(ClassicTunnelBaseConfig):
     full_config: dict = None
     additional_config: InitVar[dict] = {}
 
@@ -99,6 +115,10 @@ class QuickTunnelConfig(BaseConfig):
         object.__setattr__(self, 'full_config',
                            self.merge_config(additional_config))
 
+    def get_url(self):
+        return "https://" + self.hostname
+
+
 @dataclass(frozen=True)
 class ProxyDnsConfig(BaseConfig):
     full_config = {

component-tests/config.yaml

@@ -1,8 +0,0 @@
-cloudflared_binary: "cloudflared"
-tunnel: "ae21a96c-24d1-4ce8-a6ba-962cba5976d3"
-credentials_file: "/Users/sudarsan/.cloudflared/ae21a96c-24d1-4ce8-a6ba-962cba5976d3.json"
-origincert: "/Users/sudarsan/.cloudflared/cert.pem"
-ingress:
-- hostname: named-tunnel-component-tests.example.com
-  service: hello_world
-- service: http_status:404

component-tests/conftest.py

@@ -5,14 +5,14 @@ from time import sleep
 import pytest
 import yaml
 
-from config import NamedTunnelConfig, ProxyDnsConfig, QuickTunnelConfig
+from config import NamedTunnelConfig, ClassicTunnelConfig, ProxyDnsConfig
 from constants import BACKOFF_SECS, PROXY_DNS_PORT
 from util import LOGGER
 
 
 class CfdModes(Enum):
     NAMED = auto()
-    QUICK = auto()
+    CLASSIC = auto()
     PROXY_DNS = auto()
 
 
@@ -26,7 +26,7 @@ def component_tests_config():
         config = yaml.safe_load(stream)
         LOGGER.info(f"component tests base config {config}")
 
-        def _component_tests_config(additional_config={}, cfd_mode=CfdModes.NAMED, run_proxy_dns=True, provide_ingress=True):
+        def _component_tests_config(additional_config={}, cfd_mode=CfdModes.NAMED, run_proxy_dns=True):
             if run_proxy_dns:
                 # Regression test for TUN-4177, running with proxy-dns should not prevent tunnels from running.
                 # So we run all tests with it.
@@ -36,25 +36,18 @@ def component_tests_config():
                 additional_config.pop("proxy-dns", None)
                 additional_config.pop("proxy-dns-port", None)
 
-            # Allows the ingress rules to be omitted from the provided config
-            ingress = []
-            if provide_ingress:
-                ingress = config['ingress']
-
-            # Provide the hostname to allow routing to the tunnel even if the ingress rule isn't defined in the config
-            hostname = config['ingress'][0]['hostname']
-
             if cfd_mode is CfdModes.NAMED:
                 return NamedTunnelConfig(additional_config=additional_config,
                                          cloudflared_binary=config['cloudflared_binary'],
                                          tunnel=config['tunnel'],
                                          credentials_file=config['credentials_file'],
-                                         ingress=ingress,
-                                         hostname=hostname)
+                                         ingress=config['ingress'])
+            elif cfd_mode is CfdModes.CLASSIC:
+                return ClassicTunnelConfig(
+                    additional_config=additional_config, cloudflared_binary=config['cloudflared_binary'],
+                    hostname=config['classic_hostname'], origincert=config['origincert'])
             elif cfd_mode is CfdModes.PROXY_DNS:
                 return ProxyDnsConfig(cloudflared_binary=config['cloudflared_binary'])
-            elif cfd_mode is CfdModes.QUICK:
-                return QuickTunnelConfig(additional_config=additional_config, cloudflared_binary=config['cloudflared_binary'])
             else:
                 raise Exception(f"Unknown cloudflared mode {cfd_mode}")
 

component-tests/constants.py

@@ -4,8 +4,7 @@ BACKOFF_SECS = 7
 MAX_LOG_LINES = 50
 
 PROXY_DNS_PORT = 9053
-MANAGEMENT_HOST_NAME = "management.argotunnel.com"
 
 
 def protocols():
-    return ["http2", "quic"]
+    return ["h2mux", "http2", "quic"]

component-tests/requirements.txt

@@ -1,8 +1,6 @@
-cloudflare==2.14.3
+cloudflare==2.8.15
 flaky==3.7.0
-pytest==7.3.1
-pytest-asyncio==0.21.0
-pyyaml==6.0.1
-requests==2.28.2
-retrying==1.3.4
-websockets==11.0.1
+pytest==6.2.2
+pyyaml==5.4.1
+requests==2.25.1
+retrying==1.3.3

component-tests/setup.py

@@ -74,13 +74,19 @@ def delete_tunnel(config):
 
 @retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
 def create_dns(config, hostname, type, content):
-    cf = CloudFlare.CloudFlare(debug=False, token=get_env("DNS_API_TOKEN"))
+    cf = CloudFlare.CloudFlare(debug=True, token=get_env("DNS_API_TOKEN"))
     cf.zones.dns_records.post(
         config["zone_tag"],
         data={'name': hostname, 'type': type, 'content': content, 'proxied': True}
     )
 
 
+def create_classic_dns(config, random_uuid):
+    classic_hostname = "classic-" + random_uuid + "." + config["zone_domain"]
+    create_dns(config, classic_hostname, "AAAA", "fd10:aec2:5dae::")
+    return classic_hostname
+
+
 def create_named_dns(config, random_uuid):
     hostname = "named-" + random_uuid + "." + config["zone_domain"]
     create_dns(config, hostname, "CNAME", config["tunnel"] + ".cfargotunnel.com")
@@ -89,7 +95,7 @@ def create_named_dns(config, random_uuid):
 
 @retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
 def delete_dns(config, hostname):
-    cf = CloudFlare.CloudFlare(debug=False, token=get_env("DNS_API_TOKEN"))
+    cf = CloudFlare.CloudFlare(debug=True, token=get_env("DNS_API_TOKEN"))
     zone_tag = config["zone_tag"]
     dns_records = cf.zones.dns_records.get(zone_tag, params={'name': hostname})
     if len(dns_records) > 0:
@@ -113,6 +119,7 @@ def create():
     Creates the necessary resources for the components test to run.
      - Creates a named tunnel with a random name.
      - Creates a random CNAME DNS entry for that tunnel.
+     - Creates a random AAAA DNS entry for a classic tunnel.
 
     Those created resources are added to the config (obtained from an environment variable).
     The resulting configuration is persisted for the tests to use.
@@ -122,6 +129,7 @@ def create():
 
     random_uuid = str(uuid.uuid4())
     config["tunnel"] = create_tunnel(config, origincert_path, random_uuid)
+    config["classic_hostname"] = create_classic_dns(config, random_uuid)
     config["ingress"] = [
         {
             "hostname": create_named_dns(config, random_uuid),
@@ -142,6 +150,7 @@ def cleanup():
     """
     config = get_config_from_file()
     delete_tunnel(config)
+    delete_dns(config, config["classic_hostname"])
     delete_dns(config, config["ingress"][0]["hostname"])
 
 

component-tests/test_config.py

@@ -36,17 +36,17 @@ class TestConfig:
         _ = start_cloudflared(tmp_path, config, validate_args)
 
         self.match_rule(tmp_path, config,
-                        "http://example.com/index.html", 0)
+                        "http://example.com/index.html", 1)
         self.match_rule(tmp_path, config,
-                        "https://example.com/index.html", 0)
+                        "https://example.com/index.html", 1)
         self.match_rule(tmp_path, config,
-                        "https://api.example.com/login", 1)
+                        "https://api.example.com/login", 2)
         self.match_rule(tmp_path, config,
-                        "https://wss.example.com", 2)
+                        "https://wss.example.com", 3)
         self.match_rule(tmp_path, config,
-                        "https://ssh.example.com", 3)
+                        "https://ssh.example.com", 4)
         self.match_rule(tmp_path, config,
-                        "https://api.example.com", 4)
+                        "https://api.example.com", 5)
 
     # This is used to check that the command tunnel ingress url <url> matches rule number <rule_num>. Note that rule number uses 1-based indexing
 

component-tests/test_logging.py

@@ -74,7 +74,7 @@ def assert_log_to_dir(config, log_dir):
 class TestLogging:
     def test_logging_to_terminal(self, tmp_path, component_tests_config):
         config = component_tests_config()
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True) as cloudflared:
+        with start_cloudflared(tmp_path, config, new_process=True) as cloudflared:
             wait_tunnel_ready(tunnel_url=config.get_url())
             assert_log_to_terminal(cloudflared)
 
@@ -85,7 +85,7 @@ class TestLogging:
             "logfile": str(log_file),
         }
         config = component_tests_config(extra_config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True, capture_output=False):
+        with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
             wait_tunnel_ready(tunnel_url=config.get_url(), cfd_logs=str(log_file))
             assert_log_in_file(log_file)
             assert_json_log(log_file)
@@ -98,6 +98,6 @@ class TestLogging:
             "log-directory": str(log_dir),
         }
         config = component_tests_config(extra_config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True, capture_output=False):
+        with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
             wait_tunnel_ready(tunnel_url=config.get_url(), cfd_logs=str(log_dir))
             assert_log_to_dir(config, log_dir)

component-tests/test_management.py

@@ -1,113 +0,0 @@
-#!/usr/bin/env python
-import requests
-from conftest import CfdModes
-from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
-from retrying import retry
-from cli import CloudflaredCli
-from util import LOGGER, write_config, start_cloudflared, wait_tunnel_ready, send_requests
-import platform
-
-"""
-Each test in TestManagement will:
-1. Acquire a management token from Cloudflare public API
-2. Make a request against the management service for the running tunnel
-"""
-class TestManagement:
-    """
-        test_get_host_details does the following:
-        1. It gets a management token from Tunnelstore using cloudflared tail token <tunnel_id>
-        2. It gets the connector_id after starting a cloudflare tunnel
-        3. It sends a request to the management host with the connector_id and management token
-        4. Asserts that the response has a hostname and ip.
-    """
-    def test_get_host_details(self, tmp_path, component_tests_config):
-        # TUN-7377 : wait_tunnel_ready does not work properly in windows.
-        # Skipping this test for windows for now and will address it as part of tun-7377
-        if platform.system() == "Windows":
-            return
-        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
-        LOGGER.debug(config)
-        headers = {}
-        headers["Content-Type"] = "application/json"
-        config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--label" , "test"], cfd_args=["run", "--hello-world"], new_process=True):
-            wait_tunnel_ready(tunnel_url=config.get_url(),
-                              require_min_connections=1)
-            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
-            connector_id = cfd_cli.get_connector_id(config)[0]
-            url = cfd_cli.get_management_url("host_details", config, config_path)
-            resp = send_request(url, headers=headers)
-            
-            # Assert response json.
-            assert resp.status_code == 200, "Expected cloudflared to return 200 for host details"
-            assert resp.json()["hostname"] == "custom:test", "Expected cloudflared to return hostname"
-            assert resp.json()["ip"] != "", "Expected cloudflared to return ip"
-            assert resp.json()["connector_id"] == connector_id, "Expected cloudflared to return connector_id"
-    
-    """
-        test_get_metrics will verify that the /metrics endpoint returns the prometheus metrics dump
-    """
-    def test_get_metrics(self, tmp_path, component_tests_config):
-        # TUN-7377 : wait_tunnel_ready does not work properly in windows.
-        # Skipping this test for windows for now and will address it as part of tun-7377
-        if platform.system() == "Windows":
-            return
-        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
-        LOGGER.debug(config)
-        config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--management-diagnostics"], new_process=True):
-            wait_tunnel_ready(require_min_connections=1)
-            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
-            url = cfd_cli.get_management_url("metrics", config, config_path)
-            resp = send_request(url)
-            
-            # Assert response.
-            assert resp.status_code == 200, "Expected cloudflared to return 200 for /metrics"
-            assert "# HELP build_info Build and version information" in resp.text, "Expected /metrics to have with the build_info details"
-
-    """
-        test_get_pprof_heap will verify that the /debug/pprof/heap endpoint returns a pprof/heap dump response
-    """
-    def test_get_pprof_heap(self, tmp_path, component_tests_config):
-        # TUN-7377 : wait_tunnel_ready does not work properly in windows.
-        # Skipping this test for windows for now and will address it as part of tun-7377
-        if platform.system() == "Windows":
-            return
-        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
-        LOGGER.debug(config)
-        config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1", "--management-diagnostics"], new_process=True):
-            wait_tunnel_ready(require_min_connections=1)
-            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
-            url = cfd_cli.get_management_url("debug/pprof/heap", config, config_path)
-            resp = send_request(url)
-            
-            # Assert response.
-            assert resp.status_code == 200, "Expected cloudflared to return 200 for /debug/pprof/heap"
-            assert resp.headers["Content-Type"] == "application/octet-stream", "Expected /debug/pprof/heap to have return a binary response"
-    
-    """
-        test_get_metrics_when_disabled will verify that diagnostic endpoints (such as /metrics) return 404 and are unmounted. 
-    """
-    def test_get_metrics_when_disabled(self, tmp_path, component_tests_config):
-        # TUN-7377 : wait_tunnel_ready does not work properly in windows.
-        # Skipping this test for windows for now and will address it as part of tun-7377
-        if platform.system() == "Windows":
-            return
-        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
-        LOGGER.debug(config)
-        config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True):
-            wait_tunnel_ready(require_min_connections=1)
-            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
-            url = cfd_cli.get_management_url("metrics", config, config_path)
-            resp = send_request(url)
-            
-            # Assert response.
-            assert resp.status_code == 404, "Expected cloudflared to return 404 for /metrics"
-
-
-@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
-def send_request(url, headers={}):
-    with requests.Session() as s:
-        return s.get(url, timeout=BACKOFF_SECS, headers=headers)

component-tests/test_pq.py

@@ -12,6 +12,6 @@ class TestPostQuantum:
     def test_post_quantum(self, tmp_path, component_tests_config):
         config = component_tests_config(self._extra_config())
         LOGGER.debug(config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], cfd_args=["run", "--post-quantum"], new_process=True):
+        with start_cloudflared(tmp_path, config, cfd_args=["run", "--post-quantum"], new_process=True):
             wait_tunnel_ready(tunnel_url=config.get_url(),
-                              require_min_connections=1)
+                              require_min_connections=4)

component-tests/test_proxy_dns.py

@@ -25,7 +25,7 @@ def run_test_scenario(tmp_path, component_tests_config, cfd_mode, run_proxy_dns)
 
     if cfd_mode == CfdModes.NAMED:
         expect_tunnel = True
-        pre_args = ["tunnel", "--ha-connections", "1"]
+        pre_args = ["tunnel"]
         args = ["run"]
     elif cfd_mode == CfdModes.PROXY_DNS:
         expect_proxy_dns = True

component-tests/test_quicktunnels.py

@@ -1,33 +0,0 @@
-#!/usr/bin/env python
-from conftest import CfdModes
-from constants import METRICS_PORT
-from util import LOGGER, start_cloudflared, wait_tunnel_ready, get_quicktunnel_url, send_requests
-
-class TestQuickTunnels:
-    def test_quick_tunnel(self, tmp_path, component_tests_config):
-        config = component_tests_config(cfd_mode=CfdModes.QUICK, run_proxy_dns=False)
-        LOGGER.debug(config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], cfd_args=["--hello-world"], new_process=True):
-            wait_tunnel_ready(require_min_connections=1)
-            url = get_quicktunnel_url()
-            send_requests(url, 3, True)
-    
-    def test_quick_tunnel_url(self, tmp_path, component_tests_config):
-        config = component_tests_config(cfd_mode=CfdModes.QUICK, run_proxy_dns=False)
-        LOGGER.debug(config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], cfd_args=["--url", f"http://localhost:{METRICS_PORT}/"], new_process=True):
-            wait_tunnel_ready(require_min_connections=1)
-            url = get_quicktunnel_url()
-            send_requests(url+"/ready", 3, True)
-
-    def test_quick_tunnel_proxy_dns_url(self, tmp_path, component_tests_config):
-        config = component_tests_config(cfd_mode=CfdModes.QUICK, run_proxy_dns=True)
-        LOGGER.debug(config)
-        failed_start = start_cloudflared(tmp_path, config, cfd_args=["--url", f"http://localhost:{METRICS_PORT}/"], expect_success=False)
-        assert failed_start.returncode == 1, "Expected cloudflared to fail to run with `proxy-dns` and `hello-world`"
-    
-    def test_quick_tunnel_proxy_dns_hello_world(self, tmp_path, component_tests_config):
-        config = component_tests_config(cfd_mode=CfdModes.QUICK, run_proxy_dns=True)
-        LOGGER.debug(config)
-        failed_start = start_cloudflared(tmp_path, config, cfd_args=["--hello-world"], expect_success=False)
-        assert failed_start.returncode == 1, "Expected cloudflared to fail to run with `proxy-dns` and `url`"

component-tests/test_reconnect.py

@@ -13,7 +13,7 @@ from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connecte
 
 @flaky(max_runs=3, min_passes=1)
 class TestReconnect:
-    default_ha_conns = 1
+    default_ha_conns = 4
     default_reconnect_secs = 15
     extra_config = {
         "stdin-control": True,
@@ -29,7 +29,7 @@ class TestReconnect:
     @pytest.mark.parametrize("protocol", protocols())
     def test_named_reconnect(self, tmp_path, component_tests_config, protocol):
         config = component_tests_config(self._extra_config(protocol))
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True, allow_input=True, capture_output=False) as cloudflared:
+        with start_cloudflared(tmp_path, config, new_process=True, allow_input=True, capture_output=False) as cloudflared:
             # Repeat the test multiple times because some issues only occur after multiple reconnects
             self.assert_reconnect(config, cloudflared, 5)
 

component-tests/test_tail.py

@@ -1,182 +0,0 @@
-#!/usr/bin/env python
-import asyncio
-import json
-import pytest
-import requests
-import websockets
-from websockets.client import connect, WebSocketClientProtocol
-from conftest import CfdModes
-from constants import MAX_RETRIES, BACKOFF_SECS
-from retrying import retry
-from cli import CloudflaredCli
-from util import LOGGER, start_cloudflared, write_config, wait_tunnel_ready
-
-class TestTail:
-    @pytest.mark.asyncio
-    async def test_start_stop_streaming(self, tmp_path, component_tests_config):
-        """
-        Validates that a websocket connection to management.argotunnel.com/logs can be opened
-        with the access token and start and stop streaming on-demand.
-        """
-        print("test_start_stop_streaming")
-        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
-        LOGGER.debug(config)
-        config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_args=["run", "--hello-world"], new_process=True):
-            wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=1)
-            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
-            url = cfd_cli.get_management_wsurl("logs", config, config_path)
-            async with connect(url, open_timeout=5, close_timeout=3) as websocket:
-                await websocket.send('{"type": "start_streaming"}')
-                await websocket.send('{"type": "stop_streaming"}')
-                await websocket.send('{"type": "start_streaming"}')
-                await websocket.send('{"type": "stop_streaming"}')
-
-    @pytest.mark.asyncio
-    async def test_streaming_logs(self, tmp_path, component_tests_config):
-        """
-        Validates that a streaming logs connection will stream logs
-        """
-        print("test_streaming_logs")
-        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
-        LOGGER.debug(config)
-        config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_args=["run", "--hello-world"], new_process=True):
-            wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=1)
-            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
-            url = cfd_cli.get_management_wsurl("logs", config, config_path)
-            async with connect(url, open_timeout=5, close_timeout=5) as websocket:
-                # send start_streaming
-                await websocket.send('{"type": "start_streaming"}')
-                # send some http requests to the tunnel to trigger some logs
-                await generate_and_validate_http_events(websocket, config.get_url(), 10)
-                # send stop_streaming
-                await websocket.send('{"type": "stop_streaming"}')
-
-    @pytest.mark.asyncio
-    async def test_streaming_logs_filters(self, tmp_path, component_tests_config):
-        """
-        Validates that a streaming logs connection will stream logs 
-        but not http when filters applied.
-        """
-        print("test_streaming_logs_filters")
-        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
-        LOGGER.debug(config)
-        config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_args=["run", "--hello-world"], new_process=True):
-            wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=1)
-            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
-            url = cfd_cli.get_management_wsurl("logs", config, config_path)
-            async with connect(url, open_timeout=5, close_timeout=5) as websocket:
-                # send start_streaming with tcp logs only
-                await websocket.send(json.dumps({
-                    "type": "start_streaming",
-                    "filters": {
-                        "events": ["tcp"],
-                        "level": "debug"
-                    }
-                }))
-                # don't expect any http logs
-                await generate_and_validate_no_log_event(websocket, config.get_url())
-                # send stop_streaming
-                await websocket.send('{"type": "stop_streaming"}')
-    
-    @pytest.mark.asyncio
-    async def test_streaming_logs_sampling(self, tmp_path, component_tests_config):
-        """
-        Validates that a streaming logs connection will stream logs with sampling.
-        """
-        print("test_streaming_logs_sampling")
-        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
-        LOGGER.debug(config)
-        config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_args=["run", "--hello-world"], new_process=True):
-            wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=1)
-            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
-            url = cfd_cli.get_management_wsurl("logs", config, config_path)
-            async with connect(url, open_timeout=5, close_timeout=5) as websocket:
-                # send start_streaming with info logs only
-                await websocket.send(json.dumps({
-                    "type": "start_streaming",
-                    "filters": {
-                        "sampling": 0.5
-                    }
-                }))
-                # don't expect any http logs
-                count = await generate_and_validate_http_events(websocket, config.get_url(), 10)
-                assert count < (10 * 2) # There are typically always two log lines for http requests (request and response)
-                # send stop_streaming
-                await websocket.send('{"type": "stop_streaming"}')
-
-    @pytest.mark.asyncio
-    async def test_streaming_logs_actor_override(self, tmp_path, component_tests_config):
-        """
-        Validates that a streaming logs session can be overriden by the same actor 
-        """
-        print("test_streaming_logs_actor_override")
-        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
-        LOGGER.debug(config)
-        config_path = write_config(tmp_path, config.full_config)
-        with start_cloudflared(tmp_path, config, cfd_args=["run", "--hello-world"], new_process=True):
-            wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=1)
-            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
-            url = cfd_cli.get_management_wsurl("logs", config, config_path)
-            task = asyncio.ensure_future(start_streaming_to_be_remotely_closed(url))
-            override_task = asyncio.ensure_future(start_streaming_override(url))
-            await asyncio.wait([task, override_task])
-            assert task.exception() == None, task.exception()
-            assert override_task.exception() == None, override_task.exception()
-
-async def start_streaming_to_be_remotely_closed(url):
-    async with connect(url, open_timeout=5, close_timeout=5) as websocket:
-        try:
-            await websocket.send(json.dumps({"type": "start_streaming"}))
-            await asyncio.sleep(10)
-            assert websocket.closed, "expected this request to be forcibly closed by the override"
-        except websockets.ConnectionClosed:
-            # we expect the request to be closed
-            pass
-
-async def start_streaming_override(url):
-    # wait for the first connection to be established
-    await asyncio.sleep(1)
-    async with connect(url, open_timeout=5, close_timeout=5) as websocket:
-        await websocket.send(json.dumps({"type": "start_streaming"}))
-        await asyncio.sleep(1)
-        await websocket.send(json.dumps({"type": "stop_streaming"}))
-        await asyncio.sleep(1)
-
-# Every http request has two log lines sent
-async def generate_and_validate_http_events(websocket: WebSocketClientProtocol, url: str, count_send: int):
-    for i in range(count_send):
-        send_request(url)
-    # There are typically always two log lines for http requests (request and response)
-    count = 0
-    while True:
-        try:
-            req_line = await asyncio.wait_for(websocket.recv(), 2)
-            log_line = json.loads(req_line)
-            assert log_line["type"] == "logs"
-            assert log_line["logs"][0]["event"] == "http"
-            count += 1
-        except asyncio.TimeoutError:
-            # ignore timeout from waiting for recv
-            break
-    return count
-
-# Every http request has two log lines sent
-async def generate_and_validate_no_log_event(websocket: WebSocketClientProtocol, url: str):
-    send_request(url)
-    try:
-        # wait for 5 seconds and make sure we hit the timeout and not recv any events
-        req_line = await asyncio.wait_for(websocket.recv(), 5)
-        assert req_line == None, "expected no logs for the specified filters"
-    except asyncio.TimeoutError:
-        pass
-
-@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
-def send_request(url, headers={}):
-    with requests.Session() as s:
-        resp = s.get(url, timeout=BACKOFF_SECS, headers=headers)
-        assert resp.status_code == 200, f"{url} returned {resp}"
-        return resp.status_code == 200

component-tests/test_termination.py

@@ -34,7 +34,7 @@ class TestTermination:
     def test_graceful_shutdown(self, tmp_path, component_tests_config, signal, protocol):
         config = component_tests_config(self._extra_config(protocol))
         with start_cloudflared(
-                tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"],  new_process=True, capture_output=False) as cloudflared:
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
             wait_tunnel_ready(tunnel_url=config.get_url())
 
             connected = threading.Condition()
@@ -56,7 +56,7 @@ class TestTermination:
     def test_shutdown_once_no_connection(self, tmp_path, component_tests_config, signal, protocol):
         config = component_tests_config(self._extra_config(protocol))
         with start_cloudflared(
-                tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True, capture_output=False) as cloudflared:
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
             wait_tunnel_ready(tunnel_url=config.get_url())
 
             connected = threading.Condition()
@@ -76,7 +76,7 @@ class TestTermination:
     def test_no_connection_shutdown(self, tmp_path, component_tests_config, signal, protocol):
         config = component_tests_config(self._extra_config(protocol))
         with start_cloudflared(
-                tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"], new_process=True, capture_output=False) as cloudflared:
+                tmp_path, config, new_process=True, capture_output=False) as cloudflared:
             wait_tunnel_ready(tunnel_url=config.get_url())
             with self.within_grace_period():
                 self.terminate_by_signal(cloudflared, signal)

component-tests/test_tunnel.py

@@ -1,45 +0,0 @@
-#!/usr/bin/env python
-import requests
-from conftest import CfdModes
-from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
-from retrying import retry
-from cli import CloudflaredCli
-from util import LOGGER, write_config, start_cloudflared, wait_tunnel_ready, send_requests
-import platform
-
-class TestTunnel:
-    '''Test tunnels with no ingress rules from config.yaml but ingress rules from CLI only'''
-
-    def test_tunnel_hello_world(self, tmp_path, component_tests_config):
-        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
-        LOGGER.debug(config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"],  cfd_args=["run", "--hello-world"], new_process=True):
-            wait_tunnel_ready(tunnel_url=config.get_url(),
-                              require_min_connections=1)
-    
-    def test_tunnel_url(self, tmp_path, component_tests_config):
-        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
-        LOGGER.debug(config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"],  cfd_args=["run", "--url", f"http://localhost:{METRICS_PORT}/"], new_process=True):
-            wait_tunnel_ready(require_min_connections=1)
-            send_requests(config.get_url()+"/ready", 3, True)
-
-    def test_tunnel_no_ingress(self, tmp_path, component_tests_config):
-        '''
-        Running a tunnel with no ingress rules provided from either config.yaml or CLI will still work but return 503
-        for all incoming requests.
-        '''
-        config = component_tests_config(cfd_mode=CfdModes.NAMED, run_proxy_dns=False, provide_ingress=False)
-        LOGGER.debug(config)
-        with start_cloudflared(tmp_path, config, cfd_pre_args=["tunnel", "--ha-connections", "1"],  cfd_args=["run"], new_process=True):
-            wait_tunnel_ready(require_min_connections=1)
-            resp = send_request(config.get_url()+"/")
-            assert resp.status_code == 503, "Expected cloudflared to return 503 for all requests with no ingress defined"
-            resp = send_request(config.get_url()+"/test")
-            assert resp.status_code == 503, "Expected cloudflared to return 503 for all requests with no ingress defined"
-
-
-@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
-def send_request(url, headers={}):
-    with requests.Session() as s:
-        return s.get(url, timeout=BACKOFF_SECS, headers=headers)

component-tests/util.py

@@ -4,25 +4,17 @@ import platform
 import subprocess
 from contextlib import contextmanager
 from time import sleep
-import sys
 
 import pytest
 
 import requests
 import yaml
-import json
 from retrying import retry
 
 from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
 
-def configure_logger():
-    logger = logging.getLogger(__name__)
-    logger.setLevel(logging.DEBUG)
-    handler = logging.StreamHandler(sys.stdout)
-    logger.addHandler(handler)
-    return logger
+LOGGER = logging.getLogger(__name__)
 
-LOGGER = configure_logger()
 
 def select_platform(plat):
     return pytest.mark.skipif(
@@ -43,7 +35,7 @@ def write_config(directory, config):
 
 
 def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False,
-                      allow_input=False, capture_output=True, root=False, skip_config_flag=False, expect_success=True):
+                      allow_input=False, capture_output=True, root=False, skip_config_flag=False):
 
     config_path = None
     if not skip_config_flag:
@@ -54,7 +46,8 @@ def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel
     if new_process:
         return run_cloudflared_background(cmd, allow_input, capture_output)
     # By setting check=True, it will raise an exception if the process exits with non-zero exit code
-    return subprocess.run(cmd, check=expect_success, capture_output=capture_output)
+    return subprocess.run(cmd, check=True, capture_output=capture_output)
+
 
 def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
     cmd = []
@@ -84,18 +77,7 @@ def run_cloudflared_background(cmd, allow_input, capture_output):
             cfd.terminate()
             if capture_output:
                 LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
-    
 
-def get_quicktunnel_url():
-    quicktunnel_url = f'http://localhost:{METRICS_PORT}/quicktunnel'
-    with requests.Session() as s:
-        resp = send_request(s, quicktunnel_url, True)
-
-        hostname = resp.json()["hostname"]
-        assert hostname, \
-            f"Quicktunnel endpoint returned {hostname} but we expected a url"
-
-        return f"https://{hostname}"
 
 def wait_tunnel_ready(tunnel_url=None, require_min_connections=1, cfd_logs=None):
     try:
@@ -113,14 +95,13 @@ def inner_wait_tunnel_ready(tunnel_url=None, require_min_connections=1):
     with requests.Session() as s:
         resp = send_request(s, metrics_url, True)
 
-        ready_connections = resp.json()["readyConnections"]
-
-        assert ready_connections >= require_min_connections, \
+        assert resp.json()["readyConnections"] >= require_min_connections, \
             f"Ready endpoint returned {resp.json()} but we expect at least {require_min_connections} connections"
 
         if tunnel_url is not None:
             send_request(s, tunnel_url, True)
 
+
 def _log_cloudflared_logs(cfd_logs):
     log_file = cfd_logs
     if os.path.isdir(cfd_logs):

config/configuration.go

@@ -39,6 +39,8 @@ var (
 )
 
 const (
+	DefaultCredentialFile = "cert.pem"
+
 	// BastionFlag is to enable bastion, or jump host, operation
 	BastionFlag = "bastion"
 )
@@ -257,6 +259,7 @@ type Configuration struct {
 }
 
 type WarpRoutingConfig struct {
+	Enabled        bool            `yaml:"enabled" json:"enabled"`
 	ConnectTimeout *CustomDuration `yaml:"connectTimeout" json:"connectTimeout,omitempty"`
 	TCPKeepAlive   *CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive,omitempty"`
 }

config/configuration_test.go

@@ -23,6 +23,7 @@ func TestConfigFileSettings(t *testing.T) {
 			Service:  "https://localhost:8001",
 		}
 		warpRouting = WarpRoutingConfig{
+			Enabled:        true,
 			ConnectTimeout: &CustomDuration{Duration: 2 * time.Second},
 			TCPKeepAlive:   &CustomDuration{Duration: 10 * time.Second},
 		}

connection/connection.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"math"
-	"net"
 	"net/http"
 	"strconv"
 	"strings"
@@ -40,6 +39,7 @@ type Orchestrator interface {
 	UpdateConfig(version int32, config []byte) *pogs.UpdateConfigurationResponse
 	GetConfigJSON() ([]byte, error)
 	GetOriginProxy() (OriginProxy, error)
+	WarpRoutingEnabled() (enabled bool)
 }
 
 type NamedTunnelProperties struct {
@@ -142,7 +142,6 @@ type TCPRequest struct {
 	LBProbe   bool
 	FlowID    string
 	CfTraceID string
-	ConnIndex uint8
 }
 
 // ReadWriteAcker is a readwriter with the ability to Acknowledge to the downstream (edge) that the origin has
@@ -156,16 +155,14 @@ type ReadWriteAcker interface {
 type HTTPResponseReadWriteAcker struct {
 	r   io.Reader
 	w   ResponseWriter
-	f   http.Flusher
 	req *http.Request
 }
 
 // NewHTTPResponseReadWriterAcker returns a new instance of HTTPResponseReadWriteAcker.
-func NewHTTPResponseReadWriterAcker(w ResponseWriter, flusher http.Flusher, req *http.Request) *HTTPResponseReadWriteAcker {
+func NewHTTPResponseReadWriterAcker(w ResponseWriter, req *http.Request) *HTTPResponseReadWriteAcker {
 	return &HTTPResponseReadWriteAcker{
 		r:   req.Body,
 		w:   w,
-		f:   flusher,
 		req: req,
 	}
 }
@@ -175,11 +172,7 @@ func (h *HTTPResponseReadWriteAcker) Read(p []byte) (int, error) {
 }
 
 func (h *HTTPResponseReadWriteAcker) Write(p []byte) (int, error) {
-	n, err := h.w.Write(p)
-	if n > 0 {
-		h.f.Flush()
-	}
-	return n, err
+	return h.w.Write(p)
 }
 
 // AckConnection acks an HTTP connection by sending a switch protocols status code that enables the caller to
@@ -203,55 +196,9 @@ func (h *HTTPResponseReadWriteAcker) AckConnection(tracePropagation string) erro
 	return h.w.WriteRespHeaders(resp.StatusCode, resp.Header)
 }
 
-// localProxyConnection emulates an incoming connection to cloudflared as a net.Conn.
-// Used when handling a "hijacked" connection from connection.ResponseWriter
-type localProxyConnection struct {
-	io.ReadWriteCloser
-}
-
-func (c *localProxyConnection) Read(b []byte) (int, error) {
-	return c.ReadWriteCloser.Read(b)
-}
-
-func (c *localProxyConnection) Write(b []byte) (int, error) {
-	return c.ReadWriteCloser.Write(b)
-}
-
-func (c *localProxyConnection) Close() error {
-	return c.ReadWriteCloser.Close()
-}
-
-func (c *localProxyConnection) LocalAddr() net.Addr {
-	// Unused LocalAddr
-	return &net.TCPAddr{IP: net.IPv6loopback, Port: 0, Zone: ""}
-}
-
-func (c *localProxyConnection) RemoteAddr() net.Addr {
-	// Unused RemoteAddr
-	return &net.TCPAddr{IP: net.IPv6loopback, Port: 0, Zone: ""}
-}
-
-func (c *localProxyConnection) SetDeadline(t time.Time) error {
-	// ignored since we can't set the read/write Deadlines for the tunnel back to origintunneld
-	return nil
-}
-
-func (c *localProxyConnection) SetReadDeadline(t time.Time) error {
-	// ignored since we can't set the read/write Deadlines for the tunnel back to origintunneld
-	return nil
-}
-
-func (c *localProxyConnection) SetWriteDeadline(t time.Time) error {
-	// ignored since we can't set the read/write Deadlines for the tunnel back to origintunneld
-	return nil
-}
-
-// ResponseWriter is the response path for a request back through cloudflared's tunnel.
 type ResponseWriter interface {
 	WriteRespHeaders(status int, header http.Header) error
 	AddTrailer(trailerName, trailerValue string)
-	http.ResponseWriter
-	http.Hijacker
 	io.Writer
 }
 

connection/connection_test.go

@@ -130,8 +130,7 @@ func wsEchoEndpoint(w ResponseWriter, r *http.Request) error {
 	}
 	wsCtx, cancel := context.WithCancel(r.Context())
 	readPipe, writePipe := io.Pipe()
-
-	wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, w.(http.Flusher), r), &log)
+	wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, r), &log)
 	go func() {
 		select {
 		case <-wsCtx.Done():
@@ -176,7 +175,7 @@ func wsFlakyEndpoint(w ResponseWriter, r *http.Request) error {
 	}
 	wsCtx, cancel := context.WithCancel(r.Context())
 
-	wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, w.(http.Flusher), r), &log)
+	wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, r), &log)
 
 	closedAfter := time.Millisecond * time.Duration(rand.Intn(50))
 	originConn := &flakyConn{closeAt: time.Now().Add(closedAfter)}

connection/control.go

@@ -2,13 +2,13 @@ package connection
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"net"
 	"time"
 
 	"github.com/rs/zerolog"
 
-	"github.com/cloudflare/cloudflared/management"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
@@ -85,7 +85,7 @@ func (c *controlStream) ServeControlStream(
 		return err
 	}
 
-	c.observer.logConnected(registrationDetails.UUID, c.connIndex, registrationDetails.Location, c.edgeAddress, c.protocol)
+	c.observer.logServerInfo(c.connIndex, registrationDetails.Location, c.edgeAddress, fmt.Sprintf("Connection %s registered with protocol: %s", registrationDetails.UUID, c.protocol))
 	c.observer.sendConnectedEvent(c.connIndex, c.protocol, registrationDetails.Location)
 	c.connectedFuse.Connected()
 
@@ -116,11 +116,7 @@ func (c *controlStream) waitForUnregister(ctx context.Context, rpcClient NamedTu
 
 	c.observer.sendUnregisteringEvent(c.connIndex)
 	rpcClient.GracefulShutdown(ctx, c.gracePeriod)
-	c.observer.log.Info().
-		Int(management.EventTypeKey, int(management.Cloudflared)).
-		Uint8(LogFieldConnIndex, c.connIndex).
-		IPAddr(LogFieldIPAddress, c.edgeAddress).
-		Msg("Unregistered tunnel connection")
+	c.observer.log.Info().Uint8(LogFieldConnIndex, c.connIndex).Msg("Unregistered tunnel connection")
 }
 
 func (c *controlStream) IsStopped() bool {

connection/h2mux.go

@@ -1,17 +1,46 @@
 package connection
 
 import (
+	"context"
+	"io"
+	"net"
+	"net/http"
 	"time"
 
+	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"golang.org/x/sync/errgroup"
 
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/tracing"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
 )
 
 const (
-	muxerTimeout = 5 * time.Second
+	muxerTimeout      = 5 * time.Second
+	openStreamTimeout = 30 * time.Second
 )
 
+type h2muxConnection struct {
+	orchestrator Orchestrator
+	gracePeriod  time.Duration
+	muxerConfig  *MuxerConfig
+	muxer        *h2mux.Muxer
+	// connectionID is only used by metrics, and prometheus requires labels to be string
+	connIndexStr string
+	connIndex    uint8
+
+	observer          *Observer
+	gracefulShutdownC <-chan struct{}
+	stoppedGracefully bool
+
+	log *zerolog.Logger
+
+	// newRPCClientFunc allows us to mock RPCs during testing
+	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
+}
+
 type MuxerConfig struct {
 	HeartbeatInterval  time.Duration
 	MaxHeartbeats      uint64
@@ -30,3 +59,180 @@ func (mc *MuxerConfig) H2MuxerConfig(h h2mux.MuxedStreamHandler, log *zerolog.Lo
 		CompressionQuality: mc.CompressionSetting,
 	}
 }
+
+// NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
+func NewH2muxConnection(
+	orchestrator Orchestrator,
+	gracePeriod time.Duration,
+	muxerConfig *MuxerConfig,
+	edgeConn net.Conn,
+	connIndex uint8,
+	observer *Observer,
+	gracefulShutdownC <-chan struct{},
+	log *zerolog.Logger,
+) (*h2muxConnection, error, bool) {
+	h := &h2muxConnection{
+		orchestrator:      orchestrator,
+		gracePeriod:       gracePeriod,
+		muxerConfig:       muxerConfig,
+		connIndexStr:      uint8ToString(connIndex),
+		connIndex:         connIndex,
+		observer:          observer,
+		gracefulShutdownC: gracefulShutdownC,
+		newRPCClientFunc:  newRegistrationRPCClient,
+		log:               log,
+	}
+
+	// Establish a muxed connection with the edge
+	// Client mux handshake with agent server
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig.H2MuxerConfig(h, observer.logTransport), h2mux.ActiveStreams)
+	if err != nil {
+		recoverable := isHandshakeErrRecoverable(err, connIndex, observer)
+		return nil, err, recoverable
+	}
+	h.muxer = muxer
+	return h, nil, false
+}
+
+func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelProperties, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() error {
+		if err := h.registerNamedTunnel(serveCtx, namedTunnel, connOptions); err != nil {
+			return err
+		}
+		connectedFuse.Connected()
+		return nil
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, true)
+		return nil
+	})
+
+	err := errGroup.Wait()
+	if err == errMuxerStopped {
+		if h.stoppedGracefully {
+			return nil
+		}
+		h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unexpected muxer shutdown")
+	}
+	return err
+}
+
+func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
+	// All routines should stop when muxer finish serving. When muxer is shutdown
+	// gracefully, it doesn't return an error, so we need to return errMuxerShutdown
+	// here to notify other routines to stop
+	err := h.muxer.Serve(ctx)
+	if err == nil {
+		return errMuxerStopped
+	}
+	return err
+}
+
+func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse ConnectedFuse, isNamedTunnel bool) {
+	updateMetricsTicker := time.NewTicker(h.muxerConfig.MetricsUpdateFreq)
+	defer updateMetricsTicker.Stop()
+	var shutdownCompleted <-chan struct{}
+	for {
+		select {
+		case <-h.gracefulShutdownC:
+			if connectedFuse.IsConnected() {
+				h.unregister(isNamedTunnel)
+			}
+			h.stoppedGracefully = true
+			h.gracefulShutdownC = nil
+			shutdownCompleted = h.muxer.Shutdown()
+
+		case <-shutdownCompleted:
+			return
+
+		case <-ctx.Done():
+			// UnregisterTunnel blocks until the RPC call returns
+			if !h.stoppedGracefully && connectedFuse.IsConnected() {
+				h.unregister(isNamedTunnel)
+			}
+			h.muxer.Shutdown()
+			// don't wait for shutdown to finish when context is closed, this is the hard termination path
+			return
+
+		case <-updateMetricsTicker.C:
+			h.observer.metrics.updateMuxerMetrics(h.connIndexStr, h.muxer.Metrics())
+		}
+	}
+}
+
+func (h *h2muxConnection) newRPCStream(ctx context.Context, rpcName rpcName) (*h2mux.MuxedStream, error) {
+	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer openStreamCancel()
+	stream, err := h.muxer.OpenRPCStream(openStreamCtx)
+	if err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
+
+func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
+	respWriter := &h2muxRespWriter{stream}
+
+	req, reqErr := h.newRequest(stream)
+	if reqErr != nil {
+		respWriter.WriteErrorResponse()
+		return reqErr
+	}
+
+	var sourceConnectionType = TypeHTTP
+	if websocket.IsWebSocketUpgrade(req) {
+		sourceConnectionType = TypeWebsocket
+	}
+
+	originProxy, err := h.orchestrator.GetOriginProxy()
+	if err != nil {
+		respWriter.WriteErrorResponse()
+		return err
+	}
+
+	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, h.log), sourceConnectionType == TypeWebsocket)
+	if err != nil {
+		respWriter.WriteErrorResponse()
+	}
+	return err
+}
+
+func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request, error) {
+	req, err := http.NewRequest("GET", "http://localhost:8080", h2mux.MuxedStreamReader{MuxedStream: stream})
+	if err != nil {
+		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
+	}
+	err = H2RequestHeadersToH1Request(stream.Headers, req)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid request received")
+	}
+	return req, nil
+}
+
+type h2muxRespWriter struct {
+	*h2mux.MuxedStream
+}
+
+func (rp *h2muxRespWriter) AddTrailer(trailerName, trailerValue string) {
+	// do nothing. we don't support trailers over h2mux
+}
+
+func (rp *h2muxRespWriter) WriteRespHeaders(status int, header http.Header) error {
+	headers := H1ResponseToH2ResponseHeaders(status, header)
+	headers = append(headers, h2mux.Header{Name: ResponseMetaHeader, Value: responseMetaHeaderOrigin})
+	return rp.WriteHeaders(headers)
+}
+
+func (rp *h2muxRespWriter) WriteErrorResponse() {
+	_ = rp.WriteHeaders([]h2mux.Header{
+		{Name: ":status", Value: "502"},
+		{Name: ResponseMetaHeader, Value: responseMetaHeaderCfd},
+	})
+	_, _ = rp.Write([]byte("502 Bad Gateway"))
+}

connection/h2mux_test.go

@@ -0,0 +1,301 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/gobwas/ws/wsutil"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+)
+
+var (
+	testMuxerConfig = &MuxerConfig{
+		HeartbeatInterval:  time.Second * 5,
+		MaxHeartbeats:      5,
+		CompressionSetting: 0,
+		MetricsUpdateFreq:  time.Second * 5,
+	}
+)
+
+func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
+	edgeConn, originConn := net.Pipe()
+	edgeMuxChan := make(chan *h2mux.Muxer)
+	go func() {
+		edgeMuxConfig := h2mux.MuxerConfig{
+			Log: &log,
+			Handler: h2mux.MuxedStreamFunc(func(stream *h2mux.MuxedStream) error {
+				// we only expect RPC traffic in client->edge direction, provide minimal support for mocking
+				require.True(t, stream.IsRPCStream())
+				return stream.WriteHeaders([]h2mux.Header{
+					{Name: ":status", Value: "200"},
+				})
+			}),
+		}
+		edgeMux, err := h2mux.Handshake(edgeConn, edgeConn, edgeMuxConfig, h2mux.ActiveStreams)
+		require.NoError(t, err)
+		edgeMuxChan <- edgeMux
+	}()
+	var connIndex = uint8(0)
+	testObserver := NewObserver(&log, &log)
+	h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil, &log)
+	require.NoError(t, err)
+	return h2muxConn, <-edgeMuxChan
+}
+
+func TestServeStreamHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "/ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "/large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "/400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "/500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "/error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		_ = edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	for _, test := range tests {
+		headers := []h2mux.Header{
+			{
+				Name:  ":path",
+				Value: test.endpoint,
+			},
+		}
+		stream, err := edgeMux.OpenStream(ctx, headers, nil)
+		require.NoError(t, err)
+		require.True(t, hasHeader(stream, ":status", strconv.Itoa(test.expectedStatus)))
+
+		if test.isProxyError {
+			assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderCfd))
+		} else {
+			assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
+			body := make([]byte, len(test.expectedBody))
+			_, err = stream.Read(body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, body)
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+func TestServeStreamWS(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: "/ws/echo",
+		},
+		{
+			Name:  "connection",
+			Value: "upgrade",
+		},
+		{
+			Name:  "upgrade",
+			Value: "websocket",
+		},
+	}
+
+	readPipe, writePipe := io.Pipe()
+	stream, err := edgeMux.OpenStream(ctx, headers, readPipe)
+	require.NoError(t, err)
+
+	require.True(t, hasHeader(stream, ":status", strconv.Itoa(http.StatusSwitchingProtocols)))
+	assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientBinary(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerBinary(stream)
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	wg.Wait()
+}
+
+func TestGracefulShutdownH2Mux(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	h2muxConn, edgeMux := newH2MuxConnection(t)
+
+	shutdownC := make(chan struct{})
+	unregisteredC := make(chan struct{})
+	h2muxConn.gracefulShutdownC = shutdownC
+	h2muxConn.newRPCClientFunc = func(_ context.Context, _ io.ReadWriteCloser, _ *zerolog.Logger) NamedTunnelRPCClient {
+		return &mockNamedTunnelRPCClient{
+			registered:   nil,
+			unregistered: unregisteredC,
+		}
+	}
+
+	var wg sync.WaitGroup
+	wg.Add(3)
+	go func() {
+		defer wg.Done()
+		_ = edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		_ = h2muxConn.serveMuxer(ctx)
+	}()
+
+	go func() {
+		defer wg.Done()
+		h2muxConn.controlLoop(ctx, &mockConnectedFuse{}, true)
+	}()
+
+	time.Sleep(100 * time.Millisecond)
+	close(shutdownC)
+
+	select {
+	case <-unregisteredC:
+		break // ok
+	case <-time.Tick(time.Second):
+		assert.Fail(t, "timed out waiting for control loop to unregister")
+	}
+
+	cancel()
+	wg.Wait()
+
+	assert.True(t, h2muxConn.stoppedGracefully)
+	assert.Nil(t, h2muxConn.gracefulShutdownC)
+}
+
+func hasHeader(stream *h2mux.MuxedStream, name, val string) bool {
+	for _, header := range stream.Headers {
+		if header.Name == name && header.Value == val {
+			return true
+		}
+	}
+	return false
+}
+
+func benchmarkServeStreamHTTPSimple(b *testing.B, test testRequest) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(b)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(b, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: test.endpoint,
+		},
+	}
+
+	body := make([]byte, len(test.expectedBody))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		stream, openstreamErr := edgeMux.OpenStream(ctx, headers, nil)
+		_, readBodyErr := stream.Read(body)
+		b.StopTimer()
+
+		require.NoError(b, openstreamErr)
+		assert.True(b, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
+		require.True(b, hasHeader(stream, ":status", strconv.Itoa(http.StatusOK)))
+		require.NoError(b, readBodyErr)
+		require.Equal(b, test.expectedBody, body)
+	}
+
+	cancel()
+	wg.Wait()
+}
+
+func BenchmarkServeStreamHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "/ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}
+
+func BenchmarkServeStreamHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "/large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}

connection/http2.go

@@ -1,7 +1,6 @@
 package connection
 
 import (
-	"bufio"
 	"context"
 	gojson "encoding/json"
 	"fmt"
@@ -130,7 +129,7 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	case TypeWebsocket, TypeHTTP:
 		stripWebsocketUpgradeHeader(r)
 		// Check for tracing on request
-		tr := tracing.NewTracedHTTPRequest(r, c.connIndex, c.log)
+		tr := tracing.NewTracedHTTPRequest(r, c.log)
 		if err := originProxy.ProxyHTTP(respWriter, tr, connType == TypeWebsocket); err != nil {
 			requestErr = fmt.Errorf("Failed to proxy HTTP: %w", err)
 		}
@@ -142,13 +141,12 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			break
 		}
 
-		rws := NewHTTPResponseReadWriterAcker(respWriter, respWriter, r)
+		rws := NewHTTPResponseReadWriterAcker(respWriter, r)
 		requestErr = originProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
 			Dest:      host,
 			CFRay:     FindCfRayHeader(r),
 			LBProbe:   IsLBProbeRequest(r),
 			CfTraceID: r.Header.Get(tracing.TracerContextName),
-			ConnIndex: c.connIndex,
 		})
 
 	default:
@@ -198,9 +196,6 @@ type http2RespWriter struct {
 	flusher       http.Flusher
 	shouldFlush   bool
 	statusWritten bool
-	respHeaders   http.Header
-	hijackedMutex sync.Mutex
-	hijackedv     bool
 	log           *zerolog.Logger
 }
 
@@ -221,7 +216,6 @@ func NewHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type, l
 		w:           w,
 		flusher:     flusher,
 		shouldFlush: connType.shouldFlush(),
-		respHeaders: make(http.Header),
 		log:         log,
 	}, nil
 }
@@ -236,10 +230,6 @@ func (rp *http2RespWriter) AddTrailer(trailerName, trailerValue string) {
 }
 
 func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) error {
-	if rp.hijacked() {
-		rp.log.Warn().Msg("WriteRespHeaders after hijack")
-		return nil
-	}
 	dest := rp.w.Header()
 	userHeaders := make(http.Header, len(header))
 	for name, values := range header {
@@ -285,52 +275,6 @@ func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) erro
 	return nil
 }
 
-func (rp *http2RespWriter) Header() http.Header {
-	return rp.respHeaders
-}
-
-func (rp *http2RespWriter) Flush() {
-	rp.flusher.Flush()
-}
-
-func (rp *http2RespWriter) WriteHeader(status int) {
-	if rp.hijacked() {
-		rp.log.Warn().Msg("WriteHeader after hijack")
-		return
-	}
-	rp.WriteRespHeaders(status, rp.respHeaders)
-}
-
-func (rp *http2RespWriter) hijacked() bool {
-	rp.hijackedMutex.Lock()
-	defer rp.hijackedMutex.Unlock()
-	return rp.hijackedv
-}
-
-func (rp *http2RespWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
-	if !rp.statusWritten {
-		return nil, nil, fmt.Errorf("status not yet written before attempting to hijack connection")
-	}
-	// Make sure to flush anything left in the buffer before hijacking
-	if rp.shouldFlush {
-		rp.flusher.Flush()
-	}
-	rp.hijackedMutex.Lock()
-	defer rp.hijackedMutex.Unlock()
-	if rp.hijackedv {
-		return nil, nil, http.ErrHijacked
-	}
-	rp.hijackedv = true
-	conn := &localProxyConnection{rp}
-	// We return the http2RespWriter here because we want to make sure that we flush after every write
-	// otherwise the HTTP2 write buffer waits a few seconds before sending.
-	readWriter := bufio.NewReadWriter(
-		bufio.NewReader(rp),
-		bufio.NewWriter(rp),
-	)
-	return conn, readWriter, nil
-}
-
 func (rp *http2RespWriter) WriteErrorResponse() bool {
 	if rp.statusWritten {
 		return false

connection/http2_test.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httptest"
@@ -83,7 +84,7 @@ func TestHTTP2ConfigurationSet(t *testing.T) {
 	resp, err := edgeHTTP2Conn.RoundTrip(req)
 	require.NoError(t, err)
 	require.Equal(t, http.StatusOK, resp.StatusCode)
-	bdy, err := io.ReadAll(resp.Body)
+	bdy, err := ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
 	assert.Equal(t, `{"lastAppliedVersion":2,"err":null}`, string(bdy))
 	cancel()
@@ -148,7 +149,7 @@ func TestServeHTTP(t *testing.T) {
 		require.NoError(t, err)
 		require.Equal(t, test.expectedStatus, resp.StatusCode)
 		if test.expectedBody != nil {
-			respBody, err := io.ReadAll(resp.Body)
+			respBody, err := ioutil.ReadAll(resp.Body)
 			require.NoError(t, err)
 			require.Equal(t, test.expectedBody, respBody)
 		}
@@ -545,7 +546,7 @@ func benchmarkServeHTTP(b *testing.B, test testRequest) {
 		require.NoError(b, err)
 		require.Equal(b, test.expectedStatus, resp.StatusCode)
 		if test.expectedBody != nil {
-			respBody, err := io.ReadAll(resp.Body)
+			respBody, err := ioutil.ReadAll(resp.Body)
 			require.NoError(b, err)
 			require.Equal(b, test.expectedBody, respBody)
 		}

connection/observer.go

@@ -4,17 +4,12 @@ import (
 	"net"
 	"strings"
 
-	"github.com/google/uuid"
 	"github.com/rs/zerolog"
-
-	"github.com/cloudflare/cloudflared/management"
 )
 
 const (
-	LogFieldConnectionID      = "connection"
 	LogFieldLocation          = "location"
 	LogFieldIPAddress         = "ip"
-	LogFieldProtocol          = "protocol"
 	observerChannelBufferSize = 16
 )
 
@@ -46,16 +41,13 @@ func (o *Observer) RegisterSink(sink EventSink) {
 	o.addSinkChan <- sink
 }
 
-func (o *Observer) logConnected(connectionID uuid.UUID, connIndex uint8, location string, address net.IP, protocol Protocol) {
+func (o *Observer) logServerInfo(connIndex uint8, location string, address net.IP, msg string) {
 	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
 	o.log.Info().
-		Int(management.EventTypeKey, int(management.Cloudflared)).
-		Str(LogFieldConnectionID, connectionID.String()).
 		Uint8(LogFieldConnIndex, connIndex).
 		Str(LogFieldLocation, location).
 		IPAddr(LogFieldIPAddress, address).
-		Str(LogFieldProtocol, protocol.String()).
-		Msg("Registered tunnel connection")
+		Msg(msg)
 	o.metrics.registerServerLocation(uint8ToString(connIndex), location)
 }
 

connection/protocol.go

@@ -1,6 +1,7 @@
 package connection
 
 import (
+	"errors"
 	"fmt"
 	"hash/fnv"
 	"sync"
@@ -12,7 +13,7 @@ import (
 )
 
 const (
-	AvailableProtocolFlagMessage = "Available protocols: 'auto' - automatically chooses the best protocol over time (the default; and also the recommended one); 'quic' - based on QUIC, relying on UDP egress to Cloudflare edge; 'http2' - using Go's HTTP2 library, relying on TCP egress to Cloudflare edge"
+	AvailableProtocolFlagMessage = "Available protocols: 'auto' - automatically chooses the best protocol over time (the default; and also the recommended one); 'quic' - based on QUIC, relying on UDP egress to Cloudflare edge; 'http2' - using Go's HTTP2 library, relying on TCP egress to Cloudflare edge; 'h2mux' - Cloudflare's implementation of HTTP/2, deprecated"
 	// edgeH2muxTLSServerName is the server name to establish h2mux connection with edge
 	edgeH2muxTLSServerName = "cftunnel.com"
 	// edgeH2TLSServerName is the server name to establish http2 connection with edge
@@ -20,32 +21,43 @@ const (
 	// edgeQUICServerName is the server name to establish quic connection with edge.
 	edgeQUICServerName = "quic.cftunnel.com"
 	AutoSelectFlag     = "auto"
-	// SRV and TXT record resolution TTL
-	ResolveTTL = time.Hour
 )
 
 var (
-	// ProtocolList represents a list of supported protocols for communication with the edge
-	// in order of precedence for remote percentage fetcher.
-	ProtocolList = []Protocol{QUIC, HTTP2}
+	// ProtocolList represents a list of supported protocols for communication with the edge.
+	ProtocolList = []Protocol{H2mux, HTTP2, HTTP2Warp, QUIC, QUICWarp}
 )
 
 type Protocol int64
 
 const (
-	// HTTP2 using golang HTTP2 library for edge connections.
-	HTTP2 Protocol = iota
-	// QUIC using quic-go for edge connections.
+	// H2mux protocol can be used both with Classic and Named Tunnels. .
+	H2mux Protocol = iota
+	// HTTP2 is used only with named tunnels. It's more efficient than H2mux for L4 proxying.
+	HTTP2
+	// QUIC is used only with named tunnels.
 	QUIC
+	// HTTP2Warp is used only with named tunnels. It's useful for warp-routing where we don't want to fallback to
+	// H2mux on HTTP2 failure to connect.
+	HTTP2Warp
+	//QUICWarp is used only with named tunnels. It's useful for warp-routing where we want to fallback to HTTP2 but
+	// don't want HTTP2 to fallback to H2mux
+	QUICWarp
 )
 
 // Fallback returns the fallback protocol and whether the protocol has a fallback
 func (p Protocol) fallback() (Protocol, bool) {
 	switch p {
+	case H2mux:
+		return 0, false
 	case HTTP2:
+		return H2mux, true
+	case HTTP2Warp:
 		return 0, false
 	case QUIC:
 		return HTTP2, true
+	case QUICWarp:
+		return HTTP2Warp, true
 	default:
 		return 0, false
 	}
@@ -53,9 +65,11 @@ func (p Protocol) fallback() (Protocol, bool) {
 
 func (p Protocol) String() string {
 	switch p {
-	case HTTP2:
+	case H2mux:
+		return "h2mux"
+	case HTTP2, HTTP2Warp:
 		return "http2"
-	case QUIC:
+	case QUIC, QUICWarp:
 		return "quic"
 	default:
 		return fmt.Sprintf("unknown protocol")
@@ -64,11 +78,15 @@ func (p Protocol) String() string {
 
 func (p Protocol) TLSSettings() *TLSSettings {
 	switch p {
-	case HTTP2:
+	case H2mux:
+		return &TLSSettings{
+			ServerName: edgeH2muxTLSServerName,
+		}
+	case HTTP2, HTTP2Warp:
 		return &TLSSettings{
 			ServerName: edgeH2TLSServerName,
 		}
-	case QUIC:
+	case QUIC, QUICWarp:
 		return &TLSSettings{
 			ServerName: edgeQUICServerName,
 			NextProtos: []string{"argotunnel"},
@@ -88,7 +106,6 @@ type ProtocolSelector interface {
 	Fallback() (Protocol, bool)
 }
 
-// staticProtocolSelector will not provide a different protocol for Fallback
 type staticProtocolSelector struct {
 	current Protocol
 }
@@ -98,11 +115,10 @@ func (s *staticProtocolSelector) Current() Protocol {
 }
 
 func (s *staticProtocolSelector) Fallback() (Protocol, bool) {
-	return s.current, false
+	return 0, false
 }
 
-// remoteProtocolSelector will fetch a list of remote protocols to provide for edge discovery
-type remoteProtocolSelector struct {
+type autoProtocolSelector struct {
 	lock sync.RWMutex
 
 	current Protocol
@@ -111,21 +127,23 @@ type remoteProtocolSelector struct {
 	protocolPool []Protocol
 
 	switchThreshold int32
-	fetchFunc       edgediscovery.PercentageFetcher
+	fetchFunc       PercentageFetcher
 	refreshAfter    time.Time
 	ttl             time.Duration
 	log             *zerolog.Logger
+	needPQ          bool
 }
 
-func newRemoteProtocolSelector(
+func newAutoProtocolSelector(
 	current Protocol,
 	protocolPool []Protocol,
 	switchThreshold int32,
-	fetchFunc edgediscovery.PercentageFetcher,
+	fetchFunc PercentageFetcher,
 	ttl time.Duration,
 	log *zerolog.Logger,
-) *remoteProtocolSelector {
-	return &remoteProtocolSelector{
+	needPQ bool,
+) *autoProtocolSelector {
+	return &autoProtocolSelector{
 		current:         current,
 		protocolPool:    protocolPool,
 		switchThreshold: switchThreshold,
@@ -133,10 +151,11 @@ func newRemoteProtocolSelector(
 		refreshAfter:    time.Now().Add(ttl),
 		ttl:             ttl,
 		log:             log,
+		needPQ:          needPQ,
 	}
 }
 
-func (s *remoteProtocolSelector) Current() Protocol {
+func (s *autoProtocolSelector) Current() Protocol {
 	s.lock.Lock()
 	defer s.lock.Unlock()
 	if time.Now().Before(s.refreshAfter) {
@@ -154,13 +173,7 @@ func (s *remoteProtocolSelector) Current() Protocol {
 	return s.current
 }
 
-func (s *remoteProtocolSelector) Fallback() (Protocol, bool) {
-	s.lock.RLock()
-	defer s.lock.RUnlock()
-	return s.current.fallback()
-}
-
-func getProtocol(protocolPool []Protocol, fetchFunc edgediscovery.PercentageFetcher, switchThreshold int32) (Protocol, error) {
+func getProtocol(protocolPool []Protocol, fetchFunc PercentageFetcher, switchThreshold int32) (Protocol, error) {
 	protocolPercentages, err := fetchFunc()
 	if err != nil {
 		return 0, err
@@ -172,78 +185,112 @@ func getProtocol(protocolPool []Protocol, fetchFunc edgediscovery.PercentageFetc
 		}
 	}
 
-	// Default to first index in protocolPool list
-	return protocolPool[0], nil
+	return protocolPool[len(protocolPool)-1], nil
 }
 
-// defaultProtocolSelector will allow for a protocol to have a fallback
-type defaultProtocolSelector struct {
-	lock    sync.RWMutex
-	current Protocol
-}
-
-func newDefaultProtocolSelector(
-	current Protocol,
-) *defaultProtocolSelector {
-	return &defaultProtocolSelector{
-		current: current,
-	}
-}
-
-func (s *defaultProtocolSelector) Current() Protocol {
-	s.lock.Lock()
-	defer s.lock.Unlock()
-	return s.current
-}
-
-func (s *defaultProtocolSelector) Fallback() (Protocol, bool) {
+func (s *autoProtocolSelector) Fallback() (Protocol, bool) {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
+	if s.needPQ {
+		return 0, false
+	}
 	return s.current.fallback()
 }
 
+type PercentageFetcher func() (edgediscovery.ProtocolPercents, error)
+
 func NewProtocolSelector(
 	protocolFlag string,
-	accountTag string,
-	tunnelTokenProvided bool,
-	needPQ bool,
-	protocolFetcher edgediscovery.PercentageFetcher,
-	resolveTTL time.Duration,
+	warpRoutingEnabled bool,
+	namedTunnel *NamedTunnelProperties,
+	fetchFunc PercentageFetcher,
+	ttl time.Duration,
 	log *zerolog.Logger,
+	needPQ bool,
 ) (ProtocolSelector, error) {
-	// With --post-quantum, we force quic
-	if needPQ {
+	// Classic tunnel is only supported with h2mux
+	if namedTunnel == nil {
+		if needPQ {
+			return nil, errors.New("Classic tunnel does not support post-quantum")
+		}
+
 		return &staticProtocolSelector{
-			current: QUIC,
+			current: H2mux,
 		}, nil
 	}
 
-	threshold := switchThreshold(accountTag)
-	fetchedProtocol, err := getProtocol(ProtocolList, protocolFetcher, threshold)
-	log.Debug().Msgf("Fetched protocol: %s", fetchedProtocol)
-	if err != nil {
-		log.Warn().Msg("Unable to lookup protocol percentage.")
-		// Falling through here since 'auto' is handled in the switch and failing
-		// to do the protocol lookup isn't a failure since it can be triggered again
-		// after the TTL.
+	threshold := switchThreshold(namedTunnel.Credentials.AccountTag)
+	fetchedProtocol, err := getProtocol([]Protocol{QUIC, HTTP2}, fetchFunc, threshold)
+	if err != nil && protocolFlag == "auto" {
+		log.Err(err).Msg("Unable to lookup protocol. Defaulting to `http2`. If this fails, you can attempt `--protocol quic` instead.")
+		if needPQ {
+			return nil, errors.New("http2 does not support post-quantum")
+		}
+		return &staticProtocolSelector{
+			current: HTTP2,
+		}, nil
+	}
+	if warpRoutingEnabled {
+		if protocolFlag == H2mux.String() || fetchedProtocol == H2mux {
+			log.Warn().Msg("Warp routing is not supported in h2mux protocol. Upgrading to http2 to allow it.")
+			protocolFlag = HTTP2.String()
+			fetchedProtocol = HTTP2Warp
+		}
+		return selectWarpRoutingProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol, needPQ)
 	}
 
+	return selectNamedTunnelProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol, needPQ)
+}
+
+func selectNamedTunnelProtocols(
+	protocolFlag string,
+	fetchFunc PercentageFetcher,
+	ttl time.Duration,
+	log *zerolog.Logger,
+	threshold int32,
+	protocol Protocol,
+	needPQ bool,
+) (ProtocolSelector, error) {
 	// If the user picks a protocol, then we stick to it no matter what.
 	switch protocolFlag {
-	case "h2mux":
-		// Any users still requesting h2mux will be upgraded to http2 instead
-		log.Warn().Msg("h2mux is no longer a supported protocol: upgrading edge connection to http2. Please remove '--protocol h2mux' from runtime arguments to remove this warning.")
-		return &staticProtocolSelector{current: HTTP2}, nil
+	case H2mux.String():
+		return &staticProtocolSelector{current: H2mux}, nil
 	case QUIC.String():
 		return &staticProtocolSelector{current: QUIC}, nil
 	case HTTP2.String():
 		return &staticProtocolSelector{current: HTTP2}, nil
-	case AutoSelectFlag:
-		// When a --token is provided, we want to start with QUIC but have fallback to HTTP2
-		if tunnelTokenProvided {
-			return newDefaultProtocolSelector(QUIC), nil
-		}
-		return newRemoteProtocolSelector(fetchedProtocol, ProtocolList, threshold, protocolFetcher, resolveTTL, log), nil
+	}
+
+	// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
+	// fallback on failures.
+	if protocolFlag == AutoSelectFlag {
+		return newAutoProtocolSelector(protocol, []Protocol{QUIC, HTTP2, H2mux}, threshold, fetchFunc, ttl, log, needPQ), nil
+	}
+
+	return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
+}
+
+func selectWarpRoutingProtocols(
+	protocolFlag string,
+	fetchFunc PercentageFetcher,
+	ttl time.Duration,
+	log *zerolog.Logger,
+	threshold int32,
+	protocol Protocol,
+	needPQ bool,
+) (ProtocolSelector, error) {
+	// If the user picks a protocol, then we stick to it no matter what.
+	switch protocolFlag {
+	case QUIC.String():
+		return &staticProtocolSelector{current: QUICWarp}, nil
+	case HTTP2.String():
+		return &staticProtocolSelector{current: HTTP2Warp}, nil
+	}
+
+	// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
+	// fallback on failures.
+	if protocolFlag == AutoSelectFlag {
+		return newAutoProtocolSelector(protocol, []Protocol{QUICWarp, HTTP2Warp}, threshold, fetchFunc, ttl, log, needPQ), nil
 	}
 
 	return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)

connection/protocol_test.go

@@ -3,6 +3,7 @@ package connection
 import (
 	"fmt"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 
@@ -10,11 +11,19 @@ import (
 )
 
 const (
-	testNoTTL      = 0
-	testAccountTag = "testAccountTag"
+	testNoTTL            = 0
+	noWarpRoutingEnabled = false
 )
 
-func mockFetcher(getError bool, protocolPercent ...edgediscovery.ProtocolPercent) edgediscovery.PercentageFetcher {
+var (
+	testNamedTunnelProperties = &NamedTunnelProperties{
+		Credentials: Credentials{
+			AccountTag: "testAccountTag",
+		},
+	}
+)
+
+func mockFetcher(getError bool, protocolPercent ...edgediscovery.ProtocolPercent) PercentageFetcher {
 	return func() (edgediscovery.ProtocolPercents, error) {
 		if getError {
 			return nil, fmt.Errorf("failed to fetch percentage")
@@ -28,7 +37,7 @@ type dynamicMockFetcher struct {
 	err              error
 }
 
-func (dmf *dynamicMockFetcher) fetch() edgediscovery.PercentageFetcher {
+func (dmf *dynamicMockFetcher) fetch() PercentageFetcher {
 	return func() (edgediscovery.ProtocolPercents, error) {
 		return dmf.protocolPercents, dmf.err
 	}
@@ -36,58 +45,181 @@ func (dmf *dynamicMockFetcher) fetch() edgediscovery.PercentageFetcher {
 
 func TestNewProtocolSelector(t *testing.T) {
 	tests := []struct {
-		name                string
-		protocol            string
-		tunnelTokenProvided bool
-		needPQ              bool
-		expectedProtocol    Protocol
-		hasFallback         bool
-		expectedFallback    Protocol
-		wantErr             bool
+		name               string
+		protocol           string
+		expectedProtocol   Protocol
+		hasFallback        bool
+		expectedFallback   Protocol
+		warpRoutingEnabled bool
+		namedTunnelConfig  *NamedTunnelProperties
+		fetchFunc          PercentageFetcher
+		wantErr            bool
 	}{
 		{
-			name:     "named tunnel with unknown protocol",
-			protocol: "unknown",
-			wantErr:  true,
+			name:              "classic tunnel",
+			protocol:          "h2mux",
+			expectedProtocol:  H2mux,
+			namedTunnelConfig: nil,
 		},
 		{
-			name:             "named tunnel with h2mux: force to http2",
-			protocol:         "h2mux",
-			expectedProtocol: HTTP2,
+			name:              "named tunnel over h2mux",
+			protocol:          "h2mux",
+			expectedProtocol:  H2mux,
+			fetchFunc:         func() (edgediscovery.ProtocolPercents, error) { return nil, nil },
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
-			name:             "named tunnel with http2: no fallback",
-			protocol:         "http2",
-			expectedProtocol: HTTP2,
+			name:              "named tunnel over http2",
+			protocol:          "http2",
+			expectedProtocol:  HTTP2,
+			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}),
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
-			name:             "named tunnel with auto: quic",
+			name:              "named tunnel http2 disabled still gets http2 because it is manually picked",
+			protocol:          "http2",
+			expectedProtocol:  HTTP2,
+			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
+			namedTunnelConfig: testNamedTunnelProperties,
+		},
+		{
+			name:              "named tunnel quic disabled still gets quic because it is manually picked",
+			protocol:          "quic",
+			expectedProtocol:  QUIC,
+			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
+			namedTunnelConfig: testNamedTunnelProperties,
+		},
+		{
+			name:              "named tunnel quic and http2 disabled",
+			protocol:          AutoSelectFlag,
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
+			namedTunnelConfig: testNamedTunnelProperties,
+		},
+		{
+			name:             "named tunnel quic disabled",
 			protocol:         AutoSelectFlag,
-			expectedProtocol: QUIC,
-			hasFallback:      true,
-			expectedFallback: HTTP2,
+			expectedProtocol: HTTP2,
+			// Hasfallback true is because if http2 fails, then we further fallback to h2mux.
+			hasFallback:       true,
+			expectedFallback:  H2mux,
+			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
-			name:             "named tunnel (post quantum)",
-			protocol:         AutoSelectFlag,
-			needPQ:           true,
-			expectedProtocol: QUIC,
+			name:              "named tunnel auto all http2 disabled",
+			protocol:          AutoSelectFlag,
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
-			name:             "named tunnel (post quantum) w/http2",
-			protocol:         "http2",
-			needPQ:           true,
-			expectedProtocol: QUIC,
+			name:              "named tunnel auto to h2mux",
+			protocol:          AutoSelectFlag,
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}),
+			namedTunnelConfig: testNamedTunnelProperties,
+		},
+		{
+			name:              "named tunnel auto to http2",
+			protocol:          AutoSelectFlag,
+			expectedProtocol:  HTTP2,
+			hasFallback:       true,
+			expectedFallback:  H2mux,
+			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
+			namedTunnelConfig: testNamedTunnelProperties,
+		},
+		{
+			name:              "named tunnel auto to quic",
+			protocol:          AutoSelectFlag,
+			expectedProtocol:  QUIC,
+			hasFallback:       true,
+			expectedFallback:  HTTP2,
+			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
+			namedTunnelConfig: testNamedTunnelProperties,
+		},
+		{
+			name:               "warp routing requesting h2mux",
+			protocol:           "h2mux",
+			expectedProtocol:   HTTP2Warp,
+			hasFallback:        false,
+			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
+			warpRoutingEnabled: true,
+			namedTunnelConfig:  testNamedTunnelProperties,
+		},
+		{
+			name:               "warp routing requesting h2mux picks HTTP2 even if http2 percent is -1",
+			protocol:           "h2mux",
+			expectedProtocol:   HTTP2Warp,
+			hasFallback:        false,
+			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
+			warpRoutingEnabled: true,
+			namedTunnelConfig:  testNamedTunnelProperties,
+		},
+		{
+			name:               "warp routing http2",
+			protocol:           "http2",
+			expectedProtocol:   HTTP2Warp,
+			hasFallback:        false,
+			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
+			warpRoutingEnabled: true,
+			namedTunnelConfig:  testNamedTunnelProperties,
+		},
+		{
+			name:               "warp routing quic",
+			protocol:           AutoSelectFlag,
+			expectedProtocol:   QUICWarp,
+			hasFallback:        true,
+			expectedFallback:   HTTP2Warp,
+			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
+			warpRoutingEnabled: true,
+			namedTunnelConfig:  testNamedTunnelProperties,
+		},
+		{
+			name:               "warp routing auto",
+			protocol:           AutoSelectFlag,
+			expectedProtocol:   HTTP2Warp,
+			hasFallback:        false,
+			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
+			warpRoutingEnabled: true,
+			namedTunnelConfig:  testNamedTunnelProperties,
+		},
+		{
+			name:               "warp routing auto- quic",
+			protocol:           AutoSelectFlag,
+			expectedProtocol:   QUICWarp,
+			hasFallback:        true,
+			expectedFallback:   HTTP2Warp,
+			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
+			warpRoutingEnabled: true,
+			namedTunnelConfig:  testNamedTunnelProperties,
+		},
+		{
+			// None named tunnel can only use h2mux, so specifying an unknown protocol is not an error
+			name:             "classic tunnel unknown protocol",
+			protocol:         "unknown",
+			expectedProtocol: H2mux,
+		},
+		{
+			name:              "named tunnel unknown protocol",
+			protocol:          "unknown",
+			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
+			namedTunnelConfig: testNamedTunnelProperties,
+			wantErr:           true,
+		},
+		{
+			name:              "named tunnel fetch error",
+			protocol:          AutoSelectFlag,
+			fetchFunc:         mockFetcher(true),
+			namedTunnelConfig: testNamedTunnelProperties,
+			expectedProtocol:  HTTP2,
+			wantErr:           false,
 		},
-	}
-
-	fetcher := dynamicMockFetcher{
-		protocolPercents: edgediscovery.ProtocolPercents{},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			selector, err := NewProtocolSelector(test.protocol, testAccountTag, test.tunnelTokenProvided, test.needPQ, fetcher.fetch(), ResolveTTL, &log)
+			selector, err := NewProtocolSelector(test.protocol, test.warpRoutingEnabled, test.namedTunnelConfig, test.fetchFunc, testNoTTL, &log, false)
 			if test.wantErr {
 				assert.Error(t, err, fmt.Sprintf("test %s failed", test.name))
 			} else {
@@ -105,15 +237,15 @@ func TestNewProtocolSelector(t *testing.T) {
 
 func TestAutoProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
-	selector, err := NewProtocolSelector(AutoSelectFlag, testAccountTag, false, false, fetcher.fetch(), testNoTTL, &log)
+	selector, err := NewProtocolSelector(AutoSelectFlag, noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log, false)
 	assert.NoError(t, err)
-	assert.Equal(t, QUIC, selector.Current())
+	assert.Equal(t, H2mux, selector.Current())
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}}
 	assert.Equal(t, HTTP2, selector.Current())
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}}
-	assert.Equal(t, QUIC, selector.Current())
+	assert.Equal(t, H2mux, selector.Current())
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}}
 	assert.Equal(t, HTTP2, selector.Current())
@@ -123,10 +255,10 @@ func TestAutoProtocolSelectorRefresh(t *testing.T) {
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}}
 	fetcher.err = nil
-	assert.Equal(t, QUIC, selector.Current())
+	assert.Equal(t, H2mux, selector.Current())
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}}
-	assert.Equal(t, QUIC, selector.Current())
+	assert.Equal(t, H2mux, selector.Current())
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}}
 	assert.Equal(t, QUIC, selector.Current())
@@ -135,7 +267,7 @@ func TestAutoProtocolSelectorRefresh(t *testing.T) {
 func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
 	// Since the user chooses http2 on purpose, we always stick to it.
-	selector, err := NewProtocolSelector(HTTP2.String(), testAccountTag, false, false, fetcher.fetch(), testNoTTL, &log)
+	selector, err := NewProtocolSelector("http2", noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log, false)
 	assert.NoError(t, err)
 	assert.Equal(t, HTTP2, selector.Current())
 
@@ -162,12 +294,13 @@ func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 	assert.Equal(t, HTTP2, selector.Current())
 }
 
-func TestAutoProtocolSelectorNoRefreshWithToken(t *testing.T) {
+func TestProtocolSelectorRefreshTTL(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
-	selector, err := NewProtocolSelector(AutoSelectFlag, testAccountTag, true, false, fetcher.fetch(), testNoTTL, &log)
+	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}}
+	selector, err := NewProtocolSelector(AutoSelectFlag, noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), time.Hour, &log, false)
 	assert.NoError(t, err)
 	assert.Equal(t, QUIC, selector.Current())
 
-	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}}
+	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 0}}
 	assert.Equal(t, QUIC, selector.Current())
 }

connection/quic.go

@@ -1,7 +1,6 @@
 package connection
 
 import (
-	"bufio"
 	"context"
 	"crypto/tls"
 	"fmt"
@@ -9,7 +8,6 @@ import (
 	"net"
 	"net/http"
 	"net/netip"
-	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -17,8 +15,8 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/lucas-clemente/quic-go"
 	"github.com/pkg/errors"
-	"github.com/quic-go/quic-go"
 	"github.com/rs/zerolog"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
@@ -26,7 +24,6 @@ import (
 
 	"github.com/cloudflare/cloudflared/datagramsession"
 	"github.com/cloudflare/cloudflared/ingress"
-	"github.com/cloudflare/cloudflared/management"
 	"github.com/cloudflare/cloudflared/packet"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/tracing"
@@ -63,17 +60,12 @@ type QUICConnection struct {
 	packetRouter         *ingress.PacketRouter
 	controlStreamHandler ControlStreamHandler
 	connOptions          *tunnelpogs.ConnectionOptions
-	connIndex            uint8
-
-	udpUnregisterTimeout time.Duration
 }
 
 // NewQUICConnection returns a new instance of QUICConnection.
 func NewQUICConnection(
-	ctx context.Context,
 	quicConfig *quic.Config,
 	edgeAddr net.Addr,
-	localAddr net.IP,
 	connIndex uint8,
 	tlsConfig *tls.Config,
 	orchestrator Orchestrator,
@@ -81,14 +73,13 @@ func NewQUICConnection(
 	controlStreamHandler ControlStreamHandler,
 	logger *zerolog.Logger,
 	packetRouterConfig *ingress.GlobalRouterConfig,
-	udpUnregisterTimeout time.Duration,
 ) (*QUICConnection, error) {
-	udpConn, err := createUDPConnForConnIndex(connIndex, localAddr, logger)
+	udpConn, err := createUDPConnForConnIndex(connIndex, logger)
 	if err != nil {
 		return nil, err
 	}
 
-	session, err := quic.Dial(ctx, udpConn, edgeAddr, tlsConfig, quicConfig)
+	session, err := quic.Dial(udpConn, edgeAddr, edgeAddr.String(), tlsConfig, quicConfig)
 	if err != nil {
 		// close the udp server socket in case of error connecting to the edge
 		udpConn.Close()
@@ -104,7 +95,7 @@ func NewQUICConnection(
 	sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
 	datagramMuxer := quicpogs.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
 	sessionManager := datagramsession.NewManager(logger, datagramMuxer.SendToSession, sessionDemuxChan)
-	packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger)
+	packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger, orchestrator.WarpRoutingEnabled)
 
 	return &QUICConnection{
 		session:              session,
@@ -115,8 +106,6 @@ func NewQUICConnection(
 		packetRouter:         packetRouter,
 		controlStreamHandler: controlStreamHandler,
 		connOptions:          connOptions,
-		connIndex:            connIndex,
-		udpUnregisterTimeout: udpUnregisterTimeout,
 	}, nil
 }
 
@@ -269,7 +258,7 @@ func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.R
 
 	switch request.Type {
 	case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
-		tracedReq, err := buildHTTPRequest(ctx, request, stream, q.connIndex, q.logger)
+		tracedReq, err := buildHTTPRequest(ctx, request, stream, q.logger)
 		if err != nil {
 			return err, false
 		}
@@ -283,7 +272,6 @@ func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.R
 			Dest:      request.Dest,
 			FlowID:    metadata[QUICMetadataFlowID],
 			CfTraceID: metadata[tracing.TracerContextName],
-			ConnIndex: q.connIndex,
 		}), rwa.connectResponseSent
 	default:
 		return errors.Errorf("unsupported error type: %s", request.Type), false
@@ -305,12 +293,11 @@ func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.
 		attribute.String("session-id", sessionID.String()),
 		attribute.String("dst", fmt.Sprintf("%s:%d", dstIP, dstPort)),
 	))
-	log := q.logger.With().Int(management.EventTypeKey, int(management.UDP)).Logger()
 	// Each session is a series of datagram from an eyeball to a dstIP:dstPort.
 	// (src port, dst IP, dst port) uniquely identifies a session, so it needs a dedicated connected socket.
 	originProxy, err := ingress.DialUDP(dstIP, dstPort)
 	if err != nil {
-		log.Err(err).Msgf("Failed to create udp proxy to %s:%d", dstIP, dstPort)
+		q.logger.Err(err).Msgf("Failed to create udp proxy to %s:%d", dstIP, dstPort)
 		tracing.EndWithErrorStatus(registerSpan, err)
 		return nil, err
 	}
@@ -321,18 +308,14 @@ func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.
 
 	session, err := q.sessionManager.RegisterSession(ctx, sessionID, originProxy)
 	if err != nil {
-		log.Err(err).Str("sessionID", sessionID.String()).Msgf("Failed to register udp session")
+		q.logger.Err(err).Str("sessionID", sessionID.String()).Msgf("Failed to register udp session")
 		tracing.EndWithErrorStatus(registerSpan, err)
 		return nil, err
 	}
 
 	go q.serveUDPSession(session, closeAfterIdleHint)
 
-	log.Debug().
-		Str("sessionID", sessionID.String()).
-		Str("src", originProxy.LocalAddr().String()).
-		Str("dst", fmt.Sprintf("%s:%d", dstIP, dstPort)).
-		Msgf("Registered session")
+	q.logger.Debug().Str("sessionID", sessionID.String()).Str("src", originProxy.LocalAddr().String()).Str("dst", fmt.Sprintf("%s:%d", dstIP, dstPort)).Msgf("Registered session")
 	tracing.End(registerSpan)
 
 	resp := tunnelpogs.RegisterUdpSessionResponse{
@@ -353,29 +336,21 @@ func (q *QUICConnection) serveUDPSession(session *datagramsession.Session, close
 			q.closeUDPSession(ctx, session.ID, "terminated without error")
 		}
 	}
-	q.logger.Debug().Err(err).
-		Int(management.EventTypeKey, int(management.UDP)).
-		Str("sessionID", session.ID.String()).
-		Msg("Session terminated")
+	q.logger.Debug().Err(err).Str("sessionID", session.ID.String()).Msg("Session terminated")
 }
 
 // closeUDPSession first unregisters the session from session manager, then it tries to unregister from edge
 func (q *QUICConnection) closeUDPSession(ctx context.Context, sessionID uuid.UUID, message string) {
 	q.sessionManager.UnregisterSession(ctx, sessionID, message, false)
-	quicStream, err := q.session.OpenStream()
+	stream, err := q.session.OpenStream()
 	if err != nil {
 		// Log this at debug because this is not an error if session was closed due to lost connection
 		// with edge
-		q.logger.Debug().Err(err).
-			Int(management.EventTypeKey, int(management.UDP)).
-			Str("sessionID", sessionID.String()).
+		q.logger.Debug().Err(err).Str("sessionID", sessionID.String()).
 			Msgf("Failed to open quic stream to unregister udp session with edge")
 		return
 	}
-
-	stream := quicpogs.NewSafeStreamCloser(quicStream)
-	defer stream.Close()
-	rpcClientStream, err := quicpogs.NewRPCClientStream(ctx, stream, q.udpUnregisterTimeout, q.logger)
+	rpcClientStream, err := quicpogs.NewRPCClientStream(ctx, stream, q.logger)
 	if err != nil {
 		// Log this at debug because this is not an error if session was closed due to lost connection
 		// with edge
@@ -383,8 +358,6 @@ func (q *QUICConnection) closeUDPSession(ctx context.Context, sessionID uuid.UUI
 			Msgf("Failed to open rpc stream to unregister udp session with edge")
 		return
 	}
-	defer rpcClientStream.Close()
-
 	if err := rpcClientStream.UnregisterUdpSession(ctx, sessionID, message); err != nil {
 		q.logger.Err(err).Str("sessionID", sessionID.String()).
 			Msgf("Failed to unregister udp session with edge")
@@ -410,27 +383,22 @@ type streamReadWriteAcker struct {
 
 // AckConnection acks response back to the proxy.
 func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
-	metadata := []quicpogs.Metadata{}
-	// Only add tracing if provided by origintunneld
-	if tracePropagation != "" {
-		metadata = append(metadata, quicpogs.Metadata{
-			Key: tracing.CanonicalCloudflaredTracingHeader,
-			Val: tracePropagation,
-		})
+	metadata := quicpogs.Metadata{
+		Key: tracing.CanonicalCloudflaredTracingHeader,
+		Val: tracePropagation,
 	}
 	s.connectResponseSent = true
-	return s.WriteConnectResponseData(nil, metadata...)
+	return s.WriteConnectResponseData(nil, metadata)
 }
 
 // httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
 type httpResponseAdapter struct {
 	*quicpogs.RequestServerStream
-	headers             http.Header
 	connectResponseSent bool
 }
 
 func newHTTPResponseAdapter(s *quicpogs.RequestServerStream) httpResponseAdapter {
-	return httpResponseAdapter{RequestServerStream: s, headers: make(http.Header)}
+	return httpResponseAdapter{RequestServerStream: s}
 }
 
 func (hrw *httpResponseAdapter) AddTrailer(trailerName, trailerValue string) {
@@ -450,34 +418,6 @@ func (hrw *httpResponseAdapter) WriteRespHeaders(status int, header http.Header)
 	return hrw.WriteConnectResponseData(nil, metadata...)
 }
 
-func (hrw *httpResponseAdapter) Write(p []byte) (int, error) {
-	// Make sure to send WriteHeader response if not called yet
-	if !hrw.connectResponseSent {
-		hrw.WriteRespHeaders(http.StatusOK, hrw.headers)
-	}
-	return hrw.RequestServerStream.Write(p)
-}
-
-func (hrw *httpResponseAdapter) Header() http.Header {
-	return hrw.headers
-}
-
-// This is a no-op Flush because this adapter is over a quic.Stream and we don't need Flush here.
-func (hrw *httpResponseAdapter) Flush() {}
-
-func (hrw *httpResponseAdapter) WriteHeader(status int) {
-	hrw.WriteRespHeaders(status, hrw.headers)
-}
-
-func (hrw *httpResponseAdapter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
-	conn := &localProxyConnection{hrw.ReadWriteCloser}
-	readWriter := bufio.NewReadWriter(
-		bufio.NewReader(hrw.ReadWriteCloser),
-		bufio.NewWriter(hrw.ReadWriteCloser),
-	)
-	return conn, readWriter, nil
-}
-
 func (hrw *httpResponseAdapter) WriteErrorResponse(err error) {
 	hrw.WriteConnectResponseData(err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
 }
@@ -491,7 +431,6 @@ func buildHTTPRequest(
 	ctx context.Context,
 	connectRequest *quicpogs.ConnectRequest,
 	body io.ReadCloser,
-	connIndex uint8,
 	log *zerolog.Logger,
 ) (*tracing.TracedHTTPRequest, error) {
 	metadata := connectRequest.MetadataMap()
@@ -535,7 +474,7 @@ func buildHTTPRequest(
 	stripWebsocketUpgradeHeader(req)
 
 	// Check for tracing on request
-	tracedReq := tracing.NewTracedHTTPRequest(req, connIndex, log)
+	tracedReq := tracing.NewTracedHTTPRequest(req, log)
 	return tracedReq, err
 }
 
@@ -616,27 +555,13 @@ func (rp *muxerWrapper) Close() error {
 	return nil
 }
 
-func createUDPConnForConnIndex(connIndex uint8, localIP net.IP, logger *zerolog.Logger) (*net.UDPConn, error) {
+func createUDPConnForConnIndex(connIndex uint8, logger *zerolog.Logger) (*net.UDPConn, error) {
 	portMapMutex.Lock()
 	defer portMapMutex.Unlock()
 
-	if localIP == nil {
-		localIP = net.IPv4zero
-	}
-
-	listenNetwork := "udp"
-	// https://github.com/quic-go/quic-go/issues/3793 DF bit cannot be set for dual stack listener on OSX
-	if runtime.GOOS == "darwin" {
-		if localIP.To4() != nil {
-			listenNetwork = "udp4"
-		} else {
-			listenNetwork = "udp6"
-		}
-	}
-
 	// if port was not set yet, it will be zero, so bind will randomly allocate one.
 	if port, ok := portForConnIndex[connIndex]; ok {
-		udpConn, err := net.ListenUDP(listenNetwork, &net.UDPAddr{IP: localIP, Port: port})
+		udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: port})
 		// if there wasn't an error, or if port was 0 (independently of error or not, just return)
 		if err == nil {
 			return udpConn, nil
@@ -646,7 +571,7 @@ func createUDPConnForConnIndex(connIndex uint8, localIP net.IP, logger *zerolog.
 	}
 
 	// if we reached here, then there was an error or port as not been allocated it.
-	udpConn, err := net.ListenUDP(listenNetwork, &net.UDPAddr{IP: localIP, Port: 0})
+	udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
 	if err == nil {
 		udpAddr, ok := (udpConn.LocalAddr()).(*net.UDPAddr)
 		if !ok {

connection/quic_test.go

@@ -16,8 +16,8 @@ import (
 
 	"github.com/gobwas/ws/wsutil"
 	"github.com/google/uuid"
+	"github.com/lucas-clemente/quic-go"
 	"github.com/pkg/errors"
-	"github.com/quic-go/quic-go"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -32,8 +32,9 @@ import (
 var (
 	testTLSServerConfig = quicpogs.GenerateTLSConfig()
 	testQUICConfig      = &quic.Config{
-		KeepAlivePeriod: 5 * time.Second,
-		EnableDatagrams: true,
+		ConnectionIDLength: 16,
+		KeepAlivePeriod:    5 * time.Second,
+		EnableDatagrams:    true,
 	}
 )
 
@@ -42,6 +43,13 @@ var _ ReadWriteAcker = (*streamReadWriteAcker)(nil)
 // TestQUICServer tests if a quic server accepts and responds to a quic client with the acceptance protocol.
 // It also serves as a demonstration for communication with the QUIC connection started by a cloudflared.
 func TestQUICServer(t *testing.T) {
+	// Start a UDP Listener for QUIC.
+	udpAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0")
+	require.NoError(t, err)
+	udpListener, err := net.ListenUDP(udpAddr.Network(), udpAddr)
+	require.NoError(t, err)
+	defer udpListener.Close()
+
 	// This is simply a sample websocket frame message.
 	wsBuf := &bytes.Buffer{}
 	wsutil.WriteClientBinary(wsBuf, []byte("Hello"))
@@ -137,14 +145,8 @@ func TestQUICServer(t *testing.T) {
 		test := test // capture range variable
 		t.Run(test.desc, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
-			// Start a UDP Listener for QUIC.
-			udpAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0")
-			require.NoError(t, err)
-			udpListener, err := net.ListenUDP(udpAddr.Network(), udpAddr)
-			require.NoError(t, err)
-			defer udpListener.Close()
-			quicTransport := &quic.Transport{Conn: udpListener, ConnectionIDLength: 16}
-			quicListener, err := quicTransport.Listen(testTLSServerConfig, testQUICConfig)
+
+			quicListener, err := quic.Listen(udpListener, testTLSServerConfig, testQUICConfig)
 			require.NoError(t, err)
 
 			serverDone := make(chan struct{})
@@ -185,7 +187,7 @@ func (fakeControlStream) IsStopped() bool {
 func quicServer(
 	ctx context.Context,
 	t *testing.T,
-	listener *quic.Listener,
+	listener quic.Listener,
 	dest string,
 	connectionType quicpogs.ConnectionType,
 	metadata []quicpogs.Metadata,
@@ -483,7 +485,7 @@ func TestBuildHTTPRequest(t *testing.T) {
 	for _, test := range tests {
 		test := test // capture range variable
 		t.Run(test.name, func(t *testing.T) {
-			req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body, 0, &log)
+			req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body, &log)
 			assert.NoError(t, err)
 			test.req = test.req.WithContext(req.Context())
 			assert.Equal(t, test.req, req.Request)
@@ -570,7 +572,7 @@ func TestNopCloserReadWriterCloseAfterEOF(t *testing.T) {
 
 func TestCreateUDPConnReuseSourcePort(t *testing.T) {
 	logger := zerolog.Nop()
-	conn, err := createUDPConnForConnIndex(0, nil, &logger)
+	conn, err := createUDPConnForConnIndex(0, &logger)
 	require.NoError(t, err)
 
 	getPortFunc := func(conn *net.UDPConn) int {
@@ -584,17 +586,17 @@ func TestCreateUDPConnReuseSourcePort(t *testing.T) {
 	conn.Close()
 
 	// should get the same port as before.
-	conn, err = createUDPConnForConnIndex(0, nil, &logger)
+	conn, err = createUDPConnForConnIndex(0, &logger)
 	require.NoError(t, err)
 	require.Equal(t, initialPort, getPortFunc(conn))
 
 	// new index, should get a different port
-	conn1, err := createUDPConnForConnIndex(1, nil, &logger)
+	conn1, err := createUDPConnForConnIndex(1, &logger)
 	require.NoError(t, err)
 	require.NotEqual(t, initialPort, getPortFunc(conn1))
 
 	// not closing the conn and trying to obtain a new conn for same index should give a different random port
-	conn, err = createUDPConnForConnIndex(0, nil, &logger)
+	conn, err = createUDPConnForConnIndex(0, &logger)
 	require.NoError(t, err)
 	require.NotEqual(t, initialPort, getPortFunc(conn))
 }
@@ -621,7 +623,7 @@ func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.
 	muxedPayload, err = quicpogs.SuffixType(muxedPayload, quicpogs.DatagramTypeUDP)
 	require.NoError(t, err)
 
-	err = edgeQUICSession.SendDatagram(muxedPayload)
+	err = edgeQUICSession.SendMessage(muxedPayload)
 	require.NoError(t, err)
 
 	readBuffer := make([]byte, len(payload)+1)
@@ -711,13 +713,9 @@ func testQUICConnection(udpListenerAddr net.Addr, t *testing.T, index uint8) *QU
 	}
 	// Start a mock httpProxy
 	log := zerolog.New(os.Stdout)
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
 	qc, err := NewQUICConnection(
-		ctx,
 		testQUICConfig,
 		udpListenerAddr,
-		nil,
 		index,
 		tlsClientConfig,
 		&mockOrchestrator{originProxy: &mockOriginProxyWithRequest{}},
@@ -725,7 +723,6 @@ func testQUICConnection(udpListenerAddr net.Addr, t *testing.T, index uint8) *QU
 		fakeControlStream{},
 		&log,
 		nil,
-		5*time.Second,
 	)
 	require.NoError(t, err)
 	return qc