Release 2022.8.2

This change seeks to push an arm64 built image to dockerhub for arm users to run. This should spin cloudflared on arm machines without the warning
WARNING: The requested image's platform (linux/amd64) does not match the detected host platform (linux/arm64/v8) and no specific platform was requested

.docker-images

@@ -1,6 +1,6 @@
 images:
   - name: cloudflared
-    dockerfile: Dockerfile
+    dockerfile: Dockerfile.$ARCH
     context: .
     versions: 
       - latest
@@ -8,3 +8,6 @@ images:
     - name: docker.io/cloudflare
       user: env:DOCKER_USER
       password: env:DOCKER_PASSWORD
+    architectures:
+    - amd64
+    - arm64

.github/ISSUE_TEMPLATE/---bug-report.md

@@ -1,9 +1,8 @@
 ---
-name: "Bug report \U0001F41B"
+name: "\U0001F41B Bug report"
 about: Create a report to help us improve cloudflared
-title: ''
-labels: awaiting reply, bug
-assignees: ''
+title: "\U0001F41B"
+labels: 'Priority: Normal, Type: Bug'
 
 ---
 

.github/ISSUE_TEMPLATE/---documentation.md

@@ -0,0 +1,16 @@
+---
+name: "\U0001F4DD Documentation"
+about: Request new or updated documentation for cloudflared
+title: "\U0001F4DD"
+labels: 'Priority: Normal, Type: Documentation'
+
+---
+
+**Available Documentation**
+A link to the documentation that is available today and the areas which could be improved. 
+
+**Suggested Documentation**
+A clear and concise description of the documentation, tutorial, or guide that should be added. 
+
+**Additional context**
+Add any other context or screenshots about the documentation request here.

.github/ISSUE_TEMPLATE/---feature-request.md

@@ -1,9 +1,8 @@
 ---
-name: "Feature request \U0001F4A1"
+name: "\U0001F4A1 Feature request"
 about: Suggest a feature or enhancement for cloudflared
-title: ''
-labels: awaiting reply, feature-request
-assignees: ''
+title: "\U0001F4A1"
+labels: 'Priority: Normal, Type: Feature Request'
 
 ---
 

.github/workflows/check.yaml

@@ -12,8 +12,6 @@ jobs:
       uses: actions/setup-go@v2
       with:
         go-version: ${{ matrix.go-version }}
-    - name: Install go-sumtype
-      run: go get github.com/sudarshan-reddy/go-sumtype
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Test

.teamcity/update-homebrew-core.sh

@@ -19,5 +19,8 @@ eval "$(tmp/homebrew/bin/brew shellenv)"
 brew update --force --quiet
 chmod -R go-w "$(brew --prefix)/share/zsh"
 
+git config --global user.name "cloudflare-warp-bot"
+git config --global user.email "warp-bot@cloudflare.com"
+
 # bump formula pr
-brew bump-formula-pr cloudflared
+brew bump-formula-pr cloudflared --version="$VERSION" --no-browse

CHANGES.md

@@ -1,3 +1,15 @@
+## 2022.8.1
+### New Features
+- cloudflared now remembers if it connected to a certain protocol successfully. If it did, it does not fall back to a lower
+  protocol on connection failures.
+
+## 2022.7.1
+### New Features
+- It is now possible to connect cloudflared tunnel to Cloudflare Global Network with IPv6. See `cloudflared tunnel --help` and look for `edge-ip-version` for more information. For now, the default behavior is to still connect with IPv4 only.
+
+### Bug Fixes
+- Several bug fixes related with QUIC transport (used between cloudflared tunnel and Cloudflare Global Network). Updating to this version is highly recommended.
+
 ## 2022.4.0
 ### Bug Fixes
 - `cloudflared tunnel run` no longer logs the Tunnel token or JSON credentials in clear text as those are the secret

Dockerfile

@@ -1,11 +1,13 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.17.1 as builder
+FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \
     TARGET_GOARCH=${TARGET_GOARCH}
+    
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
 
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
@@ -16,7 +18,7 @@ COPY . .
 RUN make cloudflared
 
 # use a distroless base image with glibc
-FROM gcr.io/distroless/base-debian10:nonroot
+FROM gcr.io/distroless/base-debian11:nonroot
 
 # copy our compiled binary
 COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/

Dockerfile.amd64

@@ -0,0 +1,27 @@
+# use a builder image for building cloudflare
+FROM golang:1.19 as builder
+ENV GO111MODULE=on \
+    CGO_ENABLED=0 
+    
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
+WORKDIR /go/src/github.com/cloudflare/cloudflared/
+
+# copy our sources into the builder image
+COPY . .
+
+# compile cloudflared
+RUN GOOS=linux GOARCH=amd64 make cloudflared
+
+# use a distroless base image with glibc
+FROM gcr.io/distroless/base-debian11:nonroot
+
+# copy our compiled binary
+COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+
+# run as non-privileged user
+USER nonroot
+
+# command / entrypoint of container
+ENTRYPOINT ["cloudflared", "--no-autoupdate"]
+CMD ["version"]

Dockerfile.arm64

@@ -0,0 +1,27 @@
+# use a builder image for building cloudflare
+FROM golang:1.19 as builder
+ENV GO111MODULE=on \
+    CGO_ENABLED=0 
+    
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
+WORKDIR /go/src/github.com/cloudflare/cloudflared/
+
+# copy our sources into the builder image
+COPY . .
+
+# compile cloudflared
+RUN GOOS=linux GOARCH=arm64 make cloudflared
+
+# use a distroless base image with glibc
+FROM gcr.io/distroless/base-debian11:nonroot-arm64
+
+# copy our compiled binary
+COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+
+# run as non-privileged user
+USER nonroot
+
+# command / entrypoint of container
+ENTRYPOINT ["cloudflared", "--no-autoupdate"]
+CMD ["version"]

Makefile

@@ -40,6 +40,10 @@ ifneq ($(GO_BUILD_TAGS),)
 	GO_BUILD_TAGS := -tags "$(GO_BUILD_TAGS)"
 endif
 
+ifeq ($(debug), 1)
+	GO_BUILD_TAGS += -gcflags="all=-N -l"
+endif
+
 IMPORT_PATH    := github.com/cloudflare/cloudflared
 PACKAGE_DIR    := $(CURDIR)/packaging
 PREFIX         := /usr
@@ -94,6 +98,16 @@ else
 	TARGET_PUBLIC_REPO ?= $(FLAVOR)
 endif
 
+ifneq ($(TARGET_ARM), )
+	ARM_COMMAND := GOARM=$(TARGET_ARM)
+endif
+
+ifeq ($(TARGET_ARM), 7) 
+	PACKAGE_ARCH := armhf
+else
+	PACKAGE_ARCH := $(TARGET_ARCH)
+endif
+
 .PHONY: all
 all: cloudflared test
 
@@ -107,7 +121,7 @@ ifeq ($(FIPS), true)
 	$(info Building cloudflared with go-fips)
 	cp -f fips/fips.go.linux-amd64 cmd/cloudflared/fips.go
 endif
-	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(LDFLAGS) $(IMPORT_PATH)/cmd/cloudflared
+	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) $(ARM_COMMAND) go build -v -mod=vendor $(GO_BUILD_TAGS) $(LDFLAGS) $(IMPORT_PATH)/cmd/cloudflared
 ifeq ($(FIPS), true)
 	rm -f cmd/cloudflared/fips.go
 	./check-fips.sh cloudflared
@@ -167,7 +181,7 @@ define build_package
 		--license 'Apache License Version 2.0' \
 		--url 'https://github.com/cloudflare/cloudflared' \
 		-m 'Cloudflare <support@cloudflare.com>' \
-		-a $(TARGET_ARCH) -v $(VERSION) -n $(DEB_PACKAGE_NAME) $(NIGHTLY_FLAGS) --after-install postinst.sh --after-remove postrm.sh \
+	    -a $(PACKAGE_ARCH) -v $(VERSION) -n $(DEB_PACKAGE_NAME) $(NIGHTLY_FLAGS) --after-install postinst.sh --after-remove postrm.sh \
 		cloudflared=$(INSTALL_BINDIR) cloudflared.1=$(INSTALL_MANDIR)
 endef
 
@@ -261,6 +275,10 @@ github-release: cloudflared
 github-release-built-pkgs:
 	python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
 
+.PHONY: release-pkgs-linux
+release-pkgs-linux:
+	python3 ./release_pkgs.py
+
 .PHONY: github-message
 github-message:
 	python3 github_message.py --release-version $(VERSION)
@@ -285,11 +303,6 @@ quic-deps:
 .PHONY: vet
 vet:
 	go vet -mod=vendor ./...
-	# go get github.com/sudarshan-reddy/go-sumtype (don't do this in build directory or this will cause vendor issues)
-	# Note: If you have github.com/BurntSushi/go-sumtype then you might have to use the repo above instead
-	# for now because it uses an older version of golang.org/x/tools.
-	which go-sumtype
-	go-sumtype $$(go list -mod=vendor ./...)
 
 .PHONY: goimports
 goimports:

README.md

@@ -43,7 +43,7 @@ Once installed, you can authenticate `cloudflared` into your Cloudflare account
 * Route traffic to that Tunnel:
   * Via public [DNS records in Cloudflare](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/dns)
   * Or via a public hostname guided by a [Cloudflare Load Balancer](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/lb)
-  * Or from [WARP client private traffic](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/private-networks)
+  * Or from [WARP client private traffic](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/private-net/)
 
 
 ## TryCloudflare

RELEASE_NOTES

@@ -1,3 +1,132 @@
+2022.8.2
+- 2022-08-16 TUN-6656: Docker for arm64 should not be deployed in an amd64 container
+
+2022.8.1
+- 2022-08-15 TUN-6617: Updated CHANGES.md for protocol stickiness
+- 2022-08-12 EDGEPLAT-3918: bump go and go-boring to 1.18.5
+- 2022-08-12 TUN-6652: Publish dockerfile for both amd64 and arm64
+- 2022-08-11 TUN-6617: Dont fallback to http2 if QUIC conn was successful.
+- 2022-08-11 TUN-6617: Dont fallback to http2 if QUIC conn was successful.
+- 2022-08-11 Revert "TUN-6617: Dont fallback to http2 if QUIC conn was successful."
+- 2022-08-11 TUN-6617: Dont fallback to http2 if QUIC conn was successful.
+- 2022-08-01 TUN-6584: Define QUIC datagram v2 format to support proxying IP packets
+
+2022.8.0
+- 2022-08-10 TUN-6637: Upgrade quic-go
+- 2022-08-10 TUN-6646: Add support to SafeStreamCloser to close only write side of stream
+- 2022-08-09 TUN-6642: Fix unexpected close of quic stream triggered by upstream origin close
+- 2022-08-09 TUN-6639: Validate cyclic ingress configuration
+- 2022-08-08 TUN-6637: Upgrade go version and quic-go
+- 2022-08-08 TUN-6639: Validate cyclic ingress configuration
+- 2022-08-04 EDGEPLAT-3918: build cloudflared for Bookworm
+- 2022-08-02 Revert "TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span"
+- 2022-07-27 TUN-6601: Update gopkg.in/yaml.v3 references in modules
+- 2022-07-26 TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span
+- 2022-07-26 TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span
+- 2022-07-25 TUN-6598: Remove auto assignees on github issues
+- 2022-07-20 TUN-6583: Remove legacy --ui flag
+- 2022-07-20 cURL supports stdin and uses os pipes directly without copying
+- 2022-07-07 TUN-6517: Use QUIC stream context while proxying HTTP requests and TCP connections
+2022.7.1
+- 2022-07-06 TUN-6503: Fix transport fallback from QUIC in face of dial error "no network activity"
+
+2022.7.0
+- 2022-07-05 TUN-6499: Remove log that is per datagram
+- 2022-06-24 TUN-6460: Rename metric label location to edge_location
+- 2022-06-24 TUN-6459: Add cloudflared user-agent to access calls
+- 2022-06-17 TUN-6427: Differentiate between upstream request closed/canceled and failed origin requests
+- 2022-06-17 TUN-6388: Fix first tunnel connection not retrying
+- 2022-06-13 TUN-6384: Correct duplicate connection error to fetch new IP first
+- 2022-06-13 TUN-6373: Add edge-ip-version to remotely pushed configuration
+- 2022-06-07 TUN-6010: Add component tests for --edge-ip-version
+- 2022-05-20 TUN-6007: Implement new edge discovery algorithm
+- 2022-02-18 Ensure service install directories are created before writing file
+
+2022.6.3
+- 2022-06-20 TUN-6362: Add armhf support to cloudflare packaging
+
+2022.6.2
+- 2022-06-13 TUN-6381: Write error data on QUIC stream when we fail to talk to the origin; separate logging for protocol errors vs. origin errors.
+- 2022-06-17 TUN-6414: Remove go-sumtype from cloudflared build process
+- 2022-06-01 Add Http2Origin option to force HTTP/2 origin connections
+- 2022-06-02 fix ingress rules unit test
+- 2022-06-09 Update remaining OriginRequestConfig functions for Http2Origins
+- 2022-05-31 Add image source label to docker container.
+- 2022-05-10 Warp Private Network link updated
+
+2022.6.1
+- 2022-06-14 TUN-6395: Fix writing RPM repo data
+
+2022.6.0
+- 2022-06-14 Revert "TUN-6010: Add component tests for --edge-ip-version"
+- 2022-06-14 Revert "TUN-6373: Add edge-ip-version to remotely pushed configuration"
+- 2022-06-14 Revert "TUN-6384: Correct duplicate connection error to fetch new IP first"
+- 2022-06-14 Revert "TUN-6007: Implement new edge discovery algorithm"
+- 2022-06-13 TUN-6385: Don't share err between acceptStream loop and per-stream goroutines
+- 2022-06-13 TUN-6384: Correct duplicate connection error to fetch new IP first
+- 2022-06-13 TUN-6373: Add edge-ip-version to remotely pushed configuration
+- 2022-06-13 TUN-6380: Enforce connect and keep-alive timeouts for TCP connections in both WARP routing and websocket based TCP proxy.
+- 2022-06-11 Update issue templates
+- 2022-06-11 Amendment to previous PR
+- 2022-06-09 TUN-6347: Add TCP stream logs with FlowID
+- 2022-06-08 TUN-6361: Add cloudflared arm builds to pkging as well
+- 2022-06-07 TUN-6357: Add connector id to ready check endpoint
+- 2022-06-07 TUN-6010: Add component tests for --edge-ip-version
+- 2022-06-06 TUN-6191: Update quic-go to v0.27.1 and with custom patch to allow keep alive period to be configurable
+- 2022-06-03 TUN-6343: Fix QUIC->HTTP2 fallback
+- 2022-06-02 TUN-6339: Add config for IPv6 support
+- 2022-06-02 TUN-6341: Fix default config value for edge-ip-version
+- 2022-06-01 TUN-6323: Add Xenial and Trusty for Ubuntu pkging
+- 2022-05-31 TUN-6210: Add cloudflared.repo to make it easy for yum installs
+- 2022-05-30 TUN-6293: Update yaml v3 to latest hotfix
+- 2022-05-20 TUN-6007: Implement new edge discovery algorithm
+2022.5.3
+- 2022-05-30 TUN-6308: Add debug logs to see if packets are sent/received from edge
+- 2022-05-30 TUN-6301: Allow to update logger used by UDP session manager
+
+2022.5.2
+- 2022-05-23 TUN-6270: Import gpg keys from environment variables
+- 2022-05-24 TUN-6209: Improve feedback process if release_pkgs to deb and rpm fail
+- 2022-05-24 TUN-6280: Don't wrap qlog connection tracer for gatethering QUIC metrics since we're not writing qlog files.
+- 2022-05-25 TUN-6209: Sign RPM packages
+- 2022-05-25 TUN-6285: Upload pkg assets to repos when cloudflared is released.
+- 2022-05-24 TUN-6282: Upgrade golang to 1.17.10, go-boring to 1.17.9
+- 2022-05-26 TUN-6292: Debug builds for cloudflared
+- 2022-05-28 TUN-6304: Fixed some file permission issues
+- 2022-05-11 TUN-6197: Publish to brew core should not try to open the browser
+- 2022-05-12 TUN-5943: Add RPM support
+- 2022-05-18 TUN-6248: Fix panic in cloudflared during tracing when origin doesn't provide header map
+- 2022-05-18 TUN-6250: Add upstream response status code to tracing span attributes
+
+2022.5.1
+- 2022-05-06 TUN-6146: Release_pkgs is now a generic command line script
+- 2022-05-06 TUN-6185: Fix tcpOverWSOriginService not using original scheme for String representation
+- 2022-05-05 TUN-6175: Simply debian packaging by structural upload
+- 2022-05-05 TUN-5945: Added support for Ubuntu releases
+- 2022-05-04 TUN-6054: Create and upload deb packages to R2
+- 2022-05-03 TUN-6161: Set git user/email for brew core release
+- 2022-05-03 TUN-6166: Fix mocked QUIC transport for UDP proxy manager to return expected error
+- 2022-04-27 TUN-6016: Push local managed tunnels configuration to the edge
+2022.5.0
+- 2022-05-02 TUN-6158: Update golang.org/x/crypto
+- 2022-04-20 VULN-8383 Bump yaml.v2 to yaml.v3
+- 2022-04-21 TUN-6123: For a given connection with edge, close all datagram sessions through this connection when it's closed
+- 2022-04-20 TUN-6015: Add RPC method for pushing local config
+- 2022-04-21 TUN-6130: Fix vendoring due to case sensitive typo in package
+- 2022-04-27 TUN-6142: Add tunnel details support to RPC
+- 2022-04-28 TUN-6014: Add remote config flag as default feature
+- 2022-04-12 TUN-6000: Another fix for publishing to brew core
+- 2022-04-11 TUN-5990: Add otlp span export to response header
+- 2022-04-19 TUN-6070: First connection retries other edge IPs if the error is quic timeout(likely due to firewall blocking UDP)
+- 2022-04-11 TUN-6030: Add ttfb span for origin http request
+
+2022.4.1
+- 2022-04-11 TUN-6035: Reduce buffer size when proxying data
+- 2022-04-11 TUN-6038: Reduce buffer size used for proxying data
+- 2022-04-11 TUN-6043: Allow UI-managed Tunnels to fallback from QUIC but warn about that
+- 2022-04-07 TUN-6000 add version argument to bump-formula-pr
+- 2022-04-06 TUN-5989: Add in-memory otlp exporter
+
 2022.4.0
 - 2022-04-01 TUN-5973: Add backoff for non-recoverable errors as well
 - 2022-04-05 TUN-5992: Use QUIC protocol for remotely managed tunnels when protocol is unspecified

build-packages.sh

@@ -17,10 +17,18 @@ for arch in ${windowsArchs[@]}; do
 done
 
 
-linuxArchs=("386" "amd64" "arm" "arm64")
+linuxArchs=("386" "amd64" "arm" "armhf" "arm64")
 export TARGET_OS=linux
 for arch in ${linuxArchs[@]}; do
+    unset TARGET_ARM
     export TARGET_ARCH=$arch
+    
+    ## Support for armhf builds 
+    if [[ $arch == armhf ]] ; then
+        export TARGET_ARCH=arm
+        export TARGET_ARM=7 
+    fi
+    
     make cloudflared-deb
     mv cloudflared\_$VERSION\_$arch.deb $ARTIFACT_DIR/cloudflared-linux-$arch.deb
 

cfio/copy.go

@@ -0,0 +1,27 @@
+package cfio
+
+import (
+	"io"
+	"sync"
+)
+
+const defaultBufferSize = 16 * 1024
+
+var bufferPool = sync.Pool{
+	New: func() interface{} {
+		return make([]byte, defaultBufferSize)
+	},
+}
+
+func Copy(dst io.Writer, src io.Reader) (written int64, err error) {
+	_, okWriteTo := src.(io.WriterTo)
+	_, okReadFrom := dst.(io.ReaderFrom)
+	var buffer []byte = nil
+
+	if !(okWriteTo || okReadFrom) {
+		buffer = bufferPool.Get().([]byte)
+		defer bufferPool.Put(buffer)
+	}
+
+	return io.CopyBuffer(dst, src, buffer)
+}

cfsetup.yaml

@@ -1,8 +1,8 @@
-pinned_go: &pinned_go go=1.17.5-1
-pinned_go_fips: &pinned_go_fips go-boring=1.17.5-1
+pinned_go: &pinned_go go=1.18.5-1
+pinned_go_fips: &pinned_go_fips go-boring=1.18.5-2
 
 build_dir: &build_dir /cfsetup_build
-default-flavor: buster
+default-flavor: bullseye
 stretch: &stretch
   build:
     build_dir: *build_dir
@@ -40,16 +40,22 @@ stretch: &stretch
       - libffi-dev
       - python3-setuptools
       - python3-pip
+      - reprepro
+      - createrepo
     pre-cache: &github_release_pkgs_pre_cache
       - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
       - chmod a+x /usr/local/bin/wixl
       - pip3 install pynacl==1.4.0
       - pip3 install pygithub==1.55
+      - pip3 install boto3==1.22.9
+      - pip3 install python-gnupg==0.4.9
     post-cache:
       # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts
       - ./build-packages.sh
       # release the packages built and moved to /cfsetup/built_artifacts
       - make github-release-built-pkgs
+      # publish packages to linux repos
+      - make release-pkgs-linux
   # handle FIPS separately so that we built with gofips compiler
   github-fips-release-pkgs:
     build_dir: *build_dir
@@ -146,8 +152,7 @@ stretch: &stretch
       - build-essential
       - gotest-to-teamcity
     pre-cache: &test_pre_cache
-      - go get golang.org/x/tools/cmd/goimports
-      - go get github.com/sudarshan-reddy/go-sumtype@v0.0.0-20210827105221-82eca7e5abb1
+      - go install golang.org/x/tools/cmd/goimports@latest
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -241,6 +246,7 @@ stretch: &stretch
 
 buster: *stretch
 bullseye: *stretch
+bookworm: *stretch
 centos-7:
   publish-rpm:
     build_dir: *build_dir
@@ -249,10 +255,10 @@ centos-7:
     pre-cache:
       - yum install -y fakeroot
       - yum upgrade -y binutils-2.27-44.base.el7.x86_64
-      - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.17.5.linux-amd64.tar.gz
+      - wget https://go.dev/dl/go1.18.5.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.18.5.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
       - export GOARCH=amd64
-      - make publish-rpm
+      - make publish-rpm

cmd/cloudflared/access/cmd.go

@@ -56,11 +56,13 @@ const sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b@sentry.io/189878"
 
 var (
 	shutdownC chan struct{}
+	userAgent = "DEV"
 )
 
 // Init will initialize and store vars from the main program
-func Init(shutdown chan struct{}) {
+func Init(shutdown chan struct{}, version string) {
 	shutdownC = shutdown
+	userAgent = fmt.Sprintf("cloudflared/%s", version)
 }
 
 // Flags return the global flags for Access related commands (hopefully none)
@@ -294,6 +296,7 @@ func curl(c *cli.Context) error {
 // run kicks off a shell task and pipe the results to the respective std pipes
 func run(cmd string, args ...string) error {
 	c := exec.Command(cmd, args...)
+	c.Stdin = os.Stdin
 	stderr, err := c.StderrPipe()
 	if err != nil {
 		return err
@@ -505,7 +508,7 @@ func isTokenValid(options *carrier.StartOptions, log *zerolog.Logger) (bool, err
 	if err != nil {
 		return false, errors.Wrap(err, "Could not create access request")
 	}
-
+	req.Header.Set("User-Agent", userAgent)
 	// Do not follow redirects
 	client := &http.Client{
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {

cmd/cloudflared/main.go

@@ -21,6 +21,8 @@ import (
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/overwatch"
+	"github.com/cloudflare/cloudflared/token"
+	"github.com/cloudflare/cloudflared/tracing"
 	"github.com/cloudflare/cloudflared/watcher"
 )
 
@@ -84,8 +86,10 @@ func main() {
 	app.Commands = commands(cli.ShowVersion)
 
 	tunnel.Init(bInfo, graceShutdownC) // we need this to support the tunnel sub command...
-	access.Init(graceShutdownC)
+	access.Init(graceShutdownC, Version)
 	updater.Init(Version)
+	tracing.Init(Version)
+	token.Init(Version)
 	runApp(app, graceShutdownC)
 }
 

cmd/cloudflared/service_template.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path"
 	"text/template"
 
 	homedir "github.com/mitchellh/go-homedir"
@@ -52,10 +53,17 @@ func (st *ServiceTemplate) Generate(args *ServiceTemplateArgs) error {
 	if err != nil {
 		return fmt.Errorf("error generating %s: %v", st.Path, err)
 	}
-	fileMode := os.FileMode(0644)
+	fileMode := os.FileMode(0o644)
 	if st.FileMode != 0 {
 		fileMode = st.FileMode
 	}
+
+	plistFolder := path.Dir(resolvedPath)
+	err = os.MkdirAll(plistFolder, 0o755)
+	if err != nil {
+		return fmt.Errorf("error creating %s: %v", plistFolder, err)
+	}
+
 	err = ioutil.WriteFile(resolvedPath, buffer.Bytes(), fileMode)
 	if err != nil {
 		return fmt.Errorf("error writing %s: %v", resolvedPath, err)

cmd/cloudflared/tunnel/cmd.go

@@ -15,6 +15,7 @@ import (
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/getsentry/raven-go"
+	"github.com/google/uuid"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -24,7 +25,6 @@ import (
 	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
@@ -217,7 +217,7 @@ func runAdhocNamedTunnel(sc *subcommandContext, name, credentialsOutputPath stri
 
 // runClassicTunnel creates a "classic" non-named tunnel
 func runClassicTunnel(sc *subcommandContext) error {
-	return StartServer(sc.c, buildInfo, nil, sc.log, sc.isUIEnabled)
+	return StartServer(sc.c, buildInfo, nil, sc.log)
 }
 
 func routeFromFlag(c *cli.Context) (route cfapi.HostnameRoute, ok bool) {
@@ -235,7 +235,6 @@ func StartServer(
 	info *cliutil.BuildInfo,
 	namedTunnel *connection.NamedTunnelProperties,
 	log *zerolog.Logger,
-	isUIEnabled bool,
 ) error {
 	_ = raven.SetDSN(sentryDSN)
 	var wg sync.WaitGroup
@@ -330,9 +329,9 @@ func StartServer(
 		return fmt.Errorf(errText)
 	}
 
-	logTransport := logger.CreateTransportLoggerFromContext(c, isUIEnabled)
+	logTransport := logger.CreateTransportLoggerFromContext(c, logger.EnableTerminalLog)
 
-	observer := connection.NewObserver(log, logTransport, isUIEnabled)
+	observer := connection.NewObserver(log, logTransport)
 
 	// Send Quick Tunnel URL to UI if applicable
 	var quickTunnelURL string
@@ -343,13 +342,21 @@ func StartServer(
 		observer.SendURL(quickTunnelURL)
 	}
 
-	tunnelConfig, dynamicConfig, err := prepareTunnelConfig(c, info, log, logTransport, observer, namedTunnel)
+	tunnelConfig, orchestratorConfig, err := prepareTunnelConfig(c, info, log, logTransport, observer, namedTunnel)
 	if err != nil {
 		log.Err(err).Msg("Couldn't start tunnel")
 		return err
 	}
+	var clientID uuid.UUID
+	if tunnelConfig.NamedTunnel != nil {
+		clientID, err = uuid.FromBytes(tunnelConfig.NamedTunnel.Client.ClientID)
+		if err != nil {
+			// set to nil for classic tunnels
+			clientID = uuid.Nil
+		}
+	}
 
-	orchestrator, err := orchestration.NewOrchestrator(ctx, dynamicConfig, tunnelConfig.Tags, tunnelConfig.Log)
+	orchestrator, err := orchestration.NewOrchestrator(ctx, orchestratorConfig, tunnelConfig.Tags, tunnelConfig.Log)
 	if err != nil {
 		return err
 	}
@@ -363,7 +370,7 @@ func StartServer(
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		readinessServer := metrics.NewReadyServer(log)
+		readinessServer := metrics.NewReadyServer(log, clientID)
 		observer.RegisterSink(readinessServer)
 		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, quickTunnelURL, orchestrator, log)
 	}()
@@ -383,18 +390,6 @@ func StartServer(
 		errC <- supervisor.StartTunnelDaemon(ctx, tunnelConfig, orchestrator, connectedSignal, reconnectCh, graceShutdownC)
 	}()
 
-	if isUIEnabled {
-		tunnelUI := ui.NewUIModel(
-			info.Version(),
-			hostname,
-			metricsListener.Addr().String(),
-			dynamicConfig.Ingress,
-			tunnelConfig.HAConnections,
-		)
-		app := tunnelUI.Launch(ctx, log, logTransport)
-		observer.RegisterSink(app)
-	}
-
 	gracePeriod, err := gracePeriod(c)
 	if err != nil {
 		return err
@@ -514,6 +509,13 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Usage:   "Cloudflare Edge region to connect to. Omit or set to empty to connect to the global region.",
 			EnvVars: []string{"TUNNEL_REGION"},
 		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    "edge-ip-version",
+			Usage:   "Cloudflare Edge ip address version to connect with. {4, 6, auto}",
+			EnvVars: []string{"TUNNEL_EDGE_IP_VERSION"},
+			Value:   "4",
+			Hidden:  false,
+		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    tlsconfig.CaCertFlag,
 			Usage:   "Certificate Authority authenticating connections with Cloudflare's edge network.",
@@ -647,9 +649,9 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   uiFlag,
-			Usage:  "Launch tunnel UI. Tunnel logs are scrollable via 'j', 'k', or arrow keys.",
+			Usage:  "(depreciated) Launch tunnel UI. Tunnel logs are scrollable via 'j', 'k', or arrow keys.",
 			Value:  false,
-			Hidden: shouldHide,
+			Hidden: true,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:   "quick-service",
@@ -821,6 +823,13 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_NO_CHUNKED_ENCODING"},
 			Hidden:  shouldHide,
 		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    ingress.Http2OriginFlag,
+			Usage:   "Enables HTTP/2 origin servers.",
+			EnvVars: []string{"TUNNEL_ORIGIN_ENABLE_HTTP2"},
+			Hidden:  shouldHide,
+			Value:   false,
+		}),
 	}
 	return append(flags, sshFlags(shouldHide)...)
 }

cmd/cloudflared/tunnel/configuration.go

@@ -18,6 +18,7 @@ import (
 	"golang.org/x/crypto/ssh/terminal"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
@@ -41,7 +42,10 @@ var (
 
 	LogFieldHostname = "hostname"
 
-	secretFlags = [2]*altsrc.StringFlag{credentialsContentsFlag, tunnelTokenFlag}
+	secretFlags     = [2]*altsrc.StringFlag{credentialsContentsFlag, tunnelTokenFlag}
+	defaultFeatures = []string{supervisor.FeatureAllowRemoteConfig, supervisor.FeatureSerializedHeaders}
+
+	configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq", "edge-ip-version"}
 )
 
 // returns the first path that contains a cert.pem file. If none of the DefaultConfigSearchDirectories
@@ -216,6 +220,7 @@ func prepareTunnelConfig(
 	)
 
 	transportProtocol := c.String("protocol")
+	protocolFetcher := edgediscovery.ProtocolPercentage
 
 	cfg := config.GetConfiguration()
 	if isNamedTunnel {
@@ -224,12 +229,24 @@ func prepareTunnelConfig(
 			return nil, nil, errors.Wrap(err, "can't generate connector UUID")
 		}
 		log.Info().Msgf("Generated Connector ID: %s", clientUUID)
-		features := append(c.StringSlice("features"), supervisor.FeatureSerializedHeaders)
+		features := append(c.StringSlice("features"), defaultFeatures...)
 		if c.IsSet(TunnelTokenFlag) {
 			if transportProtocol == connection.AutoSelectFlag {
-				transportProtocol = connection.QUIC.String()
+				protocolFetcher = func() (edgediscovery.ProtocolPercents, error) {
+					// If the Tunnel is remotely managed and no protocol is set, we prefer QUIC, but still allow fall-back.
+					preferQuic := []edgediscovery.ProtocolPercent{
+						{
+							Protocol:   connection.QUIC.String(),
+							Percentage: 100,
+						},
+						{
+							Protocol:   connection.HTTP2.String(),
+							Percentage: 100,
+						},
+					}
+					return preferQuic, nil
+				}
 			}
-			features = append(features, supervisor.FeatureAllowRemoteConfig)
 			log.Info().Msg("Will be fetching remotely managed configuration from Cloudflare API. Defaulting to protocol: quic")
 		}
 		namedTunnel.Client = tunnelpogs.ClientInfo{
@@ -274,7 +291,7 @@ func prepareTunnelConfig(
 	}
 
 	warpRoutingEnabled := isWarpRoutingEnabled(cfg.WarpRouting, isNamedTunnel)
-	protocolSelector, err := connection.NewProtocolSelector(transportProtocol, warpRoutingEnabled, namedTunnel, edgediscovery.ProtocolPercentage, supervisor.ResolveTTL, log)
+	protocolSelector, err := connection.NewProtocolSelector(transportProtocol, warpRoutingEnabled, namedTunnel, protocolFetcher, supervisor.ResolveTTL, log)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -308,6 +325,10 @@ func prepareTunnelConfig(
 		CompressionSetting: h2mux.CompressionSetting(uint64(c.Int("compression-quality"))),
 		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
 	}
+	edgeIPVersion, err := parseConfigIPVersion(c.String("edge-ip-version"))
+	if err != nil {
+		return nil, nil, err
+	}
 
 	tunnelConfig := &supervisor.TunnelConfig{
 		GracePeriod:     gracePeriod,
@@ -316,6 +337,7 @@ func prepareTunnelConfig(
 		ClientID:        clientID,
 		EdgeAddrs:       c.StringSlice("edge"),
 		Region:          c.String("region"),
+		EdgeIPVersion:   edgeIPVersion,
 		HAConnections:   c.Int("ha-connections"),
 		IncidentLookup:  supervisor.NewIncidentLookup(),
 		IsAutoupdated:   c.Bool("is-autoupdated"),
@@ -334,11 +356,24 @@ func prepareTunnelConfig(
 		ProtocolSelector: protocolSelector,
 		EdgeTLSConfigs:   edgeTLSConfigs,
 	}
-	dynamicConfig := &orchestration.Config{
+	orchestratorConfig := &orchestration.Config{
 		Ingress:            &ingressRules,
-		WarpRoutingEnabled: warpRoutingEnabled,
+		WarpRouting:        ingress.NewWarpRoutingConfig(&cfg.WarpRouting),
+		ConfigurationFlags: parseConfigFlags(c),
 	}
-	return tunnelConfig, dynamicConfig, nil
+	return tunnelConfig, orchestratorConfig, nil
+}
+
+func parseConfigFlags(c *cli.Context) map[string]string {
+	result := make(map[string]string)
+
+	for _, flag := range configFlags {
+		if v := c.String(flag); c.IsSet(flag) && v != "" {
+			result[flag] = v
+		}
+	}
+
+	return result
 }
 
 func gracePeriod(c *cli.Context) (time.Duration, error) {
@@ -375,3 +410,18 @@ func dedup(slice []string) []string {
 	}
 	return keys
 }
+
+// ParseConfigIPVersion returns the IP version from possible expected values from config
+func parseConfigIPVersion(version string) (v allregions.ConfigIPVersion, err error) {
+	switch version {
+	case "4":
+		v = allregions.IPv4Only
+	case "6":
+		v = allregions.IPv6Only
+	case "auto":
+		v = allregions.Auto
+	default: // unspecified or invalid
+		err = fmt.Errorf("invalid value for edge-ip-version: %s", version)
+	}
+	return
+}

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -78,7 +78,6 @@ func RunQuickTunnel(sc *subcommandContext) error {
 		buildInfo,
 		&connection.NamedTunnelProperties{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
 		sc.log,
-		sc.isUIEnabled,
 	)
 }
 

cmd/cloudflared/tunnel/subcommand_context.go

@@ -31,10 +31,9 @@ func (e errInvalidJSONCredential) Error() string {
 // subcommandContext carries structs shared between subcommands, to reduce number of arguments needed to
 // pass between subcommands, and make sure they are only initialized once
 type subcommandContext struct {
-	c           *cli.Context
-	log         *zerolog.Logger
-	isUIEnabled bool
-	fs          fileSystem
+	c   *cli.Context
+	log *zerolog.Logger
+	fs  fileSystem
 
 	// These fields should be accessed using their respective Getter
 	tunnelstoreClient cfapi.Client
@@ -42,16 +41,10 @@ type subcommandContext struct {
 }
 
 func newSubcommandContext(c *cli.Context) (*subcommandContext, error) {
-	isUIEnabled := c.IsSet(uiFlag) && c.String("name") != ""
-
-	// If UI is enabled, terminal log output should be disabled -- log should be written into a UI log window instead
-	log := logger.CreateLoggerFromContext(c, isUIEnabled)
-
 	return &subcommandContext{
-		c:           c,
-		log:         log,
-		isUIEnabled: isUIEnabled,
-		fs:          realFileSystem{},
+		c:   c,
+		log: logger.CreateLoggerFromContext(c, logger.EnableTerminalLog),
+		fs:  realFileSystem{},
 	}, nil
 }
 
@@ -312,7 +305,6 @@ func (sc *subcommandContext) runWithCredentials(credentials connection.Credentia
 		buildInfo,
 		&connection.NamedTunnelProperties{Credentials: credentials},
 		sc.log,
-		sc.isUIEnabled,
 	)
 }
 

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -35,7 +35,6 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 	type fields struct {
 		c                 *cli.Context
 		log               *zerolog.Logger
-		isUIEnabled       bool
 		fs                fileSystem
 		tunnelstoreClient cfapi.Client
 		userCredential    *userCredential
@@ -168,7 +167,6 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 			sc := &subcommandContext{
 				c:                 tt.fields.c,
 				log:               tt.fields.log,
-				isUIEnabled:       tt.fields.isUIEnabled,
 				fs:                tt.fields.fs,
 				tunnelstoreClient: tt.fields.tunnelstoreClient,
 				userCredential:    tt.fields.userCredential,
@@ -307,7 +305,6 @@ func Test_subcommandContext_Delete(t *testing.T) {
 			sc := &subcommandContext{
 				c:                 tt.fields.c,
 				log:               tt.fields.log,
-				isUIEnabled:       tt.fields.isUIEnabled,
 				fs:                tt.fields.fs,
 				tunnelstoreClient: tt.fields.tunnelstoreClient,
 				userCredential:    tt.fields.userCredential,

cmd/cloudflared/tunnel/subcommands.go

@@ -20,7 +20,7 @@ import (
 	"github.com/urfave/cli/v2"
 	"github.com/urfave/cli/v2/altsrc"
 	"golang.org/x/net/idna"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"

cmd/cloudflared/ui/launch_ui.go

@@ -1,223 +0,0 @@
-package ui
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"github.com/gdamore/tcell"
-	"github.com/rivo/tview"
-	"github.com/rs/zerolog"
-
-	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/ingress"
-)
-
-type connState struct {
-	location string
-}
-
-type uiModel struct {
-	version       string
-	edgeURL       string
-	metricsURL    string
-	localServices []string
-	connections   []connState
-}
-
-type palette struct {
-	url          string
-	connected    string
-	defaultText  string
-	disconnected string
-	reconnecting string
-	unregistered string
-}
-
-func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haConnections int) *uiModel {
-	localServices := make([]string, len(ing.Rules))
-	for i, rule := range ing.Rules {
-		localServices[i] = rule.Service.String()
-	}
-	return &uiModel{
-		version:       version,
-		edgeURL:       hostname,
-		metricsURL:    metricsURL,
-		localServices: localServices,
-		connections:   make([]connState, haConnections),
-	}
-}
-
-func (data *uiModel) Launch(
-	ctx context.Context,
-	log, transportLog *zerolog.Logger,
-) connection.EventSink {
-	// Configure the logger to stream logs into the textview
-
-	// Add TextView as a group to write output to
-	logTextView := NewDynamicColorTextView()
-	// TODO: Format log for UI
-	//log.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
-	//transportLog.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
-
-	// Construct the UI
-	palette := palette{
-		url:          "lightblue",
-		connected:    "lime",
-		defaultText:  "white",
-		disconnected: "red",
-		reconnecting: "orange",
-		unregistered: "orange",
-	}
-
-	app := tview.NewApplication()
-
-	grid := tview.NewGrid().SetGap(1, 0)
-	frame := tview.NewFrame(grid)
-	header := fmt.Sprintf("cloudflared [::b]%s", data.version)
-
-	frame.AddText(header, true, tview.AlignLeft, tcell.ColorWhite)
-
-	// Create table to store connection info and status
-	connTable := tview.NewTable()
-	// SetColumns takes a value for each column, representing the size of the column
-	// Numbers <= 0 represent proportional widths and positive numbers represent absolute widths
-	grid.SetColumns(20, 0)
-
-	// SetRows takes a value for each row, representing the size of the row
-	grid.SetRows(1, 1, len(data.connections), 1, 0)
-
-	// AddItem takes a primitive tview type, row, column, rowSpan, columnSpan, minGridHeight, minGridWidth, and focus
-	grid.AddItem(tview.NewTextView().SetText("Tunnel:"), 0, 0, 1, 1, 0, 0, false)
-	grid.AddItem(tview.NewTextView().SetText("Status:"), 1, 0, 1, 1, 0, 0, false)
-	grid.AddItem(tview.NewTextView().SetText("Connections:"), 2, 0, 1, 1, 0, 0, false)
-
-	grid.AddItem(tview.NewTextView().SetText("Metrics:"), 3, 0, 1, 1, 0, 0, false)
-
-	tunnelHostText := tview.NewTextView().SetText(data.edgeURL)
-
-	grid.AddItem(tunnelHostText, 0, 1, 1, 1, 0, 0, false)
-	status := fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, strings.Join(data.localServices, ", "))
-	grid.AddItem(NewDynamicColorTextView().SetText(status), 1, 1, 1, 1, 0, 0, false)
-
-	grid.AddItem(connTable, 2, 1, 1, 1, 0, 0, false)
-
-	grid.AddItem(NewDynamicColorTextView().SetText(fmt.Sprintf("Metrics at [%s::b]http://%s/metrics", palette.url, data.metricsURL)), 3, 1, 1, 1, 0, 0, false)
-
-	// Add TextView to stream logs
-	// Logs are displayed in a new grid so a border can be set around them
-	logGrid := tview.NewGrid().SetBorders(true).AddItem(logTextView.SetChangedFunc(handleNewText(app, logTextView)), 0, 0, 5, 2, 0, 0, false)
-	// LogFrame holds the Logs header as well as the grid with the textView for streamed logs
-	logFrame := tview.NewFrame(logGrid).AddText("[::b]Logs:[::-]", true, tview.AlignLeft, tcell.ColorWhite).SetBorders(0, 0, 0, 0, 0, 0)
-	// Footer for log frame
-	logFrame.AddText("[::d]Use Ctrl+C to exit[::-]", false, tview.AlignRight, tcell.ColorWhite)
-	grid.AddItem(logFrame, 4, 0, 5, 2, 0, 0, false)
-
-	go func() {
-		<-ctx.Done()
-		app.Stop()
-		return
-	}()
-
-	go func() {
-		if err := app.SetRoot(frame, true).Run(); err != nil {
-			log.Error().Msgf("Error launching UI: %s", err)
-		}
-	}()
-
-	return connection.EventSinkFunc(func(event connection.Event) {
-		switch event.EventType {
-		case connection.Connected:
-			data.setConnTableCell(event, connTable, palette)
-		case connection.Disconnected, connection.Reconnecting, connection.Unregistering:
-			data.changeConnStatus(event, connTable, log, palette)
-		case connection.SetURL:
-			tunnelHostText.SetText(event.URL)
-			data.edgeURL = event.URL
-		case connection.RegisteringTunnel:
-			if data.edgeURL == "" {
-				tunnelHostText.SetText(fmt.Sprintf("Registering tunnel connection %d...", event.Index))
-			}
-		}
-		app.Draw()
-	})
-}
-
-func NewDynamicColorTextView() *tview.TextView {
-	return tview.NewTextView().SetDynamicColors(true)
-}
-
-// Re-draws application when new logs are streamed to UI
-func handleNewText(app *tview.Application, logTextView *tview.TextView) func() {
-	return func() {
-		app.Draw()
-		// SetFocus to enable scrolling in textview
-		app.SetFocus(logTextView)
-	}
-}
-
-func (data *uiModel) changeConnStatus(event connection.Event, table *tview.Table, log *zerolog.Logger, palette palette) {
-	index := int(event.Index)
-	// Get connection location and state
-	connState := data.getConnState(index)
-	// Check if connection is already displayed in UI
-	if connState == nil {
-		log.Info().Msg("Connection is not in the UI table")
-		return
-	}
-
-	locationState := event.Location
-
-	if event.EventType == connection.Reconnecting {
-		locationState = "Reconnecting..."
-	}
-
-	connectionNum := index + 1
-	// Get table cell
-	cell := table.GetCell(index, 0)
-	// Change dot color in front of text as well as location state
-	text := newCellText(palette, connectionNum, locationState, event.EventType)
-	cell.SetText(text)
-}
-
-// Return connection location and row in UI table
-func (data *uiModel) getConnState(connID int) *connState {
-	if connID < len(data.connections) {
-		return &data.connections[connID]
-	}
-
-	return nil
-}
-
-func (data *uiModel) setConnTableCell(event connection.Event, table *tview.Table, palette palette) {
-	index := int(event.Index)
-	connectionNum := index + 1
-
-	// Update slice to keep track of connection location and state in UI table
-	data.connections[index].location = event.Location
-
-	// Update text in table cell to show disconnected state
-	text := newCellText(palette, connectionNum, event.Location, event.EventType)
-	cell := tview.NewTableCell(text)
-	table.SetCell(index, 0, cell)
-}
-
-func newCellText(palette palette, connectionNum int, location string, connectedStatus connection.Status) string {
-	// HA connection indicator formatted as: "• #<CONNECTION_INDEX>: <COLO>",
-	//  where the left middle dot's color depends on the status of the connection
-	const connFmtString = "[%s]\u2022[%s] #%d: %s"
-
-	var dotColor string
-	switch connectedStatus {
-	case connection.Connected:
-		dotColor = palette.connected
-	case connection.Disconnected:
-		dotColor = palette.disconnected
-	case connection.Reconnecting:
-		dotColor = palette.reconnecting
-	case connection.Unregistering:
-		dotColor = palette.unregistered
-	}
-
-	return fmt.Sprintf(connFmtString, dotColor, palette.defaultText, connectionNum, location)
-}

component-tests/cli.py

@@ -0,0 +1,95 @@
+import json
+import subprocess
+from time import sleep
+
+from setup import get_config_from_file
+
+SINGLE_CASE_TIMEOUT = 600
+
+class CloudflaredCli:
+    def __init__(self, config, config_path, logger):
+        self.basecmd = [config.cloudflared_binary, "tunnel"]
+        if config_path is not None:
+            self.basecmd += ["--config", str(config_path)]
+        origincert = get_config_from_file()["origincert"]
+        if origincert:
+            self.basecmd += ["--origincert", origincert]
+        self.logger = logger
+
+    def _run_command(self, subcmd, subcmd_name, needs_to_pass=True):
+        cmd = self.basecmd + subcmd
+        # timeout limits the time a subprocess can run. This is useful to guard against running a tunnel when
+        # command/args are in wrong order.
+        result = run_subprocess(cmd, subcmd_name, self.logger, check=needs_to_pass, capture_output=True, timeout=15)
+        return result
+
+    def list_tunnels(self):
+        cmd_args = ["list", "--output", "json"]
+        listed = self._run_command(cmd_args, "list")
+        return json.loads(listed.stdout)
+
+    def get_tunnel_info(self, tunnel_id):
+        info = self._run_command(["info", "--output", "json", tunnel_id], "info")
+        return json.loads(info.stdout)
+
+    def __enter__(self):
+        self.basecmd += ["run"]
+        self.process = subprocess.Popen(self.basecmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        self.logger.info(f"Run cmd {self.basecmd}")
+        return self.process
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        terminate_gracefully(self.process, self.logger, self.basecmd)
+        self.logger.debug(f"{self.basecmd} logs: {self.process.stderr.read()}")
+
+
+def terminate_gracefully(process, logger, cmd):
+    process.terminate()
+    process_terminated = wait_for_terminate(process)
+    if not process_terminated:
+        process.kill()
+        logger.warning(f"{cmd}: cloudflared did not terminate within wait period. Killing process. logs: \
+                stdout: {process.stdout.read()}, stderr: {process.stderr.read()}")
+
+
+def wait_for_terminate(opened_subprocess, attempts=10, poll_interval=1):
+    """
+        wait_for_terminate polls the opened_subprocess every x seconds for a given number of attempts.
+        It returns true if the subprocess was terminated and false if it didn't.
+    """
+    for _ in range(attempts):
+        if _is_process_stopped(opened_subprocess):
+            return True
+        sleep(poll_interval)
+    return False
+
+
+def _is_process_stopped(process):
+    return process.poll() is not None
+
+
+def cert_path():
+    return get_config_from_file()["origincert"]
+
+
+class SubprocessError(Exception):
+    def __init__(self, program, exit_code, cause):
+        self.program = program
+        self.exit_code = exit_code
+        self.cause = cause
+
+
+def run_subprocess(cmd, cmd_name, logger, timeout=SINGLE_CASE_TIMEOUT, **kargs):
+    kargs["timeout"] = timeout
+    try:
+        result = subprocess.run(cmd, **kargs)
+        logger.debug(f"{cmd} log: {result.stdout}", extra={"cmd": cmd_name})
+        return result
+    except subprocess.CalledProcessError as e:
+        err = f"{cmd} return exit code {e.returncode}, stderr" + e.stderr.decode("utf-8")
+        logger.error(err, extra={"cmd": cmd_name, "return_code": e.returncode})
+        raise SubprocessError(cmd[0], e.returncode, e)
+    except subprocess.TimeoutExpired as e:
+        err = f"{cmd} timeout after {e.timeout} seconds, stdout: {e.stdout}, stderr: {e.stderr}"
+        logger.error(err, extra={"cmd": cmd_name, "return_code": "timeout"})
+        raise e

component-tests/test_edge_discovery.py

@@ -0,0 +1,165 @@
+import ipaddress
+import socket
+
+import pytest
+
+from constants import protocols
+from cli import CloudflaredCli
+from util import get_tunnel_connector_id, LOGGER, wait_tunnel_ready, write_config
+
+
+class TestEdgeDiscovery:
+    def _extra_config(self, protocol, edge_ip_version):
+        config = {
+            "protocol": protocol,
+        }
+        if edge_ip_version:
+            config["edge-ip-version"] = edge_ip_version
+        return config
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_default_only(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via IPv4-only edge addresses (default is unset "--edge-ip-version 4")
+        """
+        if self.has_ipv6_only():
+            pytest.skip("Host has IPv6 only support and current default is IPv4 only")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, None, self.expect_ipv4_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_ipv4_only(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via IPv4-only edge addresses
+        """
+        if self.has_ipv6_only():
+            pytest.skip("Host has IPv6 only support")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "4", self.expect_ipv4_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_ipv6_only(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via IPv6-only edge addresses
+        """
+        if self.has_ipv4_only():
+            pytest.skip("Host has IPv4 only support")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "6", self.expect_ipv6_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_auto_ip64(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via auto with a preference of IPv6 then IPv4 addresses for a dual stack host
+
+        This test also assumes that the host has IPv6 preference.
+        """
+        if not self.has_dual_stack(address_family_preference=socket.AddressFamily.AF_INET6):
+            pytest.skip("Host does not support dual stack with IPv6 preference")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "auto", self.expect_ipv6_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_auto_ip46(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via auto with a preference of IPv4 then IPv6 addresses for a dual stack host
+
+        This test also assumes that the host has IPv4 preference.
+        """
+        if not self.has_dual_stack(address_family_preference=socket.AddressFamily.AF_INET):
+            pytest.skip("Host does not support dual stack with IPv4 preference")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "auto", self.expect_ipv4_address)
+
+    def expect_address_connections(self, tmp_path, component_tests_config, protocol, edge_ip_version, assert_address_type):
+        config = component_tests_config(
+            self._extra_config(protocol, edge_ip_version))
+        config_path = write_config(tmp_path, config.full_config)
+        LOGGER.debug(config)
+        with CloudflaredCli(config, config_path, LOGGER):
+            wait_tunnel_ready(tunnel_url=config.get_url(),
+                              require_min_connections=4)
+            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
+            tunnel_id = config.get_tunnel_id()
+            info = cfd_cli.get_tunnel_info(tunnel_id)
+            connector_id = get_tunnel_connector_id()
+            connector = next(
+                (c for c in info["conns"] if c["id"] == connector_id), None)
+            assert connector, f"Expected connection info from get tunnel info for the connected instance: {info}"
+            conns = connector["conns"]
+            assert conns == None or len(
+                conns) == 4, f"There should be 4 connections registered: {conns}"
+            for conn in conns:
+                origin_ip = conn["origin_ip"]
+                assert origin_ip, f"No available origin_ip for this connection: {conn}"
+                assert_address_type(origin_ip)
+
+    def expect_ipv4_address(self, address):
+        assert type(ipaddress.ip_address(
+            address)) is ipaddress.IPv4Address, f"Expected connection from origin to be a valid IPv4 address: {address}"
+
+    def expect_ipv6_address(self, address):
+        assert type(ipaddress.ip_address(
+            address)) is ipaddress.IPv6Address, f"Expected connection from origin to be a valid IPv6 address: {address}"
+
+    def get_addresses(self):
+        """
+        Returns a list of addresses for the host.
+        """
+        host_addresses = socket.getaddrinfo(
+            "region1.v2.argotunnel.com", 7844, socket.AF_UNSPEC, socket.SOCK_STREAM)
+        assert len(
+            host_addresses) > 0, "No addresses returned from getaddrinfo"
+        return host_addresses
+
+    def has_dual_stack(self, address_family_preference=None):
+        """
+        Returns true if the host has dual stack support and can optionally check 
+        the provided IP family preference.
+        """
+        dual_stack = not self.has_ipv6_only() and not self.has_ipv4_only()
+        if address_family_preference:
+            address = self.get_addresses()[0]
+            return dual_stack and address[0] == address_family_preference
+
+        return dual_stack
+
+    def has_ipv6_only(self):
+        """
+        Returns True if the host has only IPv6 address support.
+        """
+        return self.attempt_connection(socket.AddressFamily.AF_INET6) and not self.attempt_connection(socket.AddressFamily.AF_INET)
+
+    def has_ipv4_only(self):
+        """
+        Returns True if the host has only IPv4 address support.
+        """
+        return self.attempt_connection(socket.AddressFamily.AF_INET) and not self.attempt_connection(socket.AddressFamily.AF_INET6)
+
+    def attempt_connection(self, address_family):
+        """
+        Returns True if a successful socket connection can be made to the 
+        remote host with the provided address family to validate host support 
+        for the provided address family.
+        """
+        address = None
+        for a in self.get_addresses():
+            if a[0] == address_family:
+                address = a
+                break
+        if address is None:
+            # Couldn't even lookup the address family so we can't connect
+            return False
+        af, socktype, proto, canonname, sockaddr = address
+        s = None
+        try:
+            s = socket.socket(af, socktype, proto)
+        except OSError:
+            return False
+        try:
+            s.connect(sockaddr)
+        except OSError:
+            s.close()
+            return False
+        s.close()
+        return True

component-tests/test_service.py

@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 import os
 import pathlib
-import platform
 import subprocess
 from contextlib import contextmanager
 from pathlib import Path
@@ -10,12 +9,7 @@ import pytest
 
 import test_logging
 from conftest import CfdModes
-from util import start_cloudflared, wait_tunnel_ready, write_config
-
-
-def select_platform(plat):
-    return pytest.mark.skipif(
-        platform.system() != plat, reason=f"Only runs on {plat}")
+from util import select_platform, start_cloudflared, wait_tunnel_ready, write_config
 
 
 def default_config_dir():

component-tests/util.py

@@ -1,9 +1,12 @@
 import logging
 import os
+import platform
 import subprocess
 from contextlib import contextmanager
 from time import sleep
 
+import pytest
+
 import requests
 import yaml
 from retrying import retry
@@ -12,6 +15,10 @@ from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
 
 LOGGER = logging.getLogger(__name__)
 
+def select_platform(plat):
+    return pytest.mark.skipif(
+        platform.system() != plat, reason=f"Only runs on {plat}")
+
 
 def write_config(directory, config):
     config_path = directory / "config.yml"
@@ -54,13 +61,15 @@ def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
 def run_cloudflared_background(cmd, allow_input, capture_output):
     output = subprocess.PIPE if capture_output else subprocess.DEVNULL
     stdin = subprocess.PIPE if allow_input else None
+    cfd = None
     try:
         cfd = subprocess.Popen(cmd, stdin=stdin, stdout=output, stderr=output)
         yield cfd
     finally:
-        cfd.terminate()
-        if capture_output:
-            LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
+        if cfd:
+            cfd.terminate()
+            if capture_output:
+                LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
 
 
 def wait_tunnel_ready(tunnel_url=None, require_min_connections=1, cfd_logs=None):
@@ -111,6 +120,17 @@ def check_tunnel_not_connected():
         LOGGER.warning(f"Failed to connect to {url}, error: {e}")
 
 
+def get_tunnel_connector_id():
+    url = f'http://localhost:{METRICS_PORT}/ready'
+
+    try:
+        resp = requests.get(url, timeout=1)
+        return resp.json()["connectorId"]
+    # cloudflared might already terminated
+    except requests.exceptions.ConnectionError as e:
+        LOGGER.warning(f"Failed to connect to {url}, error: {e}")
+
+
 # In some cases we don't need to check response status, such as when sending batch requests to generate logs
 def send_requests(url, count, require_ok=True):
     errors = 0

config/configuration.go

@@ -15,7 +15,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/cloudflare/cloudflared/validation"
 )
@@ -177,9 +177,9 @@ func ValidateUrl(c *cli.Context, allowURLFromArgs bool) (*url.URL, error) {
 }
 
 type UnvalidatedIngressRule struct {
-	Hostname      string              `json:"hostname"`
-	Path          string              `json:"path"`
-	Service       string              `json:"service"`
+	Hostname      string              `json:"hostname,omitempty"`
+	Path          string              `json:"path,omitempty"`
+	Service       string              `json:"service,omitempty"`
 	OriginRequest OriginRequestConfig `yaml:"originRequest" json:"originRequest"`
 }
 
@@ -192,41 +192,43 @@ type UnvalidatedIngressRule struct {
 // - To specify a time.Duration in json, use int64 of the nanoseconds
 type OriginRequestConfig struct {
 	// HTTP proxy timeout for establishing a new connection
-	ConnectTimeout *CustomDuration `yaml:"connectTimeout" json:"connectTimeout"`
+	ConnectTimeout *CustomDuration `yaml:"connectTimeout" json:"connectTimeout,omitempty"`
 	// HTTP proxy timeout for completing a TLS handshake
-	TLSTimeout *CustomDuration `yaml:"tlsTimeout" json:"tlsTimeout"`
+	TLSTimeout *CustomDuration `yaml:"tlsTimeout" json:"tlsTimeout,omitempty"`
 	// HTTP proxy TCP keepalive duration
-	TCPKeepAlive *CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive"`
+	TCPKeepAlive *CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive,omitempty"`
 	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
-	NoHappyEyeballs *bool `yaml:"noHappyEyeballs" json:"noHappyEyeballs"`
+	NoHappyEyeballs *bool `yaml:"noHappyEyeballs" json:"noHappyEyeballs,omitempty"`
 	// HTTP proxy maximum keepalive connection pool size
-	KeepAliveConnections *int `yaml:"keepAliveConnections" json:"keepAliveConnections"`
+	KeepAliveConnections *int `yaml:"keepAliveConnections" json:"keepAliveConnections,omitempty"`
 	// HTTP proxy timeout for closing an idle connection
-	KeepAliveTimeout *CustomDuration `yaml:"keepAliveTimeout" json:"keepAliveTimeout"`
+	KeepAliveTimeout *CustomDuration `yaml:"keepAliveTimeout" json:"keepAliveTimeout,omitempty"`
 	// Sets the HTTP Host header for the local webserver.
-	HTTPHostHeader *string `yaml:"httpHostHeader" json:"httpHostHeader"`
+	HTTPHostHeader *string `yaml:"httpHostHeader" json:"httpHostHeader,omitempty"`
 	// Hostname on the origin server certificate.
-	OriginServerName *string `yaml:"originServerName" json:"originServerName"`
+	OriginServerName *string `yaml:"originServerName" json:"originServerName,omitempty"`
 	// Path to the CA for the certificate of your origin.
 	// This option should be used only if your certificate is not signed by Cloudflare.
-	CAPool *string `yaml:"caPool" json:"caPool"`
+	CAPool *string `yaml:"caPool" json:"caPool,omitempty"`
 	// Disables TLS verification of the certificate presented by your origin.
 	// Will allow any certificate from the origin to be accepted.
 	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
-	NoTLSVerify *bool `yaml:"noTLSVerify" json:"noTLSVerify"`
+	NoTLSVerify *bool `yaml:"noTLSVerify" json:"noTLSVerify,omitempty"`
 	// Disables chunked transfer encoding.
 	// Useful if you are running a WSGI server.
-	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding" json:"disableChunkedEncoding"`
+	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding" json:"disableChunkedEncoding,omitempty"`
 	// Runs as jump host
-	BastionMode *bool `yaml:"bastionMode" json:"bastionMode"`
+	BastionMode *bool `yaml:"bastionMode" json:"bastionMode,omitempty"`
 	// Listen address for the proxy.
-	ProxyAddress *string `yaml:"proxyAddress" json:"proxyAddress"`
+	ProxyAddress *string `yaml:"proxyAddress" json:"proxyAddress,omitempty"`
 	// Listen port for the proxy.
-	ProxyPort *uint `yaml:"proxyPort" json:"proxyPort"`
+	ProxyPort *uint `yaml:"proxyPort" json:"proxyPort,omitempty"`
 	// Valid options are 'socks' or empty.
-	ProxyType *string `yaml:"proxyType" json:"proxyType"`
+	ProxyType *string `yaml:"proxyType" json:"proxyType,omitempty"`
 	// IP rules for the proxy service
-	IPRules []IngressIPRule `yaml:"ipRules" json:"ipRules"`
+	IPRules []IngressIPRule `yaml:"ipRules" json:"ipRules,omitempty"`
+	// Attempt to connect to origin with HTTP/2
+	Http2Origin *bool `yaml:"http2Origin" json:"http2Origin,omitempty"`
 }
 
 type IngressIPRule struct {
@@ -244,7 +246,9 @@ type Configuration struct {
 }
 
 type WarpRoutingConfig struct {
-	Enabled bool `yaml:"enabled" json:"enabled"`
+	Enabled        bool            `yaml:"enabled" json:"enabled"`
+	ConnectTimeout *CustomDuration `yaml:"connectTimeout" json:"connectTimeout,omitempty"`
+	TCPKeepAlive   *CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive,omitempty"`
 }
 
 type configFileSettings struct {
@@ -392,7 +396,7 @@ func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (settings *configFileSe
 	// Parse it again, with strict mode, to find warnings.
 	if file, err := os.Open(configFile); err == nil {
 		decoder := yaml.NewDecoder(file)
-		decoder.SetStrict(true)
+		decoder.KnownFields(true)
 		var unusedConfig configFileSettings
 		if err := decoder.Decode(&unusedConfig); err != nil {
 			warnings = err.Error()

config/configuration_test.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 )
 
 func TestConfigFileSettings(t *testing.T) {
@@ -23,7 +23,9 @@ func TestConfigFileSettings(t *testing.T) {
 			Service:  "https://localhost:8001",
 		}
 		warpRouting = WarpRoutingConfig{
-			Enabled: true,
+			Enabled:        true,
+			ConnectTimeout: &CustomDuration{Duration: 2 * time.Second},
+			TCPKeepAlive:   &CustomDuration{Duration: 10 * time.Second},
 		}
 	)
 	rawYAML := `
@@ -48,6 +50,9 @@ ingress:
    service: https://localhost:8001
 warp-routing: 
   enabled: true
+  connectTimeout: 2s
+  tcpKeepAlive: 10s
+
 retries: 5
 grace-period: 30s
 percentage: 3.14
@@ -111,7 +116,7 @@ counters:
 
 }
 
-var rawConfig = []byte(`
+var rawJsonConfig = []byte(`
 {
 	"connectTimeout": 10,
 	"tlsTimeout": 30,
@@ -139,7 +144,8 @@ var rawConfig = []byte(`
 			"ports": [443, 4443],
 			"allow": true
 		}
-	]
+	],
+	"http2Origin": true
 }
 `)
 
@@ -148,15 +154,14 @@ func TestMarshalUnmarshalOriginRequest(t *testing.T) {
 		name          string
 		marshalFunc   func(in interface{}) (out []byte, err error)
 		unMarshalFunc func(in []byte, out interface{}) (err error)
-		baseUnit      time.Duration
 	}{
-		{"json", json.Marshal, json.Unmarshal, time.Second},
-		{"yaml", yaml.Marshal, yaml.Unmarshal, time.Nanosecond},
+		{"json", json.Marshal, json.Unmarshal},
+		{"yaml", yaml.Marshal, yaml.Unmarshal},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			assertConfig(t, tc.marshalFunc, tc.unMarshalFunc, tc.baseUnit)
+			assertConfig(t, tc.marshalFunc, tc.unMarshalFunc)
 		})
 	}
 }
@@ -165,18 +170,17 @@ func assertConfig(
 	t *testing.T,
 	marshalFunc func(in interface{}) (out []byte, err error),
 	unMarshalFunc func(in []byte, out interface{}) (err error),
-	baseUnit time.Duration,
 ) {
 	var config OriginRequestConfig
 	var config2 OriginRequestConfig
 
-	assert.NoError(t, unMarshalFunc(rawConfig, &config))
+	assert.NoError(t, json.Unmarshal(rawJsonConfig, &config))
 
-	assert.Equal(t, baseUnit*10, config.ConnectTimeout.Duration)
-	assert.Equal(t, baseUnit*30, config.TLSTimeout.Duration)
-	assert.Equal(t, baseUnit*30, config.TCPKeepAlive.Duration)
+	assert.Equal(t, time.Second*10, config.ConnectTimeout.Duration)
+	assert.Equal(t, time.Second*30, config.TLSTimeout.Duration)
+	assert.Equal(t, time.Second*30, config.TCPKeepAlive.Duration)
 	assert.Equal(t, true, *config.NoHappyEyeballs)
-	assert.Equal(t, baseUnit*60, config.KeepAliveTimeout.Duration)
+	assert.Equal(t, time.Second*60, config.KeepAliveTimeout.Duration)
 	assert.Equal(t, 10, *config.KeepAliveConnections)
 	assert.Equal(t, "app.tunnel.com", *config.HTTPHostHeader)
 	assert.Equal(t, "app.tunnel.com", *config.OriginServerName)
@@ -188,6 +192,7 @@ func assertConfig(
 	assert.Equal(t, true, *config.NoTLSVerify)
 	assert.Equal(t, uint(9000), *config.ProxyPort)
 	assert.Equal(t, "socks", *config.ProxyType)
+	assert.Equal(t, true, *config.Http2Origin)
 
 	privateV4 := "10.0.0.0/8"
 	privateV6 := "fc00::/7"

config/manager.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/cloudflare/cloudflared/watcher"
 )

connection/connection.go

@@ -14,6 +14,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 
+	"github.com/cloudflare/cloudflared/tracing"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/websocket"
 )
@@ -29,6 +30,7 @@ var switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols,
 
 type Orchestrator interface {
 	UpdateConfig(version int32, config []byte) *pogs.UpdateConfigurationResponse
+	GetConfigJSON() ([]byte, error)
 	GetOriginProxy() (OriginProxy, error)
 }
 
@@ -121,22 +123,24 @@ func (t Type) String() string {
 
 // OriginProxy is how data flows from cloudflared to the origin services running behind it.
 type OriginProxy interface {
-	ProxyHTTP(w ResponseWriter, req *http.Request, isWebsocket bool) error
+	ProxyHTTP(w ResponseWriter, tr *tracing.TracedHTTPRequest, isWebsocket bool) error
 	ProxyTCP(ctx context.Context, rwa ReadWriteAcker, req *TCPRequest) error
 }
 
 // TCPRequest defines the input format needed to perform a TCP proxy.
 type TCPRequest struct {
-	Dest    string
-	CFRay   string
-	LBProbe bool
+	Dest      string
+	CFRay     string
+	LBProbe   bool
+	FlowID    string
+	CfTraceID string
 }
 
 // ReadWriteAcker is a readwriter with the ability to Acknowledge to the downstream (edge) that the origin has
 // accepted the connection.
 type ReadWriteAcker interface {
 	io.ReadWriter
-	AckConnection() error
+	AckConnection(tracePropagation string) error
 }
 
 // HTTPResponseReadWriteAcker is an HTTP implementation of ReadWriteAcker.
@@ -165,7 +169,7 @@ func (h *HTTPResponseReadWriteAcker) Write(p []byte) (int, error) {
 
 // AckConnection acks an HTTP connection by sending a switch protocols status code that enables the caller to
 // upgrade to streams.
-func (h *HTTPResponseReadWriteAcker) AckConnection() error {
+func (h *HTTPResponseReadWriteAcker) AckConnection(tracePropagation string) error {
 	resp := &http.Response{
 		Status:        switchingProtocolText,
 		StatusCode:    http.StatusSwitchingProtocols,
@@ -176,6 +180,10 @@ func (h *HTTPResponseReadWriteAcker) AckConnection() error {
 		resp.Header = websocket.NewResponseHeader(h.req)
 	}
 
+	if tracePropagation != "" {
+		resp.Header.Add(tracing.CanonicalCloudflaredTracingHeader, tracePropagation)
+	}
+
 	return h.w.WriteRespHeaders(resp.StatusCode, resp.Header)
 }
 

connection/connection_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/websocket"
 )
@@ -29,6 +30,8 @@ var (
 	testLargeResp = make([]byte, largeFileSize)
 )
 
+var _ ReadWriteAcker = (*HTTPResponseReadWriteAcker)(nil)
+
 type testRequest struct {
 	name           string
 	endpoint       string
@@ -41,6 +44,10 @@ type mockOrchestrator struct {
 	originProxy OriginProxy
 }
 
+func (mcr *mockOrchestrator) GetConfigJSON() ([]byte, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
 func (*mockOrchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
 	return &tunnelpogs.UpdateConfigurationResponse{
 		LastAppliedVersion: version,
@@ -55,9 +62,10 @@ type mockOriginProxy struct{}
 
 func (moc *mockOriginProxy) ProxyHTTP(
 	w ResponseWriter,
-	req *http.Request,
+	tr *tracing.TracedHTTPRequest,
 	isWebsocket bool,
 ) error {
+	req := tr.Request
 	if isWebsocket {
 		switch req.URL.Path {
 		case "/ws/echo":

connection/control.go

@@ -2,7 +2,9 @@ package connection
 
 import (
 	"context"
+	"fmt"
 	"io"
+	"net"
 	"time"
 
 	"github.com/rs/zerolog"
@@ -19,6 +21,8 @@ type controlStream struct {
 	connectedFuse         ConnectedFuse
 	namedTunnelProperties *NamedTunnelProperties
 	connIndex             uint8
+	edgeAddress           net.IP
+	protocol              Protocol
 
 	newRPCClientFunc RPCClientFunc
 
@@ -30,20 +34,26 @@ type controlStream struct {
 // ControlStreamHandler registers connections with origintunneld and initiates graceful shutdown.
 type ControlStreamHandler interface {
 	// ServeControlStream handles the control plane of the transport in the current goroutine calling this
-	ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions) error
+	ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions, tunnelConfigGetter TunnelConfigJSONGetter) error
 	// IsStopped tells whether the method above has finished
 	IsStopped() bool
 }
 
+type TunnelConfigJSONGetter interface {
+	GetConfigJSON() ([]byte, error)
+}
+
 // NewControlStream returns a new instance of ControlStreamHandler
 func NewControlStream(
 	observer *Observer,
 	connectedFuse ConnectedFuse,
 	namedTunnelConfig *NamedTunnelProperties,
 	connIndex uint8,
+	edgeAddress net.IP,
 	newRPCClientFunc RPCClientFunc,
 	gracefulShutdownC <-chan struct{},
 	gracePeriod time.Duration,
+	protocol Protocol,
 ) ControlStreamHandler {
 	if newRPCClientFunc == nil {
 		newRPCClientFunc = newRegistrationRPCClient
@@ -54,8 +64,10 @@ func NewControlStream(
 		namedTunnelProperties: namedTunnelConfig,
 		newRPCClientFunc:      newRPCClientFunc,
 		connIndex:             connIndex,
+		edgeAddress:           edgeAddress,
 		gracefulShutdownC:     gracefulShutdownC,
 		gracePeriod:           gracePeriod,
+		protocol:              protocol,
 	}
 }
 
@@ -63,15 +75,31 @@ func (c *controlStream) ServeControlStream(
 	ctx context.Context,
 	rw io.ReadWriteCloser,
 	connOptions *tunnelpogs.ConnectionOptions,
+	tunnelConfigGetter TunnelConfigJSONGetter,
 ) error {
 	rpcClient := c.newRPCClientFunc(ctx, rw, c.observer.log)
 
-	if err := rpcClient.RegisterConnection(ctx, c.namedTunnelProperties, connOptions, c.connIndex, c.observer); err != nil {
+	registrationDetails, err := rpcClient.RegisterConnection(ctx, c.namedTunnelProperties, connOptions, c.connIndex, c.edgeAddress, c.observer)
+	if err != nil {
 		rpcClient.Close()
 		return err
 	}
+
+	c.observer.logServerInfo(c.connIndex, registrationDetails.Location, c.edgeAddress, fmt.Sprintf("Connection %s registered", registrationDetails.UUID))
+	c.observer.sendConnectedEvent(c.connIndex, c.protocol, registrationDetails.Location)
 	c.connectedFuse.Connected()
 
+	// if conn index is 0 and tunnel is not remotely managed, then send local ingress rules configuration
+	if c.connIndex == 0 && !registrationDetails.TunnelIsRemotelyManaged {
+		if tunnelConfig, err := tunnelConfigGetter.GetConfigJSON(); err == nil {
+			if err := rpcClient.SendLocalConfiguration(ctx, tunnelConfig, c.observer); err != nil {
+				c.observer.log.Err(err).Msg("unable to send local configuration")
+			}
+		} else {
+			c.observer.log.Err(err).Msg("failed to obtain current configuration")
+		}
+	}
+
 	c.waitForUnregister(ctx, rpcClient)
 	return nil
 }

connection/errors.go

@@ -18,6 +18,15 @@ func (e DupConnRegisterTunnelError) Error() string {
 	return "already connected to this server, trying another address"
 }
 
+// Dial to edge server with quic failed
+type EdgeQuicDialError struct {
+	Cause error
+}
+
+func (e *EdgeQuicDialError) Error() string {
+	return "failed to dial to edge with quic: " + e.Cause.Error()
+}
+
 // RegisterTunnel error from server
 type ServerRegisterTunnelError struct {
 	Cause     error

connection/event.go

@@ -5,6 +5,7 @@ type Event struct {
 	Index     uint8
 	EventType Status
 	Location  string
+	Protocol  Protocol
 	URL       string
 }
 

connection/h2mux.go

@@ -12,6 +12,7 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/websocket"
 )
@@ -68,6 +69,7 @@ func NewH2muxConnection(
 	connIndex uint8,
 	observer *Observer,
 	gracefulShutdownC <-chan struct{},
+	log *zerolog.Logger,
 ) (*h2muxConnection, error, bool) {
 	h := &h2muxConnection{
 		orchestrator:      orchestrator,
@@ -78,6 +80,7 @@ func NewH2muxConnection(
 		observer:          observer,
 		gracefulShutdownC: gracefulShutdownC,
 		newRPCClientFunc:  newRegistrationRPCClient,
+		log:               log,
 	}
 
 	// Establish a muxed connection with the edge
@@ -233,7 +236,7 @@ func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
 		return err
 	}
 
-	err = originProxy.ProxyHTTP(respWriter, req, sourceConnectionType == TypeWebsocket)
+	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, h.log), sourceConnectionType == TypeWebsocket)
 	if err != nil {
 		respWriter.WriteErrorResponse()
 	}

connection/h2mux_test.go

@@ -47,8 +47,8 @@ func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
 		edgeMuxChan <- edgeMux
 	}()
 	var connIndex = uint8(0)
-	testObserver := NewObserver(&log, &log, false)
-	h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil)
+	testObserver := NewObserver(&log, &log)
+	h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil, &log)
 	require.NoError(t, err)
 	return h2muxConn, <-edgeMuxChan
 }

connection/http2.go

@@ -15,6 +15,7 @@ import (
 	"github.com/rs/zerolog"
 	"golang.org/x/net/http2"
 
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
@@ -116,7 +117,7 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	switch connType {
 	case TypeControlStream:
-		if err := c.controlStreamHandler.ServeControlStream(r.Context(), respWriter, c.connOptions); err != nil {
+		if err := c.controlStreamHandler.ServeControlStream(r.Context(), respWriter, c.connOptions, c.orchestrator); err != nil {
 			c.controlStreamErr = err
 			c.log.Error().Err(err)
 			respWriter.WriteErrorResponse()
@@ -130,7 +131,9 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	case TypeWebsocket, TypeHTTP:
 		stripWebsocketUpgradeHeader(r)
-		if err := originProxy.ProxyHTTP(respWriter, r, connType == TypeWebsocket); err != nil {
+		// Check for tracing on request
+		tr := tracing.NewTracedHTTPRequest(r, c.log)
+		if err := originProxy.ProxyHTTP(respWriter, tr, connType == TypeWebsocket); err != nil {
 			err := fmt.Errorf("Failed to proxy HTTP: %w", err)
 			c.log.Error().Err(err)
 			respWriter.WriteErrorResponse()
@@ -228,6 +231,12 @@ func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) erro
 			dest[name] = values
 		}
 
+		if h2name == tracing.IntCloudflaredTracingHeader {
+			// Add cf-int-cloudflared-tracing header outside of serialized userHeaders
+			rp.w.Header()[tracing.CanonicalCloudflaredTracingHeader] = values
+			continue
+		}
+
 		if !IsControlResponseHeader(h2name) || IsWebsocketClientHeader(h2name) {
 			// User headers, on the other hand, must all be serialized so that
 			// HTTP/2 header validation won't be applied to HTTP/1 header values

connection/http2_test.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/gobwas/ws/wsutil"
+	"github.com/google/uuid"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -32,7 +33,7 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 	edgeConn, cfdConn := net.Pipe()
 	var connIndex = uint8(0)
 	log := zerolog.Nop()
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
@@ -40,7 +41,9 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 		connIndex,
 		nil,
 		nil,
+		nil,
 		1*time.Second,
+		HTTP2,
 	)
 	return NewHTTP2Connection(
 		cfdConn,
@@ -166,18 +169,27 @@ type mockNamedTunnelRPCClient struct {
 	unregistered chan struct{}
 }
 
+func (mc mockNamedTunnelRPCClient) SendLocalConfiguration(c context.Context, config []byte, observer *Observer) error {
+	return nil
+}
+
 func (mc mockNamedTunnelRPCClient) RegisterConnection(
 	c context.Context,
 	properties *NamedTunnelProperties,
 	options *tunnelpogs.ConnectionOptions,
 	connIndex uint8,
+	edgeAddress net.IP,
 	observer *Observer,
-) error {
+) (*tunnelpogs.ConnectionDetails, error) {
 	if mc.shouldFail != nil {
-		return mc.shouldFail
+		return nil, mc.shouldFail
 	}
 	close(mc.registered)
-	return nil
+	return &tunnelpogs.ConnectionDetails{
+		Location:                "LIS",
+		UUID:                    uuid.New(),
+		TunnelIsRemotelyManaged: false,
+	}, nil
 }
 
 func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
@@ -345,15 +357,17 @@ func TestServeControlStream(t *testing.T) {
 		unregistered: make(chan struct{}),
 	}
 
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
 		&NamedTunnelProperties{},
 		1,
+		nil,
 		rpcClientFactory.newMockRPCClient,
 		nil,
 		1*time.Second,
+		HTTP2,
 	)
 	http2Conn.controlStreamHandler = controlStream
 
@@ -395,15 +409,17 @@ func TestFailRegistration(t *testing.T) {
 		unregistered: make(chan struct{}),
 	}
 
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
 		&NamedTunnelProperties{},
 		http2Conn.connIndex,
+		nil,
 		rpcClientFactory.newMockRPCClient,
 		nil,
 		1*time.Second,
+		HTTP2,
 	)
 	http2Conn.controlStreamHandler = controlStream
 
@@ -440,16 +456,18 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 	events := &eventCollectorSink{}
 
 	shutdownC := make(chan struct{})
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	obs.RegisterSink(events)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
 		&NamedTunnelProperties{},
 		http2Conn.connIndex,
+		nil,
 		rpcClientFactory.newMockRPCClient,
 		shutdownC,
 		1*time.Second,
+		HTTP2,
 	)
 
 	http2Conn.controlStreamHandler = controlStream
@@ -477,7 +495,7 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 
 	select {
 	case <-rpcClientFactory.registered:
-		break //ok
+		break // ok
 	case <-time.Tick(time.Second):
 		t.Fatal("timeout out waiting for registration")
 	}
@@ -487,7 +505,7 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 
 	select {
 	case <-rpcClientFactory.unregistered:
-		break //ok
+		break // ok
 	case <-time.Tick(time.Second):
 		t.Fatal("timeout out waiting for unregistered signal")
 	}

connection/metrics.go

@@ -13,6 +13,7 @@ const (
 	MetricsNamespace = "cloudflared"
 	TunnelSubsystem  = "tunnel"
 	muxerSubsystem   = "muxer"
+	configSubsystem  = "config"
 )
 
 type muxerMetrics struct {
@@ -36,6 +37,11 @@ type muxerMetrics struct {
 	compRateAve      *prometheus.GaugeVec
 }
 
+type localConfigMetrics struct {
+	pushes       prometheus.Counter
+	pushesErrors prometheus.Counter
+}
+
 type tunnelMetrics struct {
 	timerRetries    prometheus.Gauge
 	serverLocations *prometheus.GaugeVec
@@ -51,6 +57,39 @@ type tunnelMetrics struct {
 	muxerMetrics        *muxerMetrics
 	tunnelsHA           tunnelsForHA
 	userHostnamesCounts *prometheus.CounterVec
+
+	localConfigMetrics *localConfigMetrics
+}
+
+func newLocalConfigMetrics() *localConfigMetrics {
+
+	pushesMetric := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: configSubsystem,
+			Name:      "local_config_pushes",
+			Help:      "Number of local configuration pushes to the edge",
+		},
+	)
+
+	pushesErrorsMetric := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: configSubsystem,
+			Name:      "local_config_pushes_errors",
+			Help:      "Number of errors occurred during local configuration pushes",
+		},
+	)
+
+	prometheus.MustRegister(
+		pushesMetric,
+		pushesErrorsMetric,
+	)
+
+	return &localConfigMetrics{
+		pushes:       pushesMetric,
+		pushesErrors: pushesErrorsMetric,
+	}
 }
 
 func newMuxerMetrics() *muxerMetrics {
@@ -328,7 +367,7 @@ func initTunnelMetrics() *tunnelMetrics {
 			Name:      "server_locations",
 			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
 		},
-		[]string{"connection_id", "location"},
+		[]string{"connection_id", "edge_location"},
 	)
 	prometheus.MustRegister(serverLocations)
 
@@ -386,6 +425,7 @@ func initTunnelMetrics() *tunnelMetrics {
 		regFail:             registerFail,
 		rpcFail:             rpcFail,
 		userHostnamesCounts: userHostnamesCounts,
+		localConfigMetrics:  newLocalConfigMetrics(),
 	}
 }
 

connection/observer.go

@@ -1,6 +1,7 @@
 package connection
 
 import (
+	"net"
 	"strings"
 
 	"github.com/rs/zerolog"
@@ -8,6 +9,7 @@ import (
 
 const (
 	LogFieldLocation          = "location"
+	LogFieldIPAddress         = "ip"
 	observerChannelBufferSize = 16
 )
 
@@ -16,7 +18,6 @@ type Observer struct {
 	logTransport    *zerolog.Logger
 	metrics         *tunnelMetrics
 	tunnelEventChan chan Event
-	uiEnabled       bool
 	addSinkChan     chan EventSink
 }
 
@@ -24,12 +25,11 @@ type EventSink interface {
 	OnTunnelEvent(event Event)
 }
 
-func NewObserver(log, logTransport *zerolog.Logger, uiEnabled bool) *Observer {
+func NewObserver(log, logTransport *zerolog.Logger) *Observer {
 	o := &Observer{
 		log:             log,
 		logTransport:    logTransport,
 		metrics:         newTunnelMetrics(),
-		uiEnabled:       uiEnabled,
 		tunnelEventChan: make(chan Event, observerChannelBufferSize),
 		addSinkChan:     make(chan EventSink, observerChannelBufferSize),
 	}
@@ -41,11 +41,12 @@ func (o *Observer) RegisterSink(sink EventSink) {
 	o.addSinkChan <- sink
 }
 
-func (o *Observer) logServerInfo(connIndex uint8, location, msg string) {
+func (o *Observer) logServerInfo(connIndex uint8, location string, address net.IP, msg string) {
 	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
 	o.log.Info().
 		Uint8(LogFieldConnIndex, connIndex).
 		Str(LogFieldLocation, location).
+		IPAddr(LogFieldIPAddress, address).
 		Msg(msg)
 	o.metrics.registerServerLocation(uint8ToString(connIndex), location)
 }
@@ -54,8 +55,8 @@ func (o *Observer) sendRegisteringEvent(connIndex uint8) {
 	o.sendEvent(Event{Index: connIndex, EventType: RegisteringTunnel})
 }
 
-func (o *Observer) sendConnectedEvent(connIndex uint8, location string) {
-	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
+func (o *Observer) sendConnectedEvent(connIndex uint8, protocol Protocol, location string) {
+	o.sendEvent(Event{Index: connIndex, EventType: Connected, Protocol: protocol, Location: location})
 }
 
 func (o *Observer) SendURL(url string) {

connection/observer_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestSendUrl(t *testing.T) {
-	observer := NewObserver(&log, &log, false)
+	observer := NewObserver(&log, &log)
 
 	observer.SendURL("my-url.com")
 	assert.Equal(t, 1.0, getCounterValue(t, observer.metrics.userHostnamesCounts, "https://my-url.com"))
@@ -63,7 +63,7 @@ func TestRegisterServerLocation(t *testing.T) {
 }
 
 func TestObserverEventsDontBlock(t *testing.T) {
-	observer := NewObserver(&log, &log, false)
+	observer := NewObserver(&log, &log)
 	var mu sync.Mutex
 	observer.RegisterSink(EventSinkFunc(func(_ Event) {
 		// callback will block if lock is already held

connection/quic.go

@@ -20,6 +20,7 @@ import (
 	"github.com/cloudflare/cloudflared/datagramsession"
 	"github.com/cloudflare/cloudflared/ingress"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
@@ -30,14 +31,21 @@ const (
 	HTTPMethodKey = "HttpMethod"
 	// HTTPHostKey is used to get or set http Method in QUIC ALPN if the underlying proxy connection type is HTTP.
 	HTTPHostKey = "HttpHost"
+
+	QUICMetadataFlowID = "FlowID"
+	// emperically this capacity has been working well
+	demuxChanCapacity = 16
 )
 
 // QUICConnection represents the type that facilitates Proxying via QUIC streams.
 type QUICConnection struct {
-	session              quic.Session
-	logger               *zerolog.Logger
-	orchestrator         Orchestrator
-	sessionManager       datagramsession.Manager
+	session      quic.Connection
+	logger       *zerolog.Logger
+	orchestrator Orchestrator
+	// sessionManager tracks active sessions. It receives datagrams from quic connection via datagramMuxer
+	sessionManager datagramsession.Manager
+	// datagramMuxer mux/demux datagrams from quic connection
+	datagramMuxer        *quicpogs.DatagramMuxer
 	controlStreamHandler ControlStreamHandler
 	connOptions          *tunnelpogs.ConnectionOptions
 }
@@ -54,21 +62,19 @@ func NewQUICConnection(
 ) (*QUICConnection, error) {
 	session, err := quic.DialAddr(edgeAddr.String(), tlsConfig, quicConfig)
 	if err != nil {
-		return nil, fmt.Errorf("failed to dial to edge: %w", err)
+		return nil, &EdgeQuicDialError{Cause: err}
 	}
 
-	datagramMuxer, err := quicpogs.NewDatagramMuxer(session)
-	if err != nil {
-		return nil, err
-	}
-
-	sessionManager := datagramsession.NewManager(datagramMuxer, logger)
+	demuxChan := make(chan *quicpogs.SessionDatagram, demuxChanCapacity)
+	datagramMuxer := quicpogs.NewDatagramMuxer(session, logger, demuxChan)
+	sessionManager := datagramsession.NewManager(logger, datagramMuxer.MuxSession, demuxChan)
 
 	return &QUICConnection{
 		session:              session,
 		orchestrator:         orchestrator,
 		logger:               logger,
 		sessionManager:       sessionManager,
+		datagramMuxer:        datagramMuxer,
 		controlStreamHandler: controlStreamHandler,
 		connOptions:          connOptions,
 	}, nil
@@ -105,12 +111,17 @@ func (q *QUICConnection) Serve(ctx context.Context) error {
 		return q.sessionManager.Serve(ctx)
 	})
 
+	errGroup.Go(func() error {
+		defer cancel()
+		return q.datagramMuxer.ServeReceive(ctx)
+	})
+
 	return errGroup.Wait()
 }
 
 func (q *QUICConnection) serveControlStream(ctx context.Context, controlStream quic.Stream) error {
 	// This blocks until the control plane is done.
-	err := q.controlStreamHandler.ServeControlStream(ctx, controlStream, q.connOptions)
+	err := q.controlStreamHandler.ServeControlStream(ctx, controlStream, q.connOptions, q.orchestrator)
 	if err != nil {
 		// Not wrapping error here to be consistent with the http2 message.
 		return err
@@ -119,6 +130,11 @@ func (q *QUICConnection) serveControlStream(ctx context.Context, controlStream q
 	return nil
 }
 
+// Close closes the session with no errors specified.
+func (q *QUICConnection) Close() {
+	q.session.CloseWithError(0, "")
+}
+
 func (q *QUICConnection) acceptStream(ctx context.Context) error {
 	defer q.Close()
 	for {
@@ -130,23 +146,25 @@ func (q *QUICConnection) acceptStream(ctx context.Context) error {
 			}
 			return fmt.Errorf("failed to accept QUIC stream: %w", err)
 		}
-		go func() {
-			stream := quicpogs.NewSafeStreamCloser(quicStream)
-			defer stream.Close()
-
-			if err = q.handleStream(stream); err != nil {
-				q.logger.Err(err).Msg("Failed to handle QUIC stream")
-			}
-		}()
+		go q.runStream(quicStream)
 	}
 }
 
-// Close closes the session with no errors specified.
-func (q *QUICConnection) Close() {
-	q.session.CloseWithError(0, "")
+func (q *QUICConnection) runStream(quicStream quic.Stream) {
+	ctx := quicStream.Context()
+	stream := quicpogs.NewSafeStreamCloser(quicStream)
+	defer stream.Close()
+
+	// we are going to fuse readers/writers from stream <- cloudflared -> origin, and we want to guarantee that
+	// code executed in the code path of handleStream don't trigger an earlier close to the downstream stream.
+	// So, we wrap the stream with a no-op closer and only this method can actually close the stream.
+	noCloseStream := &nopCloserReadWriter{stream}
+	if err := q.handleStream(ctx, noCloseStream); err != nil {
+		q.logger.Err(err).Msg("Failed to handle QUIC stream")
+	}
 }
 
-func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
+func (q *QUICConnection) handleStream(ctx context.Context, stream io.ReadWriteCloser) error {
 	signature, err := quicpogs.DetermineProtocol(stream)
 	if err != nil {
 		return err
@@ -155,9 +173,9 @@ func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
 	case quicpogs.DataStreamProtocolSignature:
 		reqServerStream, err := quicpogs.NewRequestServerStream(stream, signature)
 		if err != nil {
-			return nil
+			return err
 		}
-		return q.handleDataStream(reqServerStream)
+		return q.handleDataStream(ctx, reqServerStream)
 	case quicpogs.RPCStreamProtocolSignature:
 		rpcStream, err := quicpogs.NewRPCServerStream(stream, signature)
 		if err != nil {
@@ -169,28 +187,43 @@ func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
 	}
 }
 
-func (q *QUICConnection) handleDataStream(stream *quicpogs.RequestServerStream) error {
-	connectRequest, err := stream.ReadConnectRequestData()
+func (q *QUICConnection) handleDataStream(ctx context.Context, stream *quicpogs.RequestServerStream) error {
+	request, err := stream.ReadConnectRequestData()
 	if err != nil {
 		return err
 	}
 
+	if err := q.dispatchRequest(ctx, stream, err, request); err != nil {
+		_ = stream.WriteConnectResponseData(err)
+		q.logger.Err(err).Str("type", request.Type.String()).Str("dest", request.Dest).Msg("Request failed")
+	}
+
+	return nil
+}
+
+func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.RequestServerStream, err error, request *quicpogs.ConnectRequest) error {
 	originProxy, err := q.orchestrator.GetOriginProxy()
 	if err != nil {
 		return err
 	}
-	switch connectRequest.Type {
+
+	switch request.Type {
 	case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
-		req, err := buildHTTPRequest(connectRequest, stream)
+		tracedReq, err := buildHTTPRequest(ctx, request, stream, q.logger)
 		if err != nil {
 			return err
 		}
-
 		w := newHTTPResponseAdapter(stream)
-		return originProxy.ProxyHTTP(w, req, connectRequest.Type == quicpogs.ConnectionTypeWebsocket)
+		return originProxy.ProxyHTTP(w, tracedReq, request.Type == quicpogs.ConnectionTypeWebsocket)
+
 	case quicpogs.ConnectionTypeTCP:
 		rwa := &streamReadWriteAcker{stream}
-		return originProxy.ProxyTCP(context.Background(), rwa, &TCPRequest{Dest: connectRequest.Dest})
+		metadata := request.MetadataMap()
+		return originProxy.ProxyTCP(ctx, rwa, &TCPRequest{
+			Dest:      request.Dest,
+			FlowID:    metadata[QUICMetadataFlowID],
+			CfTraceID: metadata[tracing.TracerContextName],
+		})
 	}
 	return nil
 }
@@ -216,7 +249,7 @@ func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.
 
 	go q.serveUDPSession(session, closeAfterIdleHint)
 
-	q.logger.Debug().Msgf("Registered session %v, %v, %v", sessionID, dstIP, dstPort)
+	q.logger.Debug().Str("sessionID", sessionID.String()).Str("src", originProxy.LocalAddr().String()).Str("dst", fmt.Sprintf("%s:%d", dstIP, dstPort)).Msgf("Registered session")
 	return nil
 }
 
@@ -276,8 +309,12 @@ type streamReadWriteAcker struct {
 }
 
 // AckConnection acks response back to the proxy.
-func (s *streamReadWriteAcker) AckConnection() error {
-	return s.WriteConnectResponseData(nil)
+func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
+	metadata := quicpogs.Metadata{
+		Key: tracing.CanonicalCloudflaredTracingHeader,
+		Val: tracePropagation,
+	}
+	return s.WriteConnectResponseData(nil, metadata)
 }
 
 // httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
@@ -305,14 +342,19 @@ func (hrw httpResponseAdapter) WriteErrorResponse(err error) {
 	hrw.WriteConnectResponseData(err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
 }
 
-func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadCloser) (*http.Request, error) {
+func buildHTTPRequest(
+	ctx context.Context,
+	connectRequest *quicpogs.ConnectRequest,
+	body io.ReadCloser,
+	log *zerolog.Logger,
+) (*tracing.TracedHTTPRequest, error) {
 	metadata := connectRequest.MetadataMap()
 	dest := connectRequest.Dest
 	method := metadata[HTTPMethodKey]
 	host := metadata[HTTPHostKey]
 	isWebsocket := connectRequest.Type == quicpogs.ConnectionTypeWebsocket
 
-	req, err := http.NewRequest(method, dest, body)
+	req, err := http.NewRequestWithContext(ctx, method, dest, body)
 	if err != nil {
 		return nil, err
 	}
@@ -345,7 +387,10 @@ func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadClose
 		req.Body = http.NoBody
 	}
 	stripWebsocketUpgradeHeader(req)
-	return req, err
+
+	// Check for tracing on request
+	tracedReq := tracing.NewTracedHTTPRequest(req, log)
+	return tracedReq, err
 }
 
 func setContentLength(req *http.Request) error {
@@ -362,3 +407,11 @@ func isTransferEncodingChunked(req *http.Request) bool {
 	// separated value as well.
 	return strings.Contains(strings.ToLower(transferEncodingVal), "chunked")
 }
+
+type nopCloserReadWriter struct {
+	io.ReadWriteCloser
+}
+
+func (n *nopCloserReadWriter) Close() error {
+	return nil
+}

connection/quic_test.go

@@ -24,17 +24,21 @@ import (
 
 	"github.com/cloudflare/cloudflared/datagramsession"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
 	testTLSServerConfig = quicpogs.GenerateTLSConfig()
 	testQUICConfig      = &quic.Config{
-		KeepAlive:       true,
-		EnableDatagrams: true,
+		ConnectionIDLength: 16,
+		KeepAlivePeriod:    5 * time.Second,
+		EnableDatagrams:    true,
 	}
 )
 
+var _ ReadWriteAcker = (*streamReadWriteAcker)(nil)
+
 // TestQUICServer tests if a quic server accepts and responds to a quic client with the acceptance protocol.
 // It also serves as a demonstration for communication with the QUIC connection started by a cloudflared.
 func TestQUICServer(t *testing.T) {
@@ -162,7 +166,7 @@ type fakeControlStream struct {
 	ControlStreamHandler
 }
 
-func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions) error {
+func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions, tunnelConfigGetter TunnelConfigJSONGetter) error {
 	<-ctx.Done()
 	return nil
 }
@@ -219,9 +223,10 @@ func quicServer(
 
 type mockOriginProxyWithRequest struct{}
 
-func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, r *http.Request, isWebsocket bool) error {
+func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, tr *tracing.TracedHTTPRequest, isWebsocket bool) error {
 	// These are a series of crude tests to ensure the headers and http related data is transferred from
 	// metadata.
+	r := tr.Request
 	if r.Method == "" {
 		return errors.New("method not sent")
 	}
@@ -473,18 +478,19 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 	}
 
+	log := zerolog.Nop()
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			req, err := buildHTTPRequest(test.connectRequest, test.body)
+			req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body, &log)
 			assert.NoError(t, err)
 			test.req = test.req.WithContext(req.Context())
-			assert.Equal(t, test.req, req)
+			assert.Equal(t, test.req, req.Request)
 		})
 	}
 }
 
 func (moc *mockOriginProxyWithRequest) ProxyTCP(ctx context.Context, rwa ReadWriteAcker, tcpRequest *TCPRequest) error {
-	rwa.AckConnection()
+	rwa.AckConnection("")
 	io.Copy(rwa, rwa)
 	return nil
 }
@@ -498,9 +504,10 @@ func TestServeUDPSession(t *testing.T) {
 	defer udpListener.Close()
 
 	ctx, cancel := context.WithCancel(context.Background())
+	val := udpListener.LocalAddr()
 
 	// Establish QUIC connection with edge
-	edgeQUICSessionChan := make(chan quic.Session)
+	edgeQUICSessionChan := make(chan quic.Connection)
 	go func() {
 		earlyListener, err := quic.Listen(udpListener, testTLSServerConfig, testQUICConfig)
 		require.NoError(t, err)
@@ -510,7 +517,7 @@ func TestServeUDPSession(t *testing.T) {
 		edgeQUICSessionChan <- edgeQUICSession
 	}()
 
-	qc := testQUICConnection(udpListener.LocalAddr(), t)
+	qc := testQUICConnection(val, t)
 	go qc.Serve(ctx)
 
 	edgeQUICSession := <-edgeQUICSessionChan
@@ -520,7 +527,7 @@ func TestServeUDPSession(t *testing.T) {
 	cancel()
 }
 
-func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.Session, closeType closeReason, expectedReason string, t *testing.T) {
+func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.Connection, closeType closeReason, expectedReason string, t *testing.T) {
 	var (
 		payload = []byte(t.Name())
 	)
@@ -581,7 +588,7 @@ const (
 	closedByTimeout
 )
 
-func runRPCServer(ctx context.Context, session quic.Session, sessionRPCServer tunnelpogs.SessionManager, configRPCServer tunnelpogs.ConfigurationManager, t *testing.T) {
+func runRPCServer(ctx context.Context, session quic.Connection, sessionRPCServer tunnelpogs.SessionManager, configRPCServer tunnelpogs.ConfigurationManager, t *testing.T) {
 	stream, err := session.AcceptStream(ctx)
 	require.NoError(t, err)
 

connection/rpc.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net"
 	"time"
 
 	"github.com/rs/zerolog"
@@ -57,6 +58,12 @@ type NamedTunnelRPCClient interface {
 		config *NamedTunnelProperties,
 		options *tunnelpogs.ConnectionOptions,
 		connIndex uint8,
+		edgeAddress net.IP,
+		observer *Observer,
+	) (*tunnelpogs.ConnectionDetails, error)
+	SendLocalConfiguration(
+		c context.Context,
+		config []byte,
 		observer *Observer,
 	) error
 	GracefulShutdown(ctx context.Context, gracePeriod time.Duration)
@@ -89,8 +96,9 @@ func (rsc *registrationServerClient) RegisterConnection(
 	properties *NamedTunnelProperties,
 	options *tunnelpogs.ConnectionOptions,
 	connIndex uint8,
+	edgeAddress net.IP,
 	observer *Observer,
-) error {
+) (*tunnelpogs.ConnectionDetails, error) {
 	conn, err := rsc.client.RegisterConnection(
 		ctx,
 		properties.Credentials.Auth(),
@@ -101,18 +109,26 @@ func (rsc *registrationServerClient) RegisterConnection(
 	if err != nil {
 		if err.Error() == DuplicateConnectionError {
 			observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
-			return errDuplicationConnection
+			return nil, errDuplicationConnection
 		}
 		observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
-		return serverRegistrationErrorFromRPC(err)
+		return nil, serverRegistrationErrorFromRPC(err)
 	}
 
 	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
 
-	observer.logServerInfo(connIndex, conn.Location, fmt.Sprintf("Connection %s registered", conn.UUID))
-	observer.sendConnectedEvent(connIndex, conn.Location)
+	return conn, nil
+}
 
-	return nil
+func (rsc *registrationServerClient) SendLocalConfiguration(ctx context.Context, config []byte, observer *Observer) (err error) {
+	observer.metrics.localConfigMetrics.pushes.Inc()
+	defer func() {
+		if err != nil {
+			observer.metrics.localConfigMetrics.pushesErrors.Inc()
+		}
+	}()
+
+	return rsc.client.SendLocalConfiguration(ctx, config)
 }
 
 func (rsc *registrationServerClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
@@ -258,7 +274,7 @@ func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelSe
 		h.observer.log.Err(err).Msg("Failed to retrieve server information")
 		return err
 	}
-	h.observer.logServerInfo(h.connIndex, serverInfo.LocationName, "Connection established")
+	h.observer.logServerInfo(h.connIndex, serverInfo.LocationName, net.IP{}, "Connection established")
 	return nil
 }
 
@@ -274,9 +290,13 @@ func (h *h2muxConnection) registerNamedTunnel(
 	rpcClient := h.newRPCClientFunc(ctx, stream, h.observer.log)
 	defer rpcClient.Close()
 
-	if err = rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+	registrationDetails, err := rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, nil, h.observer)
+	if err != nil {
 		return err
 	}
+	h.observer.logServerInfo(h.connIndex, registrationDetails.Location, nil, fmt.Sprintf("Connection %s registered", registrationDetails.UUID))
+	h.observer.sendConnectedEvent(h.connIndex, H2mux, registrationDetails.Location)
+
 	return nil
 }
 

datagramsession/event.go

@@ -42,9 +42,3 @@ func (sc *errClosedSession) Error() string {
 		return fmt.Sprintf("session closed by local due to %s", sc.message)
 	}
 }
-
-// newDatagram is an event when transport receives new datagram
-type newDatagram struct {
-	sessionID uuid.UUID
-	payload   []byte
-}

datagramsession/manager.go

@@ -2,13 +2,14 @@ package datagramsession
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/lucas-clemente/quic-go"
 	"github.com/rs/zerolog"
-	"golang.org/x/sync/errgroup"
+
+	quicpogs "github.com/cloudflare/cloudflared/quic"
 )
 
 const (
@@ -16,6 +17,10 @@ const (
 	defaultReqTimeout   = time.Second * 5
 )
 
+var (
+	errSessionManagerClosed = fmt.Errorf("session manager closed")
+)
+
 // Manager defines the APIs to manage sessions from the same transport.
 type Manager interface {
 	// Serve starts the event loop
@@ -24,73 +29,70 @@ type Manager interface {
 	RegisterSession(ctx context.Context, sessionID uuid.UUID, dstConn io.ReadWriteCloser) (*Session, error)
 	// UnregisterSession stops tracking the session and terminates it
 	UnregisterSession(ctx context.Context, sessionID uuid.UUID, message string, byRemote bool) error
+	// UpdateLogger updates the logger used by the Manager
+	UpdateLogger(log *zerolog.Logger)
 }
 
 type manager struct {
 	registrationChan   chan *registerSessionEvent
 	unregistrationChan chan *unregisterSessionEvent
-	datagramChan       chan *newDatagram
-	transport          transport
+	sendFunc           transportSender
+	receiveChan        <-chan *quicpogs.SessionDatagram
+	closedChan         <-chan struct{}
 	sessions           map[uuid.UUID]*Session
 	log                *zerolog.Logger
 	// timeout waiting for an API to finish. This can be overriden in test
 	timeout time.Duration
 }
 
-func NewManager(transport transport, log *zerolog.Logger) *manager {
+func NewManager(log *zerolog.Logger, sendF transportSender, receiveChan <-chan *quicpogs.SessionDatagram) *manager {
 	return &manager{
 		registrationChan:   make(chan *registerSessionEvent),
 		unregistrationChan: make(chan *unregisterSessionEvent),
-		// datagramChan is buffered, so it can read more datagrams from transport while the event loop is processing other events
-		datagramChan: make(chan *newDatagram, requestChanCapacity),
-		transport:    transport,
-		sessions:     make(map[uuid.UUID]*Session),
-		log:          log,
-		timeout:      defaultReqTimeout,
+		sendFunc:           sendF,
+		receiveChan:        receiveChan,
+		closedChan:         make(chan struct{}),
+		sessions:           make(map[uuid.UUID]*Session),
+		log:                log,
+		timeout:            defaultReqTimeout,
 	}
 }
 
+func (m *manager) UpdateLogger(log *zerolog.Logger) {
+	// Benign data race, no problem if the old pointer is read or not concurrently.
+	m.log = log
+}
+
 func (m *manager) Serve(ctx context.Context) error {
-	errGroup, ctx := errgroup.WithContext(ctx)
-	errGroup.Go(func() error {
-		for {
-			sessionID, payload, err := m.transport.ReceiveFrom()
-			if err != nil {
-				if aerr, ok := err.(*quic.ApplicationError); ok && uint64(aerr.ErrorCode) == uint64(quic.NoError) {
-					return nil
-				} else {
-					return err
-				}
-			}
-			datagram := &newDatagram{
-				sessionID: sessionID,
-				payload:   payload,
-			}
-			select {
-			case <-ctx.Done():
-				return ctx.Err()
-			// Only the event loop routine can update/lookup the sessions map to avoid concurrent access
-			// Send the datagram to the event loop. It will find the session to send to
-			case m.datagramChan <- datagram:
-			}
+	for {
+		select {
+		case <-ctx.Done():
+			m.shutdownSessions(ctx.Err())
+			return ctx.Err()
+		// receiveChan is buffered, so the transport can read more datagrams from transport while the event loop is
+		// processing other events
+		case datagram := <-m.receiveChan:
+			m.sendToSession(datagram)
+		case registration := <-m.registrationChan:
+			m.registerSession(ctx, registration)
+		case unregistration := <-m.unregistrationChan:
+			m.unregisterSession(unregistration)
 		}
-	})
-	errGroup.Go(func() error {
-		for {
-			select {
-			case <-ctx.Done():
-				return nil
-			case datagram := <-m.datagramChan:
-				m.sendToSession(datagram)
-			case registration := <-m.registrationChan:
-				m.registerSession(ctx, registration)
-			// TODO: TUN-5422: Unregister inactive session upon timeout
-			case unregistration := <-m.unregistrationChan:
-				m.unregisterSession(unregistration)
-			}
-		}
-	})
-	return errGroup.Wait()
+	}
+}
+
+func (m *manager) shutdownSessions(err error) {
+	if err == nil {
+		err = errSessionManagerClosed
+	}
+	closeSessionErr := &errClosedSession{
+		message: err.Error(),
+		// Usually connection with remote has been closed, so set this to true to skip unregistering from remote
+		byRemote: true,
+	}
+	for _, s := range m.sessions {
+		s.close(closeSessionErr)
+	}
 }
 
 func (m *manager) RegisterSession(ctx context.Context, sessionID uuid.UUID, originProxy io.ReadWriteCloser) (*Session, error) {
@@ -104,15 +106,34 @@ func (m *manager) RegisterSession(ctx context.Context, sessionID uuid.UUID, orig
 	case m.registrationChan <- event:
 		session := <-event.resultChan
 		return session, nil
+	// Once closedChan is closed, manager won't accept more registration because nothing is
+	// reading from registrationChan and it's an unbuffered channel
+	case <-m.closedChan:
+		return nil, errSessionManagerClosed
 	}
 }
 
 func (m *manager) registerSession(ctx context.Context, registration *registerSessionEvent) {
-	session := newSession(registration.sessionID, m.transport, registration.originProxy, m.log)
+	session := m.newSession(registration.sessionID, registration.originProxy)
 	m.sessions[registration.sessionID] = session
 	registration.resultChan <- session
 }
 
+func (m *manager) newSession(id uuid.UUID, dstConn io.ReadWriteCloser) *Session {
+	logger := m.log.With().Str("sessionID", id.String()).Logger()
+	return &Session{
+		ID:       id,
+		sendFunc: m.sendFunc,
+		dstConn:  dstConn,
+		// activeAtChan has low capacity. It can be full when there are many concurrent read/write. markActive() will
+		// drop instead of blocking because last active time only needs to be an approximation
+		activeAtChan: make(chan time.Time, 2),
+		// capacity is 2 because close() and dstToTransport routine in Serve() can write to this channel
+		closeChan: make(chan error, 2),
+		log:       &logger,
+	}
+}
+
 func (m *manager) UnregisterSession(ctx context.Context, sessionID uuid.UUID, message string, byRemote bool) error {
 	ctx, cancel := context.WithTimeout(ctx, m.timeout)
 	defer cancel()
@@ -129,6 +150,8 @@ func (m *manager) UnregisterSession(ctx context.Context, sessionID uuid.UUID, me
 		return ctx.Err()
 	case m.unregistrationChan <- event:
 		return nil
+	case <-m.closedChan:
+		return errSessionManagerClosed
 	}
 }
 
@@ -140,16 +163,13 @@ func (m *manager) unregisterSession(unregistration *unregisterSessionEvent) {
 	}
 }
 
-func (m *manager) sendToSession(datagram *newDatagram) {
-	session, ok := m.sessions[datagram.sessionID]
+func (m *manager) sendToSession(datagram *quicpogs.SessionDatagram) {
+	session, ok := m.sessions[datagram.ID]
 	if !ok {
-		m.log.Error().Str("sessionID", datagram.sessionID.String()).Msg("session not found")
+		m.log.Error().Str("sessionID", datagram.ID.String()).Msg("session not found")
 		return
 	}
 	// session writes to destination over a connected UDP socket, which should not be blocking, so this call doesn't
 	// need to run in another go routine
-	_, err := session.transportToDst(datagram.payload)
-	if err != nil {
-		m.log.Err(err).Str("sessionID", datagram.sessionID.String()).Msg("Failed to write payload to session")
-	}
+	session.transportToDst(datagram.Payload)
 }

datagramsession/manager_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"sync"
 	"testing"
 	"time"
 
@@ -13,26 +14,30 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
+
+	quicpogs "github.com/cloudflare/cloudflared/quic"
+)
+
+var (
+	nopLogger = zerolog.Nop()
 )
 
 func TestManagerServe(t *testing.T) {
 	const (
-		sessions            = 20
-		msgs                = 50
+		sessions            = 2
+		msgs                = 5
 		remoteUnregisterMsg = "eyeball closed connection"
 	)
-	log := zerolog.Nop()
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(1),
-		respChan: newDatagramChannel(1),
-	}
-	mg := NewManager(transport, &log)
 
-	eyeballTracker := make(map[uuid.UUID]*datagramChannel)
-	for i := 0; i < sessions; i++ {
-		sessionID := uuid.New()
-		eyeballTracker[sessionID] = newDatagramChannel(1)
+	requestChan := make(chan *quicpogs.SessionDatagram)
+	transport := mockQUICTransport{
+		sessions: make(map[uuid.UUID]chan []byte),
 	}
+	for i := 0; i < sessions; i++ {
+		transport.sessions[uuid.New()] = make(chan []byte)
+	}
+
+	mg := NewManager(&nopLogger, transport.MuxSession, requestChan)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	serveDone := make(chan struct{})
@@ -41,53 +46,42 @@ func TestManagerServe(t *testing.T) {
 		close(serveDone)
 	}(ctx)
 
-	go func(ctx context.Context) {
-		for {
-			sessionID, payload, err := transport.respChan.Receive(ctx)
-			if err != nil {
-				require.Equal(t, context.Canceled, err)
-				return
-			}
-			respChan := eyeballTracker[sessionID]
-			require.NoError(t, respChan.Send(ctx, sessionID, payload))
-		}
-	}(ctx)
-
 	errGroup, ctx := errgroup.WithContext(ctx)
-	for sID, receiver := range eyeballTracker {
+	for sessionID, eyeballRespChan := range transport.sessions {
+		// Assign loop variables to local variables
+		sID := sessionID
+		payload := testPayload(sID)
+		expectResp := testResponse(payload)
+
+		cfdConn, originConn := net.Pipe()
+
+		origin := mockOrigin{
+			expectMsgCount: msgs,
+			expectedMsg:    payload,
+			expectedResp:   expectResp,
+			conn:           originConn,
+		}
+
+		eyeball := mockEyeballSession{
+			id:               sID,
+			expectedMsgCount: msgs,
+			expectedMsg:      payload,
+			expectedResponse: expectResp,
+			respReceiver:     eyeballRespChan,
+		}
+
 		// Assign loop variables to local variables
-		sessionID := sID
-		eyeballRespReceiver := receiver
 		errGroup.Go(func() error {
-			payload := testPayload(sessionID)
-			expectResp := testResponse(payload)
-
-			cfdConn, originConn := net.Pipe()
-
-			origin := mockOrigin{
-				expectMsgCount: msgs,
-				expectedMsg:    payload,
-				expectedResp:   expectResp,
-				conn:           originConn,
-			}
-			eyeball := mockEyeball{
-				expectMsgCount:  msgs,
-				expectedMsg:     expectResp,
-				expectSessionID: sessionID,
-				respReceiver:    eyeballRespReceiver,
-			}
-
+			session, err := mg.RegisterSession(ctx, sID, cfdConn)
+			require.NoError(t, err)
 			reqErrGroup, reqCtx := errgroup.WithContext(ctx)
 			reqErrGroup.Go(func() error {
 				return origin.serve()
 			})
 			reqErrGroup.Go(func() error {
-				return eyeball.serve(reqCtx)
+				return eyeball.serve(reqCtx, requestChan)
 			})
 
-			session, err := mg.RegisterSession(ctx, sessionID, cfdConn)
-			require.NoError(t, err)
-
 			sessionDone := make(chan struct{})
 			go func() {
 				closedByRemote, err := session.Serve(ctx, time.Minute*2)
@@ -100,23 +94,17 @@ func TestManagerServe(t *testing.T) {
 				close(sessionDone)
 			}()
 
-			for i := 0; i < msgs; i++ {
-				require.NoError(t, transport.newRequest(ctx, sessionID, testPayload(sessionID)))
-			}
-
 			// Make sure eyeball and origin have received all messages before unregistering the session
 			require.NoError(t, reqErrGroup.Wait())
 
-			require.NoError(t, mg.UnregisterSession(ctx, sessionID, remoteUnregisterMsg, true))
+			require.NoError(t, mg.UnregisterSession(ctx, sID, remoteUnregisterMsg, true))
 			<-sessionDone
-
 			return nil
 		})
 	}
 
 	require.NoError(t, errGroup.Wait())
 	cancel()
-	transport.close()
 	<-serveDone
 }
 
@@ -124,12 +112,8 @@ func TestTimeout(t *testing.T) {
 	const (
 		testTimeout = time.Millisecond * 50
 	)
-	log := zerolog.Nop()
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(1),
-		respChan: newDatagramChannel(1),
-	}
-	mg := NewManager(transport, &log)
+
+	mg := NewManager(&nopLogger, nil, nil)
 	mg.timeout = testTimeout
 	ctx := context.Background()
 	sessionID := uuid.New()
@@ -142,6 +126,80 @@ func TestTimeout(t *testing.T) {
 	require.ErrorIs(t, err, context.DeadlineExceeded)
 }
 
+func TestUnregisterSessionCloseSession(t *testing.T) {
+	sessionID := uuid.New()
+	payload := []byte(t.Name())
+	sender := newMockTransportSender(sessionID, payload)
+	mg := NewManager(&nopLogger, sender.muxSession, nil)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	managerDone := make(chan struct{})
+	go func() {
+		err := mg.Serve(ctx)
+		require.Error(t, err)
+		close(managerDone)
+	}()
+
+	cfdConn, originConn := net.Pipe()
+	session, err := mg.RegisterSession(ctx, sessionID, cfdConn)
+	require.NoError(t, err)
+	require.NotNil(t, session)
+
+	unregisteredChan := make(chan struct{})
+	go func() {
+		_, err := originConn.Write(payload)
+		require.NoError(t, err)
+
+		err = mg.UnregisterSession(ctx, sessionID, "eyeball closed session", true)
+		require.NoError(t, err)
+
+		close(unregisteredChan)
+	}()
+
+	closedByRemote, err := session.Serve(ctx, time.Minute)
+	require.True(t, closedByRemote)
+	require.Error(t, err)
+
+	<-unregisteredChan
+	cancel()
+	<-managerDone
+}
+
+func TestManagerCtxDoneCloseSessions(t *testing.T) {
+	sessionID := uuid.New()
+	payload := []byte(t.Name())
+	sender := newMockTransportSender(sessionID, payload)
+	mg := NewManager(&nopLogger, sender.muxSession, nil)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		err := mg.Serve(ctx)
+		require.Error(t, err)
+	}()
+
+	cfdConn, originConn := net.Pipe()
+	session, err := mg.RegisterSession(ctx, sessionID, cfdConn)
+	require.NoError(t, err)
+	require.NotNil(t, session)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, err := originConn.Write(payload)
+		require.NoError(t, err)
+		cancel()
+	}()
+
+	closedByRemote, err := session.Serve(ctx, time.Minute)
+	require.False(t, closedByRemote)
+	require.Error(t, err)
+
+	wg.Wait()
+}
+
 type mockOrigin struct {
 	expectMsgCount int
 	expectedMsg    []byte
@@ -163,7 +221,6 @@ func (mo *mockOrigin) serve() error {
 		if !bytes.Equal(readBuffer[:n], mo.expectedMsg) {
 			return fmt.Errorf("Expect %v, read %v", mo.expectedMsg, readBuffer[:n])
 		}
-
 		_, err = mo.conn.Write(mo.expectedResp)
 		if err != nil {
 			return err
@@ -180,65 +237,35 @@ func testResponse(msg []byte) []byte {
 	return []byte(fmt.Sprintf("Response to %v", msg))
 }
 
-type mockEyeball struct {
-	expectMsgCount  int
-	expectedMsg     []byte
-	expectSessionID uuid.UUID
-	respReceiver    *datagramChannel
+type mockQUICTransport struct {
+	sessions map[uuid.UUID]chan []byte
 }
 
-func (me *mockEyeball) serve(ctx context.Context) error {
-	for i := 0; i < me.expectMsgCount; i++ {
-		sessionID, msg, err := me.respReceiver.Receive(ctx)
-		if err != nil {
-			return err
-		}
-		if sessionID != me.expectSessionID {
-			return fmt.Errorf("Expect session %s, got %s", me.expectSessionID, sessionID)
-		}
-		if !bytes.Equal(msg, me.expectedMsg) {
-			return fmt.Errorf("Expect %v, read %v", me.expectedMsg, msg)
-		}
-	}
+func (me *mockQUICTransport) MuxSession(id uuid.UUID, payload []byte) error {
+	session := me.sessions[id]
+	session <- payload
 	return nil
 }
 
-// datagramChannel is a channel for Datagram with wrapper to send/receive with context
-type datagramChannel struct {
-	datagramChan chan *newDatagram
-	closedChan   chan struct{}
+type mockEyeballSession struct {
+	id               uuid.UUID
+	expectedMsgCount int
+	expectedMsg      []byte
+	expectedResponse []byte
+	respReceiver     <-chan []byte
 }
 
-func newDatagramChannel(capacity uint) *datagramChannel {
-	return &datagramChannel{
-		datagramChan: make(chan *newDatagram, capacity),
-		closedChan:   make(chan struct{}),
+func (me *mockEyeballSession) serve(ctx context.Context, requestChan chan *quicpogs.SessionDatagram) error {
+	for i := 0; i < me.expectedMsgCount; i++ {
+		requestChan <- &quicpogs.SessionDatagram{
+			ID:      me.id,
+			Payload: me.expectedMsg,
+		}
+		resp := <-me.respReceiver
+		if !bytes.Equal(resp, me.expectedResponse) {
+			return fmt.Errorf("Expect %v, read %v", me.expectedResponse, resp)
+		}
+		fmt.Println("Resp", resp)
 	}
-}
-
-func (rc *datagramChannel) Send(ctx context.Context, sessionID uuid.UUID, payload []byte) error {
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	case <-rc.closedChan:
-		return fmt.Errorf("datagram channel closed")
-	case rc.datagramChan <- &newDatagram{sessionID: sessionID, payload: payload}:
-		return nil
-	}
-}
-
-func (rc *datagramChannel) Receive(ctx context.Context) (uuid.UUID, []byte, error) {
-	select {
-	case <-ctx.Done():
-		return uuid.Nil, nil, ctx.Err()
-	case <-rc.closedChan:
-		return uuid.Nil, nil, fmt.Errorf("datagram channel closed")
-	case msg := <-rc.datagramChan:
-		return msg.sessionID, msg.payload, nil
-	}
-}
-
-func (rc *datagramChannel) Close() {
-	// No need to close msgChan, it will be garbage collect once there is no reference to it
-	close(rc.closedChan)
+	return nil
 }

datagramsession/session.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net"
 	"time"
 
 	"github.com/google/uuid"
@@ -18,41 +19,26 @@ func SessionIdleErr(timeout time.Duration) error {
 	return fmt.Errorf("session idle for %v", timeout)
 }
 
+type transportSender func(sessionID uuid.UUID, payload []byte) error
+
 // Session is a bidirectional pipe of datagrams between transport and dstConn
-// Currently the only implementation of transport is quic DatagramMuxer
 // Destination can be a connection with origin or with eyeball
 // When the destination is origin:
-// - Datagrams from edge are read by Manager from the transport. Manager finds the corresponding Session and calls the
-//   write method of the Session to send to origin
-// - Datagrams from origin are read from conn and SentTo transport. Transport will return them to eyeball
+// - Manager receives datagrams from receiveChan and calls the transportToDst method of the Session to send to origin
+// - Datagrams from origin are read from conn and Send to transport using the transportSender callback. Transport will return them to eyeball
 // When the destination is eyeball:
-// - Datagrams from eyeball are read from conn and SentTo transport. Transport will send them to cloudflared
-// - Datagrams from cloudflared are read by Manager from the transport. Manager finds the corresponding Session and calls the
-//   write method of the Session to send to eyeball
+// - Datagrams from eyeball are read from conn and Send to transport. Transport will send them to cloudflared using the transportSender callback.
+// - Manager receives datagrams from receiveChan and calls the transportToDst method of the Session to send to the eyeball
 type Session struct {
-	ID        uuid.UUID
-	transport transport
-	dstConn   io.ReadWriteCloser
+	ID       uuid.UUID
+	sendFunc transportSender
+	dstConn  io.ReadWriteCloser
 	// activeAtChan is used to communicate the last read/write time
 	activeAtChan chan time.Time
 	closeChan    chan error
 	log          *zerolog.Logger
 }
 
-func newSession(id uuid.UUID, transport transport, dstConn io.ReadWriteCloser, log *zerolog.Logger) *Session {
-	return &Session{
-		ID:        id,
-		transport: transport,
-		dstConn:   dstConn,
-		// activeAtChan has low capacity. It can be full when there are many concurrent read/write. markActive() will
-		// drop instead of blocking because last active time only needs to be an approximation
-		activeAtChan: make(chan time.Time, 2),
-		// capacity is 2 because close() and dstToTransport routine in Serve() can write to this channel
-		closeChan: make(chan error, 2),
-		log:       log,
-	}
-}
-
 func (s *Session) Serve(ctx context.Context, closeAfterIdle time.Duration) (closedByRemote bool, err error) {
 	go func() {
 		// QUIC implementation copies data to another buffer before returning https://github.com/lucas-clemente/quic-go/blob/v0.24.0/session.go#L1967-L1975
@@ -60,9 +46,16 @@ func (s *Session) Serve(ctx context.Context, closeAfterIdle time.Duration) (clos
 		const maxPacketSize = 1500
 		readBuffer := make([]byte, maxPacketSize)
 		for {
-			if err := s.dstToTransport(readBuffer); err != nil {
-				s.closeChan <- err
-				return
+			if closeSession, err := s.dstToTransport(readBuffer); err != nil {
+				if err != net.ErrClosed {
+					s.log.Error().Err(err).Msg("Failed to send session payload from destination to transport")
+				} else {
+					s.log.Debug().Msg("Session cannot read from destination because the connection is closed")
+				}
+				if closeSession {
+					s.closeChan <- err
+					return
+				}
 			}
 		}
 	}()
@@ -103,32 +96,25 @@ func (s *Session) waitForCloseCondition(ctx context.Context, closeAfterIdle time
 	}
 }
 
-func (s *Session) dstToTransport(buffer []byte) error {
+func (s *Session) dstToTransport(buffer []byte) (closeSession bool, err error) {
 	n, err := s.dstConn.Read(buffer)
 	s.markActive()
 	// https://pkg.go.dev/io#Reader suggests caller should always process n > 0 bytes
-	if n > 0 {
-		if n <= int(s.transport.MTU()) {
-			err = s.transport.SendTo(s.ID, buffer[:n])
-		} else {
-			// drop packet for now, eventually reply with ICMP for PMTUD
-			s.log.Debug().
-				Str("session", s.ID.String()).
-				Int("len", n).
-				Int("mtu", s.transport.MTU()).
-				Msg("dropped packet exceeding MTU")
+	if n > 0 || err == nil {
+		if sendErr := s.sendFunc(s.ID, buffer[:n]); sendErr != nil {
+			return false, sendErr
 		}
 	}
-	// Some UDP application might send 0-size payload.
-	if err == nil && n == 0 {
-		err = s.transport.SendTo(s.ID, []byte{})
-	}
-	return err
+	return err != nil, err
 }
 
 func (s *Session) transportToDst(payload []byte) (int, error) {
 	s.markActive()
-	return s.dstConn.Write(payload)
+	n, err := s.dstConn.Write(payload)
+	if err != nil {
+		s.log.Err(err).Msg("Failed to write payload to session")
+	}
+	return n, err
 }
 
 // Sends the last active time to the idle checker loop without blocking. activeAtChan will only be full when there

datagramsession/session_test.go

@@ -14,6 +14,8 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
+
+	quicpogs "github.com/cloudflare/cloudflared/quic"
 )
 
 // TestCloseSession makes sure a session will stop after context is done
@@ -41,12 +43,10 @@ func testSessionReturns(t *testing.T, closeBy closeMethod, closeAfterIdle time.D
 	sessionID := uuid.New()
 	cfdConn, originConn := net.Pipe()
 	payload := testPayload(sessionID)
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(1),
-		respChan: newDatagramChannel(1),
-	}
+
 	log := zerolog.Nop()
-	session := newSession(sessionID, transport, cfdConn, &log)
+	mg := NewManager(&log, nil, nil)
+	session := mg.newSession(sessionID, cfdConn)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	sessionDone := make(chan struct{})
@@ -117,12 +117,11 @@ func testActiveSessionNotClosed(t *testing.T, readFromDst bool, writeToDst bool)
 	sessionID := uuid.New()
 	cfdConn, originConn := net.Pipe()
 	payload := testPayload(sessionID)
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(100),
-		respChan: newDatagramChannel(100),
-	}
-	log := zerolog.Nop()
-	session := newSession(sessionID, transport, cfdConn, &log)
+
+	respChan := make(chan *quicpogs.SessionDatagram)
+	sender := newMockTransportSender(sessionID, payload)
+	mg := NewManager(&nopLogger, sender.muxSession, respChan)
+	session := mg.newSession(sessionID, cfdConn)
 
 	startTime := time.Now()
 	activeUntil := startTime.Add(activeTime)
@@ -184,7 +183,8 @@ func testActiveSessionNotClosed(t *testing.T, readFromDst bool, writeToDst bool)
 
 func TestMarkActiveNotBlocking(t *testing.T) {
 	const concurrentCalls = 50
-	session := newSession(uuid.New(), nil, nil, nil)
+	mg := NewManager(&nopLogger, nil, nil)
+	session := mg.newSession(uuid.New(), nil)
 	var wg sync.WaitGroup
 	wg.Add(concurrentCalls)
 	for i := 0; i < concurrentCalls; i++ {
@@ -196,15 +196,17 @@ func TestMarkActiveNotBlocking(t *testing.T) {
 	wg.Wait()
 }
 
+// Some UDP application might send 0-size payload.
 func TestZeroBytePayload(t *testing.T) {
 	sessionID := uuid.New()
 	cfdConn, originConn := net.Pipe()
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(1),
-		respChan: newDatagramChannel(1),
+
+	sender := sendOnceTransportSender{
+		baseSender: newMockTransportSender(sessionID, make([]byte, 0)),
+		sentChan:   make(chan struct{}),
 	}
-	log := zerolog.Nop()
-	session := newSession(sessionID, transport, cfdConn, &log)
+	mg := NewManager(&nopLogger, sender.muxSession, nil)
+	session := mg.newSession(sessionID, cfdConn)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	errGroup, ctx := errgroup.WithContext(ctx)
@@ -224,11 +226,39 @@ func TestZeroBytePayload(t *testing.T) {
 		return nil
 	})
 
-	receivedSessionID, payload, err := transport.respChan.Receive(ctx)
-	require.NoError(t, err)
-	require.Len(t, payload, 0)
-	require.Equal(t, sessionID, receivedSessionID)
-
+	<-sender.sentChan
 	cancel()
 	require.NoError(t, errGroup.Wait())
 }
+
+type mockTransportSender struct {
+	expectedSessionID uuid.UUID
+	expectedPayload   []byte
+}
+
+func newMockTransportSender(expectedSessionID uuid.UUID, expectedPayload []byte) *mockTransportSender {
+	return &mockTransportSender{
+		expectedSessionID: expectedSessionID,
+		expectedPayload:   expectedPayload,
+	}
+}
+
+func (mts *mockTransportSender) muxSession(sessionID uuid.UUID, payload []byte) error {
+	if sessionID != mts.expectedSessionID {
+		return fmt.Errorf("Expect session %s, got %s", mts.expectedSessionID, sessionID)
+	}
+	if !bytes.Equal(payload, mts.expectedPayload) {
+		return fmt.Errorf("Expect %v, read %v", mts.expectedPayload, payload)
+	}
+	return nil
+}
+
+type sendOnceTransportSender struct {
+	baseSender *mockTransportSender
+	sentChan   chan struct{}
+}
+
+func (sots *sendOnceTransportSender) muxSession(sessionID uuid.UUID, payload []byte) error {
+	defer close(sots.sentChan)
+	return sots.baseSender.muxSession(sessionID, payload)
+}

datagramsession/transport.go

@@ -1,13 +0,0 @@
-package datagramsession
-
-import "github.com/google/uuid"
-
-// Transport is a connection between cloudflared and edge that can multiplex datagrams from multiple sessions
-type transport interface {
-	// SendTo writes payload for a session to the transport
-	SendTo(sessionID uuid.UUID, payload []byte) error
-	// ReceiveFrom reads the next datagram from the transport
-	ReceiveFrom() (uuid.UUID, []byte, error)
-	// Max transmission unit to receive from the transport
-	MTU() int
-}

datagramsession/transport_test.go

@@ -1,36 +0,0 @@
-package datagramsession
-
-import (
-	"context"
-
-	"github.com/google/uuid"
-)
-
-type mockQUICTransport struct {
-	reqChan  *datagramChannel
-	respChan *datagramChannel
-}
-
-func (mt *mockQUICTransport) SendTo(sessionID uuid.UUID, payload []byte) error {
-	buf := make([]byte, len(payload))
-	// The QUIC implementation copies data to another buffer before returning https://github.com/lucas-clemente/quic-go/blob/v0.24.0/session.go#L1967-L1975
-	copy(buf, payload)
-	return mt.respChan.Send(context.Background(), sessionID, buf)
-}
-
-func (mt *mockQUICTransport) ReceiveFrom() (uuid.UUID, []byte, error) {
-	return mt.reqChan.Receive(context.Background())
-}
-
-func (mt *mockQUICTransport) MTU() int {
-	return 1280
-}
-
-func (mt *mockQUICTransport) newRequest(ctx context.Context, sessionID uuid.UUID, payload []byte) error {
-	return mt.reqChan.Send(ctx, sessionID, payload)
-}
-
-func (mt *mockQUICTransport) close() {
-	mt.reqChan.Close()
-	mt.respChan.Close()
-}

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17.5 as builder
+FROM golang:1.18.1 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
@@ -6,4 +6,4 @@ RUN apt-get update
 COPY . .
 # compile cloudflared
 RUN make cloudflared
-RUN cp /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+RUN cp /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/

edgediscovery/allregions/address.go

@@ -0,0 +1,64 @@
+package allregions
+
+// Region contains cloudflared edge addresses. The edge is partitioned into several regions for
+// redundancy purposes.
+type AddrSet map[*EdgeAddr]UsedBy
+
+// AddrUsedBy finds the address used by the given connection in this region.
+// Returns nil if the connection isn't using any IP.
+func (a AddrSet) AddrUsedBy(connID int) *EdgeAddr {
+	for addr, used := range a {
+		if used.Used && used.ConnID == connID {
+			return addr
+		}
+	}
+	return nil
+}
+
+// AvailableAddrs counts how many unused addresses this region contains.
+func (a AddrSet) AvailableAddrs() int {
+	n := 0
+	for _, usedby := range a {
+		if !usedby.Used {
+			n++
+		}
+	}
+	return n
+}
+
+// GetUnusedIP returns a random unused address in this region.
+// Returns nil if all addresses are in use.
+func (a AddrSet) GetUnusedIP(excluding *EdgeAddr) *EdgeAddr {
+	for addr, usedby := range a {
+		if !usedby.Used && addr != excluding {
+			return addr
+		}
+	}
+	return nil
+}
+
+// Use the address, assigning it to a proxy connection.
+func (a AddrSet) Use(addr *EdgeAddr, connID int) {
+	if addr == nil {
+		return
+	}
+	a[addr] = InUse(connID)
+}
+
+// GetAnyAddress returns an arbitrary address from the region.
+func (a AddrSet) GetAnyAddress() *EdgeAddr {
+	for addr := range a {
+		return addr
+	}
+	return nil
+}
+
+// GiveBack the address, ensuring it is no longer assigned to an IP.
+// Returns true if the address is in this region.
+func (a AddrSet) GiveBack(addr *EdgeAddr) (ok bool) {
+	if _, ok := a[addr]; !ok {
+		return false
+	}
+	a[addr] = Unused()
+	return true
+}

edgediscovery/allregions/address_test.go

@@ -0,0 +1,247 @@
+package allregions
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestAddrSet_AddrUsedBy(t *testing.T) {
+	type args struct {
+		connID int
+	}
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		args    args
+		want    *EdgeAddr
+	}{
+		{
+			name: "happy trivial test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+			},
+			args: args{connID: 0},
+			want: &addr0,
+		},
+		{
+			name: "sad trivial test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+			},
+			args: args{connID: 1},
+			want: nil,
+		},
+		{
+			name: "sad test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{connID: 3},
+			want: nil,
+		},
+		{
+			name: "happy test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{connID: 1},
+			want: &addr1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.AddrUsedBy(tt.args.connID); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Region.AddrUsedBy() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAddrSet_AvailableAddrs(t *testing.T) {
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		want    int
+	}{
+		{
+			name: "contains addresses",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			},
+			want: 1,
+		},
+		{
+			name: "all free",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+				&addr1: Unused(),
+				&addr2: Unused(),
+			},
+			want: 3,
+		},
+		{
+			name: "all used",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			want: 0,
+		},
+		{
+			name:    "empty",
+			addrSet: AddrSet{},
+			want:    0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.AvailableAddrs(); got != tt.want {
+				t.Errorf("Region.AvailableAddrs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAddrSet_GetUnusedIP(t *testing.T) {
+	type args struct {
+		excluding *EdgeAddr
+	}
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		args    args
+		want    *EdgeAddr
+	}{
+		{
+			name: "happy test with excluding set",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: &addr0},
+			want: &addr1,
+		},
+		{
+			name: "happy test with no excluding",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: nil},
+			want: &addr1,
+		},
+		{
+			name: "sad test with no excluding",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: nil},
+			want: nil,
+		},
+		{
+			name: "sad test with excluding",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: &addr0},
+			want: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.GetUnusedIP(tt.args.excluding); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Region.GetUnusedIP() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAddrSet_GiveBack(t *testing.T) {
+	type args struct {
+		addr *EdgeAddr
+	}
+	tests := []struct {
+		name           string
+		addrSet        AddrSet
+		args           args
+		wantOk         bool
+		availableAfter int
+	}{
+		{
+			name: "sad test with excluding",
+			addrSet: AddrSet{
+				&addr1: InUse(1),
+			},
+			args:           args{addr: &addr1},
+			wantOk:         true,
+			availableAfter: 1,
+		},
+		{
+			name: "sad test with excluding",
+			addrSet: AddrSet{
+				&addr1: InUse(1),
+			},
+			args:           args{addr: &addr2},
+			wantOk:         false,
+			availableAfter: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if gotOk := tt.addrSet.GiveBack(tt.args.addr); gotOk != tt.wantOk {
+				t.Errorf("Region.GiveBack() = %v, want %v", gotOk, tt.wantOk)
+			}
+			if tt.availableAfter != tt.addrSet.AvailableAddrs() {
+				t.Errorf("Region.AvailableAddrs() = %v, want %v", tt.addrSet.AvailableAddrs(), tt.availableAfter)
+			}
+		})
+	}
+}
+
+func TestAddrSet_GetAnyAddress(t *testing.T) {
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		wantNil bool
+	}{
+		{
+			name:    "Sad test -- GetAnyAddress should only fail if the region is empty",
+			addrSet: AddrSet{},
+			wantNil: true,
+		},
+		{
+			name: "Happy test (all addresses unused)",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+			},
+			wantNil: false,
+		},
+		{
+			name: "Happy test (GetAnyAddress can still return addresses used by proxy conns)",
+			addrSet: AddrSet{
+				&addr0: InUse(2),
+			},
+			wantNil: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.GetAnyAddress(); tt.wantNil != (got == nil) {
+				t.Errorf("Region.GetAnyAddress() = %v, but should it return nil? %v", got, tt.wantNil)
+			}
+		})
+	}
+}

edgediscovery/allregions/discovery.go

@@ -13,7 +13,7 @@ import (
 
 const (
 	// Used to discover HA origintunneld servers
-	srvService = "origintunneld"
+	srvService = "v2-origintunneld"
 	srvProto   = "tcp"
 	srvName    = "argotunnel.com"
 
@@ -32,10 +32,40 @@ var (
 	netLookupIP  = net.LookupIP
 )
 
+// ConfigIPVersion is the selection of IP versions from config
+type ConfigIPVersion int8
+
+const (
+	Auto     ConfigIPVersion = 2
+	IPv4Only ConfigIPVersion = 4
+	IPv6Only ConfigIPVersion = 6
+)
+
+// IPVersion is the IP version of an EdgeAddr
+type EdgeIPVersion int8
+
+const (
+	V4 EdgeIPVersion = 4
+	V6 EdgeIPVersion = 6
+)
+
+// String returns the enum's constant name.
+func (c EdgeIPVersion) String() string {
+	switch c {
+	case V4:
+		return "4"
+	case V6:
+		return "6"
+	default:
+		return ""
+	}
+}
+
 // EdgeAddr is a representation of possible ways to refer an edge location.
 type EdgeAddr struct {
-	TCP *net.TCPAddr
-	UDP *net.UDPAddr
+	TCP       *net.TCPAddr
+	UDP       *net.UDPAddr
+	IPVersion EdgeIPVersion
 }
 
 // If the call to net.LookupSRV fails, try to fall back to DoT from Cloudflare directly.
@@ -85,6 +115,9 @@ func edgeDiscovery(log *zerolog.Logger, srvService string) ([][]*EdgeAddr, error
 		if err != nil {
 			return nil, err
 		}
+		for _, e := range edgeAddrs {
+			log.Debug().Msgf("Edge Address: %+v", *e)
+		}
 		resolvedAddrPerCNAME = append(resolvedAddrPerCNAME, edgeAddrs)
 	}
 
@@ -120,9 +153,14 @@ func resolveSRV(srv *net.SRV) ([]*EdgeAddr, error) {
 	}
 	addrs := make([]*EdgeAddr, len(ips))
 	for i, ip := range ips {
+		version := V6
+		if ip.To4() != nil {
+			version = V4
+		}
 		addrs[i] = &EdgeAddr{
-			TCP: &net.TCPAddr{IP: ip, Port: int(srv.Port)},
-			UDP: &net.UDPAddr{IP: ip, Port: int(srv.Port)},
+			TCP:       &net.TCPAddr{IP: ip, Port: int(srv.Port)},
+			UDP:       &net.UDPAddr{IP: ip, Port: int(srv.Port)},
+			IPVersion: version,
 		}
 	}
 	return addrs, nil
@@ -143,11 +181,15 @@ func ResolveAddrs(addrs []string, log *zerolog.Logger) (resolved []*EdgeAddr) {
 			log.Error().Str(logFieldAddress, addr).Err(err).Msg("failed to resolve to UDP address")
 			continue
 		}
+		version := V6
+		if udpAddr.IP.To4() != nil {
+			version = V4
+		}
 		resolved = append(resolved, &EdgeAddr{
-			TCP: tcpAddr,
-			UDP: udpAddr,
+			TCP:       tcpAddr,
+			UDP:       udpAddr,
+			IPVersion: version,
 		})
-
 	}
 	return
 }

edgediscovery/allregions/mocks_for_test.go

@@ -9,6 +9,115 @@ import (
 	"testing/quick"
 )
 
+var (
+	v4Addrs = []*EdgeAddr{&addr0, &addr1, &addr2, &addr3}
+	v6Addrs = []*EdgeAddr{&addr4, &addr5, &addr6, &addr7}
+	addr0   = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr1 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.1"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.1"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr2 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.2"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.2"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr3 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.3"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.3"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr4 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+	addr5 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+	addr6 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+	addr7 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+)
+
 type mockAddrs struct {
 	// a set of synthetic SRV records
 	addrMap map[net.SRV][]*EdgeAddr
@@ -36,7 +145,7 @@ func newMockAddrs(port uint16, numRegions uint8, numAddrsPerRegion uint8) mockAd
 				IP:   net.ParseIP(fmt.Sprintf("10.0.%v.%v", r, a)),
 				Port: int(port),
 			}
-			addrs = append(addrs, &EdgeAddr{tcpAddr, udpAddr})
+			addrs = append(addrs, &EdgeAddr{tcpAddr, udpAddr, V4})
 		}
 		addrMap[srv] = addrs
 		numAddrs += len(addrs)

edgediscovery/allregions/region.go

@@ -1,79 +1,155 @@
 package allregions
 
+import "time"
+
+const (
+	timeoutDuration = 10 * time.Minute
+)
+
 // Region contains cloudflared edge addresses. The edge is partitioned into several regions for
 // redundancy purposes.
 type Region struct {
-	connFor map[*EdgeAddr]UsedBy
+	primaryIsActive bool
+	active          AddrSet
+	primary         AddrSet
+	secondary       AddrSet
+	primaryTimeout  time.Time
+	timeoutDuration time.Duration
 }
 
 // NewRegion creates a region with the given addresses, which are all unused.
-func NewRegion(addrs []*EdgeAddr) Region {
+func NewRegion(addrs []*EdgeAddr, overrideIPVersion ConfigIPVersion) Region {
 	// The zero value of UsedBy is Unused(), so we can just initialize the map's values with their
 	// zero values.
-	connFor := make(map[*EdgeAddr]UsedBy)
-	for _, addr := range addrs {
-		connFor[addr] = Unused()
+	connForv4 := make(AddrSet)
+	connForv6 := make(AddrSet)
+	systemPreference := V6
+	for i, addr := range addrs {
+		if i == 0 {
+			// First family of IPs returned is system preference of IP
+			systemPreference = addr.IPVersion
+		}
+		switch addr.IPVersion {
+		case V4:
+			connForv4[addr] = Unused()
+		case V6:
+			connForv6[addr] = Unused()
+		}
 	}
+
+	// Process as system preference
+	var primary AddrSet
+	var secondary AddrSet
+	switch systemPreference {
+	case V4:
+		primary = connForv4
+		secondary = connForv6
+	case V6:
+		primary = connForv6
+		secondary = connForv4
+	}
+
+	// Override with provided preference
+	switch overrideIPVersion {
+	case IPv4Only:
+		primary = connForv4
+		secondary = make(AddrSet) // empty
+	case IPv6Only:
+		primary = connForv6
+		secondary = make(AddrSet) // empty
+	case Auto:
+		// no change
+	default:
+		// no change
+	}
+
 	return Region{
-		connFor: connFor,
+		primaryIsActive: true,
+		active:          primary,
+		primary:         primary,
+		secondary:       secondary,
+		timeoutDuration: timeoutDuration,
 	}
 }
 
 // AddrUsedBy finds the address used by the given connection in this region.
 // Returns nil if the connection isn't using any IP.
 func (r *Region) AddrUsedBy(connID int) *EdgeAddr {
-	for addr, used := range r.connFor {
-		if used.Used && used.ConnID == connID {
-			return addr
-		}
+	edgeAddr := r.primary.AddrUsedBy(connID)
+	if edgeAddr == nil {
+		edgeAddr = r.secondary.AddrUsedBy(connID)
 	}
-	return nil
+	return edgeAddr
 }
 
 // AvailableAddrs counts how many unused addresses this region contains.
 func (r Region) AvailableAddrs() int {
-	n := 0
-	for _, usedby := range r.connFor {
-		if !usedby.Used {
-			n++
-		}
-	}
-	return n
+	return r.active.AvailableAddrs()
 }
 
-// GetUnusedIP returns a random unused address in this region.
-// Returns nil if all addresses are in use.
-func (r Region) GetUnusedIP(excluding *EdgeAddr) *EdgeAddr {
-	for addr, usedby := range r.connFor {
-		if !usedby.Used && addr != excluding {
-			return addr
-		}
-	}
-	return nil
-}
-
-// Use the address, assigning it to a proxy connection.
-func (r Region) Use(addr *EdgeAddr, connID int) {
-	if addr == nil {
-		return
-	}
-	r.connFor[addr] = InUse(connID)
-}
-
-// GetAnyAddress returns an arbitrary address from the region.
-func (r Region) GetAnyAddress() *EdgeAddr {
-	for addr := range r.connFor {
+// AssignAnyAddress returns a random unused address in this region now
+// assigned to the connID excluding the provided EdgeAddr.
+// Returns nil if all addresses are in use for the region.
+func (r Region) AssignAnyAddress(connID int, excluding *EdgeAddr) *EdgeAddr {
+	if addr := r.active.GetUnusedIP(excluding); addr != nil {
+		r.active.Use(addr, connID)
 		return addr
 	}
 	return nil
 }
 
+// GetAnyAddress returns an arbitrary address from the region.
+func (r Region) GetAnyAddress() *EdgeAddr {
+	return r.active.GetAnyAddress()
+}
+
 // GiveBack the address, ensuring it is no longer assigned to an IP.
 // Returns true if the address is in this region.
-func (r Region) GiveBack(addr *EdgeAddr) (ok bool) {
-	if _, ok := r.connFor[addr]; !ok {
-		return false
+func (r *Region) GiveBack(addr *EdgeAddr, hasConnectivityError bool) (ok bool) {
+	if ok = r.primary.GiveBack(addr); !ok {
+		// Attempt to give back the address in the secondary set
+		if ok = r.secondary.GiveBack(addr); !ok {
+			// Address is not in this region
+			return
+		}
 	}
-	r.connFor[addr] = Unused()
-	return true
+
+	// No connectivity error: no worry
+	if !hasConnectivityError {
+		return
+	}
+
+	// If using primary and returned address is IPv6 and secondary is available
+	if r.primaryIsActive && addr.IPVersion == V6 && len(r.secondary) > 0 {
+		r.active = r.secondary
+		r.primaryIsActive = false
+		r.primaryTimeout = time.Now().Add(r.timeoutDuration)
+		return
+	}
+
+	// Do nothing for IPv4 or if secondary is empty
+	if r.primaryIsActive {
+		return
+	}
+
+	// Immediately return to primary pool, regardless of current primary timeout
+	if addr.IPVersion == V4 {
+		activatePrimary(r)
+		return
+	}
+
+	// Timeout exceeded and can be reset to primary pool
+	if r.primaryTimeout.Before(time.Now()) {
+		activatePrimary(r)
+		return
+	}
+
+	return
+}
+
+// activatePrimary sets the primary set to the active set and resets the timeout.
+func activatePrimary(r *Region) {
+	r.active = r.primary
+	r.primaryIsActive = true
+	r.primaryTimeout = time.Now() // reset timeout
 }

edgediscovery/allregions/region_test.go

@@ -1,284 +1,357 @@
 package allregions
 
 import (
-	"reflect"
+	"net"
 	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
 )
 
+func makeAddrSet(addrs []*EdgeAddr) AddrSet {
+	addrSet := make(AddrSet, len(addrs))
+	for _, addr := range addrs {
+		addrSet[addr] = Unused()
+	}
+	return addrSet
+}
+
 func TestRegion_New(t *testing.T) {
-	r := NewRegion([]*EdgeAddr{&addr0, &addr1, &addr2})
-	if r.AvailableAddrs() != 3 {
-		t.Errorf("r.AvailableAddrs() == %v but want 3", r.AvailableAddrs())
-	}
-}
-
-func TestRegion_AddrUsedBy(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	type args struct {
-		connID int
-	}
 	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   *EdgeAddr
+		name          string
+		addrs         []*EdgeAddr
+		mode          ConfigIPVersion
+		expectedAddrs int
+		primary       AddrSet
+		secondary     AddrSet
 	}{
 		{
-			name: "happy trivial test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-			}},
-			args: args{connID: 0},
-			want: &addr0,
+			name:          "IPv4 addresses with IPv4Only",
+			addrs:         v4Addrs,
+			mode:          IPv4Only,
+			expectedAddrs: len(v4Addrs),
+			primary:       makeAddrSet(v4Addrs),
+			secondary:     AddrSet{},
 		},
 		{
-			name: "sad trivial test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-			}},
-			args: args{connID: 1},
-			want: nil,
+			name:          "IPv6 addresses with IPv4Only",
+			addrs:         v6Addrs,
+			mode:          IPv4Only,
+			expectedAddrs: 0,
+			primary:       AddrSet{},
+			secondary:     AddrSet{},
 		},
 		{
-			name: "sad test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{connID: 3},
-			want: nil,
+			name:          "IPv6 addresses with IPv6Only",
+			addrs:         v6Addrs,
+			mode:          IPv6Only,
+			expectedAddrs: len(v6Addrs),
+			primary:       makeAddrSet(v6Addrs),
+			secondary:     AddrSet{},
 		},
 		{
-			name: "happy test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{connID: 1},
-			want: &addr1,
+			name:          "IPv6 addresses with IPv4Only",
+			addrs:         v6Addrs,
+			mode:          IPv4Only,
+			expectedAddrs: 0,
+			primary:       AddrSet{},
+			secondary:     AddrSet{},
+		},
+		{
+			name:          "IPv4 (first) and IPv6 addresses with Auto",
+			addrs:         append(v4Addrs, v6Addrs...),
+			mode:          Auto,
+			expectedAddrs: len(v4Addrs),
+			primary:       makeAddrSet(v4Addrs),
+			secondary:     makeAddrSet(v6Addrs),
+		},
+		{
+			name:          "IPv6 (first) and IPv4 addresses with Auto",
+			addrs:         append(v6Addrs, v4Addrs...),
+			mode:          Auto,
+			expectedAddrs: len(v6Addrs),
+			primary:       makeAddrSet(v6Addrs),
+			secondary:     makeAddrSet(v4Addrs),
+		},
+		{
+			name:          "IPv4 addresses with Auto",
+			addrs:         v4Addrs,
+			mode:          Auto,
+			expectedAddrs: len(v4Addrs),
+			primary:       makeAddrSet(v4Addrs),
+			secondary:     AddrSet{},
+		},
+		{
+			name:          "IPv6 addresses with Auto",
+			addrs:         v6Addrs,
+			mode:          Auto,
+			expectedAddrs: len(v6Addrs),
+			primary:       makeAddrSet(v6Addrs),
+			secondary:     AddrSet{},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			r := &Region{
-				connFor: tt.fields.connFor,
+			r := NewRegion(tt.addrs, tt.mode)
+			assert.Equal(t, tt.expectedAddrs, r.AvailableAddrs())
+			assert.Equal(t, tt.primary, r.primary)
+			assert.Equal(t, tt.secondary, r.secondary)
+		})
+	}
+}
+
+func TestRegion_AnyAddress_EmptyActiveSet(t *testing.T) {
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv6 addresses with IPv4Only",
+			addrs: v6Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv4 addresses with IPv6Only",
+			addrs: v4Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			addr := r.GetAnyAddress()
+			assert.Nil(t, addr)
+			addr = r.AssignAnyAddress(0, nil)
+			assert.Nil(t, addr)
+		})
+	}
+}
+
+func TestRegion_AssignAnyAddress_FullyUsedActiveSet(t *testing.T) {
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			total := r.active.AvailableAddrs()
+			for i := 0; i < total; i++ {
+				addr := r.AssignAnyAddress(i, nil)
+				assert.NotNil(t, addr)
 			}
-			if got := r.AddrUsedBy(tt.args.connID); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("Region.AddrUsedBy() = %v, want %v", got, tt.want)
+			addr := r.AssignAnyAddress(9, nil)
+			assert.Nil(t, addr)
+		})
+	}
+}
+
+var giveBackTests = []struct {
+	name          string
+	addrs         []*EdgeAddr
+	mode          ConfigIPVersion
+	expectedAddrs int
+	primary       AddrSet
+	secondary     AddrSet
+	primarySwap   bool
+}{
+	{
+		name:          "IPv4 addresses with IPv4Only",
+		addrs:         v4Addrs,
+		mode:          IPv4Only,
+		expectedAddrs: len(v4Addrs),
+		primary:       makeAddrSet(v4Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv6 addresses with IPv6Only",
+		addrs:         v6Addrs,
+		mode:          IPv6Only,
+		expectedAddrs: len(v6Addrs),
+		primary:       makeAddrSet(v6Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv4 (first) and IPv6 addresses with Auto",
+		addrs:         append(v4Addrs, v6Addrs...),
+		mode:          Auto,
+		expectedAddrs: len(v4Addrs),
+		primary:       makeAddrSet(v4Addrs),
+		secondary:     makeAddrSet(v6Addrs),
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv6 (first) and IPv4 addresses with Auto",
+		addrs:         append(v6Addrs, v4Addrs...),
+		mode:          Auto,
+		expectedAddrs: len(v6Addrs),
+		primary:       makeAddrSet(v6Addrs),
+		secondary:     makeAddrSet(v4Addrs),
+		primarySwap:   true,
+	},
+	{
+		name:          "IPv4 addresses with Auto",
+		addrs:         v4Addrs,
+		mode:          Auto,
+		expectedAddrs: len(v4Addrs),
+		primary:       makeAddrSet(v4Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv6 addresses with Auto",
+		addrs:         v6Addrs,
+		mode:          Auto,
+		expectedAddrs: len(v6Addrs),
+		primary:       makeAddrSet(v6Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+}
+
+func TestRegion_GiveBack_NoConnectivityError(t *testing.T) {
+	for _, tt := range giveBackTests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			addr := r.AssignAnyAddress(0, nil)
+			assert.NotNil(t, addr)
+			assert.True(t, r.GiveBack(addr, false))
+		})
+	}
+}
+
+func TestRegion_GiveBack_ForeignAddr(t *testing.T) {
+	invalid := EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	for _, tt := range giveBackTests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			assert.False(t, r.GiveBack(&invalid, false))
+			assert.False(t, r.GiveBack(&invalid, true))
+		})
+	}
+}
+
+func TestRegion_GiveBack_SwapPrimary(t *testing.T) {
+	for _, tt := range giveBackTests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			addr := r.AssignAnyAddress(0, nil)
+			assert.NotNil(t, addr)
+			assert.True(t, r.GiveBack(addr, true))
+			assert.Equal(t, tt.primarySwap, !r.primaryIsActive)
+			if tt.primarySwap {
+				assert.Equal(t, r.secondary, r.active)
+				assert.False(t, r.primaryTimeout.IsZero())
+			} else {
+				assert.Equal(t, r.primary, r.active)
+				assert.True(t, r.primaryTimeout.IsZero())
 			}
 		})
 	}
 }
 
-func TestRegion_AvailableAddrs(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		want   int
-	}{
-		{
-			name: "contains addresses",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: Unused(),
-				&addr2: InUse(2),
-			}},
-			want: 1,
-		},
-		{
-			name: "all free",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-				&addr1: Unused(),
-				&addr2: Unused(),
-			}},
-			want: 3,
-		},
-		{
-			name: "all used",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			want: 0,
-		},
-		{
-			name:   "empty",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{}},
-			want:   0,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if got := r.AvailableAddrs(); got != tt.want {
-				t.Errorf("Region.AvailableAddrs() = %v, want %v", got, tt.want)
-			}
-		})
-	}
+func TestRegion_GiveBack_IPv4_ResetPrimary(t *testing.T) {
+	r := NewRegion(append(v6Addrs, v4Addrs...), Auto)
+	// Exhaust all IPv6 addresses
+	a0 := r.AssignAnyAddress(0, nil)
+	a1 := r.AssignAnyAddress(1, nil)
+	a2 := r.AssignAnyAddress(2, nil)
+	a3 := r.AssignAnyAddress(3, nil)
+	assert.NotNil(t, a0)
+	assert.NotNil(t, a1)
+	assert.NotNil(t, a2)
+	assert.NotNil(t, a3)
+	// Give back the first IPv6 address to fallback to secondary IPv4 address set
+	assert.True(t, r.GiveBack(a0, true))
+	assert.False(t, r.primaryIsActive)
+	// Give back another IPv6 address
+	assert.True(t, r.GiveBack(a1, true))
+	// Primary shouldn't change
+	assert.False(t, r.primaryIsActive)
+	// Request an address (should be IPv4 from secondary)
+	a4_v4 := r.AssignAnyAddress(4, nil)
+	assert.NotNil(t, a4_v4)
+	assert.Equal(t, V4, a4_v4.IPVersion)
+	a5_v4 := r.AssignAnyAddress(5, nil)
+	assert.NotNil(t, a5_v4)
+	assert.Equal(t, V4, a5_v4.IPVersion)
+	a6_v4 := r.AssignAnyAddress(6, nil)
+	assert.NotNil(t, a6_v4)
+	assert.Equal(t, V4, a6_v4.IPVersion)
+	// Return IPv4 address (without failure)
+	// Primary shouldn't change because it is not a connectivity failure
+	assert.True(t, r.GiveBack(a4_v4, false))
+	assert.False(t, r.primaryIsActive)
+	// Return IPv4 address (with failure)
+	// Primary should change because it is a connectivity failure
+	assert.True(t, r.GiveBack(a5_v4, true))
+	assert.True(t, r.primaryIsActive)
+	// Return IPv4 address (with failure)
+	// Primary shouldn't change because the address is returned to the inactive
+	// secondary address set
+	assert.True(t, r.GiveBack(a6_v4, true))
+	assert.True(t, r.primaryIsActive)
+	// Return IPv6 address (without failure)
+	// Primary shoudn't change because it is not a connectivity failure
+	assert.True(t, r.GiveBack(a2, false))
+	assert.True(t, r.primaryIsActive)
 }
 
-func TestRegion_GetUnusedIP(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	type args struct {
-		excluding *EdgeAddr
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   *EdgeAddr
-	}{
-		{
-			name: "happy test with excluding set",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-				&addr1: Unused(),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: &addr0},
-			want: &addr1,
-		},
-		{
-			name: "happy test with no excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: Unused(),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: nil},
-			want: &addr1,
-		},
-		{
-			name: "sad test with no excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: nil},
-			want: nil,
-		},
-		{
-			name: "sad test with excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: &addr0},
-			want: nil,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if got := r.GetUnusedIP(tt.args.excluding); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("Region.GetUnusedIP() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func TestRegion_GiveBack(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	type args struct {
-		addr *EdgeAddr
-	}
-	tests := []struct {
-		name           string
-		fields         fields
-		args           args
-		wantOk         bool
-		availableAfter int
-	}{
-		{
-			name: "sad test with excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr1: InUse(1),
-			}},
-			args:           args{addr: &addr1},
-			wantOk:         true,
-			availableAfter: 1,
-		},
-		{
-			name: "sad test with excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr1: InUse(1),
-			}},
-			args:           args{addr: &addr2},
-			wantOk:         false,
-			availableAfter: 0,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if gotOk := r.GiveBack(tt.args.addr); gotOk != tt.wantOk {
-				t.Errorf("Region.GiveBack() = %v, want %v", gotOk, tt.wantOk)
-			}
-			if tt.availableAfter != r.AvailableAddrs() {
-				t.Errorf("Region.AvailableAddrs() = %v, want %v", r.AvailableAddrs(), tt.availableAfter)
-			}
-		})
-	}
-}
-
-func TestRegion_GetAnyAddress(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	tests := []struct {
-		name    string
-		fields  fields
-		wantNil bool
-	}{
-		{
-			name:    "Sad test -- GetAnyAddress should only fail if the region is empty",
-			fields:  fields{connFor: map[*EdgeAddr]UsedBy{}},
-			wantNil: true,
-		},
-		{
-			name: "Happy test (all addresses unused)",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-			}},
-			wantNil: false,
-		},
-		{
-			name: "Happy test (GetAnyAddress can still return addresses used by proxy conns)",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(2),
-			}},
-			wantNil: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if got := r.GetAnyAddress(); tt.wantNil != (got == nil) {
-				t.Errorf("Region.GetAnyAddress() = %v, but should it return nil? %v", got, tt.wantNil)
-			}
-		})
-	}
+func TestRegion_GiveBack_Timeout(t *testing.T) {
+	r := NewRegion(append(v6Addrs, v4Addrs...), Auto)
+	a0 := r.AssignAnyAddress(0, nil)
+	a1 := r.AssignAnyAddress(1, nil)
+	a2 := r.AssignAnyAddress(2, nil)
+	assert.NotNil(t, a0)
+	assert.NotNil(t, a1)
+	assert.NotNil(t, a2)
+	// Give back IPv6 address to set timeout
+	assert.True(t, r.GiveBack(a0, true))
+	assert.False(t, r.primaryIsActive)
+	assert.False(t, r.primaryTimeout.IsZero())
+	// Request an address (should be IPv4 from secondary)
+	a3_v4 := r.AssignAnyAddress(3, nil)
+	assert.NotNil(t, a3_v4)
+	assert.Equal(t, V4, a3_v4.IPVersion)
+	assert.False(t, r.primaryIsActive)
+	// Give back IPv6 address inside timeout (no change)
+	assert.True(t, r.GiveBack(a2, true))
+	assert.False(t, r.primaryIsActive)
+	assert.False(t, r.primaryTimeout.IsZero())
+	// Accelerate timeout
+	r.primaryTimeout = time.Now().Add(-time.Minute)
+	// Return IPv6 address
+	assert.True(t, r.GiveBack(a1, true))
+	assert.True(t, r.primaryIsActive)
+	// Returning an IPv4 address after primary is active shouldn't change primary
+	// even with a connectivity error
+	assert.True(t, r.GiveBack(a3_v4, true))
+	assert.True(t, r.primaryIsActive)
 }

edgediscovery/allregions/regions.go

@@ -18,7 +18,7 @@ type Regions struct {
 // ------------------------------------
 
 // ResolveEdge resolves the Cloudflare edge, returning all regions discovered.
-func ResolveEdge(log *zerolog.Logger, region string) (*Regions, error) {
+func ResolveEdge(log *zerolog.Logger, region string, overrideIPVersion ConfigIPVersion) (*Regions, error) {
 	edgeAddrs, err := edgeDiscovery(log, getRegionalServiceName(region))
 	if err != nil {
 		return nil, err
@@ -27,8 +27,8 @@ func ResolveEdge(log *zerolog.Logger, region string) (*Regions, error) {
 		return nil, fmt.Errorf("expected at least 2 Cloudflare Regions regions, but SRV only returned %v", len(edgeAddrs))
 	}
 	return &Regions{
-		region1: NewRegion(edgeAddrs[0]),
-		region2: NewRegion(edgeAddrs[1]),
+		region1: NewRegion(edgeAddrs[0], overrideIPVersion),
+		region2: NewRegion(edgeAddrs[1], overrideIPVersion),
 	}, nil
 }
 
@@ -56,8 +56,8 @@ func NewNoResolve(addrs []*EdgeAddr) *Regions {
 	}
 
 	return &Regions{
-		region1: NewRegion(region1),
-		region2: NewRegion(region2),
+		region1: NewRegion(region1, Auto),
+		region2: NewRegion(region2, Auto),
 	}
 }
 
@@ -95,14 +95,12 @@ func (rs *Regions) GetUnusedAddr(excluding *EdgeAddr, connID int) *EdgeAddr {
 // getAddrs tries to grab address form `first` region, then `second` region
 // this is an unrolled loop over 2 element array
 func getAddrs(excluding *EdgeAddr, connID int, first *Region, second *Region) *EdgeAddr {
-	addr := first.GetUnusedIP(excluding)
+	addr := first.AssignAnyAddress(connID, excluding)
 	if addr != nil {
-		first.Use(addr, connID)
 		return addr
 	}
-	addr = second.GetUnusedIP(excluding)
+	addr = second.AssignAnyAddress(connID, excluding)
 	if addr != nil {
-		second.Use(addr, connID)
 		return addr
 	}
 
@@ -116,18 +114,18 @@ func (rs *Regions) AvailableAddrs() int {
 
 // GiveBack the address so that other connections can use it.
 // Returns true if the address is in this edge.
-func (rs *Regions) GiveBack(addr *EdgeAddr) bool {
-	if found := rs.region1.GiveBack(addr); found {
+func (rs *Regions) GiveBack(addr *EdgeAddr, hasConnectivityError bool) bool {
+	if found := rs.region1.GiveBack(addr, hasConnectivityError); found {
 		return found
 	}
-	return rs.region2.GiveBack(addr)
+	return rs.region2.GiveBack(addr, hasConnectivityError)
 }
 
 // Return regionalized service name if `region` isn't empty, otherwise return the global service name for origintunneld
 func getRegionalServiceName(region string) string {
 	if region != "" {
-		return region + "-" + srvService // Example: `us-origintunneld`
+		return region + "-" + srvService // Example: `us-v2-origintunneld`
 	}
 
-	return srvService // Global service is just `origintunneld`
+	return srvService // Global service is just `v2-origintunneld`
 }

edgediscovery/allregions/regions_test.go

@@ -1,134 +1,215 @@
 package allregions
 
 import (
-	"net"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-var (
-	addr0 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.0"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.0"),
-			Port: 8000,
-			Zone: "",
-		},
+func makeRegions(addrs []*EdgeAddr, mode ConfigIPVersion) Regions {
+	r1addrs := make([]*EdgeAddr, 0)
+	r2addrs := make([]*EdgeAddr, 0)
+	for i, addr := range addrs {
+		if i%2 == 0 {
+			r1addrs = append(r1addrs, addr)
+		} else {
+			r2addrs = append(r2addrs, addr)
+		}
 	}
-	addr1 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.1"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.1"),
-			Port: 8000,
-			Zone: "",
-		},
-	}
-	addr2 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.2"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.2"),
-			Port: 8000,
-			Zone: "",
-		},
-	}
-	addr3 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.3"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.3"),
-			Port: 8000,
-			Zone: "",
-		},
-	}
-)
-
-func makeRegions() Regions {
-	r1 := NewRegion([]*EdgeAddr{&addr0, &addr1})
-	r2 := NewRegion([]*EdgeAddr{&addr2, &addr3})
+	r1 := NewRegion(r1addrs, mode)
+	r2 := NewRegion(r2addrs, mode)
 	return Regions{region1: r1, region2: r2}
 }
 
 func TestRegions_AddrUsedBy(t *testing.T) {
-	rs := makeRegions()
-	addr1 := rs.GetUnusedAddr(nil, 1)
-	assert.Equal(t, addr1, rs.AddrUsedBy(1))
-	addr2 := rs.GetUnusedAddr(nil, 2)
-	assert.Equal(t, addr2, rs.AddrUsedBy(2))
-	addr3 := rs.GetUnusedAddr(nil, 3)
-	assert.Equal(t, addr3, rs.AddrUsedBy(3))
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			addr1 := rs.GetUnusedAddr(nil, 1)
+			assert.Equal(t, addr1, rs.AddrUsedBy(1))
+			addr2 := rs.GetUnusedAddr(nil, 2)
+			assert.Equal(t, addr2, rs.AddrUsedBy(2))
+			addr3 := rs.GetUnusedAddr(nil, 3)
+			assert.Equal(t, addr3, rs.AddrUsedBy(3))
+		})
+	}
 }
 
 func TestRegions_Giveback_Region1(t *testing.T) {
-	rs := makeRegions()
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
-	rs.region2.Use(&addr2, 2)
-	rs.region2.Use(&addr3, 3)
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			addr := rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			rs.region2.AssignAnyAddress(2, nil)
+			rs.region2.AssignAnyAddress(3, nil)
 
-	assert.Equal(t, 0, rs.AvailableAddrs())
+			assert.Equal(t, 0, rs.AvailableAddrs())
 
-	rs.GiveBack(&addr0)
-	assert.Equal(t, &addr0, rs.GetUnusedAddr(nil, 3))
+			rs.GiveBack(addr, false)
+			assert.Equal(t, addr, rs.GetUnusedAddr(nil, 0))
+		})
+	}
 }
 
 func TestRegions_Giveback_Region2(t *testing.T) {
-	rs := makeRegions()
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
-	rs.region2.Use(&addr2, 2)
-	rs.region2.Use(&addr3, 3)
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			addr := rs.region2.AssignAnyAddress(2, nil)
+			rs.region2.AssignAnyAddress(3, nil)
 
-	assert.Equal(t, 0, rs.AvailableAddrs())
+			assert.Equal(t, 0, rs.AvailableAddrs())
 
-	rs.GiveBack(&addr2)
-	assert.Equal(t, &addr2, rs.GetUnusedAddr(nil, 2))
+			rs.GiveBack(addr, false)
+			assert.Equal(t, addr, rs.GetUnusedAddr(nil, 2))
+		})
+	}
 }
 
 func TestRegions_GetUnusedAddr_OneAddrLeft(t *testing.T) {
-	rs := makeRegions()
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			rs.region2.AssignAnyAddress(2, nil)
+			addr := rs.region2.active.GetUnusedIP(nil)
 
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
-	rs.region2.Use(&addr2, 2)
-
-	assert.Equal(t, 1, rs.AvailableAddrs())
-	assert.Equal(t, &addr3, rs.GetUnusedAddr(nil, 3))
+			assert.Equal(t, 1, rs.AvailableAddrs())
+			assert.Equal(t, addr, rs.GetUnusedAddr(nil, 3))
+		})
+	}
 }
 
 func TestRegions_GetUnusedAddr_Excluding_Region1(t *testing.T) {
-	rs := makeRegions()
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
 
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
+			rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			addr := rs.region2.active.GetUnusedIP(nil)
+			a2 := rs.region2.active.GetUnusedIP(addr)
 
-	assert.Equal(t, 2, rs.AvailableAddrs())
-	assert.Equal(t, &addr3, rs.GetUnusedAddr(&addr2, 3))
+			assert.Equal(t, 2, rs.AvailableAddrs())
+			assert.Equal(t, addr, rs.GetUnusedAddr(a2, 3))
+		})
+	}
 }
 
 func TestRegions_GetUnusedAddr_Excluding_Region2(t *testing.T) {
-	rs := makeRegions()
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
 
-	rs.region2.Use(&addr2, 0)
-	rs.region2.Use(&addr3, 1)
+			rs.region2.AssignAnyAddress(0, nil)
+			rs.region2.AssignAnyAddress(1, nil)
+			addr := rs.region1.active.GetUnusedIP(nil)
+			a2 := rs.region1.active.GetUnusedIP(addr)
 
-	assert.Equal(t, 2, rs.AvailableAddrs())
-	assert.Equal(t, &addr1, rs.GetUnusedAddr(&addr0, 1))
+			assert.Equal(t, 2, rs.AvailableAddrs())
+			assert.Equal(t, addr, rs.GetUnusedAddr(a2, 1))
+		})
+	}
 }
 
 func TestNewNoResolveBalancesRegions(t *testing.T) {

edgediscovery/edgediscovery.go

@@ -1,7 +1,6 @@
 package edgediscovery
 
 import (
-	"fmt"
 	"sync"
 
 	"github.com/rs/zerolog"
@@ -11,9 +10,16 @@ import (
 
 const (
 	LogFieldConnIndex = "connIndex"
+	LogFieldIPAddress = "ip"
 )
 
-var errNoAddressesLeft = fmt.Errorf("there are no free edge addresses left")
+var errNoAddressesLeft = ErrNoAddressesLeft{}
+
+type ErrNoAddressesLeft struct{}
+
+func (e ErrNoAddressesLeft) Error() string {
+	return "there are no free edge addresses left to resolve to"
+}
 
 // Edge finds addresses on the Cloudflare edge and hands them out to connections.
 type Edge struct {
@@ -28,8 +34,8 @@ type Edge struct {
 
 // ResolveEdge runs the initial discovery of the Cloudflare edge, finding Addrs that can be allocated
 // to connections.
-func ResolveEdge(log *zerolog.Logger, region string) (*Edge, error) {
-	regions, err := allregions.ResolveEdge(log, region)
+func ResolveEdge(log *zerolog.Logger, region string, edgeIpVersion allregions.ConfigIPVersion) (*Edge, error) {
+	regions, err := allregions.ResolveEdge(log, region, edgeIpVersion)
 	if err != nil {
 		return new(Edge), err
 	}
@@ -51,15 +57,6 @@ func StaticEdge(log *zerolog.Logger, hostnames []string) (*Edge, error) {
 	}, nil
 }
 
-// MockEdge creates a Cloudflare Edge from arbitrary TCP addresses. Used for testing.
-func MockEdge(log *zerolog.Logger, addrs []*allregions.EdgeAddr) *Edge {
-	regions := allregions.NewNoResolve(addrs)
-	return &Edge{
-		log:     log,
-		regions: regions,
-	}
-}
-
 // ------------------------------------
 // Methods
 // ------------------------------------
@@ -93,12 +90,15 @@ func (ed *Edge) GetAddr(connIndex int) (*allregions.EdgeAddr, error) {
 		log.Debug().Msg("edgediscovery - GetAddr: No addresses left to give proxy connection")
 		return nil, errNoAddressesLeft
 	}
-	log.Debug().Msg("edgediscovery - GetAddr: Giving connection its new address")
+	log = ed.log.With().
+		Int(LogFieldConnIndex, connIndex).
+		IPAddr(LogFieldIPAddress, addr.UDP.IP).Logger()
+	log.Debug().Msgf("edgediscovery - GetAddr: Giving connection its new address")
 	return addr, nil
 }
 
 // GetDifferentAddr gives back the proxy connection's edge Addr and uses a new one.
-func (ed *Edge) GetDifferentAddr(connIndex int) (*allregions.EdgeAddr, error) {
+func (ed *Edge) GetDifferentAddr(connIndex int, hasConnectivityError bool) (*allregions.EdgeAddr, error) {
 	log := ed.log.With().Int(LogFieldConnIndex, connIndex).Logger()
 
 	ed.Lock()
@@ -106,7 +106,7 @@ func (ed *Edge) GetDifferentAddr(connIndex int) (*allregions.EdgeAddr, error) {
 
 	oldAddr := ed.regions.AddrUsedBy(connIndex)
 	if oldAddr != nil {
-		ed.regions.GiveBack(oldAddr)
+		ed.regions.GiveBack(oldAddr, hasConnectivityError)
 	}
 	addr := ed.regions.GetUnusedAddr(oldAddr, connIndex)
 	if addr == nil {
@@ -114,8 +114,10 @@ func (ed *Edge) GetDifferentAddr(connIndex int) (*allregions.EdgeAddr, error) {
 		// note: if oldAddr were not nil, it will become available on the next iteration
 		return nil, errNoAddressesLeft
 	}
-	log.Debug().Msgf("edgediscovery - GetDifferentAddr: Giving connection its new address: %v from the address list: %v",
-		addr, ed.regions.AvailableAddrs())
+	log = ed.log.With().
+		Int(LogFieldConnIndex, connIndex).
+		IPAddr(LogFieldIPAddress, addr.UDP.IP).Logger()
+	log.Debug().Msgf("edgediscovery - GetDifferentAddr: Giving connection its new address from the address list: %v", ed.regions.AvailableAddrs())
 	return addr, nil
 }
 
@@ -128,9 +130,11 @@ func (ed *Edge) AvailableAddrs() int {
 
 // GiveBack the address so that other connections can use it.
 // Returns true if the address is in this edge.
-func (ed *Edge) GiveBack(addr *allregions.EdgeAddr) bool {
+func (ed *Edge) GiveBack(addr *allregions.EdgeAddr, hasConnectivityError bool) bool {
 	ed.Lock()
 	defer ed.Unlock()
-	ed.log.Debug().Msg("edgediscovery - GiveBack: Address now unused")
-	return ed.regions.GiveBack(addr)
+	log := ed.log.With().
+		IPAddr(LogFieldIPAddress, addr.UDP.IP).Logger()
+	log.Debug().Msgf("edgediscovery - GiveBack: Address now unused")
+	return ed.regions.GiveBack(addr, hasConnectivityError)
 }

edgediscovery/edgediscovery_test.go

@@ -11,56 +11,113 @@ import (
 )
 
 var (
-	addr0 = allregions.EdgeAddr{
+	testLogger = zerolog.Nop()
+	v4Addrs    = []*allregions.EdgeAddr{&addr0, &addr1, &addr2, &addr3}
+	v6Addrs    = []*allregions.EdgeAddr{&addr4, &addr5, &addr6, &addr7}
+	addr0      = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.0"),
+			IP:   net.ParseIP("123.4.5.0"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.0"),
+			IP:   net.ParseIP("123.4.5.0"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
 	}
 	addr1 = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.1"),
+			IP:   net.ParseIP("123.4.5.1"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.1"),
+			IP:   net.ParseIP("123.4.5.1"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
 	}
 	addr2 = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.2"),
+			IP:   net.ParseIP("123.4.5.2"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.2"),
+			IP:   net.ParseIP("123.4.5.2"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
 	}
 	addr3 = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.3"),
+			IP:   net.ParseIP("123.4.5.3"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.3"),
+			IP:   net.ParseIP("123.4.5.3"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
+	}
+	addr4 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
+	}
+	addr5 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
+	}
+	addr6 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
+	}
+	addr7 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
 	}
-
-	testLogger = zerolog.Nop()
 )
 
 func TestGiveBack(t *testing.T) {
@@ -75,7 +132,7 @@ func TestGiveBack(t *testing.T) {
 	assert.Equal(t, 3, edge.AvailableAddrs())
 
 	// Get it back
-	edge.GiveBack(addr)
+	edge.GiveBack(addr, false)
 	assert.Equal(t, 4, edge.AvailableAddrs())
 }
 
@@ -107,7 +164,7 @@ func TestGetAddrForRPC(t *testing.T) {
 	assert.Equal(t, 4, edge.AvailableAddrs())
 
 	// Get it back
-	edge.GiveBack(addr)
+	edge.GiveBack(addr, false)
 	assert.Equal(t, 4, edge.AvailableAddrs())
 }
 
@@ -122,13 +179,13 @@ func TestOnePerRegion(t *testing.T) {
 	assert.NotNil(t, a1)
 
 	// if the first address is bad, get the second one
-	a2, err := edge.GetDifferentAddr(connID)
+	a2, err := edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.NotNil(t, a2)
 	assert.NotEqual(t, a1, a2)
 
 	// now that second one is bad, get the first one again
-	a3, err := edge.GetDifferentAddr(connID)
+	a3, err := edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.Equal(t, a1, a3)
 }
@@ -144,11 +201,11 @@ func TestOnlyOneAddrLeft(t *testing.T) {
 	assert.NotNil(t, addr)
 
 	// If that edge address is "bad", there's no alternative address.
-	_, err = edge.GetDifferentAddr(connID)
+	_, err = edge.GetDifferentAddr(connID, false)
 	assert.Error(t, err)
 
 	// previously bad address should become available again on next iteration.
-	addr, err = edge.GetDifferentAddr(connID)
+	addr, err = edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.NotNil(t, addr)
 }
@@ -190,8 +247,17 @@ func TestGetDifferentAddr(t *testing.T) {
 	assert.Equal(t, 3, edge.AvailableAddrs())
 
 	// If the same connection requests another address, it should get the same one.
-	addr2, err := edge.GetDifferentAddr(connID)
+	addr2, err := edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.NotEqual(t, addr, addr2)
 	assert.Equal(t, 3, edge.AvailableAddrs())
 }
+
+// MockEdge creates a Cloudflare Edge from arbitrary TCP addresses. Used for testing.
+func MockEdge(log *zerolog.Logger, addrs []*allregions.EdgeAddr) *Edge {
+	regions := allregions.NewNoResolve(addrs)
+	return &Edge{
+		log:     log,
+		regions: regions,
+	}
+}

go.mod

@@ -1,6 +1,6 @@
 module github.com/cloudflare/cloudflared
 
-go 1.17
+go 1.18
 
 require (
 	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
@@ -9,7 +9,6 @@ require (
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
 	github.com/fsnotify/fsnotify v1.4.9
-	github.com/gdamore/tcell v1.3.0
 	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
 	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
@@ -17,27 +16,33 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/json-iterator/go v1.1.12
-	github.com/lucas-clemente/quic-go v0.24.0
+	github.com/lucas-clemente/quic-go v0.28.1
 	github.com/mattn/go-colorable v0.1.8
 	github.com/miekg/dns v1.1.45
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/client_model v0.2.0
-	github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92
 	github.com/rs/zerolog v1.20.0
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.7.1
 	github.com/urfave/cli/v2 v2.3.0
+	go.opentelemetry.io/contrib/propagators v0.22.0
+	go.opentelemetry.io/otel v1.6.3
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3
+	go.opentelemetry.io/otel/sdk v1.6.3
+	go.opentelemetry.io/otel/trace v1.6.3
+	go.opentelemetry.io/proto/otlp v0.15.0
 	go.uber.org/automaxprocs v1.4.0
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
-	golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d
+	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f
+	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
-	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
+	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211
+	google.golang.org/protobuf v1.28.0
 	gopkg.in/coreos/go-oidc.v2 v2.2.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/square/go-jose.v2 v2.6.0
-	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 	zombiezen.com/go/capnproto2 v2.18.0+incompatible
 )
 
@@ -56,20 +61,20 @@ require (
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
-	github.com/francoispqt/gojay v1.2.13 // indirect
-	github.com/gdamore/encoding v1.0.0 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
 	github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 // indirect
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
-	github.com/marten-seemann/qtls-go1-16 v0.1.4 // indirect
-	github.com/marten-seemann/qtls-go1-17 v0.1.0 // indirect
-	github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1 // indirect
+	github.com/marten-seemann/qtls-go1-16 v0.1.5 // indirect
+	github.com/marten-seemann/qtls-go1-17 v0.1.2 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect
+	github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
-	github.com/mattn/go-runewidth v0.0.8 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -80,7 +85,6 @@ require (
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/prometheus/procfs v0.7.3 // indirect
-	github.com/rivo/uniseg v0.1.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
@@ -89,15 +93,16 @@ require (
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb // indirect
-	google.golang.org/grpc v1.43.0 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	google.golang.org/grpc v1.45.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 
 replace github.com/urfave/cli/v2 => github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d
 
-replace github.com/lucas-clemente/quic-go => github.com/chungthuang/quic-go v0.24.1-0.20220110095058-981dc498cb62
+replace github.com/lucas-clemente/quic-go => github.com/chungthuang/quic-go v0.27.1-0.20220809135021-ca330f1dec9f
 
 // Avoid 'CVE-2022-21698'
 replace github.com/prometheus/golang_client => github.com/prometheus/golang_client v1.12.1
+
+replace gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1

go.sum

@@ -70,7 +70,6 @@ github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBp
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/DataDog/datadog-go v4.4.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/DataDog/gostackparse v0.5.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM=
 github.com/DataDog/sketches-go v1.0.0/go.mod h1:O+XkJHWk9w4hDwY2ZUDU31ZC9sNYlYo8DiFsxjYeo1k=
@@ -100,18 +99,18 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
+github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
-github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
-github.com/chungthuang/quic-go v0.24.1-0.20220110095058-981dc498cb62 h1:PLTB4iA6sOgAItzQY642tYdcGKfG/7i2gu93JQGgUcM=
-github.com/chungthuang/quic-go v0.24.1-0.20220110095058-981dc498cb62/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg=
+github.com/chungthuang/quic-go v0.27.1-0.20220809135021-ca330f1dec9f h1:UWC3XjwZzocdNCzzXxq9j/1SdHMZXhcTOsh/+gNRBUQ=
+github.com/chungthuang/quic-go v0.27.1-0.20220809135021-ca330f1dec9f/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -183,16 +182,11 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
-github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
 github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
-github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
-github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
-github.com/gdamore/tcell v1.3.0 h1:r35w0JBADPZCVQijYebl6YMWWtHRqVEGt7kL2eBADRM=
-github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
 github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/kkTFdWhctH96grJ7qiy7bMdiZcIvPKs=
 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
@@ -211,6 +205,11 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
+github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
@@ -233,6 +232,8 @@ github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -283,8 +284,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -336,6 +338,8 @@ github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:Fecb
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
@@ -385,29 +389,24 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s=
-github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac=
-github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
 github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
 github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc=
-github.com/marten-seemann/qtls-go1-15 v0.1.4/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
-github.com/marten-seemann/qtls-go1-16 v0.1.4 h1:xbHbOGGhrenVtII6Co8akhLEdrawwB2iHl5yhJRpnco=
-github.com/marten-seemann/qtls-go1-16 v0.1.4/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
-github.com/marten-seemann/qtls-go1-17 v0.1.0 h1:P9ggrs5xtwiqXv/FHNwntmuLMNq3KaSIG93AtAZ48xk=
-github.com/marten-seemann/qtls-go1-17 v0.1.0/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8=
-github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1 h1:EnzzN9fPUkUck/1CuY1FlzBaIYMoiBsdwTNmNGkwUUM=
-github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1/go.mod h1:PUhIQk19LoFt2174H4+an8TYvWOGjb/hHwphBeaDHwI=
+github.com/marten-seemann/qtls-go1-16 v0.1.5 h1:o9JrYPPco/Nukd/HpOHMHZoBDXQqoNtUCmny98/1uqQ=
+github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk=
+github.com/marten-seemann/qtls-go1-17 v0.1.2 h1:JADBlm0LYiVbuSySCHeY863dNkcpMmDR7s0bLKJeYlQ=
+github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s=
+github.com/marten-seemann/qtls-go1-18 v0.1.2 h1:JH6jmzbduz0ITVQ7ShevK10Av5+jBEKAHMntXmIV7kM=
+github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4=
+github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1 h1:7m/WlWcSROrcK5NxuXaxYD32BZqe/LEEnBrWcH/cOqQ=
+github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI=
 github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0=
-github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4=
@@ -503,16 +502,11 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1
 github.com/rabbitmq/amqp091-go v1.1.0/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0VTJ0kHRghqbM=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92 h1:rqaqSUdaW+OBbjnsrOoiaJv43mSRARuvsAuirmdxu7E=
-github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92/go.mod h1:6lkG1x+13OShEf0EaOCaTQYyB7d5nSbb181KtjlS+84=
-github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY=
-github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.20.0 h1:38k9hgtUBdxFwE34yS8rTHmHBa4eN16E4DJlv177LNs=
 github.com/rs/zerolog v1.20.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
-github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
@@ -561,8 +555,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ=
 github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
@@ -587,7 +582,22 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/contrib/propagators v0.22.0 h1:KGdv58M2//veiYLIhb31mofaI2LgkIPXXAZVeYVyfd8=
+go.opentelemetry.io/contrib/propagators v0.22.0/go.mod h1:xGOuXr6lLIF9BXipA4pm6UuOSI0M98U6tsI3khbOiwU=
+go.opentelemetry.io/otel v1.0.0-RC2/go.mod h1:w1thVQ7qbAy8MHb0IFj8a5Q2QU0l2ksf8u/CN8m3NOM=
+go.opentelemetry.io/otel v1.6.3 h1:FLOfo8f9JzFVFVyU+MSRJc2HdEAXQgm7pIv2uFKRSZE=
+go.opentelemetry.io/otel v1.6.3/go.mod h1:7BgNga5fNlF/iZjG06hM3yofffp0ofKCDwSXx1GC4dI=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.6.3/go.mod h1:NEu79Xo32iVb+0gVNV8PMd7GoWqnyDXRlj04yFjqz40=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3 h1:4/UjHWMVVc5VwX/KAtqJOHErKigMCH8NexChMuanb/o=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3/go.mod h1:UJmXdiVVBaZ63umRUTwJuCMAV//GCMvDiQwn703/GoY=
+go.opentelemetry.io/otel/sdk v1.6.3 h1:prSHYdwCQOX5DrsEzxowH3nLhoAzEBdZhvrR79scfLs=
+go.opentelemetry.io/otel/sdk v1.6.3/go.mod h1:A4iWF7HTXa+GWL/AaqESz28VuSBIcZ+0CV+IzJ5NMiQ=
+go.opentelemetry.io/otel/trace v1.0.0-RC2/go.mod h1:JPQ+z6nNw9mqEGT8o3eoPTdnNI+Aj5JcxEsVGREIAy4=
+go.opentelemetry.io/otel/trace v1.6.3 h1:IqN4L+5b0mPNjdXIiZ90Ni4Bl5BRkDQywePLWemd9bc=
+go.opentelemetry.io/otel/trace v1.6.3/go.mod h1:GNJQusJlUgZl9/TQBPKU/Y/ty+0iVB5fjhKeJGZPGFs=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.opentelemetry.io/proto/otlp v0.15.0 h1:h0bKrvdrT/9sBwEJ6iWUqT/N/xPcS66bL4u3isneJ6w=
+go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/automaxprocs v1.4.0 h1:CpDZl6aOlLhReez+8S3eEotD7Jx0Os++lemPlMULQP0=
 go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q=
@@ -608,8 +618,9 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc=
+golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -697,8 +708,8 @@ golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs=
-golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e h1:TsQ7F31D3bUCLeqPT0u+yjp1guoArKaNKmCr22PYgTQ=
+golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -745,7 +756,6 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -789,6 +799,7 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -802,11 +813,13 @@ golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1029,8 +1042,10 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
-google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM=
+google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
+google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
+google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -1044,8 +1059,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/DataDog/dd-trace-go.v1 v1.34.0/go.mod h1:HtrC65fyJ6lWazShCC9rlOeiTSZJ0XtZhkwjZM2WpC4=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1074,10 +1090,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

ingress/config.go

@@ -12,10 +12,11 @@ import (
 )
 
 var (
-	defaultConnectTimeout   = config.CustomDuration{Duration: 30 * time.Second}
-	defaultTLSTimeout       = config.CustomDuration{Duration: 10 * time.Second}
-	defaultTCPKeepAlive     = config.CustomDuration{Duration: 30 * time.Second}
-	defaultKeepAliveTimeout = config.CustomDuration{Duration: 90 * time.Second}
+	defaultHTTPConnectTimeout        = config.CustomDuration{Duration: 30 * time.Second}
+	defaultWarpRoutingConnectTimeout = config.CustomDuration{Duration: 5 * time.Second}
+	defaultTLSTimeout                = config.CustomDuration{Duration: 10 * time.Second}
+	defaultTCPKeepAlive              = config.CustomDuration{Duration: 30 * time.Second}
+	defaultKeepAliveTimeout          = config.CustomDuration{Duration: 90 * time.Second}
 )
 
 const (
@@ -35,47 +36,90 @@ const (
 	NoChunkedEncodingFlag         = "no-chunked-encoding"
 	ProxyAddressFlag              = "proxy-address"
 	ProxyPortFlag                 = "proxy-port"
+	Http2OriginFlag               = "http2-origin"
 )
 
 const (
 	socksProxy = "socks"
 )
 
+type WarpRoutingConfig struct {
+	Enabled        bool                  `yaml:"enabled" json:"enabled"`
+	ConnectTimeout config.CustomDuration `yaml:"connectTimeout" json:"connectTimeout,omitempty"`
+	TCPKeepAlive   config.CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive,omitempty"`
+}
+
+func NewWarpRoutingConfig(raw *config.WarpRoutingConfig) WarpRoutingConfig {
+	cfg := WarpRoutingConfig{
+		Enabled:        raw.Enabled,
+		ConnectTimeout: defaultWarpRoutingConnectTimeout,
+		TCPKeepAlive:   defaultTCPKeepAlive,
+	}
+	if raw.ConnectTimeout != nil {
+		cfg.ConnectTimeout = *raw.ConnectTimeout
+	}
+	if raw.TCPKeepAlive != nil {
+		cfg.TCPKeepAlive = *raw.TCPKeepAlive
+	}
+	return cfg
+}
+
+func (c *WarpRoutingConfig) RawConfig() config.WarpRoutingConfig {
+	raw := config.WarpRoutingConfig{
+		Enabled: c.Enabled,
+	}
+	if c.ConnectTimeout.Duration != defaultWarpRoutingConnectTimeout.Duration {
+		raw.ConnectTimeout = &c.ConnectTimeout
+	}
+	if c.TCPKeepAlive.Duration != defaultTCPKeepAlive.Duration {
+		raw.TCPKeepAlive = &c.TCPKeepAlive
+	}
+	return raw
+}
+
 // RemoteConfig models ingress settings that can be managed remotely, for example through the dashboard.
 type RemoteConfig struct {
 	Ingress     Ingress
-	WarpRouting config.WarpRoutingConfig
+	WarpRouting WarpRoutingConfig
 }
 
-type remoteConfigJSON struct {
-	GlobalOriginRequest config.OriginRequestConfig      `json:"originRequest"`
+type RemoteConfigJSON struct {
+	GlobalOriginRequest *config.OriginRequestConfig     `json:"originRequest,omitempty"`
 	IngressRules        []config.UnvalidatedIngressRule `json:"ingress"`
 	WarpRouting         config.WarpRoutingConfig        `json:"warp-routing"`
 }
 
 func (rc *RemoteConfig) UnmarshalJSON(b []byte) error {
-	var rawConfig remoteConfigJSON
+	var rawConfig RemoteConfigJSON
+
 	if err := json.Unmarshal(b, &rawConfig); err != nil {
 		return err
 	}
-	ingress, err := validateIngress(rawConfig.IngressRules, originRequestFromConfig(rawConfig.GlobalOriginRequest))
+
+	// if nil, just assume the default values.
+	globalOriginRequestConfig := rawConfig.GlobalOriginRequest
+	if globalOriginRequestConfig == nil {
+		globalOriginRequestConfig = &config.OriginRequestConfig{}
+	}
+
+	ingress, err := validateIngress(rawConfig.IngressRules, originRequestFromConfig(*globalOriginRequestConfig))
 	if err != nil {
 		return err
 	}
 
 	rc.Ingress = ingress
-	rc.WarpRouting = rawConfig.WarpRouting
+	rc.WarpRouting = NewWarpRoutingConfig(&rawConfig.WarpRouting)
 
 	return nil
 }
 
 func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
-	var connectTimeout config.CustomDuration = defaultConnectTimeout
-	var tlsTimeout config.CustomDuration = defaultTLSTimeout
-	var tcpKeepAlive config.CustomDuration = defaultTCPKeepAlive
+	var connectTimeout = defaultHTTPConnectTimeout
+	var tlsTimeout = defaultTLSTimeout
+	var tcpKeepAlive = defaultTCPKeepAlive
 	var noHappyEyeballs bool
-	var keepAliveConnections int = defaultKeepAliveConnections
-	var keepAliveTimeout config.CustomDuration = defaultKeepAliveTimeout
+	var keepAliveConnections = defaultKeepAliveConnections
+	var keepAliveTimeout = defaultKeepAliveTimeout
 	var httpHostHeader string
 	var originServerName string
 	var caPool string
@@ -85,6 +129,7 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 	var proxyAddress = defaultProxyAddress
 	var proxyPort uint
 	var proxyType string
+	var http2Origin bool
 	if flag := ProxyConnectTimeoutFlag; c.IsSet(flag) {
 		connectTimeout = config.CustomDuration{Duration: c.Duration(flag)}
 	}
@@ -128,9 +173,13 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		proxyPort = uint(c.Int(flag))
 	}
+	if flag := Http2OriginFlag; c.IsSet(flag) {
+		http2Origin = c.Bool(flag)
+	}
 	if c.IsSet(Socks5Flag) {
 		proxyType = socksProxy
 	}
+
 	return OriginRequestConfig{
 		ConnectTimeout:         connectTimeout,
 		TLSTimeout:             tlsTimeout,
@@ -147,12 +196,13 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 		ProxyAddress:           proxyAddress,
 		ProxyPort:              proxyPort,
 		ProxyType:              proxyType,
+		Http2Origin:            http2Origin,
 	}
 }
 
 func originRequestFromConfig(c config.OriginRequestConfig) OriginRequestConfig {
 	out := OriginRequestConfig{
-		ConnectTimeout:       defaultConnectTimeout,
+		ConnectTimeout:       defaultHTTPConnectTimeout,
 		TLSTimeout:           defaultTLSTimeout,
 		TCPKeepAlive:         defaultTCPKeepAlive,
 		KeepAliveConnections: defaultKeepAliveConnections,
@@ -212,6 +262,9 @@ func originRequestFromConfig(c config.OriginRequestConfig) OriginRequestConfig {
 			}
 		}
 	}
+	if c.Http2Origin != nil {
+		out.Http2Origin = *c.Http2Origin
+	}
 	return out
 }
 
@@ -255,6 +308,8 @@ type OriginRequestConfig struct {
 	ProxyType string `yaml:"proxyType" json:"proxyType"`
 	// IP rules for the proxy service
 	IPRules []ipaccess.Rule `yaml:"ipRules" json:"ipRules"`
+	// Attempt to connect to origin with HTTP/2
+	Http2Origin bool `yaml:"http2Origin" json:"http2Origin"`
 }
 
 func (defaults *OriginRequestConfig) setConnectTimeout(overrides config.OriginRequestConfig) {
@@ -360,6 +415,12 @@ func (defaults *OriginRequestConfig) setIPRules(overrides config.OriginRequestCo
 	}
 }
 
+func (defaults *OriginRequestConfig) setHttp2Origin(overrides config.OriginRequestConfig) {
+	if val := overrides.Http2Origin; val != nil {
+		defaults.Http2Origin = *val
+	}
+}
+
 // SetConfig gets config for the requests that cloudflared sends to origins.
 // Each field has a setter method which sets a value for the field by trying to find:
 //   1. The user config for this rule
@@ -385,5 +446,95 @@ func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfi
 	cfg.setProxyAddress(overrides)
 	cfg.setProxyType(overrides)
 	cfg.setIPRules(overrides)
+	cfg.setHttp2Origin(overrides)
 	return cfg
 }
+
+func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig {
+	var connectTimeout *config.CustomDuration
+	var tlsTimeout *config.CustomDuration
+	var tcpKeepAlive *config.CustomDuration
+	var keepAliveConnections *int
+	var keepAliveTimeout *config.CustomDuration
+	var proxyAddress *string
+
+	if c.ConnectTimeout != defaultHTTPConnectTimeout {
+		connectTimeout = &c.ConnectTimeout
+	}
+	if c.TLSTimeout != defaultTLSTimeout {
+		tlsTimeout = &c.TLSTimeout
+	}
+	if c.TCPKeepAlive != defaultTCPKeepAlive {
+		tcpKeepAlive = &c.TCPKeepAlive
+	}
+	if c.KeepAliveConnections != defaultKeepAliveConnections {
+		keepAliveConnections = &c.KeepAliveConnections
+	}
+	if c.KeepAliveTimeout != defaultKeepAliveTimeout {
+		keepAliveTimeout = &c.KeepAliveTimeout
+	}
+	if c.ProxyAddress != defaultProxyAddress {
+		proxyAddress = &c.ProxyAddress
+	}
+
+	return config.OriginRequestConfig{
+		ConnectTimeout:         connectTimeout,
+		TLSTimeout:             tlsTimeout,
+		TCPKeepAlive:           tcpKeepAlive,
+		NoHappyEyeballs:        defaultBoolToNil(c.NoHappyEyeballs),
+		KeepAliveConnections:   keepAliveConnections,
+		KeepAliveTimeout:       keepAliveTimeout,
+		HTTPHostHeader:         emptyStringToNil(c.HTTPHostHeader),
+		OriginServerName:       emptyStringToNil(c.OriginServerName),
+		CAPool:                 emptyStringToNil(c.CAPool),
+		NoTLSVerify:            defaultBoolToNil(c.NoTLSVerify),
+		DisableChunkedEncoding: defaultBoolToNil(c.DisableChunkedEncoding),
+		BastionMode:            defaultBoolToNil(c.BastionMode),
+		ProxyAddress:           proxyAddress,
+		ProxyPort:              zeroUIntToNil(c.ProxyPort),
+		ProxyType:              emptyStringToNil(c.ProxyType),
+		IPRules:                convertToRawIPRules(c.IPRules),
+		Http2Origin:            defaultBoolToNil(c.Http2Origin),
+	}
+}
+
+func convertToRawIPRules(ipRules []ipaccess.Rule) []config.IngressIPRule {
+	result := make([]config.IngressIPRule, 0)
+	for _, r := range ipRules {
+		cidr := r.StringCIDR()
+
+		newRule := config.IngressIPRule{
+			Prefix: &cidr,
+			Ports:  r.Ports(),
+			Allow:  r.RulePolicy(),
+		}
+
+		result = append(result, newRule)
+	}
+
+	return result
+}
+
+func defaultBoolToNil(b bool) *bool {
+	if b == false {
+		return nil
+	}
+
+	return &b
+}
+
+func emptyStringToNil(s string) *string {
+	if s == "" {
+		return nil
+	}
+
+	return &s
+}
+
+func zeroUIntToNil(v uint) *uint {
+	if v == 0 {
+		return nil
+	}
+
+	return &v
+}

ingress/config_test.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 	"github.com/urfave/cli/v2"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/ipaccess"
@@ -274,7 +274,7 @@ func TestOriginRequestConfigDefaults(t *testing.T) {
 		// Rule 0 didn't override anything, so it inherits the cloudflared defaults
 		actual0 := ing.Rules[0].Config
 		expected0 := OriginRequestConfig{
-			ConnectTimeout:       defaultConnectTimeout,
+			ConnectTimeout:       defaultHTTPConnectTimeout,
 			TLSTimeout:           defaultTLSTimeout,
 			TCPKeepAlive:         defaultTCPKeepAlive,
 			KeepAliveConnections: defaultKeepAliveConnections,
@@ -404,7 +404,7 @@ func TestDefaultConfigFromCLI(t *testing.T) {
 	c := cli.NewContext(nil, set, nil)
 
 	expected := OriginRequestConfig{
-		ConnectTimeout:       defaultConnectTimeout,
+		ConnectTimeout:       defaultHTTPConnectTimeout,
 		TLSTimeout:           defaultTLSTimeout,
 		TCPKeepAlive:         defaultTCPKeepAlive,
 		KeepAliveConnections: defaultKeepAliveConnections,

ingress/ingress.go

@@ -97,8 +97,16 @@ type WarpRoutingService struct {
 	Proxy StreamBasedOriginProxy
 }
 
-func NewWarpRoutingService() *WarpRoutingService {
-	return &WarpRoutingService{Proxy: &rawTCPService{name: ServiceWarpRouting}}
+func NewWarpRoutingService(config WarpRoutingConfig) *WarpRoutingService {
+	svc := &rawTCPService{
+		name: ServiceWarpRouting,
+		dialer: net.Dialer{
+			Timeout:   config.ConnectTimeout.Duration,
+			KeepAlive: config.TCPKeepAlive.Duration,
+		},
+	}
+
+	return &WarpRoutingService{Proxy: svc}
 }
 
 // Get a single origin service from the CLI/config.

ingress/ingress_test.go

@@ -12,7 +12,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"github.com/urfave/cli/v2"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/ipaccess"

ingress/origin_proxy.go

@@ -1,25 +1,20 @@
 package ingress
 
 import (
-	"net"
+	"context"
+	"fmt"
 	"net/http"
-
-	"github.com/pkg/errors"
-)
-
-var (
-	errUnsupportedConnectionType = errors.New("internal error: unsupported connection type")
 )
 
 // HTTPOriginProxy can be implemented by origin services that want to proxy http requests.
 type HTTPOriginProxy interface {
-	// RoundTrip is how cloudflared proxies eyeball requests to the actual origin services
+	// RoundTripper is how cloudflared proxies eyeball requests to the actual origin services
 	http.RoundTripper
 }
 
 // StreamBasedOriginProxy can be implemented by origin services that want to proxy ws/TCP.
 type StreamBasedOriginProxy interface {
-	EstablishConnection(dest string) (OriginConnection, error)
+	EstablishConnection(ctx context.Context, dest string) (OriginConnection, error)
 }
 
 func (o *unixSocketPath) RoundTrip(req *http.Request) (*http.Response, error) {
@@ -49,11 +44,17 @@ func (o *httpService) RoundTrip(req *http.Request) (*http.Response, error) {
 }
 
 func (o *statusCode) RoundTrip(_ *http.Request) (*http.Response, error) {
-	return o.resp, nil
+	resp := &http.Response{
+		StatusCode: o.code,
+		Status:     fmt.Sprintf("%d %s", o.code, http.StatusText(o.code)),
+		Body:       new(NopReadCloser),
+	}
+
+	return resp, nil
 }
 
-func (o *rawTCPService) EstablishConnection(dest string) (OriginConnection, error) {
-	conn, err := net.Dial("tcp", dest)
+func (o *rawTCPService) EstablishConnection(ctx context.Context, dest string) (OriginConnection, error) {
+	conn, err := o.dialer.DialContext(ctx, "tcp", dest)
 	if err != nil {
 		return nil, err
 	}
@@ -64,13 +65,13 @@ func (o *rawTCPService) EstablishConnection(dest string) (OriginConnection, erro
 	return originConn, nil
 }
 
-func (o *tcpOverWSService) EstablishConnection(dest string) (OriginConnection, error) {
+func (o *tcpOverWSService) EstablishConnection(ctx context.Context, dest string) (OriginConnection, error) {
 	var err error
 	if !o.isBastion {
 		dest = o.dest
 	}
 
-	conn, err := net.Dial("tcp", dest)
+	conn, err := o.dialer.DialContext(ctx, "tcp", dest)
 	if err != nil {
 		return nil, err
 	}
@@ -82,6 +83,6 @@ func (o *tcpOverWSService) EstablishConnection(dest string) (OriginConnection, e
 
 }
 
-func (o *socksProxyOverWSService) EstablishConnection(dest string) (OriginConnection, error) {
+func (o *socksProxyOverWSService) EstablishConnection(_ctx context.Context, _dest string) (OriginConnection, error) {
 	return o.conn, nil
 }

ingress/origin_proxy_test.go

@@ -36,7 +36,7 @@ func TestRawTCPServiceEstablishConnection(t *testing.T) {
 	require.NoError(t, err)
 
 	// Origin not listening for new connection, should return an error
-	_, err = rawTCPService.EstablishConnection(req.URL.String())
+	_, err = rawTCPService.EstablishConnection(context.Background(), req.URL.String())
 	require.Error(t, err)
 }
 
@@ -87,7 +87,7 @@ func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
 		t.Run(test.testCase, func(t *testing.T) {
 			if test.expectErr {
 				bastionHost, _ := carrier.ResolveBastionDest(test.req)
-				_, err := test.service.EstablishConnection(bastionHost)
+				_, err := test.service.EstablishConnection(context.Background(), bastionHost)
 				assert.Error(t, err)
 			}
 		})
@@ -99,7 +99,7 @@ func TestTCPOverWSServiceEstablishConnection(t *testing.T) {
 	for _, service := range []*tcpOverWSService{newTCPOverWSService(originURL), newBastionService()} {
 		// Origin not listening for new connection, should return an error
 		bastionHost, _ := carrier.ResolveBastionDest(bastionReq)
-		_, err := service.EstablishConnection(bastionHost)
+		_, err := service.EstablishConnection(context.Background(), bastionHost)
 		assert.Error(t, err)
 	}
 }

ingress/origin_service.go

@@ -91,7 +91,8 @@ func (o httpService) MarshalJSON() ([]byte, error) {
 // rawTCPService dials TCP to the destination specified by the client
 // It's used by warp routing
 type rawTCPService struct {
-	name string
+	name   string
+	dialer net.Dialer
 }
 
 func (o *rawTCPService) String() string {
@@ -109,9 +110,11 @@ func (o rawTCPService) MarshalJSON() ([]byte, error) {
 // tcpOverWSService models TCP origins serving eyeballs connecting over websocket, such as
 // cloudflared access commands.
 type tcpOverWSService struct {
+	scheme        string
 	dest          string
 	isBastion     bool
 	streamHandler streamHandlerFunc
+	dialer        net.Dialer
 }
 
 type socksProxyOverWSService struct {
@@ -130,7 +133,8 @@ func newTCPOverWSService(url *url.URL) *tcpOverWSService {
 		addPortIfMissing(url, 7864) // just a random port since there isn't a default in this case
 	}
 	return &tcpOverWSService{
-		dest: url.Host,
+		scheme: url.Scheme,
+		dest:   url.Host,
 	}
 }
 
@@ -160,7 +164,12 @@ func (o *tcpOverWSService) String() string {
 	if o.isBastion {
 		return ServiceBastion
 	}
-	return o.dest
+
+	if o.scheme != "" {
+		return fmt.Sprintf("%s://%s", o.scheme, o.dest)
+	} else {
+		return o.dest
+	}
 }
 
 func (o *tcpOverWSService) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
@@ -169,6 +178,8 @@ func (o *tcpOverWSService) start(log *zerolog.Logger, _ <-chan struct{}, cfg Ori
 	} else {
 		o.streamHandler = DefaultStreamHandler
 	}
+	o.dialer.Timeout = cfg.ConnectTimeout.Duration
+	o.dialer.KeepAlive = cfg.TCPKeepAlive.Duration
 	return nil
 }
 
@@ -231,20 +242,15 @@ func (o helloWorld) MarshalJSON() ([]byte, error) {
 // statusCode is an OriginService that just responds with a given HTTP status.
 // Typical use-case is "user wants the catch-all rule to just respond 404".
 type statusCode struct {
-	resp *http.Response
+	code int
 }
 
 func newStatusCode(status int) statusCode {
-	resp := &http.Response{
-		StatusCode: status,
-		Status:     fmt.Sprintf("%d %s", status, http.StatusText(status)),
-		Body:       new(NopReadCloser),
-	}
-	return statusCode{resp: resp}
+	return statusCode{code: status}
 }
 
 func (o *statusCode) String() string {
-	return fmt.Sprintf("http_status:%d", o.resp.StatusCode)
+	return fmt.Sprintf("http_status:%d", o.code)
 }
 
 func (o *statusCode) start(
@@ -284,6 +290,7 @@ func newHTTPTransport(service OriginService, cfg OriginRequestConfig, log *zerol
 		TLSHandshakeTimeout:   cfg.TLSTimeout.Duration,
 		ExpectContinueTimeout: 1 * time.Second,
 		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: cfg.NoTLSVerify},
+		ForceAttemptHTTP2:     cfg.Http2Origin,
 	}
 	if _, isHelloWorld := service.(*helloWorld); !isHelloWorld && cfg.OriginServerName != "" {
 		httpTransport.TLSClientConfig.ServerName = cfg.OriginServerName

ingress/origin_udp_proxy.go

@@ -6,11 +6,16 @@ import (
 	"net"
 )
 
-type UDPProxy struct {
+type UDPProxy interface {
 	io.ReadWriteCloser
+	LocalAddr() net.Addr
 }
 
-func DialUDP(dstIP net.IP, dstPort uint16) (*UDPProxy, error) {
+type udpProxy struct {
+	*net.UDPConn
+}
+
+func DialUDP(dstIP net.IP, dstPort uint16) (UDPProxy, error) {
 	dstAddr := &net.UDPAddr{
 		IP:   dstIP,
 		Port: int(dstPort),
@@ -23,5 +28,5 @@ func DialUDP(dstIP net.IP, dstPort uint16) (*UDPProxy, error) {
 		return nil, fmt.Errorf("unable to create UDP proxy to origin (%v:%v): %w", dstIP, dstPort, err)
 	}
 
-	return &UDPProxy{udpConn}, nil
+	return &udpProxy{udpConn}, nil
 }

ingress/rule_test.go

@@ -182,25 +182,25 @@ func TestMarshalJSON(t *testing.T) {
 		{
 			name:     "Nil",
 			path:     nil,
-			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
+			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
 			want:     true,
 		},
 		{
 			name:     "Nil regex",
 			path:     &Regexp{Regexp: nil},
-			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
+			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
 			want:     true,
 		},
 		{
 			name:     "Empty",
 			path:     &Regexp{Regexp: regexp.MustCompile("")},
-			expected: `{"hostname":"example.com","path":"","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
+			expected: `{"hostname":"example.com","path":"","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
 			want:     true,
 		},
 		{
 			name:     "Basic",
 			path:     &Regexp{Regexp: regexp.MustCompile("/echo")},
-			expected: `{"hostname":"example.com","path":"/echo","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
+			expected: `{"hostname":"example.com","path":"/echo","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null,"http2Origin":false}}`,
 			want:     true,
 		},
 	}

ipaccess/access.go

@@ -99,3 +99,15 @@ func (ipr *Rule) PortsString() string {
 	}
 	return "all"
 }
+
+func (ipr *Rule) Ports() []int {
+	return ipr.ports
+}
+
+func (ipr *Rule) RulePolicy() bool {
+	return ipr.allow
+}
+
+func (ipr *Rule) StringCIDR() string {
+	return ipr.ipNet.String()
+}

metrics/metrics.go

@@ -23,7 +23,7 @@ const (
 )
 
 type orchestrator interface {
-	GetConfigJSON() ([]byte, error)
+	GetVersionedConfigJSON() ([]byte, error)
 }
 
 func newMetricsHandler(
@@ -47,7 +47,7 @@ func newMetricsHandler(
 	})
 	if orchestrator != nil {
 		router.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) {
-			json, err := orchestrator.GetConfigJSON()
+			json, err := orchestrator.GetVersionedConfigJSON()
 			if err != nil {
 				w.WriteHeader(500)
 				_, _ = fmt.Fprintf(w, "ERR: %v", err)

metrics/readiness.go

@@ -5,21 +5,24 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/google/uuid"
+	"github.com/rs/zerolog"
+
 	conn "github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/tunnelstate"
-
-	"github.com/rs/zerolog"
 )
 
 // ReadyServer serves HTTP 200 if the tunnel can serve traffic. Intended for k8s readiness checks.
 type ReadyServer struct {
-	tracker *tunnelstate.ConnTracker
+	clientID uuid.UUID
+	tracker  *tunnelstate.ConnTracker
 }
 
 // NewReadyServer initializes a ReadyServer and starts listening for dis/connection events.
-func NewReadyServer(log *zerolog.Logger) *ReadyServer {
+func NewReadyServer(log *zerolog.Logger, clientID uuid.UUID) *ReadyServer {
 	return &ReadyServer{
-		tracker: tunnelstate.NewConnTracker(log),
+		clientID: clientID,
+		tracker:  tunnelstate.NewConnTracker(log),
 	}
 }
 
@@ -28,8 +31,9 @@ func (rs *ReadyServer) OnTunnelEvent(c conn.Event) {
 }
 
 type body struct {
-	Status           int  `json:"status"`
-	ReadyConnections uint `json:"readyConnections"`
+	Status           int       `json:"status"`
+	ReadyConnections uint      `json:"readyConnections"`
+	ConnectorID      uuid.UUID `json:"connectorId"`
 }
 
 // ServeHTTP responds with HTTP 200 if the tunnel is connected to the edge.
@@ -39,6 +43,7 @@ func (rs *ReadyServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	body := body{
 		Status:           statusCode,
 		ReadyConnections: readyConnections,
+		ConnectorID:      rs.clientID,
 	}
 	msg, err := json.Marshal(body)
 	if err != nil {

metrics/readiness_test.go

@@ -4,17 +4,17 @@ import (
 	"net/http"
 	"testing"
 
+	"github.com/google/uuid"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/cloudflare/cloudflared/tunnelstate"
-
 	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/tunnelstate"
 )
 
 func TestReadyServer_makeResponse(t *testing.T) {
 	type fields struct {
-		isConnected map[int]bool
+		isConnected map[uint8]tunnelstate.ConnectionInfo
 	}
 	tests := []struct {
 		name                 string
@@ -25,11 +25,11 @@ func TestReadyServer_makeResponse(t *testing.T) {
 		{
 			name: "One connection online => HTTP 200",
 			fields: fields{
-				isConnected: map[int]bool{
-					0: false,
-					1: false,
-					2: true,
-					3: false,
+				isConnected: map[uint8]tunnelstate.ConnectionInfo{
+					0: {IsConnected: false},
+					1: {IsConnected: false},
+					2: {IsConnected: true},
+					3: {IsConnected: false},
 				},
 			},
 			wantOK:               true,
@@ -38,11 +38,11 @@ func TestReadyServer_makeResponse(t *testing.T) {
 		{
 			name: "No connections online => no HTTP 200",
 			fields: fields{
-				isConnected: map[int]bool{
-					0: false,
-					1: false,
-					2: false,
-					3: false,
+				isConnected: map[uint8]tunnelstate.ConnectionInfo{
+					0: {IsConnected: false},
+					1: {IsConnected: false},
+					2: {IsConnected: false},
+					3: {IsConnected: false},
 				},
 			},
 			wantReadyConnections: 0,
@@ -66,7 +66,7 @@ func TestReadyServer_makeResponse(t *testing.T) {
 
 func TestReadinessEventHandling(t *testing.T) {
 	nopLogger := zerolog.Nop()
-	rs := NewReadyServer(&nopLogger)
+	rs := NewReadyServer(&nopLogger, uuid.Nil)
 
 	// start not ok
 	code, ready := rs.makeResponse()

orchestration/config.go

@@ -1,15 +1,66 @@
 package orchestration
 
 import (
+	"encoding/json"
+
+	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/ingress"
 )
 
-type newConfig struct {
+type newRemoteConfig struct {
 	ingress.RemoteConfig
 	// Add more fields when we support other settings in tunnel orchestration
 }
 
-type Config struct {
-	Ingress            *ingress.Ingress
-	WarpRoutingEnabled bool
+type newLocalConfig struct {
+	RemoteConfig       ingress.RemoteConfig
+	ConfigurationFlags map[string]string `json:"__configuration_flags,omitempty"`
+}
+
+// Config is the original config as read and parsed by cloudflared.
+type Config struct {
+	Ingress     *ingress.Ingress
+	WarpRouting ingress.WarpRoutingConfig
+
+	// Extra settings used to configure this instance but that are not eligible for remotely management
+	// ie. (--protocol, --loglevel, ...)
+	ConfigurationFlags map[string]string
+}
+
+func (rc *newLocalConfig) MarshalJSON() ([]byte, error) {
+	var r = struct {
+		ConfigurationFlags map[string]string `json:"__configuration_flags,omitempty"`
+		ingress.RemoteConfigJSON
+	}{
+		ConfigurationFlags: rc.ConfigurationFlags,
+		RemoteConfigJSON: ingress.RemoteConfigJSON{
+			// UI doesn't support top level configs, so we reconcile to individual ingress configs.
+			GlobalOriginRequest: nil,
+			IngressRules:        convertToUnvalidatedIngressRules(rc.RemoteConfig.Ingress),
+			WarpRouting:         rc.RemoteConfig.WarpRouting.RawConfig(),
+		},
+	}
+
+	return json.Marshal(r)
+}
+
+func convertToUnvalidatedIngressRules(i ingress.Ingress) []config.UnvalidatedIngressRule {
+	result := make([]config.UnvalidatedIngressRule, 0)
+	for _, rule := range i.Rules {
+		var path string
+		if rule.Path != nil {
+			path = rule.Path.String()
+		}
+
+		newRule := config.UnvalidatedIngressRule{
+			Hostname:      rule.Hostname,
+			Path:          path,
+			Service:       rule.Service.String(),
+			OriginRequest: ingress.ConvertToRawOriginConfig(rule.Config),
+		}
+
+		result = append(result, newRule)
+	}
+
+	return result
 }

orchestration/config_test.go

@@ -0,0 +1,95 @@
+package orchestration
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ingress"
+)
+
+// TestNewLocalConfig_MarshalJSON tests that we are able to converte a compiled and validated config back
+// into an "unvalidated" format which is compatible with Remote Managed configurations.
+func TestNewLocalConfig_MarshalJSON(t *testing.T) {
+	rawConfig := []byte(`
+	{
+		"originRequest": {
+					"connectTimeout": 160,
+					"httpHostHeader": "default"
+		},
+		"ingress": [
+			{
+				"hostname": "tun.example.com",
+				"service": "https://localhost:8000"
+			},
+			{
+				"hostname": "*",
+				"service": "https://localhost:8001",
+				"originRequest": {
+					"connectTimeout": 121,
+					"tlsTimeout": 2,
+					"noHappyEyeballs": false,
+					"tcpKeepAlive": 2,
+					"keepAliveConnections": 2,
+					"keepAliveTimeout": 2,
+					"httpHostHeader": "def",
+					"originServerName": "b2",
+					"caPool": "/tmp/path1",
+					"noTLSVerify": false,
+					"disableChunkedEncoding": false,
+					"bastionMode": false,
+					"proxyAddress": "interface",
+					"proxyPort": 200,
+					"proxyType": "",
+					"ipRules": [
+						{
+							"prefix": "10.0.0.0/16",
+							"ports": [3000, 3030],
+							"allow": false
+						},
+						{
+							"prefix": "192.16.0.0/24",
+							"ports": [5000, 5050],
+							"allow": true
+						}
+					]
+				}
+			}
+		],
+        "warp-routing": {
+			"enabled": true,
+			"connectTimeout": 1
+        }
+	}
+	`)
+
+	var expectedConfig ingress.RemoteConfig
+	err := json.Unmarshal(rawConfig, &expectedConfig)
+	require.NoError(t, err)
+
+	c := &newLocalConfig{
+		RemoteConfig:       expectedConfig,
+		ConfigurationFlags: nil,
+	}
+
+	jsonSerde, err := json.Marshal(c)
+	require.NoError(t, err)
+
+	var remoteConfig ingress.RemoteConfig
+	err = json.Unmarshal(jsonSerde, &remoteConfig)
+	require.NoError(t, err)
+
+	require.Equal(t, remoteConfig.WarpRouting, ingress.WarpRoutingConfig{
+		Enabled: true,
+		ConnectTimeout: config.CustomDuration{
+			Duration: time.Second,
+		},
+		TCPKeepAlive: config.CustomDuration{
+			Duration: 30 * time.Second, // default value is 30 seconds
+		},
+	})
+	require.Equal(t, remoteConfig.Ingress.Rules, expectedConfig.Ingress.Rules)
+}

orchestration/orchestrator.go

@@ -47,14 +47,14 @@ func NewOrchestrator(ctx context.Context, config *Config, tags []tunnelpogs.Tag,
 		log:            log,
 		shutdownC:      ctx.Done(),
 	}
-	if err := o.updateIngress(*config.Ingress, config.WarpRoutingEnabled); err != nil {
+	if err := o.updateIngress(*config.Ingress, config.WarpRouting); err != nil {
 		return nil, err
 	}
 	go o.waitToCloseLastProxy()
 	return o, nil
 }
 
-// Update creates a new proxy with the new ingress rules
+// UpdateConfig creates a new proxy with the new ingress rules
 func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
 	o.lock.Lock()
 	defer o.lock.Unlock()
@@ -63,12 +63,12 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 		o.log.Debug().
 			Int32("current_version", o.currentVersion).
 			Int32("received_version", version).
-			Msg("Current version is equal or newer than receivied version")
+			Msg("Current version is equal or newer than received version")
 		return &tunnelpogs.UpdateConfigurationResponse{
 			LastAppliedVersion: o.currentVersion,
 		}
 	}
-	var newConf newConfig
+	var newConf newRemoteConfig
 	if err := json.Unmarshal(config, &newConf); err != nil {
 		o.log.Err(err).
 			Int32("version", version).
@@ -80,7 +80,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 		}
 	}
 
-	if err := o.updateIngress(newConf.Ingress, newConf.WarpRouting.Enabled); err != nil {
+	if err := o.updateIngress(newConf.Ingress, newConf.WarpRouting); err != nil {
 		o.log.Err(err).
 			Int32("version", version).
 			Str("config", string(config)).
@@ -103,7 +103,7 @@ func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.Up
 }
 
 // The caller is responsible to make sure there is no concurrent access
-func (o *Orchestrator) updateIngress(ingressRules ingress.Ingress, warpRoutingEnabled bool) error {
+func (o *Orchestrator) updateIngress(ingressRules ingress.Ingress, warpRouting ingress.WarpRoutingConfig) error {
 	select {
 	case <-o.shutdownC:
 		return fmt.Errorf("cloudflared already shutdown")
@@ -118,10 +118,10 @@ func (o *Orchestrator) updateIngress(ingressRules ingress.Ingress, warpRoutingEn
 	if err := ingressRules.StartOrigins(o.log, proxyShutdownC); err != nil {
 		return errors.Wrap(err, "failed to start origin")
 	}
-	newProxy := proxy.NewOriginProxy(ingressRules, warpRoutingEnabled, o.tags, o.log)
+	newProxy := proxy.NewOriginProxy(ingressRules, warpRouting, o.tags, o.log)
 	o.proxy.Store(newProxy)
 	o.config.Ingress = &ingressRules
-	o.config.WarpRoutingEnabled = warpRoutingEnabled
+	o.config.WarpRouting = warpRouting
 
 	// If proxyShutdownC is nil, there is no previous running proxy
 	if o.proxyShutdownC != nil {
@@ -131,10 +131,26 @@ func (o *Orchestrator) updateIngress(ingressRules ingress.Ingress, warpRoutingEn
 	return nil
 }
 
-// GetConfigJSON returns the current version and configuration as JSON
+// GetConfigJSON returns the current json serialization of the config as the edge understands it
 func (o *Orchestrator) GetConfigJSON() ([]byte, error) {
 	o.lock.RLock()
 	defer o.lock.RUnlock()
+
+	c := &newLocalConfig{
+		RemoteConfig: ingress.RemoteConfig{
+			Ingress:     *o.config.Ingress,
+			WarpRouting: o.config.WarpRouting,
+		},
+		ConfigurationFlags: o.config.ConfigurationFlags,
+	}
+
+	return json.Marshal(c)
+}
+
+// GetVersionedConfigJSON returns the current version and configuration as JSON
+func (o *Orchestrator) GetVersionedConfigJSON() ([]byte, error) {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
 	var currentConfiguration = struct {
 		Version int32 `json:"version"`
 		Config  struct {
@@ -150,7 +166,7 @@ func (o *Orchestrator) GetConfigJSON() ([]byte, error) {
 			OriginRequest ingress.OriginRequestConfig `json:"originRequest"`
 		}{
 			Ingress:       o.config.Ingress.Rules,
-			WarpRouting:   config.WarpRoutingConfig{Enabled: o.config.WarpRoutingEnabled},
+			WarpRouting:   o.config.WarpRouting.RawConfig(),
 			OriginRequest: o.config.Ingress.Defaults,
 		},
 	}

orchestration/orchestrator_test.go

@@ -2,6 +2,7 @@ package orchestration
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -21,6 +22,7 @@ import (
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/proxy"
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
@@ -46,8 +48,7 @@ var (
 // - receiving an old version is noop
 func TestUpdateConfiguration(t *testing.T) {
 	initConfig := &Config{
-		Ingress:            &ingress.Ingress{},
-		WarpRoutingEnabled: false,
+		Ingress: &ingress.Ingress{},
 	}
 	orchestrator, err := NewOrchestrator(context.Background(), initConfig, testTags, &testLogger)
 	require.NoError(t, err)
@@ -85,7 +86,8 @@ func TestUpdateConfiguration(t *testing.T) {
         }
     ],
     "warp-routing": {
-        "enabled": true
+        "enabled": true,
+        "connectTimeout": 10
     }
 }	
 `)
@@ -119,7 +121,8 @@ func TestUpdateConfiguration(t *testing.T) {
 	require.Equal(t, config.CustomDuration{Duration: time.Second * 90}, configV2.Ingress.Rules[2].Config.ConnectTimeout)
 	require.Equal(t, false, configV2.Ingress.Rules[2].Config.NoTLSVerify)
 	require.Equal(t, true, configV2.Ingress.Rules[2].Config.NoHappyEyeballs)
-	require.True(t, configV2.WarpRoutingEnabled)
+	require.True(t, configV2.WarpRouting.Enabled)
+	require.Equal(t, configV2.WarpRouting.ConnectTimeout.Duration, 10*time.Second)
 
 	originProxyV2, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)
@@ -162,7 +165,7 @@ func TestUpdateConfiguration(t *testing.T) {
 	require.Len(t, configV10.Ingress.Rules, 1)
 	require.True(t, configV10.Ingress.Rules[0].Matches("blogs.tunnel.io", "/2022/02/10"))
 	require.Equal(t, ingress.HelloWorldService, configV10.Ingress.Rules[0].Service.String())
-	require.False(t, configV10.WarpRoutingEnabled)
+	require.False(t, configV10.WarpRouting.Enabled)
 
 	originProxyV10, err := orchestrator.GetOriginProxy()
 	require.NoError(t, err)
@@ -244,8 +247,7 @@ func TestConcurrentUpdateAndRead(t *testing.T) {
 		appliedV2 = make(chan struct{})
 
 		initConfig = &Config{
-			Ingress:            &ingress.Ingress{},
-			WarpRoutingEnabled: false,
+			Ingress: &ingress.Ingress{},
 		}
 	)
 
@@ -353,7 +355,7 @@ func proxyHTTP(originProxy connection.OriginProxy, hostname string) (*http.Respo
 		return nil, err
 	}
 
-	err = originProxy.ProxyHTTP(respWriter, req, false)
+	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), false)
 	if err != nil {
 		return nil, err
 	}
@@ -474,8 +476,7 @@ func TestClosePreviousProxies(t *testing.T) {
 }
 `)
 		initConfig = &Config{
-			Ingress:            &ingress.Ingress{},
-			WarpRoutingEnabled: false,
+			Ingress: &ingress.Ingress{},
 		}
 	)
 
@@ -532,8 +533,7 @@ func TestPersistentConnection(t *testing.T) {
 	)
 	msg := t.Name()
 	initConfig := &Config{
-		Ingress:            &ingress.Ingress{},
-		WarpRoutingEnabled: false,
+		Ingress: &ingress.Ingress{},
 	}
 	orchestrator, err := NewOrchestrator(context.Background(), initConfig, testTags, &testLogger)
 	require.NoError(t, err)
@@ -604,7 +604,7 @@ func TestPersistentConnection(t *testing.T) {
 		respWriter, err := connection.NewHTTP2RespWriter(req, wsRespReadWriter, connection.TypeWebsocket, &log)
 		require.NoError(t, err)
 
-		err = originProxy.ProxyHTTP(respWriter, req, true)
+		err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), true)
 		require.NoError(t, err)
 	}()
 
@@ -640,6 +640,18 @@ func TestPersistentConnection(t *testing.T) {
 	wg.Wait()
 }
 
+func TestSerializeLocalConfig(t *testing.T) {
+	c := &newLocalConfig{
+		RemoteConfig: ingress.RemoteConfig{
+			Ingress: ingress.Ingress{},
+		},
+		ConfigurationFlags: map[string]string{"a": "b"},
+	}
+
+	result, _ := json.Marshal(c)
+	fmt.Println(string(result))
+}
+
 func wsEcho(w http.ResponseWriter, r *http.Request) {
 	upgrader := gows.Upgrader{}
 

proxy/pool.go

@@ -1,29 +0,0 @@
-package proxy
-
-import (
-	"sync"
-)
-
-type bufferPool struct {
-	// A bufferPool must not be copied after first use.
-	// https://golang.org/pkg/sync/#Pool
-	buffers sync.Pool
-}
-
-func newBufferPool(bufferSize int) *bufferPool {
-	return &bufferPool{
-		buffers: sync.Pool{
-			New: func() interface{} {
-				return make([]byte, bufferSize)
-			},
-		},
-	}
-}
-
-func (p *bufferPool) Get() []byte {
-	return p.buffers.Get().([]byte)
-}
-
-func (p *bufferPool) Put(buf []byte) {
-	p.buffers.Put(buf)
-}

proxy/proxy.go

@@ -10,10 +10,14 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/cloudflare/cloudflared/carrier"
+	"github.com/cloudflare/cloudflared/cfio"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/websocket"
 )
@@ -24,6 +28,7 @@ const (
 	LogFieldCFRay         = "cfRay"
 	LogFieldRule          = "ingressRule"
 	LogFieldOriginService = "originService"
+	LogFieldFlowID        = "flowID"
 )
 
 // Proxy represents a means to Proxy between cloudflared and the origin services.
@@ -32,13 +37,12 @@ type Proxy struct {
 	warpRouting  *ingress.WarpRoutingService
 	tags         []tunnelpogs.Tag
 	log          *zerolog.Logger
-	bufferPool   *bufferPool
 }
 
 // NewOriginProxy returns a new instance of the Proxy struct.
 func NewOriginProxy(
 	ingressRules ingress.Ingress,
-	warpRoutingEnabled bool,
+	warpRouting ingress.WarpRoutingConfig,
 	tags []tunnelpogs.Tag,
 	log *zerolog.Logger,
 ) *Proxy {
@@ -46,10 +50,9 @@ func NewOriginProxy(
 		ingressRules: ingressRules,
 		tags:         tags,
 		log:          log,
-		bufferPool:   newBufferPool(512 * 1024),
 	}
-	if warpRoutingEnabled {
-		proxy.warpRouting = ingress.NewWarpRoutingService()
+	if warpRouting.Enabled {
+		proxy.warpRouting = ingress.NewWarpRoutingService(warpRouting)
 		log.Info().Msgf("Warp-routing is enabled")
 	}
 
@@ -60,16 +63,19 @@ func NewOriginProxy(
 // a simple roundtrip or a tcp/websocket dial depending on ingres rule setup.
 func (p *Proxy) ProxyHTTP(
 	w connection.ResponseWriter,
-	req *http.Request,
+	tr *tracing.TracedHTTPRequest,
 	isWebsocket bool,
 ) error {
 	incrementRequests()
 	defer decrementConcurrentRequests()
 
+	req := tr.Request
 	cfRay := connection.FindCfRayHeader(req)
 	lbProbe := connection.IsLBProbeRequest(req)
 	p.appendTagHeaders(req)
 
+	_, ruleSpan := tr.Tracer().Start(req.Context(), "ingress_match",
+		trace.WithAttributes(attribute.String("req-host", req.Host)))
 	rule, ruleNum := p.ingressRules.FindMatchingRule(req.Host, req.URL.Path)
 	logFields := logFields{
 		cfRay:   cfRay,
@@ -77,23 +83,24 @@ func (p *Proxy) ProxyHTTP(
 		rule:    ruleNum,
 	}
 	p.logRequest(req, logFields)
+	ruleSpan.SetAttributes(attribute.Int("rule-num", ruleNum))
+	ruleSpan.End()
 
 	switch originProxy := rule.Service.(type) {
 	case ingress.HTTPOriginProxy:
 		if err := p.proxyHTTPRequest(
 			w,
-			req,
+			tr,
 			originProxy,
 			isWebsocket,
 			rule.Config.DisableChunkedEncoding,
 			logFields,
 		); err != nil {
 			rule, srv := ruleField(p.ingressRules, ruleNum)
-			p.logRequestError(err, cfRay, rule, srv)
+			p.logRequestError(err, cfRay, "", rule, srv)
 			return err
 		}
 		return nil
-
 	case ingress.StreamBasedOriginProxy:
 		dest, err := getDestFromRule(rule, req)
 		if err != nil {
@@ -101,9 +108,9 @@ func (p *Proxy) ProxyHTTP(
 		}
 
 		rws := connection.NewHTTPResponseReadWriterAcker(w, req)
-		if err := p.proxyStream(req.Context(), rws, dest, originProxy, logFields); err != nil {
+		if err := p.proxyStream(tr.ToTracedContext(), rws, dest, originProxy); err != nil {
 			rule, srv := ruleField(p.ingressRules, ruleNum)
-			p.logRequestError(err, cfRay, rule, srv)
+			p.logRequestError(err, cfRay, "", rule, srv)
 			return err
 		}
 		return nil
@@ -130,17 +137,17 @@ func (p *Proxy) ProxyTCP(
 	serveCtx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	logFields := logFields{
-		cfRay:   req.CFRay,
-		lbProbe: req.LBProbe,
-		rule:    ingress.ServiceWarpRouting,
-	}
+	tracedCtx := tracing.NewTracedContext(serveCtx, req.CfTraceID, p.log)
 
-	if err := p.proxyStream(serveCtx, rwa, req.Dest, p.warpRouting.Proxy, logFields); err != nil {
-		p.logRequestError(err, req.CFRay, "", ingress.ServiceWarpRouting)
+	p.log.Debug().Str(LogFieldFlowID, req.FlowID).Msg("tcp proxy stream started")
+
+	if err := p.proxyStream(tracedCtx, rwa, req.Dest, p.warpRouting.Proxy); err != nil {
+		p.logRequestError(err, req.CFRay, req.FlowID, "", ingress.ServiceWarpRouting)
 		return err
 	}
 
+	p.log.Debug().Str(LogFieldFlowID, req.FlowID).Msg("tcp proxy stream finished successfully")
+
 	return nil
 }
 
@@ -155,15 +162,15 @@ func ruleField(ing ingress.Ingress, ruleNum int) (ruleID string, srv string) {
 // ProxyHTTPRequest proxies requests of underlying type http and websocket to the origin service.
 func (p *Proxy) proxyHTTPRequest(
 	w connection.ResponseWriter,
-	req *http.Request,
+	tr *tracing.TracedHTTPRequest,
 	httpService ingress.HTTPOriginProxy,
 	isWebsocket bool,
 	disableChunkedEncoding bool,
 	fields logFields,
 ) error {
-	roundTripReq := req
+	roundTripReq := tr.Request
 	if isWebsocket {
-		roundTripReq = req.Clone(req.Context())
+		roundTripReq = tr.Clone(tr.Request.Context())
 		roundTripReq.Header.Set("Connection", "Upgrade")
 		roundTripReq.Header.Set("Upgrade", "websocket")
 		roundTripReq.Header.Set("Sec-Websocket-Version", "13")
@@ -173,7 +180,7 @@ func (p *Proxy) proxyHTTPRequest(
 		// Support for WSGI Servers by switching transfer encoding from chunked to gzip/deflate
 		if disableChunkedEncoding {
 			roundTripReq.TransferEncoding = []string{"gzip", "deflate"}
-			cLength, err := strconv.Atoi(req.Header.Get("Content-Length"))
+			cLength, err := strconv.Atoi(tr.Request.Header.Get("Content-Length"))
 			if err == nil {
 				roundTripReq.ContentLength = int64(cLength)
 			}
@@ -187,12 +194,27 @@ func (p *Proxy) proxyHTTPRequest(
 		roundTripReq.Header.Set("User-Agent", "")
 	}
 
+	_, ttfbSpan := tr.Tracer().Start(tr.Context(), "ttfb_origin")
 	resp, err := httpService.RoundTrip(roundTripReq)
 	if err != nil {
+		tracing.EndWithErrorStatus(ttfbSpan, err)
+		if err := roundTripReq.Context().Err(); err != nil {
+			return errors.Wrap(err, "Incoming request ended abruptly")
+		}
 		return errors.Wrap(err, "Unable to reach the origin service. The service may be down or it may not be responding to traffic from cloudflared")
 	}
+
+	tracing.EndWithStatusCode(ttfbSpan, resp.StatusCode)
 	defer resp.Body.Close()
 
+	// resp headers can be nil
+	if resp.Header == nil {
+		resp.Header = make(http.Header)
+	}
+
+	// Add spans to response header (if available)
+	tr.AddSpans(resp.Header)
+
 	err = w.WriteRespHeaders(resp.StatusCode, resp.Header)
 	if err != nil {
 		return errors.Wrap(err, "Error writing response header")
@@ -207,7 +229,7 @@ func (p *Proxy) proxyHTTPRequest(
 
 		eyeballStream := &bidirectionalStream{
 			writer: w,
-			reader: req.Body,
+			reader: tr.Request.Body,
 		}
 
 		websocket.Stream(eyeballStream, rwc, p.log)
@@ -218,11 +240,7 @@ func (p *Proxy) proxyHTTPRequest(
 		p.log.Debug().Msg("Detected Server-Side Events from Origin")
 		p.writeEventStream(w, resp.Body)
 	} else {
-		// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
-		// compression generates dictionary on first write
-		buf := p.bufferPool.Get()
-		defer p.bufferPool.Put(buf)
-		_, _ = io.CopyBuffer(w, resp.Body, buf)
+		_, _ = cfio.Copy(w, resp.Body)
 	}
 
 	p.logOriginResponse(resp, fields)
@@ -232,18 +250,23 @@ func (p *Proxy) proxyHTTPRequest(
 // proxyStream proxies type TCP and other underlying types if the connection is defined as a stream oriented
 // ingress rule.
 func (p *Proxy) proxyStream(
-	ctx context.Context,
+	tr *tracing.TracedContext,
 	rwa connection.ReadWriteAcker,
 	dest string,
 	connectionProxy ingress.StreamBasedOriginProxy,
-	fields logFields,
 ) error {
-	originConn, err := connectionProxy.EstablishConnection(dest)
+	ctx := tr.Context
+	_, connectSpan := tr.Tracer().Start(ctx, "stream_connect")
+	originConn, err := connectionProxy.EstablishConnection(ctx, dest)
 	if err != nil {
+		tracing.EndWithErrorStatus(connectSpan, err)
 		return err
 	}
+	connectSpan.End()
 
-	if err := rwa.AckConnection(); err != nil {
+	encodedSpans := tr.GetSpans()
+
+	if err := rwa.AckConnection(encodedSpans); err != nil {
 		return err
 	}
 
@@ -303,6 +326,7 @@ type logFields struct {
 	cfRay   string
 	lbProbe bool
 	rule    interface{}
+	flowID  string
 }
 
 func (p *Proxy) logRequest(r *http.Request, fields logFields) {
@@ -346,12 +370,15 @@ func (p *Proxy) logOriginResponse(resp *http.Response, fields logFields) {
 	}
 }
 
-func (p *Proxy) logRequestError(err error, cfRay string, rule, service string) {
+func (p *Proxy) logRequestError(err error, cfRay string, flowID string, rule, service string) {
 	requestErrors.Inc()
 	log := p.log.Error().Err(err)
 	if cfRay != "" {
 		log = log.Str(LogFieldCFRay, cfRay)
 	}
+	if flowID != "" {
+		log = log.Str(LogFieldFlowID, flowID)
+	}
 	if rule != "" {
 		log = log.Str(LogFieldRule, rule)
 	}

proxy/proxy_test.go

@@ -27,11 +27,17 @@ import (
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
-	testTags = []tunnelpogs.Tag{tunnelpogs.Tag{Name: "Name", Value: "value"}}
+	testTags        = []tunnelpogs.Tag{tunnelpogs.Tag{Name: "Name", Value: "value"}}
+	noWarpRouting   = ingress.WarpRoutingConfig{}
+	testWarpRouting = ingress.WarpRoutingConfig{
+		Enabled:        true,
+		ConnectTimeout: config.CustomDuration{Duration: time.Second},
+	}
 )
 
 type mockHTTPRespWriter struct {
@@ -137,7 +143,7 @@ func TestProxySingleOrigin(t *testing.T) {
 
 	require.NoError(t, ingressRule.StartOrigins(&log, ctx.Done()))
 
-	proxy := NewOriginProxy(ingressRule, false, testTags, &log)
+	proxy := NewOriginProxy(ingressRule, noWarpRouting, testTags, &log)
 	t.Run("testProxyHTTP", testProxyHTTP(proxy))
 	t.Run("testProxyWebsocket", testProxyWebsocket(proxy))
 	t.Run("testProxySSE", testProxySSE(proxy))
@@ -151,7 +157,8 @@ func testProxyHTTP(proxy connection.OriginProxy) func(t *testing.T) {
 		req, err := http.NewRequest(http.MethodGet, "http://localhost:8080", nil)
 		require.NoError(t, err)
 
-		err = proxy.ProxyHTTP(responseWriter, req, false)
+		log := zerolog.Nop()
+		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
 		require.NoError(t, err)
 		for _, tag := range testTags {
 			assert.Equal(t, tag.Value, req.Header.Get(TagHeaderNamePrefix+tag.Name))
@@ -178,7 +185,8 @@ func testProxyWebsocket(proxy connection.OriginProxy) func(t *testing.T) {
 
 		errGroup, ctx := errgroup.WithContext(ctx)
 		errGroup.Go(func() error {
-			err = proxy.ProxyHTTP(responseWriter, req, true)
+			log := zerolog.Nop()
+			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), true)
 			require.NoError(t, err)
 
 			require.Equal(t, http.StatusSwitchingProtocols, responseWriter.Code)
@@ -239,7 +247,8 @@ func testProxySSE(proxy connection.OriginProxy) func(t *testing.T) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			err = proxy.ProxyHTTP(responseWriter, req, false)
+			log := zerolog.Nop()
+			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
 			require.NoError(t, err)
 
 			require.Equal(t, http.StatusOK, responseWriter.Code)
@@ -344,14 +353,14 @@ func runIngressTestScenarios(t *testing.T, unvalidatedIngress []config.Unvalidat
 	ctx, cancel := context.WithCancel(context.Background())
 	require.NoError(t, ingress.StartOrigins(&log, ctx.Done()))
 
-	proxy := NewOriginProxy(ingress, false, testTags, &log)
+	proxy := NewOriginProxy(ingress, noWarpRouting, testTags, &log)
 
 	for _, test := range tests {
 		responseWriter := newMockHTTPRespWriter()
 		req, err := http.NewRequest(http.MethodGet, test.url, nil)
 		require.NoError(t, err)
 
-		err = proxy.ProxyHTTP(responseWriter, req, false)
+		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false)
 		require.NoError(t, err)
 
 		assert.Equal(t, test.expectedStatus, responseWriter.Code)
@@ -392,13 +401,13 @@ func TestProxyError(t *testing.T) {
 
 	log := zerolog.Nop()
 
-	proxy := NewOriginProxy(ing, false, testTags, &log)
+	proxy := NewOriginProxy(ing, noWarpRouting, testTags, &log)
 
 	responseWriter := newMockHTTPRespWriter()
 	req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
 	assert.NoError(t, err)
 
-	assert.Error(t, proxy.ProxyHTTP(responseWriter, req, false))
+	assert.Error(t, proxy.ProxyHTTP(responseWriter, tracing.NewTracedHTTPRequest(req, &log), false))
 }
 
 type replayer struct {
@@ -508,7 +517,7 @@ func TestConnections(t *testing.T) {
 				originService:         runEchoTCPService,
 				eyeballResponseWriter: newTCPRespWriter(replayer),
 				eyeballRequestBody:    newTCPRequestBody([]byte("test2")),
-				warpRoutingService:    ingress.NewWarpRoutingService(),
+				warpRoutingService:    ingress.NewWarpRoutingService(testWarpRouting),
 				connectionType:        connection.TypeTCP,
 				requestHeaders: map[string][]string{
 					"Cf-Cloudflared-Proxy-Src": {"non-blank-value"},
@@ -525,7 +534,7 @@ func TestConnections(t *testing.T) {
 				originService:        runEchoWSService,
 				// eyeballResponseWriter gets set after roundtrip dial.
 				eyeballRequestBody: newPipedWSRequestBody([]byte("test3")),
-				warpRoutingService: ingress.NewWarpRoutingService(),
+				warpRoutingService: ingress.NewWarpRoutingService(testWarpRouting),
 				requestHeaders: map[string][]string{
 					"Cf-Cloudflared-Proxy-Src": {"non-blank-value"},
 				},
@@ -651,7 +660,7 @@ func TestConnections(t *testing.T) {
 
 			ingressRule := createSingleIngressConfig(t, test.args.ingressServiceScheme+ln.Addr().String())
 			ingressRule.StartOrigins(logger, ctx.Done())
-			proxy := NewOriginProxy(ingressRule, true, testTags, logger)
+			proxy := NewOriginProxy(ingressRule, testWarpRouting, testTags, logger)
 			proxy.warpRouting = test.args.warpRoutingService
 
 			dest := ln.Addr().String()
@@ -676,7 +685,8 @@ func TestConnections(t *testing.T) {
 				rwa := connection.NewHTTPResponseReadWriterAcker(respWriter, req)
 				err = proxy.ProxyTCP(ctx, rwa, &connection.TCPRequest{Dest: dest})
 			} else {
-				err = proxy.ProxyHTTP(respWriter, req, test.args.connectionType == connection.TypeWebsocket)
+				log := zerolog.Nop()
+				err = proxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, &log), test.args.connectionType == connection.TypeWebsocket)
 			}
 
 			cancel()

quic/datagram.go

@@ -1,64 +1,101 @@
 package quic
 
 import (
+	"context"
 	"fmt"
 
 	"github.com/google/uuid"
 	"github.com/lucas-clemente/quic-go"
 	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
 )
 
 const (
 	sessionIDLen = len(uuid.UUID{})
 )
 
-type DatagramMuxer struct {
+type SessionDatagram struct {
 	ID      uuid.UUID
-	session quic.Session
+	Payload []byte
 }
 
-func NewDatagramMuxer(quicSession quic.Session) (*DatagramMuxer, error) {
-	muxerID, err := uuid.NewRandom()
-	if err != nil {
-		return nil, err
-	}
+type BaseDatagramMuxer interface {
+	// MuxSession suffix the session ID to the payload so the other end of the QUIC connection can demultiplex the
+	// payload from multiple datagram sessions
+	MuxSession(sessionID uuid.UUID, payload []byte) error
+	// ServeReceive starts a loop to receive datagrams from the QUIC connection
+	ServeReceive(ctx context.Context) error
+}
+
+type DatagramMuxer struct {
+	session   quic.Connection
+	logger    *zerolog.Logger
+	demuxChan chan<- *SessionDatagram
+}
+
+func NewDatagramMuxer(quicSession quic.Connection, log *zerolog.Logger, demuxChan chan<- *SessionDatagram) *DatagramMuxer {
+	logger := log.With().Uint8("datagramVersion", 1).Logger()
 	return &DatagramMuxer{
-		ID:      muxerID,
-		session: quicSession,
-	}, nil
+		session:   quicSession,
+		logger:    &logger,
+		demuxChan: demuxChan,
+	}
 }
 
-// SendTo suffix the session ID to the payload so the other end of the QUIC session can demultiplex
-// the payload from multiple datagram sessions
-func (dm *DatagramMuxer) SendTo(sessionID uuid.UUID, payload []byte) error {
-	if len(payload) > maxDatagramPayloadSize {
+// Maximum application payload to send to / receive from QUIC datagram frame
+func (dm *DatagramMuxer) mtu() int {
+	return maxDatagramPayloadSize
+}
+
+func (dm *DatagramMuxer) MuxSession(sessionID uuid.UUID, payload []byte) error {
+	if len(payload) > dm.mtu() {
 		// TODO: TUN-5302 return ICMP packet too big message
-		return fmt.Errorf("origin UDP payload has %d bytes, which exceeds transport MTU %d", len(payload), dm.MTU())
+		// drop packet for now, eventually reply with ICMP for PMTUD
+		return fmt.Errorf("origin UDP payload has %d bytes, which exceeds transport MTU %d", len(payload), dm.mtu())
 	}
-	msgWithID, err := suffixSessionID(sessionID, payload)
+	payloadWithMetadata, err := suffixSessionID(sessionID, payload)
 	if err != nil {
 		return errors.Wrap(err, "Failed to suffix session ID to datagram, it will be dropped")
 	}
-	if err := dm.session.SendMessage(msgWithID); err != nil {
+	if err := dm.session.SendMessage(payloadWithMetadata); err != nil {
 		return errors.Wrap(err, "Failed to send datagram back to edge")
 	}
 	return nil
 }
 
-// ReceiveFrom extracts datagram session ID, then sends the session ID and payload to session manager
-// which determines how to proxy to the origin. It assumes the datagram session has already been
-// registered with session manager through other side channel
-func (dm *DatagramMuxer) ReceiveFrom() (uuid.UUID, []byte, error) {
-	msg, err := dm.session.ReceiveMessage()
-	if err != nil {
-		return uuid.Nil, nil, err
+func (dm *DatagramMuxer) ServeReceive(ctx context.Context) error {
+	for {
+		// Extracts datagram session ID, then sends the session ID and payload to receiver
+		// which determines how to proxy to the origin. It assumes the datagram session has already been
+		// registered with receiver through other side channel
+		msg, err := dm.session.ReceiveMessage()
+		if err != nil {
+			return err
+		}
+		if err := dm.demux(ctx, msg); err != nil {
+			dm.logger.Error().Err(err).Msg("Failed to demux datagram")
+			if err == context.Canceled {
+				return err
+			}
+		}
 	}
-	return extractSessionID(msg)
 }
 
-// Maximum application payload to send to / receive from QUIC datagram frame
-func (dm *DatagramMuxer) MTU() int {
-	return maxDatagramPayloadSize
+func (dm *DatagramMuxer) demux(ctx context.Context, msg []byte) error {
+	sessionID, payload, err := extractSessionID(msg)
+	if err != nil {
+		return err
+	}
+	sessionDatagram := SessionDatagram{
+		ID:      sessionID,
+		Payload: payload,
+	}
+	select {
+	case dm.demuxChan <- &sessionDatagram:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
 }
 
 // Each QUIC datagram should be suffixed with session ID.

quic/datagram_test.go

@@ -8,12 +8,14 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
+	"fmt"
 	"math/big"
 	"testing"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/lucas-clemente/quic-go"
+	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 )
@@ -51,17 +53,39 @@ func TestSuffixSessionIDError(t *testing.T) {
 	require.Error(t, err)
 }
 
-func TestMaxDatagramPayload(t *testing.T) {
-	payload := make([]byte, maxDatagramPayloadSize)
+func TestDatagram(t *testing.T) {
+	maxPayload := make([]byte, maxDatagramPayloadSize)
+	noPayloadSession := uuid.New()
+	maxPayloadSession := uuid.New()
+	sessionToPayload := []*SessionDatagram{
+		{
+			ID:      noPayloadSession,
+			Payload: make([]byte, 0),
+		},
+		{
+			ID:      maxPayloadSession,
+			Payload: maxPayload,
+		},
+	}
+	flowPayloads := [][]byte{
+		maxPayload,
+	}
 
+	testDatagram(t, 1, sessionToPayload, nil)
+	testDatagram(t, 2, sessionToPayload, flowPayloads)
+}
+
+func testDatagram(t *testing.T, version uint8, sessionToPayloads []*SessionDatagram, packetPayloads [][]byte) {
 	quicConfig := &quic.Config{
-		KeepAlive:            true,
+		KeepAlivePeriod:      5 * time.Millisecond,
 		EnableDatagrams:      true,
 		MaxDatagramFrameSize: MaxDatagramFrameSize,
 	}
 	quicListener := newQUICListener(t, quicConfig)
 	defer quicListener.Close()
 
+	logger := zerolog.Nop()
+
 	errGroup, ctx := errgroup.WithContext(context.Background())
 	// Run edge side of datagram muxer
 	errGroup.Go(func() error {
@@ -71,21 +95,32 @@ func TestMaxDatagramPayload(t *testing.T) {
 			return err
 		}
 
-		muxer, err := NewDatagramMuxer(quicSession)
-		if err != nil {
-			return err
+		sessionDemuxChan := make(chan *SessionDatagram, 16)
+
+		switch version {
+		case 1:
+			muxer := NewDatagramMuxer(quicSession, &logger, sessionDemuxChan)
+			muxer.ServeReceive(ctx)
+		case 2:
+			packetDemuxChan := make(chan []byte, len(packetPayloads))
+			muxer := NewDatagramMuxerV2(quicSession, &logger, sessionDemuxChan, packetDemuxChan)
+			muxer.ServeReceive(ctx)
+
+			for _, expectedPayload := range packetPayloads {
+				require.Equal(t, expectedPayload, <-packetDemuxChan)
+			}
+		default:
+			return fmt.Errorf("unknown datagram version %d", version)
 		}
 
-		sessionID, receivedPayload, err := muxer.ReceiveFrom()
-		if err != nil {
-			return err
+		for _, expectedPayload := range sessionToPayloads {
+			actualPayload := <-sessionDemuxChan
+			require.Equal(t, expectedPayload, actualPayload)
 		}
-		require.Equal(t, testSessionID, sessionID)
-		require.True(t, bytes.Equal(payload, receivedPayload))
-
 		return nil
 	})
 
+	largePayload := make([]byte, MaxDatagramFrameSize)
 	// Run cloudflared side of datagram muxer
 	errGroup.Go(func() error {
 		tlsClientConfig := &tls.Config{
@@ -95,23 +130,35 @@ func TestMaxDatagramPayload(t *testing.T) {
 		// Establish quic connection
 		quicSession, err := quic.DialAddrEarly(quicListener.Addr().String(), tlsClientConfig, quicConfig)
 		require.NoError(t, err)
-
-		muxer, err := NewDatagramMuxer(quicSession)
-		if err != nil {
-			return err
-		}
+		defer quicSession.CloseWithError(0, "")
 
 		// Wait a few milliseconds for MTU discovery to take place
 		time.Sleep(time.Millisecond * 100)
-		err = muxer.SendTo(testSessionID, payload)
-		if err != nil {
-			return err
+
+		var muxer BaseDatagramMuxer
+		switch version {
+		case 1:
+			muxer = NewDatagramMuxer(quicSession, &logger, nil)
+		case 2:
+			muxerV2 := NewDatagramMuxerV2(quicSession, &logger, nil, nil)
+			for _, payload := range packetPayloads {
+				require.NoError(t, muxerV2.MuxPacket(payload))
+			}
+			// Payload larger than transport MTU, should not be sent
+			require.Error(t, muxerV2.MuxPacket(largePayload))
+			muxer = muxerV2
+		default:
+			return fmt.Errorf("unknown datagram version %d", version)
 		}
 
-		// Payload larger than transport MTU, should return an error
-		largePayload := make([]byte, MaxDatagramFrameSize)
-		err = muxer.SendTo(testSessionID, largePayload)
-		require.Error(t, err)
+		for _, sessionDatagram := range sessionToPayloads {
+			require.NoError(t, muxer.MuxSession(sessionDatagram.ID, sessionDatagram.Payload))
+		}
+		// Payload larger than transport MTU, should not be sent
+		require.Error(t, muxer.MuxSession(testSessionID, largePayload))
+
+		// Wait for edge to finish receiving the messages
+		time.Sleep(time.Millisecond * 100)
 
 		return nil
 	})
@@ -151,3 +198,35 @@ func generateTLSConfig() *tls.Config {
 		NextProtos:   []string{"argotunnel"},
 	}
 }
+
+type sessionMuxer interface {
+	SendToSession(sessionID uuid.UUID, payload []byte) error
+}
+
+type mockSessionReceiver struct {
+	expectedSessionToPayload map[uuid.UUID][]byte
+	receivedCount            int
+}
+
+func (msr *mockSessionReceiver) ReceiveDatagram(sessionID uuid.UUID, payload []byte) error {
+	expectedPayload := msr.expectedSessionToPayload[sessionID]
+	if !bytes.Equal(expectedPayload, payload) {
+		return fmt.Errorf("expect %v to have payload %s, got %s", sessionID, string(expectedPayload), string(payload))
+	}
+	msr.receivedCount++
+	return nil
+}
+
+type mockFlowReceiver struct {
+	expectedPayloads [][]byte
+	receivedCount    int
+}
+
+func (mfr *mockFlowReceiver) ReceiveFlow(payload []byte) error {
+	expectedPayload := mfr.expectedPayloads[mfr.receivedCount]
+	if !bytes.Equal(expectedPayload, payload) {
+		return fmt.Errorf("expect flow %d to have payload %s, got %s", mfr.receivedCount, string(expectedPayload), string(payload))
+	}
+	mfr.receivedCount++
+	return nil
+}

quic/datagramv2.go

@@ -0,0 +1,136 @@
+package quic
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/google/uuid"
+	"github.com/lucas-clemente/quic-go"
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+)
+
+type datagramV2Type byte
+
+const (
+	udp datagramV2Type = iota
+	ip
+)
+
+func suffixType(b []byte, datagramType datagramV2Type) ([]byte, error) {
+	if len(b)+1 > MaxDatagramFrameSize {
+		return nil, fmt.Errorf("datagram size %d exceeds max frame size %d", len(b), MaxDatagramFrameSize)
+	}
+	b = append(b, byte(datagramType))
+	return b, nil
+}
+
+// Maximum application payload to send to / receive from QUIC datagram frame
+func (dm *DatagramMuxerV2) mtu() int {
+	return maxDatagramPayloadSize
+}
+
+type DatagramMuxerV2 struct {
+	session          quic.Connection
+	logger           *zerolog.Logger
+	sessionDemuxChan chan<- *SessionDatagram
+	packetDemuxChan  chan<- []byte
+}
+
+func NewDatagramMuxerV2(
+	quicSession quic.Connection,
+	log *zerolog.Logger,
+	sessionDemuxChan chan<- *SessionDatagram,
+	packetDemuxChan chan<- []byte) *DatagramMuxerV2 {
+	logger := log.With().Uint8("datagramVersion", 2).Logger()
+	return &DatagramMuxerV2{
+		session:          quicSession,
+		logger:           &logger,
+		sessionDemuxChan: sessionDemuxChan,
+		packetDemuxChan:  packetDemuxChan,
+	}
+}
+
+// MuxSession suffix the session ID and datagram version to the payload so the other end of the QUIC connection can
+// demultiplex the payload from multiple datagram sessions
+func (dm *DatagramMuxerV2) MuxSession(sessionID uuid.UUID, payload []byte) error {
+	if len(payload) > dm.mtu() {
+		// TODO: TUN-5302 return ICMP packet too big message
+		return fmt.Errorf("origin UDP payload has %d bytes, which exceeds transport MTU %d", len(payload), dm.mtu())
+	}
+	msgWithID, err := suffixSessionID(sessionID, payload)
+	if err != nil {
+		return errors.Wrap(err, "Failed to suffix session ID to datagram, it will be dropped")
+	}
+	msgWithIDAndType, err := suffixType(msgWithID, udp)
+	if err != nil {
+		return errors.Wrap(err, "Failed to suffix datagram type, it will be dropped")
+	}
+	if err := dm.session.SendMessage(msgWithIDAndType); err != nil {
+		return errors.Wrap(err, "Failed to send datagram back to edge")
+	}
+	return nil
+}
+
+// MuxPacket suffix the datagram type to the packet. The other end of the QUIC connection can demultiplex by parsing
+// the payload as IP and look at the source and destination.
+func (dm *DatagramMuxerV2) MuxPacket(packet []byte) error {
+	payloadWithVersion, err := suffixType(packet, ip)
+	if err != nil {
+		return errors.Wrap(err, "Failed to suffix datagram type, it will be dropped")
+	}
+	if err := dm.session.SendMessage(payloadWithVersion); err != nil {
+		return errors.Wrap(err, "Failed to send datagram back to edge")
+	}
+	return nil
+}
+
+// Demux reads datagrams from the QUIC connection and demuxes depending on whether it's a session or packet
+func (dm *DatagramMuxerV2) ServeReceive(ctx context.Context) error {
+	for {
+		msg, err := dm.session.ReceiveMessage()
+		if err != nil {
+			return err
+		}
+		if err := dm.demux(ctx, msg); err != nil {
+			dm.logger.Error().Err(err).Msg("Failed to demux datagram")
+			if err == context.Canceled {
+				return err
+			}
+		}
+	}
+}
+
+func (dm *DatagramMuxerV2) demux(ctx context.Context, msgWithType []byte) error {
+	if len(msgWithType) < 1 {
+		return fmt.Errorf("QUIC datagram should have at least 1 byte")
+	}
+	msgType := datagramV2Type(msgWithType[len(msgWithType)-1])
+	msg := msgWithType[0 : len(msgWithType)-1]
+	switch msgType {
+	case udp:
+		sessionID, payload, err := extractSessionID(msg)
+		if err != nil {
+			return err
+		}
+		sessionDatagram := SessionDatagram{
+			ID:      sessionID,
+			Payload: payload,
+		}
+		select {
+		case dm.sessionDemuxChan <- &sessionDatagram:
+			return nil
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	case ip:
+		select {
+		case dm.packetDemuxChan <- msg:
+			return nil
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	default:
+		return fmt.Errorf("Unexpected datagram type %d", msgType)
+	}
+}

quic/pogs.go

@@ -1,6 +1,8 @@
 package quic
 
 import (
+	"fmt"
+
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/pogs"
 
@@ -16,6 +18,18 @@ const (
 	ConnectionTypeTCP
 )
 
+func (c ConnectionType) String() string {
+	switch c {
+	case ConnectionTypeHTTP:
+		return "http"
+	case ConnectionTypeWebsocket:
+		return "ws"
+	case ConnectionTypeTCP:
+		return "tcp"
+	}
+	panic(fmt.Sprintf("invalid ConnectionType: %d", c))
+}
+
 // ConnectRequest is the representation of metadata sent at the start of a QUIC application handshake.
 type ConnectRequest struct {
 	Dest     string         `capnp:"dest"`

quic/quic_protocol.go

@@ -37,7 +37,8 @@ const (
 	protocolVersionLength = 2
 
 	HandshakeIdleTimeout = 5 * time.Second
-	MaxIdleTimeout       = 15 * time.Second
+	MaxIdleTimeout       = 5 * time.Second
+	MaxIdlePingPeriod    = 1 * time.Second
 )
 
 // RequestServerStream is a stream to serve requests

quic/safe_stream.go

@@ -41,3 +41,14 @@ func (s *SafeStreamCloser) Close() error {
 	s.stream.CancelRead(0)
 	return s.stream.Close()
 }
+
+func (s *SafeStreamCloser) CloseWrite() error {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	// As documented by the quic-go library, this doesn't actually close the entire stream.
+	// It prevents further writes, which in turn will result in an EOF signal being sent the other side of stream when
+	// reading.
+	// We can still read from this stream.
+	return s.stream.Close()
+}

quic/safe_stream_test.go

@@ -7,6 +7,7 @@ import (
 	"net"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/lucas-clemente/quic-go"
 	"github.com/stretchr/testify/require"
@@ -15,7 +16,7 @@ import (
 var (
 	testTLSServerConfig = GenerateTLSConfig()
 	testQUICConfig      = &quic.Config{
-		KeepAlive:       true,
+		KeepAlivePeriod: 5 * time.Second,
 		EnableDatagrams: true,
 	}
 	exchanges       = 1000

quic/tracing.go

@@ -3,9 +3,9 @@ package quic
 import (
 	"context"
 	"net"
+	"time"
 
 	"github.com/lucas-clemente/quic-go/logging"
-	"github.com/lucas-clemente/quic-go/qlog"
 	"github.com/rs/zerolog"
 )
 
@@ -40,75 +40,116 @@ func NewServerTracer(logger *zerolog.Logger) logging.Tracer {
 	}
 }
 
-func (t *tracer) TracerForConnection(_ context.Context, p logging.Perspective, odcid logging.ConnectionID) logging.ConnectionTracer {
-	connID := logging.ConnectionID(odcid).String()
-	ql := &quicLogger{
-		logger:       t.logger,
-		connectionID: connID,
-	}
+func (t *tracer) TracerForConnection(_ctx context.Context, _p logging.Perspective, _odcid logging.ConnectionID) logging.ConnectionTracer {
 	if t.config.isClient {
-		return newConnTracer(ql, p, odcid, newClientCollector(t.config.index))
+		return newConnTracer(newClientCollector(t.config.index))
 	}
-	return newConnTracer(ql, p, odcid, newServiceCollector())
+	return newConnTracer(newServiceCollector())
 }
 
 func (*tracer) SentPacket(net.Addr, *logging.Header, logging.ByteCount, []logging.Frame) {}
 func (*tracer) DroppedPacket(net.Addr, logging.PacketType, logging.ByteCount, logging.PacketDropReason) {
 }
 
-// connTracer is a wrapper around https://pkg.go.dev/github.com/lucas-clemente/quic-go@v0.23.0/qlog#NewConnectionTracer to collect metrics
+var _ logging.Tracer = (*tracer)(nil)
+
+// connTracer collects connection level metrics
 type connTracer struct {
-	logging.ConnectionTracer
 	metricsCollector MetricsCollector
-	connectionID     string
 }
 
-func newConnTracer(ql *quicLogger, p logging.Perspective, odcid logging.ConnectionID, metricsCollector MetricsCollector) logging.ConnectionTracer {
+var _ logging.ConnectionTracer = (*connTracer)(nil)
+
+func newConnTracer(metricsCollector MetricsCollector) logging.ConnectionTracer {
 	return &connTracer{
-		qlog.NewConnectionTracer(ql, p, odcid),
-		metricsCollector,
-		logging.ConnectionID(odcid).String(),
+		metricsCollector: metricsCollector,
 	}
 }
 
 func (ct *connTracer) StartedConnection(local, remote net.Addr, srcConnID, destConnID logging.ConnectionID) {
 	ct.metricsCollector.startedConnection()
-	ct.ConnectionTracer.StartedConnection(local, remote, srcConnID, destConnID)
 }
 
 func (ct *connTracer) ClosedConnection(err error) {
 	ct.metricsCollector.closedConnection(err)
-	ct.ConnectionTracer.ClosedConnection(err)
 }
 
 func (ct *connTracer) SentPacket(hdr *logging.ExtendedHeader, packetSize logging.ByteCount, ack *logging.AckFrame, frames []logging.Frame) {
 	ct.metricsCollector.sentPackets(packetSize)
-	ct.ConnectionTracer.SentPacket(hdr, packetSize, ack, frames)
 }
 
 func (ct *connTracer) ReceivedPacket(hdr *logging.ExtendedHeader, size logging.ByteCount, frames []logging.Frame) {
 	ct.metricsCollector.receivedPackets(size)
-	ct.ConnectionTracer.ReceivedPacket(hdr, size, frames)
 }
 
 func (ct *connTracer) BufferedPacket(pt logging.PacketType) {
 	ct.metricsCollector.bufferedPackets(pt)
-	ct.ConnectionTracer.BufferedPacket(pt)
 }
 
 func (ct *connTracer) DroppedPacket(pt logging.PacketType, size logging.ByteCount, reason logging.PacketDropReason) {
 	ct.metricsCollector.droppedPackets(pt, size, reason)
-	ct.ConnectionTracer.DroppedPacket(pt, size, reason)
 }
 
 func (ct *connTracer) LostPacket(level logging.EncryptionLevel, number logging.PacketNumber, reason logging.PacketLossReason) {
 	ct.metricsCollector.lostPackets(reason)
-	ct.ConnectionTracer.LostPacket(level, number, reason)
 }
 
 func (ct *connTracer) UpdatedMetrics(rttStats *logging.RTTStats, cwnd, bytesInFlight logging.ByteCount, packetsInFlight int) {
 	ct.metricsCollector.updatedRTT(rttStats)
-	ct.ConnectionTracer.UpdatedMetrics(rttStats, cwnd, bytesInFlight, packetsInFlight)
+}
+
+func (ct *connTracer) NegotiatedVersion(chosen logging.VersionNumber, clientVersions, serverVersions []logging.VersionNumber) {
+}
+
+func (ct *connTracer) SentTransportParameters(parameters *logging.TransportParameters) {
+}
+
+func (ct *connTracer) ReceivedTransportParameters(parameters *logging.TransportParameters) {
+}
+
+func (ct *connTracer) RestoredTransportParameters(parameters *logging.TransportParameters) {
+}
+
+func (ct *connTracer) ReceivedVersionNegotiationPacket(header *logging.Header, numbers []logging.VersionNumber) {
+}
+
+func (ct *connTracer) ReceivedRetry(header *logging.Header) {
+}
+
+func (ct *connTracer) AcknowledgedPacket(level logging.EncryptionLevel, number logging.PacketNumber) {
+}
+
+func (ct *connTracer) UpdatedCongestionState(state logging.CongestionState) {
+}
+
+func (ct *connTracer) UpdatedPTOCount(value uint32) {
+}
+
+func (ct *connTracer) UpdatedKeyFromTLS(level logging.EncryptionLevel, perspective logging.Perspective) {
+}
+
+func (ct *connTracer) UpdatedKey(generation logging.KeyPhase, remote bool) {
+}
+
+func (ct *connTracer) DroppedEncryptionLevel(level logging.EncryptionLevel) {
+}
+
+func (ct *connTracer) DroppedKey(generation logging.KeyPhase) {
+}
+
+func (ct *connTracer) SetLossTimer(timerType logging.TimerType, level logging.EncryptionLevel, time time.Time) {
+}
+
+func (ct *connTracer) LossTimerExpired(timerType logging.TimerType, level logging.EncryptionLevel) {
+}
+
+func (ct *connTracer) LossTimerCanceled() {
+}
+
+func (ct *connTracer) Close() {
+}
+
+func (ct *connTracer) Debug(name, msg string) {
 }
 
 type quicLogger struct {

release_pkgs.py

@@ -0,0 +1,391 @@
+"""
+    This is a utility for creating deb and rpm packages, signing them 
+    and uploading them to a storage and adding metadata to workers KV.
+
+    It has two over-arching responsiblities:
+    1. Create deb and yum repositories from .deb and .rpm files. 
+       This is also responsible for signing the packages and generally preparing 
+       them to be in an uploadable state.
+    2. Upload these packages to a storage in a format that apt and yum expect.
+"""
+import argparse
+import base64
+import logging
+import os
+import shutil
+from hashlib import sha256
+from pathlib import Path
+from subprocess import Popen, PIPE
+
+import boto3
+import gnupg
+from botocore.client import Config
+from botocore.exceptions import ClientError
+
+# The front facing R2 URL to access assets from.
+R2_ASSET_URL = 'https://demo-r2-worker.cloudflare-tunnel.workers.dev/'
+
+
+class PkgUploader:
+    def __init__(self, account_id, bucket_name, client_id, client_secret):
+        self.account_id = account_id
+        self.bucket_name = bucket_name
+        self.client_id = client_id
+        self.client_secret = client_secret
+
+    def upload_pkg_to_r2(self, filename, upload_file_path):
+        endpoint_url = f"https://{self.account_id}.r2.cloudflarestorage.com"
+        token_secret_hash = sha256(self.client_secret.encode()).hexdigest()
+
+        config = Config(
+            region_name='auto',
+            s3={
+                "addressing_style": "path",
+            }
+        )
+
+        r2 = boto3.client(
+            "s3",
+            endpoint_url=endpoint_url,
+            aws_access_key_id=self.client_id,
+            aws_secret_access_key=token_secret_hash,
+            config=config,
+        )
+
+        print(f"uploading asset: {filename} to {upload_file_path} in bucket{self.bucket_name}...")
+        try:
+            r2.upload_file(filename, self.bucket_name, upload_file_path)
+        except ClientError as e:
+            raise e
+
+
+class PkgCreator:
+    """
+        The distribution conf is what dictates to reprepro, the debian packaging building
+        and signing tool we use, what distros to support, what GPG key to use for signing
+        and what to call the debian binary etc. This function creates it "./conf/distributions".
+
+        origin - name of your package (String)
+        label - label of your package (could be same as the name) (String)
+        release - release you want this to be distributed for (List of Strings)
+        components - could be a channel like main/stable/beta
+        archs - Architecture (List of Strings)
+        description - (String)
+        gpg_key_id - gpg key id of what you want to use to sign the packages.(String) 
+    """
+
+    def create_distribution_conf(self,
+                                 file_path,
+                                 origin,
+                                 label,
+                                 releases,
+                                 archs,
+                                 components,
+                                 description,
+                                 gpg_key_id):
+        with open(file_path, "w+") as distributions_file:
+            for release in releases:
+                distributions_file.write(f"Origin: {origin}\n")
+                distributions_file.write(f"Label: {label}\n")
+                distributions_file.write(f"Codename: {release}\n")
+                archs_list = " ".join(archs)
+                distributions_file.write(f"Architectures: {archs_list}\n")
+                distributions_file.write(f"Components: {components}\n")
+                distributions_file.write(f"Description: {description} - {release}\n")
+                distributions_file.write(f"SignWith: {gpg_key_id}\n")
+                distributions_file.write("\n")
+        return distributions_file
+
+    """
+        Uses the reprepro tool to generate packages, sign them and create the InRelease as specified
+        by the distribution_conf file. 
+
+        This function creates three folders db, pool and dist. 
+        db and pool contain information and metadata about builds. We can ignore these.
+        dist: contains all the pkgs and signed releases that are necessary for an apt download.
+    """
+
+    def create_deb_pkgs(self, release, deb_file):
+        print(f"creating deb pkgs: {release} : {deb_file}")
+        p = Popen(["reprepro", "includedeb", release, deb_file], stdout=PIPE, stderr=PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"create deb_pkgs result => {out}, {err}")
+            raise
+
+    def create_rpm_pkgs(self, artifacts_path, gpg_key_name):
+        self._setup_rpm_pkg_directories(artifacts_path, gpg_key_name)
+        p = Popen(["createrepo", "./rpm"], stdout=PIPE, stderr=PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"create rpm_pkgs result => {out}, {err}")
+            raise
+
+        self._sign_repomd()
+
+    """
+        creates a <binary>.repo file with details like so
+        [cloudflared-stable]
+        name=cloudflared-stable
+        baseurl=https://pkg.cloudflare.com/cloudflared/rpm
+        enabled=1
+        type=rpm
+        gpgcheck=1
+        gpgkey=https://pkg.cloudflare.com/cloudflare-main.gpg
+    """
+
+    def create_repo_file(self, file_path, binary_name, baseurl, gpgkey_url):
+        with open(os.path.join(file_path, binary_name + '.repo'), "w+") as repo_file:
+            repo_file.write(f"[{binary_name}-stable]")
+            repo_file.write(f"{binary_name}-stable")
+            repo_file.write(f"baseurl={baseurl}/rpm")
+            repo_file.write("enabled=1")
+            repo_file.write("type=rpm")
+            repo_file.write("gpgcheck=1")
+            repo_file.write(f"gpgkey={gpgkey_url}")
+
+    def _sign_rpms(self, file_path):
+        p = Popen(["rpm", "--define", f"_gpg_name {gpg_key_name}", "--addsign", file_path], stdout=PIPE, stderr=PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"rpm sign result result => {out}, {err}")
+            raise
+
+    def _sign_repomd(self):
+        p = Popen(["gpg", "--batch", "--detach-sign", "--armor", "./rpm/repodata/repomd.xml"], stdout=PIPE, stderr=PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"sign repomd result => {out}, {err}")
+            raise
+
+    """
+        sets up and signs the RPM directories in the following format:
+        - rpm 
+           - aarch64
+           - x86_64
+           - 386
+
+        this assumes the assets are in the format <prefix>-<aarch64/x86_64/386>.rpm
+    """
+
+    def _setup_rpm_pkg_directories(self, artifacts_path, gpg_key_name, archs=["aarch64", "x86_64", "386"]):
+        for arch in archs:
+            for root, _, files in os.walk(artifacts_path):
+                for file in files:
+                    if file.endswith(f"{arch}.rpm"):
+                        new_dir = f"./rpm/{arch}"
+                        os.makedirs(new_dir, exist_ok=True)
+                        old_path = os.path.join(root, file)
+                        new_path = os.path.join(new_dir, file)
+                        shutil.copyfile(old_path, new_path)
+                        self._sign_rpms(new_path)
+
+    """
+        imports gpg keys into the system so reprepro and createrepo can use it to sign packages.
+        it returns the GPG ID after a successful import
+    """
+
+    def import_gpg_keys(self, private_key, public_key):
+        gpg = gnupg.GPG()
+        private_key = base64.b64decode(private_key)
+        gpg.import_keys(private_key)
+        public_key = base64.b64decode(public_key)
+        gpg.import_keys(public_key)
+        data = gpg.list_keys(secret=True)
+        return (data[0]["fingerprint"], data[0]["uids"][0])
+
+    """
+        basically rpm --import <key_file>
+        This enables us to sign rpms.
+    """
+
+    def import_rpm_key(self, public_key):
+        file_name = "pb.key"
+        with open(file_name, "wb") as f:
+            public_key = base64.b64decode(public_key)
+            f.write(public_key)
+
+        p = Popen(["rpm", "--import", file_name], stdout=PIPE, stderr=PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"create rpm import result => {out}, {err}")
+            raise
+
+
+"""
+    Walks through a directory and uploads it's assets to R2.
+    directory : root directory to walk through (String).
+    release: release string. If this value is none, a specific release path will not be created 
+              and the release will be uploaded to the default path. 
+    binary: name of the binary to upload
+"""
+
+
+def upload_from_directories(pkg_uploader, directory, release, binary):
+    for root, _, files in os.walk(directory):
+        for file in files:
+            upload_file_name = os.path.join(binary, root, file)
+            if release:
+                upload_file_name = os.path.join(release, upload_file_name)
+            filename = os.path.join(root, file)
+            try:
+                pkg_uploader.upload_pkg_to_r2(filename, upload_file_name)
+            except ClientError as e:
+                logging.error(e)
+                return
+
+
+""" 
+    1. looks into a built_artifacts folder for cloudflared debs
+    2. creates Packages.gz, InRelease (signed) files
+    3. uploads them to Cloudflare R2 
+
+    pkg_creator, pkg_uploader: are instantiations of the two classes above.
+
+    gpg_key_id: is an id indicating the key the package should be signed with. The public key of this id will be 
+    uploaded to R2 so it can be presented to apt downloaders.
+
+    release_version: is the cloudflared release version. Only provide this if you want a permanent backup.
+"""
+
+
+def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component,
+                         release_version):
+    # set configuration for package creation.
+    print(f"initialising configuration for {binary_name} , {archs}")
+    Path("./conf").mkdir(parents=True, exist_ok=True)
+    pkg_creator.create_distribution_conf(
+        "./conf/distributions",
+        binary_name,
+        binary_name,
+        releases,
+        archs,
+        package_component,
+        f"apt repository for {binary_name}",
+        gpg_key_id)
+
+    # create deb pkgs
+    for release in releases:
+        for arch in archs:
+            print(f"creating deb pkgs for {release} and {arch}...")
+            pkg_creator.create_deb_pkgs(release, f"./built_artifacts/cloudflared-linux-{arch}.deb")
+
+    print("uploading latest to r2...")
+    upload_from_directories(pkg_uploader, "dists", None, binary_name)
+    upload_from_directories(pkg_uploader, "pool", None, binary_name)
+
+    if release_version:
+        print(f"uploading versioned release {release_version} to r2...")
+        upload_from_directories(pkg_uploader, "dists", release_version, binary_name)
+        upload_from_directories(pkg_uploader, "pool", release_version, binary_name)
+
+
+def create_rpm_packaging(
+        pkg_creator,
+        pkg_uploader,
+        artifacts_path,
+        release_version,
+        binary_name,
+        gpg_key_name,
+        base_url,
+        gpg_key_url,
+):
+    print(f"creating rpm pkgs...")
+    pkg_creator.create_rpm_pkgs(artifacts_path, gpg_key_name)
+    pkg_creator.create_repo_file(artifacts_path, binary_name, base_url, gpg_key_url)
+
+    print("uploading latest to r2...")
+    upload_from_directories(pkg_uploader, "rpm", None, binary_name)
+
+    if release_version:
+        print(f"uploading versioned release {release_version} to r2...")
+        upload_from_directories(pkg_uploader, "rpm", release_version, binary_name)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Creates linux releases and uploads them in a packaged format"
+    )
+
+    parser.add_argument(
+        "--bucket", default=os.environ.get("R2_BUCKET"), help="R2 Bucket name"
+    )
+    parser.add_argument(
+        "--id", default=os.environ.get("R2_CLIENT_ID"), help="R2 Client ID"
+    )
+    parser.add_argument(
+        "--secret", default=os.environ.get("R2_CLIENT_SECRET"), help="R2 Client Secret"
+    )
+    parser.add_argument(
+        "--account", default=os.environ.get("R2_ACCOUNT_ID"), help="R2 Account Tag"
+    )
+    parser.add_argument(
+        "--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\
+            prefixed with. Leave empty if you don't want tagged release versions backed up to R2."
+    )
+
+    parser.add_argument(
+        "--binary", default=os.environ.get("BINARY_NAME"), help="The name of the binary the packages are for"
+    )
+
+    parser.add_argument(
+        "--gpg-private-key", default=os.environ.get("LINUX_SIGNING_PRIVATE_KEY"), help="GPG private key to sign the\
+            packages"
+    )
+
+    parser.add_argument(
+        "--gpg-public-key", default=os.environ.get("LINUX_SIGNING_PUBLIC_KEY"), help="GPG public key used for\
+            signing packages"
+    )
+
+    parser.add_argument(
+        "--gpg-public-key-url", default=os.environ.get("GPG_PUBLIC_KEY_URL"), help="GPG public key url that\
+            downloaders can use to verify signing"
+    )
+
+    parser.add_argument(
+        "--pkg-upload-url", default=os.environ.get("PKG_URL"), help="URL to be used by downloaders"
+    )
+
+    parser.add_argument(
+        "--deb-based-releases", default=["bookworm", "bullseye", "buster", "jammy", "impish", "focal", "bionic",
+                                         "xenial", "trusty"],
+        help="list of debian based releases that need to be packaged for"
+    )
+
+    parser.add_argument(
+        "--archs", default=["amd64", "386", "arm64", "arm", "armhf"], help="list of architectures we want to package for. Note that\
+            it is the caller's responsiblity to ensure that these debs are already present in a directory. This script\
+            will not build binaries or create their debs."
+    )
+    args = parser.parse_args()
+
+    return args
+
+
+if __name__ == "__main__":
+    try:
+        args = parse_args()
+    except Exception as e:
+        logging.exception(e)
+        exit(1)
+
+    pkg_creator = PkgCreator()
+    (gpg_key_id, gpg_key_name) = pkg_creator.import_gpg_keys(args.gpg_private_key, args.gpg_public_key)
+    pkg_creator.import_rpm_key(args.gpg_public_key)
+
+    pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret)
+    print(f"signing with gpg_key: {gpg_key_id}")
+    create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, gpg_key_id, args.binary, args.archs,
+                         "main", args.release_tag)
+
+    create_rpm_packaging(
+        pkg_creator,
+        pkg_uploader,
+        "./built_artifacts",
+        args.release_tag,
+        args.binary,
+        gpg_key_name,
+        args.gpg_public_key_url,
+        args.pkg_upload_url,
+    )

supervisor/conn_aware_logger.go

@@ -12,9 +12,9 @@ type ConnAwareLogger struct {
 	logger  *zerolog.Logger
 }
 
-func NewConnAwareLogger(logger *zerolog.Logger, observer *connection.Observer) *ConnAwareLogger {
+func NewConnAwareLogger(logger *zerolog.Logger, tracker *tunnelstate.ConnTracker, observer *connection.Observer) *ConnAwareLogger {
 	connAwareLogger := &ConnAwareLogger{
-		tracker: tunnelstate.NewConnTracker(logger),
+		tracker: tracker,
 		logger:  logger,
 	}
 

supervisor/supervisor.go

@@ -4,9 +4,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/lucas-clemente/quic-go"
 	"github.com/rs/zerolog"
 
 	"github.com/cloudflare/cloudflared/connection"
@@ -17,6 +19,7 @@ import (
 	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelstate"
 )
 
 const (
@@ -37,12 +40,14 @@ const (
 // Supervisor manages non-declarative tunnels. Establishes TCP connections with the edge, and
 // reconnects them if they disconnect.
 type Supervisor struct {
-	cloudflaredUUID   uuid.UUID
-	config            *TunnelConfig
-	orchestrator      *orchestration.Orchestrator
-	edgeIPs           *edgediscovery.Edge
-	tunnelErrors      chan tunnelError
-	tunnelsConnecting map[int]chan struct{}
+	cloudflaredUUID         uuid.UUID
+	config                  *TunnelConfig
+	orchestrator            *orchestration.Orchestrator
+	edgeIPs                 *edgediscovery.Edge
+	edgeTunnelServer        EdgeTunnelServer
+	tunnelErrors            chan tunnelError
+	tunnelsConnecting       map[int]chan struct{}
+	tunnelsProtocolFallback map[int]*protocolFallback
 	// nextConnectedIndex and nextConnectedSignal are used to wait for all
 	// currently-connecting tunnels to finish connecting so we can reset backoff timer
 	nextConnectedIndex  int
@@ -62,7 +67,6 @@ var errEarlyShutdown = errors.New("shutdown started")
 
 type tunnelError struct {
 	index int
-	addr  *allregions.EdgeAddr
 	err   error
 }
 
@@ -72,16 +76,45 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 		return nil, fmt.Errorf("failed to generate cloudflared instance ID: %w", err)
 	}
 
+	isStaticEdge := len(config.EdgeAddrs) > 0
+
 	var edgeIPs *edgediscovery.Edge
-	if len(config.EdgeAddrs) > 0 {
+	if isStaticEdge { // static edge addresses
 		edgeIPs, err = edgediscovery.StaticEdge(config.Log, config.EdgeAddrs)
 	} else {
-		edgeIPs, err = edgediscovery.ResolveEdge(config.Log, config.Region)
+		edgeIPs, err = edgediscovery.ResolveEdge(config.Log, config.Region, config.EdgeIPVersion)
 	}
 	if err != nil {
 		return nil, err
 	}
 
+	reconnectCredentialManager := newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections)
+
+	tracker := tunnelstate.NewConnTracker(config.Log)
+	log := NewConnAwareLogger(config.Log, tracker, config.Observer)
+
+	var edgeAddrHandler EdgeAddrHandler
+	if isStaticEdge { // static edge addresses
+		edgeAddrHandler = &IPAddrFallback{}
+	} else if config.EdgeIPVersion == allregions.IPv6Only || config.EdgeIPVersion == allregions.Auto {
+		edgeAddrHandler = &IPAddrFallback{}
+	} else { // IPv4Only
+		edgeAddrHandler = &DefaultAddrFallback{}
+	}
+
+	edgeTunnelServer := EdgeTunnelServer{
+		config:            config,
+		cloudflaredUUID:   cloudflaredUUID,
+		orchestrator:      orchestrator,
+		credentialManager: reconnectCredentialManager,
+		edgeAddrs:         edgeIPs,
+		edgeAddrHandler:   edgeAddrHandler,
+		tracker:           tracker,
+		reconnectCh:       reconnectCh,
+		gracefulShutdownC: gracefulShutdownC,
+		connAwareLogger:   log,
+	}
+
 	useReconnectToken := false
 	if config.ClassicTunnel != nil {
 		useReconnectToken = config.ClassicTunnel.UseReconnectToken
@@ -92,11 +125,13 @@ func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrato
 		config:                     config,
 		orchestrator:               orchestrator,
 		edgeIPs:                    edgeIPs,
+		edgeTunnelServer:           edgeTunnelServer,
 		tunnelErrors:               make(chan tunnelError),
 		tunnelsConnecting:          map[int]chan struct{}{},
-		log:                        NewConnAwareLogger(config.Log, config.Observer),
+		tunnelsProtocolFallback:    map[int]*protocolFallback{},
+		log:                        log,
 		logTransport:               config.LogTransport,
-		reconnectCredentialManager: newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections),
+		reconnectCredentialManager: reconnectCredentialManager,
 		useReconnectToken:          useReconnectToken,
 		reconnectCh:                reconnectCh,
 		gracefulShutdownC:          gracefulShutdownC,
@@ -143,11 +178,22 @@ func (s *Supervisor) Run(
 				tunnelsActive--
 			}
 			return nil
-		// startTunnel returned with error
+		// startTunnel completed with a response
 		// (note that this may also be caused by context cancellation)
 		case tunnelError := <-s.tunnelErrors:
 			tunnelsActive--
 			if tunnelError.err != nil && !shuttingDown {
+				switch tunnelError.err.(type) {
+				case ReconnectSignal:
+					// For tunnels that closed with reconnect signal, we reconnect immediately
+					go s.startTunnel(ctx, tunnelError.index, s.newConnectedTunnelSignal(tunnelError.index))
+					tunnelsActive++
+					continue
+				}
+				// Make sure we don't continue if there is no more fallback allowed
+				if _, retry := s.tunnelsProtocolFallback[tunnelError.index].GetMaxBackoffDuration(ctx); !retry {
+					continue
+				}
 				s.log.ConnAwareLogger().Err(tunnelError.err).Int(connection.LogFieldConnIndex, tunnelError.index).Msg("Connection terminated")
 				tunnelsWaiting = append(tunnelsWaiting, tunnelError.index)
 				s.waitForNextTunnel(tunnelError.index)
@@ -155,10 +201,9 @@ func (s *Supervisor) Run(
 				if backoffTimer == nil {
 					backoffTimer = backoff.BackoffTimer()
 				}
-
-				// Previously we'd mark the edge address as bad here, but now we'll just silently use another.
 			} else if tunnelsActive == 0 {
-				// all connected tunnels exited gracefully, no more work to do
+				s.log.ConnAwareLogger().Msg("no more connections active and exiting")
+				// All connected tunnels exited gracefully, no more work to do
 				return nil
 			}
 		// Backoff was set and its timer expired
@@ -192,6 +237,8 @@ func (s *Supervisor) Run(
 }
 
 // Returns nil if initialization succeeded, else the initialization error.
+// Attempts here will be made to connect one tunnel, if successful, it will
+// connect the available tunnels up to config.HAConnections.
 func (s *Supervisor) initialize(
 	ctx context.Context,
 	connectedSignal *signal.Signal,
@@ -201,8 +248,15 @@ func (s *Supervisor) initialize(
 		s.log.Logger().Info().Msgf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
 		s.config.HAConnections = availableAddrs
 	}
+	s.tunnelsProtocolFallback[0] = &protocolFallback{
+		retry.BackoffHandler{MaxRetries: s.config.Retries},
+		s.config.ProtocolSelector.Current(),
+		false,
+	}
 
 	go s.startFirstTunnel(ctx, connectedSignal)
+
+	// Wait for response from first tunnel before proceeding to attempt other HA edge tunnels
 	select {
 	case <-ctx.Done():
 		<-s.tunnelErrors
@@ -213,8 +267,14 @@ func (s *Supervisor) initialize(
 		return errEarlyShutdown
 	case <-connectedSignal.Wait():
 	}
+
 	// At least one successful connection, so start the rest
 	for i := 1; i < s.config.HAConnections; i++ {
+		s.tunnelsProtocolFallback[i] = &protocolFallback{
+			retry.BackoffHandler{MaxRetries: s.config.Retries},
+			s.config.ProtocolSelector.Current(),
+			false,
+		}
 		ch := signal.New(make(chan struct{}))
 		go s.startTunnel(ctx, i, ch)
 		time.Sleep(registrationInterval)
@@ -229,102 +289,65 @@ func (s *Supervisor) startFirstTunnel(
 	connectedSignal *signal.Signal,
 ) {
 	var (
-		addr *allregions.EdgeAddr
-		err  error
+		err error
 	)
 	const firstConnIndex = 0
+	isStaticEdge := len(s.config.EdgeAddrs) > 0
 	defer func() {
-		s.tunnelErrors <- tunnelError{index: firstConnIndex, addr: addr, err: err}
+		s.tunnelErrors <- tunnelError{index: firstConnIndex, err: err}
 	}()
 
-	addr, err = s.edgeIPs.GetAddr(firstConnIndex)
-	if err != nil {
-		return
-	}
-
-	err = ServeTunnelLoop(
-		ctx,
-		s.reconnectCredentialManager,
-		s.config,
-		s.orchestrator,
-		addr,
-		s.log,
-		firstConnIndex,
-		connectedSignal,
-		s.cloudflaredUUID,
-		s.reconnectCh,
-		s.gracefulShutdownC,
-	)
 	// If the first tunnel disconnects, keep restarting it.
-	edgeErrors := 0
-	for s.unusedIPs() {
+	for {
+		err = s.edgeTunnelServer.Serve(ctx, firstConnIndex, s.tunnelsProtocolFallback[firstConnIndex], connectedSignal)
 		if ctx.Err() != nil {
 			return
 		}
-		switch err.(type) {
-		case nil:
-			return
-		// try the next address if it was a dialError(network problem) or
-		// dupConnRegisterTunnelError
-		case edgediscovery.DialError, connection.DupConnRegisterTunnelError:
-			edgeErrors++
-		default:
+		if err == nil {
 			return
 		}
-		if edgeErrors >= 2 {
-			addr, err = s.edgeIPs.GetDifferentAddr(firstConnIndex)
-			if err != nil {
+		// Make sure we don't continue if there is no more fallback allowed
+		if _, retry := s.tunnelsProtocolFallback[firstConnIndex].GetMaxBackoffDuration(ctx); !retry {
+			return
+		}
+		// Try again for Unauthorized errors because we hope them to be
+		// transient due to edge propagation lag on new Tunnels.
+		if strings.Contains(err.Error(), "Unauthorized") {
+			continue
+		}
+		switch err.(type) {
+		case edgediscovery.ErrNoAddressesLeft:
+			// If your provided addresses are not available, we will keep trying regardless.
+			if !isStaticEdge {
 				return
 			}
+		case connection.DupConnRegisterTunnelError,
+			*quic.IdleTimeoutError,
+			edgediscovery.DialError,
+			*connection.EdgeQuicDialError:
+			// Try again for these types of errors
+		default:
+			// Uncaught errors should bail startup
+			return
 		}
-		err = ServeTunnelLoop(
-			ctx,
-			s.reconnectCredentialManager,
-			s.config,
-			s.orchestrator,
-			addr,
-			s.log,
-			firstConnIndex,
-			connectedSignal,
-			s.cloudflaredUUID,
-			s.reconnectCh,
-			s.gracefulShutdownC,
-		)
 	}
 }
 
 // startTunnel starts a new tunnel connection. The resulting error will be sent on
-// s.tunnelErrors.
+// s.tunnelError as this is expected to run in a goroutine.
 func (s *Supervisor) startTunnel(
 	ctx context.Context,
 	index int,
 	connectedSignal *signal.Signal,
 ) {
 	var (
-		addr *allregions.EdgeAddr
-		err  error
+		err error
 	)
 	defer func() {
-		s.tunnelErrors <- tunnelError{index: index, addr: addr, err: err}
+		s.tunnelErrors <- tunnelError{index: index, err: err}
 	}()
 
-	addr, err = s.edgeIPs.GetDifferentAddr(index)
-	if err != nil {
-		return
-	}
-	err = ServeTunnelLoop(
-		ctx,
-		s.reconnectCredentialManager,
-		s.config,
-		s.orchestrator,
-		addr,
-		s.log,
-		uint8(index),
-		connectedSignal,
-		s.cloudflaredUUID,
-		s.reconnectCh,
-		s.gracefulShutdownC,
-	)
+	err = s.edgeTunnelServer.Serve(ctx, uint8(index), s.tunnelsProtocolFallback[index], connectedSignal)
 }
 
 func (s *Supervisor) newConnectedTunnelSignal(index int) *signal.Signal {

supervisor/tunnel.go

@@ -26,6 +26,7 @@ import (
 	"github.com/cloudflare/cloudflared/signal"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelstate"
 )
 
 const (
@@ -33,6 +34,7 @@ const (
 	FeatureSerializedHeaders = "serialized_headers"
 	FeatureQuickReconnects   = "quick_reconnects"
 	FeatureAllowRemoteConfig = "allow_remote_config"
+	FeatureDatagramV2        = "support_datagram_v2"
 )
 
 type TunnelConfig struct {
@@ -43,6 +45,7 @@ type TunnelConfig struct {
 	CloseConnOnce   *sync.Once // Used to close connectedSignal no more than once
 	EdgeAddrs       []string
 	Region          string
+	EdgeIPVersion   allregions.ConfigIPVersion
 	HAConnections   int
 	IncidentLookup  IncidentLookup
 	IsAutoupdated   bool
@@ -121,30 +124,83 @@ func StartTunnelDaemon(
 	return s.Run(ctx, connectedSignal)
 }
 
-func ServeTunnelLoop(
-	ctx context.Context,
-	credentialManager *reconnectCredentialManager,
-	config *TunnelConfig,
-	orchestrator *orchestration.Orchestrator,
-	addr *allregions.EdgeAddr,
-	connAwareLogger *ConnAwareLogger,
-	connIndex uint8,
-	connectedSignal *signal.Signal,
-	cloudflaredUUID uuid.UUID,
-	reconnectCh chan ReconnectSignal,
-	gracefulShutdownC <-chan struct{},
-) error {
+// EdgeAddrHandler provides a mechanism switch between behaviors in ServeTunnel
+// for handling the errors when attempting to make edge connections.
+type EdgeAddrHandler interface {
+	// ShouldGetNewAddress will check the edge connection error and determine if
+	// the edge address should be replaced with a new one. Also, will return if the
+	// error should be recognized as a connectivity error, or otherwise, a general
+	// application error.
+	ShouldGetNewAddress(err error) (needsNewAddress bool, isConnectivityError bool)
+}
+
+// DefaultAddrFallback will always return false for isConnectivityError since this
+// handler is a way to provide the legacy behavior in the new edge discovery algorithm.
+type DefaultAddrFallback struct {
+	edgeErrors int
+}
+
+func (f DefaultAddrFallback) ShouldGetNewAddress(err error) (needsNewAddress bool, isConnectivityError bool) {
+	switch err.(type) {
+	case nil: // maintain current IP address
+	// DupConnRegisterTunnelError should indicate to get a new address immediately
+	case connection.DupConnRegisterTunnelError:
+		return true, false
+	// Try the next address if it was a quic.IdleTimeoutError
+	case *quic.IdleTimeoutError,
+		edgediscovery.DialError,
+		*connection.EdgeQuicDialError:
+		// Wait for two failures before falling back to a new address
+		f.edgeErrors++
+		if f.edgeErrors >= 2 {
+			f.edgeErrors = 0
+			return true, false
+		}
+	default: // maintain current IP address
+	}
+	return false, false
+}
+
+// IPAddrFallback will have more conditions to fall back to a new address for certain
+// edge connection errors. This means that this handler will return true for isConnectivityError
+// for more cases like duplicate connection register and edge quic dial errors.
+type IPAddrFallback struct{}
+
+func (f IPAddrFallback) ShouldGetNewAddress(err error) (needsNewAddress bool, isConnectivityError bool) {
+	switch err.(type) {
+	case nil: // maintain current IP address
+	// Try the next address if it was a quic.IdleTimeoutError
+	// DupConnRegisterTunnelError needs to also receive a new ip address
+	case connection.DupConnRegisterTunnelError,
+		*quic.IdleTimeoutError:
+		return true, false
+	// Network problems should be retried with new address immediately and report
+	// as connectivity error
+	case edgediscovery.DialError, *connection.EdgeQuicDialError:
+		return true, true
+	default: // maintain current IP address
+	}
+	return false, false
+}
+
+type EdgeTunnelServer struct {
+	config            *TunnelConfig
+	cloudflaredUUID   uuid.UUID
+	orchestrator      *orchestration.Orchestrator
+	credentialManager *reconnectCredentialManager
+	edgeAddrHandler   EdgeAddrHandler
+	edgeAddrs         *edgediscovery.Edge
+	reconnectCh       chan ReconnectSignal
+	gracefulShutdownC <-chan struct{}
+	tracker           *tunnelstate.ConnTracker
+
+	connAwareLogger *ConnAwareLogger
+}
+
+func (e EdgeTunnelServer) Serve(ctx context.Context, connIndex uint8, protocolFallback *protocolFallback, connectedSignal *signal.Signal) error {
 	haConnections.Inc()
 	defer haConnections.Dec()
 
-	logger := config.Log.With().Uint8(connection.LogFieldConnIndex, connIndex).Logger()
-	connLog := connAwareLogger.ReplaceLogger(&logger)
-
-	protocolFallback := &protocolFallback{
-		retry.BackoffHandler{MaxRetries: config.Retries},
-		config.ProtocolSelector.Current(),
-		false,
-	}
 	connectedFuse := h2mux.NewBooleanFuse()
 	go func() {
 		if connectedFuse.Await() {
@@ -153,54 +209,89 @@ func ServeTunnelLoop(
 	}()
 	// Ensure the above goroutine will terminate if we return without connecting
 	defer connectedFuse.Fuse(false)
+
+	// Fetch IP address to associated connection index
+	addr, err := e.edgeAddrs.GetAddr(int(connIndex))
+	switch err.(type) {
+	case nil: // no error
+	case edgediscovery.ErrNoAddressesLeft:
+		return err
+	default:
+		return err
+	}
+
+	logger := e.config.Log.With().
+		IPAddr(connection.LogFieldIPAddress, addr.UDP.IP).
+		Uint8(connection.LogFieldConnIndex, connIndex).
+		Logger()
+	connLog := e.connAwareLogger.ReplaceLogger(&logger)
 	// Each connection to keep its own copy of protocol, because individual connections might fallback
 	// to another protocol when a particular metal doesn't support new protocol
-	for {
-		err, recoverable := ServeTunnel(
-			ctx,
-			connLog,
-			credentialManager,
-			config,
-			orchestrator,
-			addr,
-			connIndex,
-			connectedFuse,
-			protocolFallback,
-			cloudflaredUUID,
-			reconnectCh,
-			protocolFallback.protocol,
-			gracefulShutdownC,
-		)
+	// Each connection can also have it's own IP version because individual connections might fallback
+	// to another IP version.
+	err, recoverable := ServeTunnel(
+		ctx,
+		connLog,
+		e.credentialManager,
+		e.config,
+		e.orchestrator,
+		addr,
+		connIndex,
+		connectedFuse,
+		protocolFallback,
+		e.cloudflaredUUID,
+		e.reconnectCh,
+		protocolFallback.protocol,
+		e.gracefulShutdownC,
+	)
 
-		if recoverable {
-			duration, ok := protocolFallback.GetMaxBackoffDuration(ctx)
-			if !ok {
-				return err
-			}
-			config.Observer.SendReconnect(connIndex)
-			connLog.Logger().Info().Msgf("Retrying connection in up to %s seconds", duration)
+	// If the connection is recoverable, we want to maintain the same IP
+	// but backoff a reconnect with some duration.
+	if recoverable {
+		duration, ok := protocolFallback.GetMaxBackoffDuration(ctx)
+		if !ok {
+			return err
 		}
+		e.config.Observer.SendReconnect(connIndex)
+		connLog.Logger().Info().Msgf("Retrying connection in up to %s seconds", duration)
+	}
 
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-gracefulShutdownC:
-			return nil
-		case <-protocolFallback.BackoffTimer():
-			if !recoverable {
-				return err
-			}
-
-			if !selectNextProtocol(
-				connLog.Logger(),
-				protocolFallback,
-				config.ProtocolSelector,
-				err,
-			) {
-				return err
-			}
+	// Check if the connection error was from an IP issue with the host or
+	// establishing a connection to the edge and if so, rotate the IP address.
+	yes, hasConnectivityError := e.edgeAddrHandler.ShouldGetNewAddress(err)
+	if yes {
+		if _, err := e.edgeAddrs.GetDifferentAddr(int(connIndex), hasConnectivityError); err != nil {
+			return err
 		}
 	}
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-e.gracefulShutdownC:
+		return nil
+	case <-protocolFallback.BackoffTimer():
+		if !recoverable {
+			return err
+		}
+
+		// If a single connection has connected with the current protocol, we know we know we don't have to fallback
+		// to a different protocol.
+		if e.tracker.HasConnectedWith(e.config.ProtocolSelector.Current()) {
+			return err
+		}
+
+		if !selectNextProtocol(
+			connLog.Logger(),
+			protocolFallback,
+			e.config.ProtocolSelector,
+			err,
+		) {
+			return err
+		}
+	}
+
+	return err
 }
 
 // protocolFallback is a wrapper around backoffHandler that will try fallback option when backoff reaches
@@ -232,9 +323,22 @@ func selectNextProtocol(
 ) bool {
 	var idleTimeoutError *quic.IdleTimeoutError
 	isNetworkActivityTimeout := errors.As(cause, &idleTimeoutError)
+	edgeQuicDialError, ok := cause.(*connection.EdgeQuicDialError)
+	if !isNetworkActivityTimeout && ok {
+		isNetworkActivityTimeout = errors.As(edgeQuicDialError.Cause, &idleTimeoutError)
+	}
 	_, hasFallback := selector.Fallback()
 
 	if protocolBackoff.ReachedMaxRetries() || (hasFallback && isNetworkActivityTimeout) {
+		if isNetworkActivityTimeout {
+			connLog.Warn().Msg("If this log occurs persistently, and cloudflared is unable to connect to " +
+				"Cloudflare Network with `quic` protocol, then most likely your machine/network is getting its egress " +
+				"UDP to port 7844 (or others) blocked or dropped. Make sure to allow egress connectivity as per " +
+				"https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/ports-and-ips/\n" +
+				"If you are using private routing to this Tunnel, then UDP (and Private DNS Resolution) will not work " +
+				"unless your cloudflared can connect with Cloudflare Network with `quic`.")
+		}
+
 		fallback, hasFallback := selector.Fallback()
 		if !hasFallback {
 			return false
@@ -316,8 +420,11 @@ func ServeTunnel(
 				connLog.ConnAwareLogger().Msg(activeIncidentsMsg(incidents))
 			}
 			return err.Cause, !err.Permanent
+		case *connection.EdgeQuicDialError:
+			return err, true
 		case ReconnectSignal:
 			connLog.Logger().Info().
+				IPAddr(connection.LogFieldIPAddress, addr.UDP.IP).
 				Uint8(connection.LogFieldConnIndex, connIndex).
 				Msgf("Restarting connection due to reconnect signal in %s", err.Delay)
 			err.DelayBeforeReconnect()
@@ -359,9 +466,11 @@ func serveTunnel(
 		connectedFuse,
 		config.NamedTunnel,
 		connIndex,
+		addr.UDP.IP,
 		nil,
 		gracefulShutdownC,
 		config.GracePeriod,
+		protocol,
 	)
 
 	switch protocol {
@@ -458,6 +567,7 @@ func ServeH2mux(
 		connIndex,
 		config.Observer,
 		gracefulShutdownC,
+		config.Log,
 	)
 	if err != nil {
 		if !recoverable {
@@ -516,6 +626,7 @@ func ServeHTTP2(
 		err := listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
 		if err != nil {
 			// forcefully break the connection (this is only used for testing)
+			connLog.Logger().Debug().Msg("Forcefully breaking http2 connection")
 			_ = tlsServerConn.Close()
 		}
 		return err
@@ -540,9 +651,9 @@ func ServeQUIC(
 	quicConfig := &quic.Config{
 		HandshakeIdleTimeout:  quicpogs.HandshakeIdleTimeout,
 		MaxIdleTimeout:        quicpogs.MaxIdleTimeout,
+		KeepAlivePeriod:       quicpogs.MaxIdlePingPeriod,
 		MaxIncomingStreams:    connection.MaxConcurrentStreams,
 		MaxIncomingUniStreams: connection.MaxConcurrentStreams,
-		KeepAlive:             true,
 		EnableDatagrams:       true,
 		MaxDatagramFrameSize:  quicpogs.MaxDatagramFrameSize,
 		Tracer:                quicpogs.NewClientTracer(connLogger.Logger(), connIndex),
@@ -574,6 +685,7 @@ func ServeQUIC(
 		err := listenReconnect(serveCtx, reconnectCh, gracefulShutdownC)
 		if err != nil {
 			// forcefully break the connection (this is only used for testing)
+			connLogger.Logger().Debug().Msg("Forcefully breaking quic connection")
 			quicConn.Close()
 		}
 		return err