Release 2022.12.0

This PR does two things:
It changes how we fallback to a lower protocol: The current state
is to try connecting with a protocol. If it fails, fall back to a
lower protocol. And try connecting with that and so on. With this PR,
if we fail to connect with a protocol, we will try to connect to other
edge addresses first. Only if we fail to connect to those will we
fall back to a lower protocol.
It fixes a behaviour where if we fail to connect to an edge addr,
we keep re-trying the same address over and over again.
This PR now switches between edge addresses on subsequent connecton attempts.
Note that through these switches, it still respects the backoff time.
(We are connecting to a different edge, but this helps to not bombard an edge
address with connect requests if a particular edge addresses stops working).

.docker-images

@@ -1,8 +1,12 @@
 images:
   - name: cloudflared
-    dockerfile: Dockerfile
+    dockerfile: Dockerfile.$ARCH
     context: .
+    version_file: versions
     registries:
     - name: docker.io/cloudflare
       user: env:DOCKER_USER
       password: env:DOCKER_PASSWORD
+    architectures:
+    - amd64
+    - arm64

.github/ISSUE_TEMPLATE/---bug-report.md

@@ -1,9 +1,8 @@
 ---
-name: "Bug report \U0001F41B"
+name: "\U0001F41B Bug report"
 about: Create a report to help us improve cloudflared
-title: ''
-labels: awaiting reply, bug
-assignees: ''
+title: "\U0001F41B"
+labels: 'Priority: Normal, Type: Bug'
 
 ---
 
@@ -16,6 +15,10 @@ Steps to reproduce the behavior:
 2. Run '....'
 3. See error
 
+If it's an issue with Cloudflare Tunnel:
+4. Tunnel ID : 
+5. cloudflared config: 
+
 **Expected behavior**
 A clear and concise description of what you expected to happen.
 

.github/ISSUE_TEMPLATE/---documentation.md

@@ -0,0 +1,16 @@
+---
+name: "\U0001F4DD Documentation"
+about: Request new or updated documentation for cloudflared
+title: "\U0001F4DD"
+labels: 'Priority: Normal, Type: Documentation'
+
+---
+
+**Available Documentation**
+A link to the documentation that is available today and the areas which could be improved. 
+
+**Suggested Documentation**
+A clear and concise description of the documentation, tutorial, or guide that should be added. 
+
+**Additional context**
+Add any other context or screenshots about the documentation request here.

.github/ISSUE_TEMPLATE/---feature-request.md

@@ -1,9 +1,8 @@
 ---
-name: "Feature request \U0001F4A1"
+name: "\U0001F4A1 Feature request"
 about: Suggest a feature or enhancement for cloudflared
-title: ''
-labels: awaiting reply, feature-request
-assignees: ''
+title: "\U0001F4A1"
+labels: 'Priority: Normal, Type: Feature Request'
 
 ---
 

.github/workflows/check.yaml

@@ -4,7 +4,7 @@ jobs:
   check:
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.19.x]
         os: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -12,8 +12,6 @@ jobs:
       uses: actions/setup-go@v2
       with:
         go-version: ${{ matrix.go-version }}
-    - name: Install go-sumtype
-      run: go get github.com/sudarshan-reddy/go-sumtype
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Test

.gitignore

@@ -10,6 +10,7 @@ cscope.*
 /cloudflared.exe
 /cloudflared.msi
 /cloudflared-x86-64*
+/cloudflared.1
 /packaging
 .DS_Store
 *-session.log

.teamcity/build-macos.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -exo pipefail
+
 if [[ "$(uname)" != "Darwin" ]] ; then
     echo "This should be run on macOS"
     exit 1
@@ -33,7 +35,9 @@ if [[ ! -z "$CFD_CODE_SIGN_KEY" ]]; then
   if [[ ! -z "$CFD_CODE_SIGN_PASS" ]]; then
     # write private key to disk and then import it keychain
     echo -n -e ${CFD_CODE_SIGN_KEY} | base64 -D > ${CODE_SIGN_PRIV}
-    out=$(security import ${CODE_SIGN_PRIV} -A -P "${CFD_CODE_SIGN_PASS}" 2>&1)
+    # we set || true  here and for every `security import invoke` because the  "duplicate SecKeychainItemImport" error 
+    # will cause set -e to exit 1. It is okay we do this because we deliberately handle this error in the lines below.
+    out=$(security import ${CODE_SIGN_PRIV} -A -P "${CFD_CODE_SIGN_PASS}" 2>&1) || true 
     exitcode=$?
     if [ -n "$out" ]; then
       if [ $exitcode -eq 0 ]; then
@@ -53,7 +57,7 @@ fi
 if [[ ! -z "$CFD_CODE_SIGN_CERT" ]]; then
   # write certificate to disk and then import it keychain
   echo -n -e ${CFD_CODE_SIGN_CERT} | base64 -D > ${CODE_SIGN_CERT}
-  out1=$(security import ${CODE_SIGN_CERT} -A 2>&1)
+  out1=$(security import ${CODE_SIGN_CERT} -A 2>&1) || true
   exitcode1=$?
   if [ -n "$out1" ]; then
     if [ $exitcode1 -eq 0 ]; then
@@ -75,7 +79,7 @@ if [[ ! -z "$CFD_INSTALLER_KEY" ]]; then
   if [[ ! -z "$CFD_INSTALLER_PASS" ]]; then
     # write private key to disk and then import it into the keychain
     echo -n -e ${CFD_INSTALLER_KEY} | base64 -D > ${INSTALLER_PRIV}
-    out2=$(security import ${INSTALLER_PRIV} -A -P "${CFD_INSTALLER_PASS}" 2>&1)
+    out2=$(security import ${INSTALLER_PRIV} -A -P "${CFD_INSTALLER_PASS}" 2>&1) || true
     exitcode2=$?
     if [ -n "$out2" ]; then
       if [ $exitcode2 -eq 0 ]; then
@@ -95,7 +99,7 @@ fi
 if [[ ! -z "$CFD_INSTALLER_CERT" ]]; then
   # write certificate to disk and then import it keychain
   echo -n -e ${CFD_INSTALLER_CERT} | base64 -D > ${INSTALLER_CERT}
-  out3=$(security import ${INSTALLER_CERT} -A 2>&1)
+  out3=$(security import ${INSTALLER_CERT} -A 2>&1) || true
   exitcode3=$?
   if [ -n "$out3" ]; then
     if [ $exitcode3 -eq 0 ]; then
@@ -138,14 +142,10 @@ fi
 if [[ ! -z "$CODE_SIGN_NAME" ]]; then
   codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME}
   
-  # notarize the binary
-  if [[ ! -z "$CFD_NOTE_PASSWORD" ]]; then
-    zip "${BINARY_NAME}.zip" ${BINARY_NAME} 
-    xcrun altool --notarize-app -f "${BINARY_NAME}.zip" -t osx -u ${CFD_NOTE_USERNAME} -p ${CFD_NOTE_PASSWORD} --primary-bundle-id ${BUNDLE_ID}
-  fi
+ # notarize the binary
+ # TODO: https://jira.cfdata.org/browse/TUN-5789
 fi
 
-
 # creating build directory
 rm -rf $TARGET_DIRECTORY
 mkdir "${TARGET_DIRECTORY}"
@@ -169,10 +169,7 @@ if [[ ! -z "$PKG_SIGN_NAME" ]]; then
       ${PKGNAME}
 
       # notarize the package
-      if [[ ! -z "$CFD_NOTE_PASSWORD" ]]; then
-        xcrun altool --notarize-app -f ${PKGNAME} -t osx -u ${CFD_NOTE_USERNAME} -p ${CFD_NOTE_PASSWORD} --primary-bundle-id ${BUNDLE_ID}
-        xcrun stapler staple ${PKGNAME}
-      fi
+      # TODO: https://jira.cfdata.org/browse/TUN-5789
 else
     pkgbuild --identifier com.cloudflare.${PRODUCT} \
       --version ${VERSION} \

.teamcity/package-windows.sh

@@ -0,0 +1,17 @@
+VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
+echo $VERSION
+
+export TARGET_OS=windows
+# This controls the directory the built artifacts go into
+export ARTIFACT_DIR=built_artifacts/
+mkdir -p $ARTIFACT_DIR
+windowsArchs=("amd64" "386")
+for arch in ${windowsArchs[@]}; do
+    export TARGET_ARCH=$arch
+    # Copy exe into final directory
+    cp ./artifacts/cloudflared-windows-$arch.exe $ARTIFACT_DIR/cloudflared-windows-$arch.exe
+    cp ./artifacts/cloudflared-windows-$arch.exe ./cloudflared.exe
+    make cloudflared-msi
+    # Copy msi into final directory
+    mv cloudflared-$VERSION-$arch.msi $ARTIFACT_DIR/cloudflared-windows-$arch.msi
+done

.teamcity/update-homebrew-core.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if ! VERSION="$(git describe --tags --exact-match 2>/dev/null)" ; then
+    echo "Skipping public release for an untagged commit."
+    echo "##teamcity[buildStatus status='SUCCESS' text='Skipped due to lack of tag']"
+    exit 0
+fi
+
+if [[ "${HOMEBREW_GITHUB_API_TOKEN:-}" == "" ]] ; then
+    echo "Missing GITHUB_API_TOKEN"
+    exit 1
+fi
+
+# "install" Homebrew
+git clone https://github.com/Homebrew/brew tmp/homebrew
+eval "$(tmp/homebrew/bin/brew shellenv)"
+brew update --force --quiet
+chmod -R go-w "$(brew --prefix)/share/zsh"
+
+git config --global user.name "cloudflare-warp-bot"
+git config --global user.email "warp-bot@cloudflare.com"
+
+# bump formula pr
+brew bump-formula-pr cloudflared --version="$VERSION" --no-browse --no-audit

CHANGES.md

@@ -1,4 +1,75 @@
-**Experimental**: This is a new format for release notes. The format and availability is subject to change.
+## 2022.9.0
+### New Features
+- cloudflared now rejects ingress rules with invalid http status codes for http_status.
+
+## 2022.8.1
+### New Features
+- cloudflared now remembers if it connected to a certain protocol successfully. If it did, it does not fall back to a lower
+  protocol on connection failures.
+
+## 2022.7.1
+### New Features
+- It is now possible to connect cloudflared tunnel to Cloudflare Global Network with IPv6. See `cloudflared tunnel --help` and look for `edge-ip-version` for more information. For now, the default behavior is to still connect with IPv4 only.
+
+### Bug Fixes
+- Several bug fixes related with QUIC transport (used between cloudflared tunnel and Cloudflare Global Network). Updating to this version is highly recommended.
+
+## 2022.4.0
+### Bug Fixes
+- `cloudflared tunnel run` no longer logs the Tunnel token or JSON credentials in clear text as those are the secret
+that allows to run the Tunnel.
+
+## 2022.3.4
+### New Features
+- It is now possible to retrieve the credentials that allow to run a Tunnel in case you forgot/lost them. This is
+achievable with: `cloudflared tunnel token --cred-file /path/to/file.json TUNNEL`. This new feature only works for
+Tunnels created with cloudflared version 2022.3.0 or more recent.
+
+### Bug Fixes
+- `cloudflared service install` now starts the underlying agent service on Linux operating system (similarly to the
+behaviour in Windows and MacOS).
+
+## 2022.3.3
+### Bug Fixes
+- `cloudflared service install` now starts the underlying agent service on Windows operating system (similarly to the
+behaviour in MacOS).
+
+## 2022.3.1
+### Bug Fixes
+- Various fixes to the reliability of `quic` protocol, including an edge case that could lead to cloudflared crashing.
+
+## 2022.3.0
+### New Features
+- It is now possible to configure Ingress Rules to point to an origin served by unix socket with either HTTP or HTTPS.
+If the origin starts with `unix:/` then we assume HTTP (existing behavior). Otherwise, the origin can start with
+`unix+tls:/` for HTTPS.
+
+## 2022.2.1
+### New Features
+- This project now has a new LICENSE that is more compliant with open source purposes.
+
+### Bug Fixes
+- Various fixes to the reliability of `quic` protocol.
+
+## 2022.1.3
+### New Features
+- New `cloudflared tunnel vnet` commands to allow for private routing to be virtualized. This means that the same CIDR
+can now be used to point to two different Tunnels with `cloudflared tunnel route ip` command. More information will be
+made available on blog.cloudflare.com and developers.cloudflare.com/cloudflare-one once the feature is globally available.
+
+### Bug Fixes
+- Correctly handle proxying UDP datagrams with no payload.
+- Bug fix for origins that use Server-Sent Events (SSE).
+
+## 2022.1.0
+### Improvements
+- If a specific `protocol` property is defined (e.g. for `quic`), cloudflared no longer falls back to an older protocol
+(such as `http2`) in face of connectivity errors. This is important because some features are only supported in a specific
+protocol (e.g. UDP proxying only works for `quic`). Hence, if a user chooses a protocol, cloudflared now adheres to it
+no matter what.
+
+### Bug Fixes
+- Stopping cloudflared running with `quic` protocol now respects graceful shutdown.
 
 ## 2021.12.2
 ### Bug Fixes

Dockerfile

@@ -1,12 +1,12 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.17.1 as builder
+FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \
     TARGET_GOARCH=${TARGET_GOARCH}
-
+    
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 
 # copy our sources into the builder image
@@ -16,7 +16,9 @@ COPY . .
 RUN make cloudflared
 
 # use a distroless base image with glibc
-FROM gcr.io/distroless/base-debian10:nonroot
+FROM gcr.io/distroless/base-debian11:nonroot
+
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
 
 # copy our compiled binary
 COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/

Dockerfile.amd64

@@ -0,0 +1,27 @@
+# use a builder image for building cloudflare
+FROM golang:1.19 as builder
+ENV GO111MODULE=on \
+    CGO_ENABLED=0 
+    
+WORKDIR /go/src/github.com/cloudflare/cloudflared/
+
+# copy our sources into the builder image
+COPY . .
+
+# compile cloudflared
+RUN GOOS=linux GOARCH=amd64 make cloudflared
+
+# use a distroless base image with glibc
+FROM gcr.io/distroless/base-debian11:nonroot
+
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
+# copy our compiled binary
+COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+
+# run as non-privileged user
+USER nonroot
+
+# command / entrypoint of container
+ENTRYPOINT ["cloudflared", "--no-autoupdate"]
+CMD ["version"]

Dockerfile.arm64

@@ -0,0 +1,27 @@
+# use a builder image for building cloudflare
+FROM golang:1.19 as builder
+ENV GO111MODULE=on \
+    CGO_ENABLED=0 
+    
+WORKDIR /go/src/github.com/cloudflare/cloudflared/
+
+# copy our sources into the builder image
+COPY . .
+
+# compile cloudflared
+RUN GOOS=linux GOARCH=arm64 make cloudflared
+
+# use a distroless base image with glibc
+FROM gcr.io/distroless/base-debian11:nonroot-arm64
+
+LABEL org.opencontainers.image.source="https://github.com/cloudflare/cloudflared"
+
+# copy our compiled binary
+COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+
+# run as non-privileged user
+USER nonroot
+
+# command / entrypoint of container
+ENTRYPOINT ["cloudflared", "--no-autoupdate"]
+CMD ["version"]

LICENSE

@@ -1,211 +1,202 @@
-Apache License
-Version 2.0, January 2004
-http://www.apache.org/licenses/
 
-TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-1. Definitions.
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-"License" shall mean the terms and conditions for use, reproduction,
-and distribution as defined by Sections 1 through 9 of this document.
+   1. Definitions.
 
-"Licensor" shall mean the copyright owner or entity authorized by
-the copyright owner that is granting the License.
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
 
-"Legal Entity" shall mean the union of the acting entity and all
-other entities that control, are controlled by, or are under common
-control with that entity. For the purposes of this definition,
-"control" means (i) the power, direct or indirect, to cause the
-direction or management of such entity, whether by contract or
-otherwise, or (ii) ownership of fifty percent (50%) or more of the
-outstanding shares, or (iii) beneficial ownership of such entity.
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
 
-"You" (or "Your") shall mean an individual or Legal Entity
-exercising permissions granted by this License.
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
 
-"Source" form shall mean the preferred form for making modifications,
-including but not limited to software source code, documentation
-source, and configuration files.
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
 
-"Object" form shall mean any form resulting from mechanical
-transformation or translation of a Source form, including but
-not limited to compiled object code, generated documentation,
-and conversions to other media types.
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
 
-"Work" shall mean the work of authorship, whether in Source or
-Object form, made available under the License, as indicated by a
-copyright notice that is included in or attached to the work
-(an example is provided in the Appendix below).
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
 
-"Derivative Works" shall mean any work, whether in Source or Object
-form, that is based on (or derived from) the Work and for which the
-editorial revisions, annotations, elaborations, or other modifications
-represent, as a whole, an original work of authorship. For the purposes
-of this License, Derivative Works shall not include works that remain
-separable from, or merely link (or bind by name) to the interfaces of,
-the Work and Derivative Works thereof.
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
 
-"Contribution" shall mean any work of authorship, including
-the original version of the Work and any modifications or additions
-to that Work or Derivative Works thereof, that is intentionally
-submitted to Licensor for inclusion in the Work by the copyright owner
-or by an individual or Legal Entity authorized to submit on behalf of
-the copyright owner. For the purposes of this definition, "submitted"
-means any form of electronic, verbal, or written communication sent
-to the Licensor or its representatives, including but not limited to
-communication on electronic mailing lists, source code control systems,
-and issue tracking systems that are managed by, or on behalf of, the
-Licensor for the purpose of discussing and improving the Work, but
-excluding communication that is conspicuously marked or otherwise
-designated in writing by the copyright owner as "Not a Contribution."
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
 
-"Contributor" shall mean Licensor and any individual or Legal Entity
-on behalf of whom a Contribution has been received by Licensor and
-subsequently incorporated within the Work.
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
 
-2. Grant of Copyright License. Subject to the terms and conditions of
-this License, each Contributor hereby grants to You a perpetual,
-worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-copyright license to reproduce, prepare Derivative Works of,
-publicly display, publicly perform, sublicense, and distribute the
-Work and such Derivative Works in Source or Object form.
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
 
-3. Grant of Patent License. Subject to the terms and conditions of
-this License, each Contributor hereby grants to You a perpetual,
-worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-(except as stated in this section) patent license to make, have made,
-use, offer to sell, sell, import, and otherwise transfer the Work,
-where such license applies only to those patent claims licensable
-by such Contributor that are necessarily infringed by their
-Contribution(s) alone or by combination of their Contribution(s)
-with the Work to which such Contribution(s) was submitted. If You
-institute patent litigation against any entity (including a
-cross-claim or counterclaim in a lawsuit) alleging that the Work
-or a Contribution incorporated within the Work constitutes direct
-or contributory patent infringement, then any patent licenses
-granted to You under this License for that Work shall terminate
-as of the date such litigation is filed.
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
 
-4. Redistribution. You may reproduce and distribute copies of the
-Work or Derivative Works thereof in any medium, with or without
-modifications, and in Source or Object form, provided that You
-meet the following conditions:
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
 
-(a) You must give any other recipients of the Work or
-Derivative Works a copy of this License; and
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
 
-(b) You must cause any modified files to carry prominent notices
-stating that You changed the files; and
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
 
-(c) You must retain, in the Source form of any Derivative Works
-that You distribute, all copyright, patent, trademark, and
-attribution notices from the Source form of the Work,
-excluding those notices that do not pertain to any part of
-the Derivative Works; and
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
 
-(d) If the Work includes a "NOTICE" text file as part of its
-distribution, then any Derivative Works that You distribute must
-include a readable copy of the attribution notices contained
-within such NOTICE file, excluding those notices that do not
-pertain to any part of the Derivative Works, in at least one
-of the following places: within a NOTICE text file distributed
-as part of the Derivative Works; within the Source form or
-documentation, if provided along with the Derivative Works; or,
-within a display generated by the Derivative Works, if and
-wherever such third-party notices normally appear. The contents
-of the NOTICE file are for informational purposes only and
-do not modify the License. You may add Your own attribution
-notices within Derivative Works that You distribute, alongside
-or as an addendum to the NOTICE text from the Work, provided
-that such additional attribution notices cannot be construed
-as modifying the License.
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
 
-You may add Your own copyright statement to Your modifications and
-may provide additional or different license terms and conditions
-for use, reproduction, or distribution of Your modifications, or
-for any such Derivative Works as a whole, provided Your use,
-reproduction, and distribution of the Work otherwise complies with
-the conditions stated in this License.
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
 
-5. Submission of Contributions. Unless You explicitly state otherwise,
-any Contribution intentionally submitted for inclusion in the Work
-by You to the Licensor shall be under the terms and conditions of
-this License, without any additional terms or conditions.
-Notwithstanding the above, nothing herein shall supersede or modify
-the terms of any separate license agreement you may have executed
-with Licensor regarding such Contributions.
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
 
-6. Trademarks. This License does not grant permission to use the trade
-names, trademarks, service marks, or product names of the Licensor,
-except as required for reasonable and customary use in describing the
-origin of the Work and reproducing the content of the NOTICE file.
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
 
-7. Disclaimer of Warranty. Unless required by applicable law or
-agreed to in writing, Licensor provides the Work (and each
-Contributor provides its Contributions) on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-implied, including, without limitation, any warranties or conditions
-of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-PARTICULAR PURPOSE. You are solely responsible for determining the
-appropriateness of using or redistributing the Work and assume any
-risks associated with Your exercise of permissions under this License.
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
 
-8. Limitation of Liability. In no event and under no legal theory,
-whether in tort (including negligence), contract, or otherwise,
-unless required by applicable law (such as deliberate and grossly
-negligent acts) or agreed to in writing, shall any Contributor be
-liable to You for damages, including any direct, indirect, special,
-incidental, or consequential damages of any character arising as a
-result of this License or out of the use or inability to use the
-Work (including but not limited to damages for loss of goodwill,
-work stoppage, computer failure or malfunction, or any and all
-other commercial damages or losses), even if such Contributor
-has been advised of the possibility of such damages.
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
 
-9. Accepting Warranty or Additional Liability. While redistributing
-the Work or Derivative Works thereof, You may choose to offer,
-and charge a fee for, acceptance of support, warranty, indemnity,
-or other liability obligations and/or rights consistent with this
-License. However, in accepting such obligations, You may act only
-on Your own behalf and on Your sole responsibility, not on behalf
-of any other Contributor, and only if You agree to indemnify,
-defend, and hold each Contributor harmless for any liability
-incurred by, or claims asserted against, such Contributor by reason
-of your accepting any such warranty or additional liability.
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
 
-END OF TERMS AND CONDITIONS
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
 
-APPENDIX: How to apply the Apache License to your work.
+   END OF TERMS AND CONDITIONS
 
-To apply the Apache License to your work, attach the following
-boilerplate notice, with the fields enclosed by brackets "[]"
-replaced with your own identifying information. (Don't include
-the brackets!)  The text should be enclosed in the appropriate
-comment syntax for the file format. We also recommend that a
-file or class name and description of purpose be included on the
-same "printed page" as the copyright notice for easier
-identification within third-party archives.
+   APPENDIX: How to apply the Apache License to your work.
 
-Copyright [yyyy] [name of copyright owner]
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+   Copyright [yyyy] [name of copyright owner]
 
-http://www.apache.org/licenses/LICENSE-2.0
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+       http://www.apache.org/licenses/LICENSE-2.0
 
-
-
-## Runtime Library Exception to the Apache 2.0 License: ##
-
-
-As an exception, if you use this Software to compile your source code and
-portions of this Software are embedded into the binary product as a result,
-you may redistribute such product without providing attribution as would
-otherwise be required by Sections 4(a), 4(b) and 4(d) of the License.
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -23,6 +23,9 @@ endif
 
 DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
 VERSION_FLAGS := -X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"
+ifdef PACKAGE_MANAGER
+	VERSION_FLAGS := $(VERSION_FLAGS) -X "github.com/cloudflare/cloudflared/cmd/cloudflared/updater.BuiltForPackageManager=$(PACKAGE_MANAGER)"
+endif
 
 LINK_FLAGS :=
 ifeq ($(FIPS), true)
@@ -37,10 +40,15 @@ ifneq ($(GO_BUILD_TAGS),)
 	GO_BUILD_TAGS := -tags "$(GO_BUILD_TAGS)"
 endif
 
-IMPORT_PATH   := github.com/cloudflare/cloudflared
-PACKAGE_DIR   := $(CURDIR)/packaging
-INSTALL_BINDIR := /usr/bin/
-MAN_DIR := /usr/share/man/man1/
+ifeq ($(debug), 1)
+	GO_BUILD_TAGS += -gcflags="all=-N -l"
+endif
+
+IMPORT_PATH    := github.com/cloudflare/cloudflared
+PACKAGE_DIR    := $(CURDIR)/packaging
+PREFIX         := /usr
+INSTALL_BINDIR := $(PREFIX)/bin/
+INSTALL_MANDIR := $(PREFIX)/share/man/man1/
 
 LOCAL_ARCH ?= $(shell uname -m)
 ifneq ($(GOARCH),)
@@ -59,6 +67,8 @@ else ifeq ($(LOCAL_ARCH),arm64)
     TARGET_ARCH ?= arm64
 else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 4),armv)
     TARGET_ARCH ?= arm
+else ifeq ($(LOCAL_ARCH),s390x)
+    TARGET_ARCH ?= s390x
 else
     $(error This system's architecture $(LOCAL_ARCH) isn't supported)
 endif
@@ -88,6 +98,16 @@ else
 	TARGET_PUBLIC_REPO ?= $(FLAVOR)
 endif
 
+ifneq ($(TARGET_ARM), )
+	ARM_COMMAND := GOARM=$(TARGET_ARM)
+endif
+
+ifeq ($(TARGET_ARM), 7) 
+	PACKAGE_ARCH := armhf
+else
+	PACKAGE_ARCH := $(TARGET_ARCH)
+endif
+
 .PHONY: all
 all: cloudflared test
 
@@ -101,7 +121,7 @@ ifeq ($(FIPS), true)
 	$(info Building cloudflared with go-fips)
 	cp -f fips/fips.go.linux-amd64 cmd/cloudflared/fips.go
 endif
-	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(LDFLAGS) $(IMPORT_PATH)/cmd/cloudflared
+	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) $(ARM_COMMAND) go build -v -mod=vendor $(GO_BUILD_TAGS) $(LDFLAGS) $(IMPORT_PATH)/cmd/cloudflared
 ifeq ($(FIPS), true)
 	rm -f cmd/cloudflared/fips.go
 	./check-fips.sh cloudflared
@@ -111,6 +131,10 @@ endif
 container:
 	docker build --build-arg=TARGET_ARCH=$(TARGET_ARCH) --build-arg=TARGET_OS=$(TARGET_OS) -t cloudflare/cloudflared-$(TARGET_OS)-$(TARGET_ARCH):"$(VERSION)" .
 
+.PHONY: generate-docker-version
+generate-docker-version:
+	echo latest $(VERSION) > versions
+
 .PHONY: test
 test: vet
 ifndef CI
@@ -125,52 +149,44 @@ endif
 test-ssh-server:
 	docker-compose -f ssh_server_tests/docker-compose.yml up
 
-define publish_package
-	chmod 664 $(BINARY_NAME)*.$(1); \
-	for HOST in $(CF_PKG_HOSTS); do \
-		ssh-keyscan -t ecdsa $$HOST >> ~/.ssh/known_hosts; \
-		scp -p -4 $(BINARY_NAME)*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/$(BINARY_NAME)/; \
-	done
-endef
+cloudflared.1: cloudflared_man_template
+	sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' cloudflared_man_template > cloudflared.1
 
-.PHONY: publish-deb
-publish-deb: cloudflared-deb
-	$(call publish_package,deb,apt)
-
-.PHONY: publish-rpm
-publish-rpm: cloudflared-rpm
-	$(call publish_package,rpm,yum)
+install: cloudflared cloudflared.1
+	mkdir -p $(DESTDIR)$(INSTALL_BINDIR) $(DESTDIR)$(INSTALL_MANDIR)
+	install -m755 cloudflared $(DESTDIR)$(INSTALL_BINDIR)/cloudflared
+	install -m644 cloudflared.1 $(DESTDIR)$(INSTALL_MANDIR)/cloudflared.1
 
 # When we build packages, the package name will be FIPS-aware.
 # But we keep the binary installed by it to be named "cloudflared" regardless.
 define build_package
 	mkdir -p $(PACKAGE_DIR)
 	cp cloudflared $(PACKAGE_DIR)/cloudflared
-	cat cloudflared_man_template | sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' > $(PACKAGE_DIR)/cloudflared.1
+	cp cloudflared.1 $(PACKAGE_DIR)/cloudflared.1
 	fakeroot fpm -C $(PACKAGE_DIR) -s dir -t $(1) \
 		--description 'Cloudflare Tunnel daemon' \
 		--vendor 'Cloudflare' \
-		--license 'Cloudflare Service Agreement' \
+		--license 'Apache License Version 2.0' \
 		--url 'https://github.com/cloudflare/cloudflared' \
 		-m 'Cloudflare <support@cloudflare.com>' \
-		-a $(TARGET_ARCH) -v $(VERSION) -n $(DEB_PACKAGE_NAME) $(NIGHTLY_FLAGS) --after-install postinst.sh --after-remove postrm.sh \
-		cloudflared=$(INSTALL_BINDIR) cloudflared.1=$(MAN_DIR)
+	    -a $(PACKAGE_ARCH) -v $(VERSION) -n $(DEB_PACKAGE_NAME) $(NIGHTLY_FLAGS) --after-install postinst.sh --after-remove postrm.sh \
+		cloudflared=$(INSTALL_BINDIR) cloudflared.1=$(INSTALL_MANDIR)
 endef
 
 .PHONY: cloudflared-deb
-cloudflared-deb: cloudflared
+cloudflared-deb: cloudflared cloudflared.1
 	$(call build_package,deb)
 
 .PHONY: cloudflared-rpm
-cloudflared-rpm: cloudflared
+cloudflared-rpm: cloudflared cloudflared.1
 	$(call build_package,rpm)
 
 .PHONY: cloudflared-pkg
-cloudflared-pkg: cloudflared
+cloudflared-pkg: cloudflared cloudflared.1
 	$(call build_package,osxpkg)
 
 .PHONY: cloudflared-msi
-cloudflared-msi: cloudflared
+cloudflared-msi:
 	wixl --define Version=$(VERSION) --define Path=$(EXECUTABLE_PATH) --output cloudflared-$(VERSION)-$(TARGET_ARCH).msi cloudflared.wxs
 
 .PHONY: cloudflared-darwin-amd64.tgz
@@ -247,6 +263,10 @@ github-release: cloudflared
 github-release-built-pkgs:
 	python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
 
+.PHONY: release-pkgs-linux
+release-pkgs-linux:
+	python3 ./release_pkgs.py
+
 .PHONY: github-message
 github-message:
 	python3 github_message.py --release-version $(VERSION)
@@ -256,10 +276,17 @@ github-mac-upload:
 	python3 github_release.py --path artifacts/cloudflared-darwin-amd64.tgz --release-version $(VERSION) --name cloudflared-darwin-amd64.tgz
 	python3 github_release.py --path artifacts/cloudflared-amd64.pkg --release-version $(VERSION) --name cloudflared-amd64.pkg
 
+.PHONY: github-windows-upload
+github-windows-upload:
+	python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.exe --release-version $(VERSION) --name cloudflared-windows-amd64.exe
+	python3 github_release.py --path built_artifacts/cloudflared-windows-amd64.msi --release-version $(VERSION) --name cloudflared-windows-amd64.msi
+	python3 github_release.py --path built_artifacts/cloudflared-windows-386.exe --release-version $(VERSION) --name cloudflared-windows-386.exe
+	python3 github_release.py --path built_artifacts/cloudflared-windows-386.msi --release-version $(VERSION) --name cloudflared-windows-386.msi
+
 .PHONY: tunnelrpc-deps
 tunnelrpc-deps:
 	which capnp  # https://capnproto.org/install.html
-	which capnpc-go  # go get zombiezen.com/go/capnproto2/capnpc-go
+	which capnpc-go  # go install zombiezen.com/go/capnproto2/capnpc-go@latest
 	capnp compile -ogo tunnelrpc/tunnelrpc.capnp
 
 .PHONY: quic-deps
@@ -270,13 +297,8 @@ quic-deps:
 
 .PHONY: vet
 vet:
-	go vet -mod=vendor ./...
-	# go get github.com/sudarshan-reddy/go-sumtype (don't do this in build directory or this will cause vendor issues)
-	# Note: If you have github.com/BurntSushi/go-sumtype then you might have to use the repo above instead
-	# for now because it uses an older version of golang.org/x/tools.
-	which go-sumtype
-	go-sumtype $$(go list -mod=vendor ./...)
+	go vet -v -mod=vendor github.com/cloudflare/cloudflared/...
 
-.PHONY: goimports
-goimports:
-	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done
+.PHONY: fmt
+fmt:
+	goimports -l -w -local github.com/cloudflare/cloudflared $$(go list -mod=vendor -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc)

README.md

@@ -25,12 +25,13 @@ routing), but for legacy reasons this requirement is still necessary:
 
 ## Installing `cloudflared`
 
-Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases here on the `cloudflared` GitHub repository.
+Downloads are available as standalone binaries, a Docker image, and Debian, RPM, and Homebrew packages. You can also find releases [here](https://github.com/cloudflare/cloudflared/releases) on the `cloudflared` GitHub repository.
 
 * You can [install on macOS](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#macos) via Homebrew or by downloading the [latest Darwin amd64 release](https://github.com/cloudflare/cloudflared/releases)
 * Binaries, Debian, and RPM packages for Linux [can be found here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#linux)
 * A Docker image of `cloudflared` is [available on DockerHub](https://hub.docker.com/r/cloudflare/cloudflared)
 * You can install on Windows machines with the [steps here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#windows)
+* Build from source with the [instructions here](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/installation#build-from-source)
 
 User documentation for Cloudflare Tunnel can be found at https://developers.cloudflare.com/cloudflare-one/connections/connect-apps
 
@@ -43,7 +44,7 @@ Once installed, you can authenticate `cloudflared` into your Cloudflare account
 * Route traffic to that Tunnel:
   * Via public [DNS records in Cloudflare](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/dns)
   * Or via a public hostname guided by a [Cloudflare Load Balancer](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/routing-to-tunnel/lb)
-  * Or from [WARP client private traffic](https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/private-networks)
+  * Or from [WARP client private traffic](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/private-net/)
 
 
 ## TryCloudflare

RELEASE_NOTES

@@ -1,3 +1,319 @@
+2022.12.0
+- 2022-12-14 TUN-6999: cloudflared should attempt other edge addresses before falling back on protocol
+- 2022-12-13 TUN-7004: Dont show local config dirs for remotely configured tuns
+- 2022-12-12 TUN-7003: Tempoarily disable erroneous notarize-app
+- 2022-12-12 TUN-7003: Add back a missing fi
+- 2022-12-07 TUN-7000: Reduce metric cardinality of closedConnections metric by removing error as tag
+- 2022-12-07 TUN-6994: Improve logging config file not found
+- 2022-12-07 TUN-7002: Randomise first region selection
+- 2022-12-07 TUN-6995: Disable quick-tunnels spin up by default
+- 2022-12-05 TUN-6984: Add bash set x to improve visibility during builds
+- 2022-12-05 TUN-6984: [CI] Ignore security import errors for code_sigining
+- 2022-12-05 TUN-6984: [CI] Don't fail on unset.
+- 2022-11-30 TUN-6984: Set euo pipefile for homebrew builds
+
+2022.11.1
+- 2022-11-29 TUN-6981: We should close UDP socket if failed to connecto to edge
+- 2022-11-25 CUSTESC-23757: Fix a bug where a wildcard ingress rule would match an host without starting with a dot
+- 2022-11-24 TUN-6970: Print newline when printing tunnel token
+- 2022-11-22 TUN-6963: Refactor Metrics service setup
+2022.11.0
+- 2022-11-16 Revert "TUN-6935: Cloudflared should use APIToken instead of serviceKey"
+- 2022-11-16 TUN-6929: Use same protocol for other connections as first one
+- 2022-11-14 TUN-6941: Reduce log level to debug when failing to proxy ICMP reply
+- 2022-11-14 TUN-6935: Cloudflared should use APIToken instead of serviceKey
+- 2022-11-14 TUN-6935: Cloudflared should use APIToken instead of serviceKey
+- 2022-11-11 TUN-6937: Bump golang.org/x/* packages to new release tags
+- 2022-11-10 ZTC-234: macOS tests
+- 2022-11-09 TUN-6927: Refactor validate access configuration to allow empty audTags only
+- 2022-11-08 ZTC-234: Replace ICMP funnels when ingress connection changes
+- 2022-11-04 TUN-6917: Bump go to 1.19.3
+- 2022-11-02 Issue #574: Better ssh config for short-lived cert (#763)
+- 2022-10-28 TUN-6898: Fix bug handling IPv6 based ingresses with missing port
+- 2022-10-28 TUN-6898: Refactor addPortIfMissing
+2022.10.3
+- 2022-10-24 TUN-6871: Add default feature to cloudflared to support EOF on QUIC connections
+- 2022-10-19 TUN-6876: Fix flaky TestTraceICMPRouterEcho by taking account request span can return before reply
+- 2022-10-18 TUN-6867: Clear spans right after they are serialized to avoid returning duplicate spans
+2022.10.2
+- 2022-10-18 TUN-6869: Fix Makefile complaining about missing GO packages
+- 2022-10-18 TUN-6864: Don't reuse port in quic unit tests
+- 2022-10-18 TUN-6868: Return left padded tracing ID when tracing identity is converted to string
+
+2022.10.1
+- 2022-10-16 TUN-6861: Trace ICMP on Windows
+- 2022-10-15 TUN-6860: Send access configuration keys to the edge
+- 2022-10-14 TUN-6858: Trace ICMP reply
+- 2022-10-13 TUN-6855: Add DatagramV2Type for IP packet with trace and tracing spans
+- 2022-10-13 TUN-6856: Refactor to lay foundation for tracing ICMP
+- 2022-10-13 TUN-6604: Trace icmp echo request on Linux and Darwin
+- 2022-10-12 Fix log message (#591)
+- 2022-10-12 TUN-6853: Reuse source port when connecting to the edge for quic connections
+- 2022-10-11 TUN-6829: Allow user of datagramsession to control logging level of errors
+- 2022-10-10 RTG-2276 Update qtls and go mod tidy
+- 2022-10-05 Add post-quantum flag to quick tunnel
+- 2022-10-05 TUN-6823: Update github release message to pull from KV
+- 2022-10-04 TUN-6825: Fix cloudflared:version images require arch hyphens
+- 2022-10-03 TUN-6806: Add ingress rule number to log when filtering due to middlware handler
+- 2022-08-17 Label correct container
+- 2022-08-16 Fix typo in help text for `cloudflared tunnel route lb`
+- 2022-07-18 drop usage of cat when sed is invoked to generate the manpage
+- 2021-03-15 update-build-readme
+- 2021-03-15 fix link
+
+2022.10.0
+- 2022-09-30 TUN-6755: Remove unused publish functions
+- 2022-09-30 TUN-6813: Only proxy ICMP packets when warp-routing is enabled
+- 2022-09-29 TUN-6811: Ping group range should be parsed as int32
+- 2022-09-29 TUN-6812: Drop IP packets if ICMP proxy is not initialized
+- 2022-09-28 TUN-6716: Document limitation of Windows ICMP proxy
+- 2022-09-28 TUN-6810: Add component test for post-quantum
+- 2022-09-27 TUN-6715: Provide suggestion to add cloudflared to ping_group_range if it failed to open ICMP socket
+- 2022-09-22 TUN-6792: Fix brew core release by not auditing the formula
+- 2022-09-22 TUN-6774: Validate OriginRequest.Access to add Ingress.Middleware
+- 2022-09-22 TUN-6775: Add middleware.Handler verification to ProxyHTTP
+- 2022-09-22 TUN-6791: Calculate ICMPv6 checksum
+- 2022-09-22 TUN-6801: Add punycode alternatives for ingress rules
+- 2022-09-21 TUN-6772: Add a JWT Validator as an ingress verifier
+- 2022-09-21 TUN-6772: Add a JWT Validator as an ingress verifier
+- 2022-09-21 TUN-6774: Validate OriginRequest.Access to add Ingress.Middleware
+- 2022-09-21 TUN-6772: Add a JWT Validator as an ingress verifier
+- 2022-09-20 TUN-6741: ICMP proxy tries to listen on specific IPv4 & IPv6 when possible
+2022.9.1
+- 2022-09-20 TUN-6777: Fix race condition in TestFunnelIdleTimeout
+- 2022-09-20 TUN-6595: Enable datagramv2 and icmp proxy by default
+- 2022-09-20 TUN-6773: Add access based configuration to ingress.OriginRequestConfig
+- 2022-09-19 TUN-6778: Cleanup logs about ICMP
+- 2022-09-19 TUN-6779: cloudflared should also use the root CAs from system pool to validate edge certificate
+- 2022-09-19 TUN-6780: Add support for certReload to also include support for client certificates
+- 2022-09-16 TUN-6767: Build ICMP proxy for Windows only when CGO is enabled
+- 2022-09-15 TUN-6590: Use Windows Teamcity agent to build binary
+- 2022-09-13 TUN-6592: Decrement TTL and return ICMP time exceed if it's 0
+- 2022-09-09 TUN-6749: Fix icmp_generic build
+- 2022-09-09 TUN-6744: On posix platforms, assign unique echo ID per (src, dst, echo ID)
+- 2022-09-08 TUN-6743: Support ICMPv6 echo on Windows
+- 2022-09-08 TUN-6689: Utilize new RegisterUDPSession to begin tracing
+- 2022-09-07 TUN-6688: Update RegisterUdpSession capnproto to include trace context
+- 2022-09-06 TUN-6740: Detect no UDP packets allowed and fallback from QUIC in that case
+- 2022-09-06 TUN-6654: Support ICMPv6 on Linux and Darwin
+- 2022-09-02 TUN-6696: Refactor flow into funnel and close idle funnels
+- 2022-09-02 TUN-6718: Bump go and go-boring 1.18.6
+- 2022-08-29 TUN-6531: Implement ICMP proxy for Windows using IcmpSendEcho
+- 2022-08-24 RTG-1339 Support post-quantum hybrid key exchange
+
+2022.9.0
+- 2022-09-05 TUN-6737: Fix datagramV2Type should be declared in its own block so it starts at 0
+- 2022-09-01 TUN-6725: Fix testProxySSEAllData
+- 2022-09-01 TUN-6726: Fix maxDatagramPayloadSize for Windows QUIC datagrams
+- 2022-09-01 TUN-6729: Fix flaky TestClosePreviousProxies
+- 2022-09-01 TUN-6728: Verify http status code ingress rule
+- 2022-08-25 TUN-6695: Implement ICMP proxy for linux
+
+2022.8.4
+- 2022-08-31 TUN-6717: Update Github action to run with Go 1.19
+- 2022-08-31 TUN-6720: Remove forcibly closing connection during reconnect signal
+- 2022-08-29 Release 2022.8.3
+
+2022.8.3
+- 2022-08-26 TUN-6708: Fix replace flow logic
+- 2022-08-25 TUN-6705: Tunnel should retry connections forever
+- 2022-08-25 TUN-6704: Honor protocol flag when edge discovery is unreachable
+- 2022-08-25 TUN-6699: Add metric for packet too big dropped
+- 2022-08-24 TUN-6691: Properly error check for net.ErrClosed
+- 2022-08-22 TUN-6679: Allow client side of quic request to close body
+- 2022-08-22 TUN-6586: Change ICMP proxy to only build for Darwin and use echo ID to track flows
+- 2022-08-18 TUN-6530: Implement ICMPv4 proxy
+- 2022-08-17 TUN-6666: Define packet package
+- 2022-08-17 TUN-6667: DatagramMuxerV2 provides a method to receive RawPacket
+- 2022-08-16 TUN-6657: Ask for Tunnel ID and Configuration on Bug Report
+- 2022-08-16 TUN-6676: Add suport for trailers in http2 connections
+- 2022-08-11 TUN-6575: Consume cf-trace-id from incoming http2 TCP requests
+
+2022.8.2
+- 2022-08-16 TUN-6656: Docker for arm64 should not be deployed in an amd64 container
+
+2022.8.1
+- 2022-08-15 TUN-6617: Updated CHANGES.md for protocol stickiness
+- 2022-08-12 EDGEPLAT-3918: bump go and go-boring to 1.18.5
+- 2022-08-12 TUN-6652: Publish dockerfile for both amd64 and arm64
+- 2022-08-11 TUN-6617: Dont fallback to http2 if QUIC conn was successful.
+- 2022-08-11 TUN-6617: Dont fallback to http2 if QUIC conn was successful.
+- 2022-08-11 Revert "TUN-6617: Dont fallback to http2 if QUIC conn was successful."
+- 2022-08-11 TUN-6617: Dont fallback to http2 if QUIC conn was successful.
+- 2022-08-01 TUN-6584: Define QUIC datagram v2 format to support proxying IP packets
+
+2022.8.0
+- 2022-08-10 TUN-6637: Upgrade quic-go
+- 2022-08-10 TUN-6646: Add support to SafeStreamCloser to close only write side of stream
+- 2022-08-09 TUN-6642: Fix unexpected close of quic stream triggered by upstream origin close
+- 2022-08-09 TUN-6639: Validate cyclic ingress configuration
+- 2022-08-08 TUN-6637: Upgrade go version and quic-go
+- 2022-08-08 TUN-6639: Validate cyclic ingress configuration
+- 2022-08-04 EDGEPLAT-3918: build cloudflared for Bookworm
+- 2022-08-02 Revert "TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span"
+- 2022-07-27 TUN-6601: Update gopkg.in/yaml.v3 references in modules
+- 2022-07-26 TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span
+- 2022-07-26 TUN-6576: Consume cf-trace-id from incoming TCP requests to create root span
+- 2022-07-25 TUN-6598: Remove auto assignees on github issues
+- 2022-07-20 TUN-6583: Remove legacy --ui flag
+- 2022-07-20 cURL supports stdin and uses os pipes directly without copying
+- 2022-07-07 TUN-6517: Use QUIC stream context while proxying HTTP requests and TCP connections
+2022.7.1
+- 2022-07-06 TUN-6503: Fix transport fallback from QUIC in face of dial error "no network activity"
+
+2022.7.0
+- 2022-07-05 TUN-6499: Remove log that is per datagram
+- 2022-06-24 TUN-6460: Rename metric label location to edge_location
+- 2022-06-24 TUN-6459: Add cloudflared user-agent to access calls
+- 2022-06-17 TUN-6427: Differentiate between upstream request closed/canceled and failed origin requests
+- 2022-06-17 TUN-6388: Fix first tunnel connection not retrying
+- 2022-06-13 TUN-6384: Correct duplicate connection error to fetch new IP first
+- 2022-06-13 TUN-6373: Add edge-ip-version to remotely pushed configuration
+- 2022-06-07 TUN-6010: Add component tests for --edge-ip-version
+- 2022-05-20 TUN-6007: Implement new edge discovery algorithm
+- 2022-02-18 Ensure service install directories are created before writing file
+
+2022.6.3
+- 2022-06-20 TUN-6362: Add armhf support to cloudflare packaging
+
+2022.6.2
+- 2022-06-13 TUN-6381: Write error data on QUIC stream when we fail to talk to the origin; separate logging for protocol errors vs. origin errors.
+- 2022-06-17 TUN-6414: Remove go-sumtype from cloudflared build process
+- 2022-06-01 Add Http2Origin option to force HTTP/2 origin connections
+- 2022-06-02 fix ingress rules unit test
+- 2022-06-09 Update remaining OriginRequestConfig functions for Http2Origins
+- 2022-05-31 Add image source label to docker container.
+- 2022-05-10 Warp Private Network link updated
+
+2022.6.1
+- 2022-06-14 TUN-6395: Fix writing RPM repo data
+
+2022.6.0
+- 2022-06-14 Revert "TUN-6010: Add component tests for --edge-ip-version"
+- 2022-06-14 Revert "TUN-6373: Add edge-ip-version to remotely pushed configuration"
+- 2022-06-14 Revert "TUN-6384: Correct duplicate connection error to fetch new IP first"
+- 2022-06-14 Revert "TUN-6007: Implement new edge discovery algorithm"
+- 2022-06-13 TUN-6385: Don't share err between acceptStream loop and per-stream goroutines
+- 2022-06-13 TUN-6384: Correct duplicate connection error to fetch new IP first
+- 2022-06-13 TUN-6373: Add edge-ip-version to remotely pushed configuration
+- 2022-06-13 TUN-6380: Enforce connect and keep-alive timeouts for TCP connections in both WARP routing and websocket based TCP proxy.
+- 2022-06-11 Update issue templates
+- 2022-06-11 Amendment to previous PR
+- 2022-06-09 TUN-6347: Add TCP stream logs with FlowID
+- 2022-06-08 TUN-6361: Add cloudflared arm builds to pkging as well
+- 2022-06-07 TUN-6357: Add connector id to ready check endpoint
+- 2022-06-07 TUN-6010: Add component tests for --edge-ip-version
+- 2022-06-06 TUN-6191: Update quic-go to v0.27.1 and with custom patch to allow keep alive period to be configurable
+- 2022-06-03 TUN-6343: Fix QUIC->HTTP2 fallback
+- 2022-06-02 TUN-6339: Add config for IPv6 support
+- 2022-06-02 TUN-6341: Fix default config value for edge-ip-version
+- 2022-06-01 TUN-6323: Add Xenial and Trusty for Ubuntu pkging
+- 2022-05-31 TUN-6210: Add cloudflared.repo to make it easy for yum installs
+- 2022-05-30 TUN-6293: Update yaml v3 to latest hotfix
+- 2022-05-20 TUN-6007: Implement new edge discovery algorithm
+2022.5.3
+- 2022-05-30 TUN-6308: Add debug logs to see if packets are sent/received from edge
+- 2022-05-30 TUN-6301: Allow to update logger used by UDP session manager
+
+2022.5.2
+- 2022-05-23 TUN-6270: Import gpg keys from environment variables
+- 2022-05-24 TUN-6209: Improve feedback process if release_pkgs to deb and rpm fail
+- 2022-05-24 TUN-6280: Don't wrap qlog connection tracer for gatethering QUIC metrics since we're not writing qlog files.
+- 2022-05-25 TUN-6209: Sign RPM packages
+- 2022-05-25 TUN-6285: Upload pkg assets to repos when cloudflared is released.
+- 2022-05-24 TUN-6282: Upgrade golang to 1.17.10, go-boring to 1.17.9
+- 2022-05-26 TUN-6292: Debug builds for cloudflared
+- 2022-05-28 TUN-6304: Fixed some file permission issues
+- 2022-05-11 TUN-6197: Publish to brew core should not try to open the browser
+- 2022-05-12 TUN-5943: Add RPM support
+- 2022-05-18 TUN-6248: Fix panic in cloudflared during tracing when origin doesn't provide header map
+- 2022-05-18 TUN-6250: Add upstream response status code to tracing span attributes
+
+2022.5.1
+- 2022-05-06 TUN-6146: Release_pkgs is now a generic command line script
+- 2022-05-06 TUN-6185: Fix tcpOverWSOriginService not using original scheme for String representation
+- 2022-05-05 TUN-6175: Simply debian packaging by structural upload
+- 2022-05-05 TUN-5945: Added support for Ubuntu releases
+- 2022-05-04 TUN-6054: Create and upload deb packages to R2
+- 2022-05-03 TUN-6161: Set git user/email for brew core release
+- 2022-05-03 TUN-6166: Fix mocked QUIC transport for UDP proxy manager to return expected error
+- 2022-04-27 TUN-6016: Push local managed tunnels configuration to the edge
+2022.5.0
+- 2022-05-02 TUN-6158: Update golang.org/x/crypto
+- 2022-04-20 VULN-8383 Bump yaml.v2 to yaml.v3
+- 2022-04-21 TUN-6123: For a given connection with edge, close all datagram sessions through this connection when it's closed
+- 2022-04-20 TUN-6015: Add RPC method for pushing local config
+- 2022-04-21 TUN-6130: Fix vendoring due to case sensitive typo in package
+- 2022-04-27 TUN-6142: Add tunnel details support to RPC
+- 2022-04-28 TUN-6014: Add remote config flag as default feature
+- 2022-04-12 TUN-6000: Another fix for publishing to brew core
+- 2022-04-11 TUN-5990: Add otlp span export to response header
+- 2022-04-19 TUN-6070: First connection retries other edge IPs if the error is quic timeout(likely due to firewall blocking UDP)
+- 2022-04-11 TUN-6030: Add ttfb span for origin http request
+
+2022.4.1
+- 2022-04-11 TUN-6035: Reduce buffer size when proxying data
+- 2022-04-11 TUN-6038: Reduce buffer size used for proxying data
+- 2022-04-11 TUN-6043: Allow UI-managed Tunnels to fallback from QUIC but warn about that
+- 2022-04-07 TUN-6000 add version argument to bump-formula-pr
+- 2022-04-06 TUN-5989: Add in-memory otlp exporter
+
+2022.4.0
+- 2022-04-01 TUN-5973: Add backoff for non-recoverable errors as well
+- 2022-04-05 TUN-5992: Use QUIC protocol for remotely managed tunnels when protocol is unspecified
+- 2022-04-06 Update Makefile
+- 2022-04-06 TUN-5995: Update prometheus to 1.12.1 to avoid vulnerabilities
+- 2022-04-07 TUN-5995: Force prometheus v1.12.1 usage
+- 2022-04-07 TUN-4130: cloudflared docker images now have a latest tag
+- 2022-03-30 TUN-5842: Fix flaky TestConcurrentUpdateAndRead by making sure resources are released
+- 2022-03-30 carrier: fix dropped errors
+- 2022-03-25 TUN-5959: tidy go.mod
+- 2022-03-25 TUN-5958: Fix release to homebrew core
+- 2022-03-28 TUN-5960: Do not log the tunnel token or json credentials
+- 2022-03-28 TUN-5956: Add timeout to session manager APIs
+
+2022.3.4
+- 2022-03-22 TUN-5918: Clean up text in cloudflared tunnel --help
+- 2022-03-22 TUN-5895 run brew bump-formula-pr on release
+- 2022-03-22 TUN-5915: New cloudflared command to allow to retrieve the token credentials for a Tunnel
+- 2022-03-24 TUN-5933: Better messaging to help user when installing service if it is already installed
+- 2022-03-25 TUN-5954: Start cloudflared service in Linux too similarly to other OSs
+- 2022-03-14 TUN-5869: Add configuration endpoint in metrics server
+
+2022.3.3
+- 2022-03-17 TUN-5893: Start windows service on install, stop on uninstall. Previously user had to manually start the service after running 'cloudflared tunnel install' and stop the service before running uninstall command.
+- 2022-03-17 Revert "CC-796: Remove dependency on unsupported version of go-oidc"
+- 2022-03-18 TUN-5881: Clarify success (or lack thereof) of (un)installing cloudflared service
+- 2022-03-18 CC-796: Remove dependency on unsupported version of go-oidc
+- 2022-03-18 TUN-5907: Change notes for 2022.3.3
+
+2022.3.2
+- 2022-03-10 TUN-5833: Create constant for allow-remote-config
+- 2022-03-15 TUN-5867: Return error if service was already installed
+- 2022-03-16 TUN-5833: Send feature `allow_remote_config` if Tunnel is run with --token
+- 2022-03-08 TUN-5849: Remove configuration debug log
+- 2022-03-08 TUN-5850: Update CHANGES.md with latest releases
+- 2022-03-08 TUN-5851: Update all references to point to Apache License 2.0
+- 2022-03-07 TUN-5853 Add "install" make target and build package manager info into executable
+- 2022-03-08 TUN-5801: Add custom wrapper for OriginConfig for JSON serde
+- 2022-03-09 TUN-5703: Add prometheus metric for current configuration version
+- 2022-02-05 CC-796: Remove dependency on unsupported version of go-oidc
+
+2022.3.1
+- 2022-03-04 TUN-5837: Log panic recovery in http2 logic with debug level log
+- 2022-03-04  TUN-5696: HTTP/2 Configuration Update
+- 2022-03-04 TUN-5836: Avoid websocket#Stream function from crashing cloudflared with unexpected memory access
+- 2022-03-05 TUN-5836: QUIC transport no longer sets body to nil in any condition
+
+2022.3.0
+- 2022-03-02 TUN-5680: Adapt component tests for new service install based on token
+- 2022-02-21 TUN-5682: Remove name field from credentials
+- 2022-02-21 TUN-5681: Add support for running tunnel using Token
+- 2022-02-28 TUN-5824: Update updater no-update-in-shell link
+- 2022-02-28 TUN-5823: Warn about legacy flags that are ignored when ingress rules are used
+- 2022-02-28 TUN-5737: Support https protocol over unix socket origin
+- 2022-02-23 TUN-5679: Add support for service install using Tunnel Token
+
 2022.2.2
 - 2022-02-22 TUN-5754: Allow ingress validate to take plaintext option
 - 2022-02-17 TUN-5678: Cloudflared uses typed tunnel API

build-packages.sh

@@ -7,20 +7,19 @@ export CGO_ENABLED=0
 # This controls the directory the built artifacts go into
 export ARTIFACT_DIR=built_artifacts/
 mkdir -p $ARTIFACT_DIR
-windowsArchs=("amd64" "386")
-export TARGET_OS=windows
-for arch in ${windowsArchs[@]}; do
-    export TARGET_ARCH=$arch
-    make cloudflared-msi
-    mv ./cloudflared.exe $ARTIFACT_DIR/cloudflared-windows-$arch.exe
-    mv cloudflared-$VERSION-$arch.msi $ARTIFACT_DIR/cloudflared-windows-$arch.msi
-done
 
-
-linuxArchs=("386" "amd64" "arm" "arm64")
+linuxArchs=("386" "amd64" "arm" "armhf" "arm64")
 export TARGET_OS=linux
 for arch in ${linuxArchs[@]}; do
+    unset TARGET_ARM
     export TARGET_ARCH=$arch
+    
+    ## Support for armhf builds 
+    if [[ $arch == armhf ]] ; then
+        export TARGET_ARCH=arm
+        export TARGET_ARM=7 
+    fi
+    
     make cloudflared-deb
     mv cloudflared\_$VERSION\_$arch.deb $ARTIFACT_DIR/cloudflared-linux-$arch.deb
 

carrier/carrier.go

@@ -1,6 +1,6 @@
-//Package carrier provides a WebSocket proxy to carry or proxy a connection
-//from the local client to the edge. See it as a wrapper around any protocol
-//that it packages up in a WebSocket connection to the edge.
+// Package carrier provides a WebSocket proxy to carry or proxy a connection
+// from the local client to the edge. See it as a wrapper around any protocol
+// that it packages up in a WebSocket connection to the edge.
 package carrier
 
 import (

carrier/websocket.go

@@ -55,6 +55,9 @@ func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebso
 	}
 
 	dump, err := httputil.DumpRequest(req, false)
+	if err != nil {
+		return nil, err
+	}
 	log.Debug().Msgf("Websocket request: %s", string(dump))
 
 	dialer := &websocket.Dialer{
@@ -182,6 +185,9 @@ func createAccessWebSocketStream(options *StartOptions, log *zerolog.Logger) (*w
 	}
 
 	dump, err := httputil.DumpRequest(req, false)
+	if err != nil {
+		return nil, nil, err
+	}
 	log.Debug().Msgf("Access Websocket request: %s", string(dump))
 
 	conn, resp, err := clientConnect(req, nil)

certutil/certutil.go

@@ -1,25 +1,21 @@
 package certutil
 
 import (
-	"crypto/x509"
 	"encoding/json"
 	"encoding/pem"
 	"fmt"
-	"strings"
 )
 
 type namedTunnelToken struct {
-	ZoneID     string `json:"zoneID"`
-	AccountID  string `json:"accountID"`
-	ServiceKey string `json:"serviceKey"`
+	ZoneID    string `json:"zoneID"`
+	AccountID string `json:"accountID"`
+	APIToken  string `json:"apiToken"`
 }
 
 type OriginCert struct {
-	PrivateKey interface{}
-	Cert       *x509.Certificate
-	ZoneID     string
-	ServiceKey string
-	AccountID  string
+	ZoneID    string
+	APIToken  string
+	AccountID string
 }
 
 func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
@@ -33,29 +29,11 @@ func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
 			break
 		}
 		switch block.Type {
-		case "PRIVATE KEY":
-			if originCert.PrivateKey != nil {
-				return nil, fmt.Errorf("Found multiple private key in the certificate")
-			}
-			// RSA private key
-			privateKey, err := x509.ParsePKCS8PrivateKey(block.Bytes)
-			if err != nil {
-				return nil, fmt.Errorf("Cannot parse private key")
-			}
-			originCert.PrivateKey = privateKey
-		case "CERTIFICATE":
-			if originCert.Cert != nil {
-				return nil, fmt.Errorf("Found multiple certificates in the certificate")
-			}
-			cert, err := x509.ParseCertificates(block.Bytes)
-			if err != nil {
-				return nil, fmt.Errorf("Cannot parse certificate")
-			} else if len(cert) > 1 {
-				return nil, fmt.Errorf("Found multiple certificates in the certificate")
-			}
-			originCert.Cert = cert[0]
-		case "WARP TOKEN", "ARGO TUNNEL TOKEN":
-			if originCert.ZoneID != "" || originCert.ServiceKey != "" {
+		case "PRIVATE KEY", "CERTIFICATE":
+			// this is for legacy purposes.
+			break
+		case "ARGO TUNNEL TOKEN":
+			if originCert.ZoneID != "" || originCert.APIToken != "" {
 				return nil, fmt.Errorf("Found multiple tokens in the certificate")
 			}
 			// The token is a string,
@@ -63,18 +41,8 @@ func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
 			ntt := namedTunnelToken{}
 			if err := json.Unmarshal(block.Bytes, &ntt); err == nil {
 				originCert.ZoneID = ntt.ZoneID
-				originCert.ServiceKey = ntt.ServiceKey
+				originCert.APIToken = ntt.APIToken
 				originCert.AccountID = ntt.AccountID
-			} else {
-				// Try the older format, where the zoneID and service key are separated by
-				// a new line character
-				token := string(block.Bytes)
-				s := strings.Split(token, "\n")
-				if len(s) != 2 {
-					return nil, fmt.Errorf("Cannot parse token")
-				}
-				originCert.ZoneID = s[0]
-				originCert.ServiceKey = s[1]
 			}
 		default:
 			return nil, fmt.Errorf("Unknown block %s in the certificate", block.Type)
@@ -82,11 +50,7 @@ func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
 		block, rest = pem.Decode(rest)
 	}
 
-	if originCert.PrivateKey == nil {
-		return nil, fmt.Errorf("Missing private key in the certificate")
-	} else if originCert.Cert == nil {
-		return nil, fmt.Errorf("Missing certificate in the certificate")
-	} else if originCert.ZoneID == "" || originCert.ServiceKey == "" {
+	if originCert.ZoneID == "" || originCert.APIToken == "" {
 		return nil, fmt.Errorf("Missing token in the certificate")
 	}
 

certutil/certutil_test.go

@@ -13,49 +13,33 @@ func TestLoadOriginCert(t *testing.T) {
 	assert.Equal(t, fmt.Errorf("Cannot decode empty certificate"), err)
 	assert.Nil(t, cert)
 
-	blocks, err := ioutil.ReadFile("test-cert-no-key.pem")
-	assert.Nil(t, err)
-	cert, err = DecodeOriginCert(blocks)
-	assert.Equal(t, fmt.Errorf("Missing private key in the certificate"), err)
-	assert.Nil(t, cert)
-
-	blocks, err = ioutil.ReadFile("test-cert-two-certificates.pem")
-	assert.Nil(t, err)
-	cert, err = DecodeOriginCert(blocks)
-	assert.Equal(t, fmt.Errorf("Found multiple certificates in the certificate"), err)
-	assert.Nil(t, cert)
-
-	blocks, err = ioutil.ReadFile("test-cert-unknown-block.pem")
+	blocks, err := ioutil.ReadFile("test-cert-unknown-block.pem")
 	assert.Nil(t, err)
 	cert, err = DecodeOriginCert(blocks)
 	assert.Equal(t, fmt.Errorf("Unknown block RSA PRIVATE KEY in the certificate"), err)
 	assert.Nil(t, cert)
-
-	blocks, err = ioutil.ReadFile("test-cert.pem")
-	assert.Nil(t, err)
-	cert, err = DecodeOriginCert(blocks)
-	assert.Nil(t, err)
-	assert.NotNil(t, cert)
-	assert.Equal(t, "7b0a4d77dfb881c1a3b7d61ea9443e19", cert.ZoneID)
-	key := "v1.0-58bd4f9e28f7b3c28e05a35ff3e80ab4fd9644ef3fece537eb0d12e2e9258217-183442fbb0bbdb3e571558fec9b5589ebd77aafc87498ee3f09f64a4ad79ffe8791edbae08b36c1d8f1d70a8670de56922dff92b15d214a524f4ebfa1958859e-7ce80f79921312a6022c5d25e2d380f82ceaefe3fbdc43dd13b080e3ef1e26f7"
-	assert.Equal(t, key, cert.ServiceKey)
 }
 
-func TestNewlineArgoTunnelToken(t *testing.T) {
-	ArgoTunnelTokenTest(t, "test-argo-tunnel-cert.pem")
+func TestJSONArgoTunnelTokenEmpty(t *testing.T) {
+	cert, err := DecodeOriginCert([]byte{})
+	blocks, err := ioutil.ReadFile("test-cert-no-token.pem")
+	assert.Nil(t, err)
+	cert, err = DecodeOriginCert(blocks)
+	assert.Equal(t, fmt.Errorf("Missing token in the certificate"), err)
+	assert.Nil(t, cert)
 }
 
 func TestJSONArgoTunnelToken(t *testing.T) {
 	// The given cert's Argo Tunnel Token was generated by base64 encoding this JSON:
 	// {
 	// "zoneID": "7b0a4d77dfb881c1a3b7d61ea9443e19",
-	// "serviceKey": "test-service-key",
+	// "apiToken": "test-service-key",
 	// "accountID": "abcdabcdabcdabcd1234567890abcdef"
 	// }
-	ArgoTunnelTokenTest(t, "test-argo-tunnel-cert-json.pem")
+	CloudflareTunnelTokenTest(t, "test-cloudflare-tunnel-cert-json.pem")
 }
 
-func ArgoTunnelTokenTest(t *testing.T, path string) {
+func CloudflareTunnelTokenTest(t *testing.T, path string) {
 	blocks, err := ioutil.ReadFile(path)
 	assert.Nil(t, err)
 	cert, err := DecodeOriginCert(blocks)
@@ -63,5 +47,5 @@ func ArgoTunnelTokenTest(t *testing.T, path string) {
 	assert.NotNil(t, cert)
 	assert.Equal(t, "7b0a4d77dfb881c1a3b7d61ea9443e19", cert.ZoneID)
 	key := "test-service-key"
-	assert.Equal(t, key, cert.ServiceKey)
+	assert.Equal(t, key, cert.APIToken)
 }

certutil/test-cert-no-key.pem

@@ -1,33 +0,0 @@
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN WARP TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
-ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
-MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
-NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
-NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
-OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
-ZWYxZTI2Zjc=
------END WARP TOKEN-----

certutil/test-cert-no-token.pem

@@ -51,7 +51,6 @@ K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
 x+Yo/cL8fGfVpPt4UM8=
 -----END CERTIFICATE-----
 -----BEGIN ARGO TUNNEL TOKEN-----
-eyJ6b25lSUQiOiAiN2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkiLCAi
-c2VydmljZUtleSI6ICJ0ZXN0LXNlcnZpY2Uta2V5IiwgImFjY291bnRJRCI6ICJh
-YmNkYWJjZGFiY2RhYmNkMTIzNDU2Nzg5MGFiY2RlZiJ9
+eyJ6b25lSUQiOiAiN2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkiLCAiYWNjb3VudElE
+IjogImFiY2RhYmNkYWJjZGFiY2QxMjM0NTY3ODkwYWJjZGVmIn0=
 -----END ARGO TUNNEL TOKEN-----

certutil/test-cert-two-certificates.pem

@@ -1,85 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
-sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
-1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
-z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
-6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
-x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
-1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
-IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
-xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
-sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
-2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
-sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
-Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
-AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
-m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
-QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
-P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
-pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
-G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
-6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
-LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
-OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
-W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
-M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
-y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
-uQicoQq3yzeQh20wtrtaXzTNmA==
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN WARP TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
-ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
-MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
-NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
-NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
-OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
-ZWYxZTI2Zjc=
------END WARP TOKEN-----

certutil/test-cert-unknown-block.pem

@@ -50,7 +50,7 @@ cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
 K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
 x+Yo/cL8fGfVpPt4UM8=
 -----END CERTIFICATE-----
------BEGIN WARP TOKEN-----
+-----BEGIN ARGO TUNNEL TOKEN-----
 N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
 ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
 MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
@@ -58,7 +58,7 @@ NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
 NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
 OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
 ZWYxZTI2Zjc=
------END WARP TOKEN-----
+-----END ARGO TUNNEL TOKEN-----
 -----BEGIN RSA PRIVATE KEY-----
 MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
 sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg

certutil/test-cert.pem

@@ -1,61 +0,0 @@
------BEGIN PRIVATE KEY-----
-MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
-sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
-1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
-z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
-6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
-x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
-1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
-IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
-xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
-sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
-2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
-sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
-Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
-AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
-m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
-QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
-P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
-pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
-G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
-6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
-LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
-OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
-W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
-M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
-y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
-uQicoQq3yzeQh20wtrtaXzTNmA==
------END PRIVATE KEY-----
------BEGIN CERTIFICATE-----
-MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
-gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
-VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
-cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
-YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
-MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
-ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
-aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
-GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
-6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
-tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
-PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
-AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
-HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
-VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
-zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
-RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
-YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
-YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
-cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
-K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
-x+Yo/cL8fGfVpPt4UM8=
------END CERTIFICATE-----
------BEGIN WARP TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
-ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
-MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
-NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
-NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
-OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
-ZWYxZTI2Zjc=
------END WARP TOKEN-----

certutil/test-cloudflare-tunnel-cert-json.pem

@@ -51,6 +51,7 @@ K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
 x+Yo/cL8fGfVpPt4UM8=
 -----END CERTIFICATE-----
 -----BEGIN ARGO TUNNEL TOKEN-----
-N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdGVzdC1zZXJ2aWNlLWtl
-eQ==
+eyJ6b25lSUQiOiAiN2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkiLCAiYXBpVG9rZW4i
+OiAidGVzdC1zZXJ2aWNlLWtleSIsICJhY2NvdW50SUQiOiAiYWJjZGFiY2RhYmNkYWJjZDEyMzQ1
+Njc4OTBhYmNkZWYifQ==
 -----END ARGO TUNNEL TOKEN-----

cfapi/base_client.go

@@ -104,7 +104,7 @@ func (r *RESTClient) sendRequest(method string, url url.URL, body interface{}) (
 	if bodyReader != nil {
 		req.Header.Set("Content-Type", jsonContentType)
 	}
-	req.Header.Add("X-Auth-User-Service-Key", r.authToken)
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", r.authToken))
 	req.Header.Add("Accept", "application/json;version=1")
 	return r.client.Do(req)
 }

cfapi/client.go

@@ -5,8 +5,9 @@ import (
 )
 
 type TunnelClient interface {
-	CreateTunnel(name string, tunnelSecret []byte) (*Tunnel, error)
+	CreateTunnel(name string, tunnelSecret []byte) (*TunnelWithToken, error)
 	GetTunnel(tunnelID uuid.UUID) (*Tunnel, error)
+	GetTunnelToken(tunnelID uuid.UUID) (string, error)
 	DeleteTunnel(tunnelID uuid.UUID) error
 	ListTunnels(filter *TunnelFilter) ([]*Tunnel, error)
 	ListActiveClients(tunnelID uuid.UUID) ([]*ActiveClient, error)

cfapi/tunnel.go

@@ -23,6 +23,11 @@ type Tunnel struct {
 	Connections []Connection `json:"connections"`
 }
 
+type TunnelWithToken struct {
+	Tunnel
+	Token string `json:"token"`
+}
+
 type Connection struct {
 	ColoName           string    `json:"colo_name"`
 	ID                 uuid.UUID `json:"id"`
@@ -63,7 +68,7 @@ func (cp CleanupParams) encode() string {
 	return cp.queryParams.Encode()
 }
 
-func (r *RESTClient) CreateTunnel(name string, tunnelSecret []byte) (*Tunnel, error) {
+func (r *RESTClient) CreateTunnel(name string, tunnelSecret []byte) (*TunnelWithToken, error) {
 	if name == "" {
 		return nil, errors.New("tunnel name required")
 	}
@@ -83,7 +88,11 @@ func (r *RESTClient) CreateTunnel(name string, tunnelSecret []byte) (*Tunnel, er
 
 	switch resp.StatusCode {
 	case http.StatusOK:
-		return unmarshalTunnel(resp.Body)
+		var tunnel TunnelWithToken
+		if serdeErr := parseResponse(resp.Body, &tunnel); err != nil {
+			return nil, serdeErr
+		}
+		return &tunnel, nil
 	case http.StatusConflict:
 		return nil, ErrTunnelNameConflict
 	}
@@ -107,6 +116,23 @@ func (r *RESTClient) GetTunnel(tunnelID uuid.UUID) (*Tunnel, error) {
 	return nil, r.statusCodeToError("get tunnel", resp)
 }
 
+func (r *RESTClient) GetTunnelToken(tunnelID uuid.UUID) (token string, err error) {
+	endpoint := r.baseEndpoints.accountLevel
+	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/token", tunnelID))
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return "", errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		err = parseResponse(resp.Body, &token)
+		return token, err
+	}
+
+	return "", r.statusCodeToError("get tunnel token", resp)
+}
+
 func (r *RESTClient) DeleteTunnel(tunnelID uuid.UUID) error {
 	endpoint := r.baseEndpoints.accountLevel
 	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v", tunnelID))

cfio/copy.go

@@ -0,0 +1,27 @@
+package cfio
+
+import (
+	"io"
+	"sync"
+)
+
+const defaultBufferSize = 16 * 1024
+
+var bufferPool = sync.Pool{
+	New: func() interface{} {
+		return make([]byte, defaultBufferSize)
+	},
+}
+
+func Copy(dst io.Writer, src io.Reader) (written int64, err error) {
+	_, okWriteTo := src.(io.WriterTo)
+	_, okReadFrom := dst.(io.ReaderFrom)
+	var buffer []byte = nil
+
+	if !(okWriteTo || okReadFrom) {
+		buffer = bufferPool.Get().([]byte)
+		defer bufferPool.Put(buffer)
+	}
+
+	return io.CopyBuffer(dst, src, buffer)
+}

cfsetup.yaml

@@ -1,23 +1,29 @@
-pinned_go: &pinned_go go=1.17.5-1
-pinned_go_fips: &pinned_go_fips go-boring=1.17.5-1
+pinned_go: &pinned_go go=1.19.3-1
+pinned_go_fips: &pinned_go_fips go-boring=1.19.3-1
 
 build_dir: &build_dir /cfsetup_build
-default-flavor: buster
+default-flavor: bullseye
 stretch: &stretch
   build:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &build_deps
       - *pinned_go
       - build-essential
+      - gotest-to-teamcity
+    pre-cache: &build_pre_cache
+      - export GOCACHE=/cfsetup_build/.cache/go-build
+      - go install golang.org/x/tools/cmd/goimports@latest
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared
   build-fips:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &build_deps_fips
       - *pinned_go_fips
       - build-essential
+      - gotest-to-teamcity
+    pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -40,16 +46,22 @@ stretch: &stretch
       - libffi-dev
       - python3-setuptools
       - python3-pip
+      - reprepro
+      - createrepo
     pre-cache: &github_release_pkgs_pre_cache
       - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
       - chmod a+x /usr/local/bin/wixl
       - pip3 install pynacl==1.4.0
       - pip3 install pygithub==1.55
+      - pip3 install boto3==1.22.9
+      - pip3 install python-gnupg==0.4.9
     post-cache:
       # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts
       - ./build-packages.sh
       # release the packages built and moved to /cfsetup/built_artifacts
       - make github-release-built-pkgs
+      # publish packages to linux repos
+      - make release-pkgs-linux
   # handle FIPS separately so that we built with gofips compiler
   github-fips-release-pkgs:
     build_dir: *build_dir
@@ -72,6 +84,13 @@ stretch: &stretch
       # same logic as above, but for FIPS packages only
       - ./build-packages-fips.sh
       - make github-release-built-pkgs
+  generate-versions-file:
+    build_dir: *build_dir
+    builddeps: 
+      - *pinned_go
+      - build-essential
+    post-cache:
+      - make generate-docker-version
   build-deb:
     build_dir: *build_dir
     builddeps: &build_deb_deps
@@ -113,21 +132,9 @@ stretch: &stretch
       - export GOOS=linux
       - export GOARCH=arm64
       - make cloudflared-deb
-  publish-deb:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - build-essential
-      - fakeroot
-      - rubygem-fpm
-      - openssh-client
-    post-cache:
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - make publish-deb
   github-release-macos-amd64:
     build_dir: *build_dir
-    builddeps:
+    builddeps: &build_pygithub
       - *pinned_go
       - build-essential
       - python3-dev
@@ -139,15 +146,31 @@ stretch: &stretch
       - pip3 install pygithub==1.55
     post-cache:
       - make github-mac-upload
-  test:
+  github-release-windows:
     build_dir: *build_dir
     builddeps:
       - *pinned_go
       - build-essential
-      - gotest-to-teamcity
-    pre-cache: &test_pre_cache
-      - go get golang.org/x/tools/cmd/goimports
-      - go get github.com/sudarshan-reddy/go-sumtype@v0.0.0-20210827105221-82eca7e5abb1
+      - python3-dev
+      - libffi-dev
+      - python3-setuptools
+      - python3-pip
+      - wget
+      # libmsi and libgcab are libraries the wixl binary depends on.
+      - libmsi-dev
+      - libgcab-dev
+    pre-cache:
+      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
+      - chmod a+x /usr/local/bin/wixl
+      - pip3 install pynacl==1.4.0
+      - pip3 install pygithub==1.55
+    post-cache:
+      - .teamcity/package-windows.sh
+      - make github-windows-upload
+  test:
+    build_dir: *build_dir
+    builddeps: *build_deps
+    pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -156,11 +179,8 @@ stretch: &stretch
       - make test | gotest-to-teamcity
   test-fips:
     build_dir: *build_dir
-    builddeps:
-      - *pinned_go_fips
-      - build-essential
-      - gotest-to-teamcity
-    pre-cache: *test_pre_cache
+    builddeps: *build_deps_fips
+    pre-cache: *build_pre_cache
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -169,6 +189,26 @@ stretch: &stretch
       - ./fmt-check.sh
       - make test | gotest-to-teamcity
   component-test:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - python3.7
+      - python3-pip
+      - python3-setuptools
+      # procps installs the ps command which is needed in test_sysv_service because the init script
+      # uses ps pid to determine if the agent is running
+      - procps
+    pre-cache-copy-paths:
+      - component-tests/requirements.txt
+    pre-cache: &component_test_pre_cache
+      - sudo pip3 install --upgrade -r component-tests/requirements.txt
+    post-cache: &component_test_post_cache
+      # Creates and routes a Named Tunnel for this build. Also constructs config file from env vars.
+      - python3 component-tests/setup.py --type create
+      - pytest component-tests -o log_cli=true --log-cli-level=INFO
+      # The Named Tunnel is deleted and its route unprovisioned here.
+      - python3 component-tests/setup.py --type cleanup
+  component-test-fips:
     build_dir: *build_dir
     builddeps:
       - *pinned_go_fips
@@ -180,29 +220,21 @@ stretch: &stretch
       - procps
     pre-cache-copy-paths:
       - component-tests/requirements.txt
-    pre-cache:
-      - sudo pip3 install --upgrade -r component-tests/requirements.txt
-    post-cache:
-      # Creates and routes a Named Tunnel for this build. Also constructs config file from env vars.
-      - python3 component-tests/setup.py --type create
-      - pytest component-tests -o log_cli=true --log-cli-level=INFO
-      # The Named Tunnel is deleted and its route unprovisioned here.
-      - python3 component-tests/setup.py --type cleanup
+    pre-cache: *component_test_pre_cache
+    post-cache: *component_test_post_cache
   update-homebrew:
     builddeps:
       - openssh-client
       - s3cmd
+      - jq
+      - build-essential
+      - procps
     post-cache:
       - .teamcity/update-homebrew.sh
+      - .teamcity/update-homebrew-core.sh
   github-message-release:
     build_dir: *build_dir
-    builddeps:
-      - *pinned_go
-      - build-essential
-      - python3-dev
-      - libffi-dev
-      - python3-setuptools
-      - python3-pip
+    builddeps: *build_pygithub
     pre-cache: *install_pygithub
     post-cache:
       - make github-message
@@ -213,13 +245,13 @@ stretch: &stretch
       - build-essential
       - python3
       - genisoimage
-      - jetez
     pre-cache:
       - ln -s /usr/bin/genisoimage /usr/bin/mkisofs
     post-cache:
+      - export CGO_ENABLED=0
       - export GOOS=freebsd
       - export GOARCH=amd64
-      - make cloudflared-junos
+      - make cloudflared
   publish-junos:
     build_dir: *build_dir
     builddeps:
@@ -238,18 +270,4 @@ stretch: &stretch
 
 buster: *stretch
 bullseye: *stretch
-centos-7:
-  publish-rpm:
-    build_dir: *build_dir
-    builddeps: &el7_builddeps
-      - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
-    pre-cache:
-      - yum install -y fakeroot
-      - yum upgrade -y binutils-2.27-44.base.el7.x86_64
-      - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.17.5.linux-amd64.tar.gz
-    post-cache:
-      - export PATH=$PATH:/usr/local/go/bin
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - make publish-rpm
+bookworm: *stretch

cmd/cloudflared/access/cmd.go

@@ -37,16 +37,13 @@ const (
 	sshConfigTemplate  = `
 Add to your {{.Home}}/.ssh/config:
 
-Host {{.Hostname}}
 {{- if .ShortLivedCerts}}
-  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1'
-
-Host cfpipe-{{.Hostname}}
-  HostName {{.Hostname}}
+Match host {{.Hostname}} exec "{{.Cloudflared}} access ssh-gen --hostname %h"
   ProxyCommand {{.Cloudflared}} access ssh --hostname %h
-  IdentityFile ~/.cloudflared/{{.Hostname}}-cf_key
-  CertificateFile ~/.cloudflared/{{.Hostname}}-cf_key-cert.pub
+  IdentityFile ~/.cloudflared/%h-cf_key
+  CertificateFile ~/.cloudflared/%h-cf_key-cert.pub
 {{- else}}
+Host {{.Hostname}}
   ProxyCommand {{.Cloudflared}} access ssh --hostname %h
 {{end}}
 `
@@ -56,11 +53,13 @@ const sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b@sentry.io/189878"
 
 var (
 	shutdownC chan struct{}
+	userAgent = "DEV"
 )
 
 // Init will initialize and store vars from the main program
-func Init(shutdown chan struct{}) {
+func Init(shutdown chan struct{}, version string) {
 	shutdownC = shutdown
+	userAgent = fmt.Sprintf("cloudflared/%s", version)
 }
 
 // Flags return the global flags for Access related commands (hopefully none)
@@ -294,6 +293,7 @@ func curl(c *cli.Context) error {
 // run kicks off a shell task and pipe the results to the respective std pipes
 func run(cmd string, args ...string) error {
 	c := exec.Command(cmd, args...)
+	c.Stdin = os.Stdin
 	stderr, err := c.StderrPipe()
 	if err != nil {
 		return err
@@ -505,7 +505,7 @@ func isTokenValid(options *carrier.StartOptions, log *zerolog.Logger) (bool, err
 	if err != nil {
 		return false, errors.Wrap(err, "Could not create access request")
 	}
-
+	req.Header.Set("User-Agent", userAgent)
 	// Do not follow redirects
 	client := &http.Client{
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {

cmd/cloudflared/common_service.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
+)
+
+func buildArgsForToken(c *cli.Context, log *zerolog.Logger) ([]string, error) {
+	token := c.Args().First()
+	if _, err := tunnel.ParseToken(token); err != nil {
+		return nil, cliutil.UsageError("Provided tunnel token is not valid (%s).", err)
+	}
+
+	return []string{
+		"tunnel", "run", "--token", token,
+	}, nil
+}
+
+func getServiceExtraArgsFromCliArgs(c *cli.Context, log *zerolog.Logger) ([]string, error) {
+	if c.NArg() > 0 {
+		// currently, we only support extra args for token
+		return buildArgsForToken(c, log)
+	} else {
+		// empty extra args
+		return make([]string, 0), nil
+	}
+}

cmd/cloudflared/linux_service.go

@@ -6,7 +6,6 @@ package main
 import (
 	"fmt"
 	"os"
-	"path/filepath"
 
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
@@ -20,22 +19,16 @@ import (
 func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
-		Usage: "Manages the Cloudflare Tunnel system service",
+		Usage: "Manages the cloudflared system service",
 		Subcommands: []*cli.Command{
 			{
 				Name:   "install",
-				Usage:  "Install Cloudflare Tunnel as a system service",
+				Usage:  "Install cloudflared as a system service",
 				Action: cliutil.ConfiguredAction(installLinuxService),
-				Flags: []cli.Flag{
-					&cli.BoolFlag{
-						Name:  "legacy",
-						Usage: "Generate service file for non-named tunnels",
-					},
-				},
 			},
 			{
 				Name:   "uninstall",
-				Usage:  "Uninstall the Cloudflare Tunnel service",
+				Usage:  "Uninstall the cloudflared service",
 				Action: cliutil.ConfiguredAction(uninstallLinuxService),
 			},
 		},
@@ -50,19 +43,20 @@ const (
 	serviceConfigFile     = "config.yml"
 	serviceCredentialFile = "cert.pem"
 	serviceConfigPath     = serviceConfigDir + "/" + serviceConfigFile
+	cloudflaredService    = "cloudflared.service"
 )
 
 var systemdTemplates = []ServiceTemplate{
 	{
-		Path: "/etc/systemd/system/cloudflared.service",
+		Path: fmt.Sprintf("/etc/systemd/system/%s", cloudflaredService),
 		Content: `[Unit]
-Description=Cloudflare Tunnel
+Description=cloudflared
 After=network.target
 
 [Service]
 TimeoutStartSec=0
 Type=notify
-ExecStart={{ .Path }} --config /etc/cloudflared/config.yml --no-autoupdate{{ range .ExtraArgs }} {{ . }}{{ end }}
+ExecStart={{ .Path }} --no-autoupdate{{ range .ExtraArgs }} {{ . }}{{ end }}
 Restart=on-failure
 RestartSec=5s
 
@@ -73,7 +67,7 @@ WantedBy=multi-user.target
 	{
 		Path: "/etc/systemd/system/cloudflared-update.service",
 		Content: `[Unit]
-Description=Update Cloudflare Tunnel
+Description=Update cloudflared
 After=network.target
 
 [Service]
@@ -83,7 +77,7 @@ ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 11 ]; then s
 	{
 		Path: "/etc/systemd/system/cloudflared-update.timer",
 		Content: `[Unit]
-Description=Update Cloudflare Tunnel
+Description=Update cloudflared
 
 [Timer]
 OnCalendar=daily
@@ -100,7 +94,7 @@ var sysvTemplate = ServiceTemplate{
 	Content: `#!/bin/sh
 # For RedHat and cousins:
 # chkconfig: 2345 99 01
-# description: Cloudflare Tunnel agent
+# description: cloudflared
 # processname: {{.Path}}
 ### BEGIN INIT INFO
 # Provides:          {{.Path}}
@@ -108,11 +102,11 @@ var sysvTemplate = ServiceTemplate{
 # Required-Stop:
 # Default-Start:     2 3 4 5
 # Default-Stop:      0 1 6
-# Short-Description: Cloudflare Tunnel
-# Description:       Cloudflare Tunnel agent
+# Short-Description: cloudflared
+# Description:       cloudflared agent
 ### END INIT INFO
 name=$(basename $(readlink -f $0))
-cmd="{{.Path}} --config /etc/cloudflared/config.yml --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s{{ range .ExtraArgs }} {{ . }}{{ end }}"
+cmd="{{.Path}} --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s{{ range .ExtraArgs }} {{ . }}{{ end }}"
 pid_file="/var/run/$name.pid"
 stdout_log="/var/log/$name.log"
 stderr_log="/var/log/$name.err"
@@ -191,27 +185,6 @@ func isSystemd() bool {
 	return false
 }
 
-func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile string, log *zerolog.Logger) error {
-	srcCredentialPath := filepath.Join(userConfigDir, userCredentialFile)
-	destCredentialPath := filepath.Join(serviceConfigDir, serviceCredentialFile)
-	if srcCredentialPath != destCredentialPath {
-		if err := copyCredential(srcCredentialPath, destCredentialPath); err != nil {
-			return err
-		}
-	}
-
-	srcConfigPath := filepath.Join(userConfigDir, userConfigFile)
-	destConfigPath := filepath.Join(serviceConfigDir, serviceConfigFile)
-	if srcConfigPath != destConfigPath {
-		if err := copyConfig(srcConfigPath, destConfigPath); err != nil {
-			return err
-		}
-		log.Info().Msgf("Copied %s to %s", srcConfigPath, destConfigPath)
-	}
-
-	return nil
-}
-
 func installLinuxService(c *cli.Context) error {
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
@@ -223,61 +196,69 @@ func installLinuxService(c *cli.Context) error {
 		Path: etPath,
 	}
 
-	if err := ensureConfigDirExists(serviceConfigDir); err != nil {
+	var extraArgsFunc func(c *cli.Context, log *zerolog.Logger) ([]string, error)
+	if c.NArg() == 0 {
+		extraArgsFunc = buildArgsForConfig
+	} else {
+		extraArgsFunc = buildArgsForToken
+	}
+
+	extraArgs, err := extraArgsFunc(c, log)
+	if err != nil {
 		return err
 	}
-	if c.Bool("legacy") {
-		userConfigDir := filepath.Dir(c.String("config"))
-		userConfigFile := filepath.Base(c.String("config"))
-		userCredentialFile := config.DefaultCredentialFile
-		if err = copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile, log); err != nil {
-			log.Err(err).Msgf("Failed to copy user configuration. Before running the service, ensure that %s contains two files, %s and %s",
-				serviceConfigDir, serviceCredentialFile, serviceConfigFile)
-			return err
-		}
-		templateArgs.ExtraArgs = []string{
-			"--origincert", serviceConfigDir + "/" + serviceCredentialFile,
-		}
-	} else {
-		src, _, err := config.ReadConfigFile(c, log)
-		if err != nil {
-			return err
-		}
 
-		// can't use context because this command doesn't define "credentials-file" flag
-		configPresent := func(s string) bool {
-			val, err := src.String(s)
-			return err == nil && val != ""
-		}
-		if src.TunnelID == "" || !configPresent(tunnel.CredFileFlag) {
-			return fmt.Errorf(`Configuration file %s must contain entries for the tunnel to run and its associated credentials:
-tunnel: TUNNEL-UUID
-credentials-file: CREDENTIALS-FILE
-`, src.Source())
-		}
-		if src.Source() != serviceConfigPath {
-			if exists, err := config.FileExists(serviceConfigPath); err != nil || exists {
-				return fmt.Errorf("Possible conflicting configuration in %[1]s and %[2]s. Either remove %[2]s or run `cloudflared --config %[2]s service install`", src.Source(), serviceConfigPath)
-			}
-
-			if err := copyFile(src.Source(), serviceConfigPath); err != nil {
-				return fmt.Errorf("failed to copy %s to %s: %w", src.Source(), serviceConfigPath, err)
-			}
-		}
-
-		templateArgs.ExtraArgs = []string{
-			"tunnel", "run",
-		}
-	}
+	templateArgs.ExtraArgs = extraArgs
 
 	switch {
 	case isSystemd():
 		log.Info().Msgf("Using Systemd")
-		return installSystemd(&templateArgs, log)
+		err = installSystemd(&templateArgs, log)
 	default:
 		log.Info().Msgf("Using SysV")
-		return installSysv(&templateArgs, log)
+		err = installSysv(&templateArgs, log)
 	}
+
+	if err == nil {
+		log.Info().Msg("Linux service for cloudflared installed successfully")
+	}
+	return err
+}
+
+func buildArgsForConfig(c *cli.Context, log *zerolog.Logger) ([]string, error) {
+	if err := ensureConfigDirExists(serviceConfigDir); err != nil {
+		return nil, err
+	}
+
+	src, _, err := config.ReadConfigFile(c, log)
+	if err != nil {
+		return nil, err
+	}
+
+	// can't use context because this command doesn't define "credentials-file" flag
+	configPresent := func(s string) bool {
+		val, err := src.String(s)
+		return err == nil && val != ""
+	}
+	if src.TunnelID == "" || !configPresent(tunnel.CredFileFlag) {
+		return nil, fmt.Errorf(`Configuration file %s must contain entries for the tunnel to run and its associated credentials:
+tunnel: TUNNEL-UUID
+credentials-file: CREDENTIALS-FILE
+`, src.Source())
+	}
+	if src.Source() != serviceConfigPath {
+		if exists, err := config.FileExists(serviceConfigPath); err != nil || exists {
+			return nil, fmt.Errorf("Possible conflicting configuration in %[1]s and %[2]s. Either remove %[2]s or run `cloudflared --config %[2]s service install`", src.Source(), serviceConfigPath)
+		}
+
+		if err := copyFile(src.Source(), serviceConfigPath); err != nil {
+			return nil, fmt.Errorf("failed to copy %s to %s: %w", src.Source(), serviceConfigPath, err)
+		}
+	}
+
+	return []string{
+		"--config", "/etc/cloudflared/config.yml", "tunnel", "run",
+	}, nil
 }
 
 func installSystemd(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
@@ -288,16 +269,19 @@ func installSystemd(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) erro
 			return err
 		}
 	}
-	if err := runCommand("systemctl", "enable", "cloudflared.service"); err != nil {
-		log.Err(err).Msg("systemctl enable cloudflared.service error")
+	if err := runCommand("systemctl", "enable", cloudflaredService); err != nil {
+		log.Err(err).Msgf("systemctl enable %s error", cloudflaredService)
 		return err
 	}
 	if err := runCommand("systemctl", "start", "cloudflared-update.timer"); err != nil {
 		log.Err(err).Msg("systemctl start cloudflared-update.timer error")
 		return err
 	}
-	log.Info().Msg("systemctl daemon-reload")
-	return runCommand("systemctl", "daemon-reload")
+	if err := runCommand("systemctl", "daemon-reload"); err != nil {
+		log.Err(err).Msg("systemctl daemon-reload error")
+		return err
+	}
+	return runCommand("systemctl", "start", cloudflaredService)
 }
 
 func installSysv(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
@@ -320,25 +304,35 @@ func installSysv(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
 			continue
 		}
 	}
-	return nil
+	return runCommand("service", "cloudflared", "start")
 }
 
 func uninstallLinuxService(c *cli.Context) error {
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
+	var err error
 	switch {
 	case isSystemd():
 		log.Info().Msg("Using Systemd")
-		return uninstallSystemd(log)
+		err = uninstallSystemd(log)
 	default:
 		log.Info().Msg("Using SysV")
-		return uninstallSysv(log)
+		err = uninstallSysv(log)
 	}
+
+	if err == nil {
+		log.Info().Msg("Linux service for cloudflared uninstalled successfully")
+	}
+	return err
 }
 
 func uninstallSystemd(log *zerolog.Logger) error {
-	if err := runCommand("systemctl", "disable", "cloudflared.service"); err != nil {
-		log.Err(err).Msg("systemctl disable cloudflared.service error")
+	if err := runCommand("systemctl", "disable", cloudflaredService); err != nil {
+		log.Err(err).Msgf("systemctl disable %s error", cloudflaredService)
+		return err
+	}
+	if err := runCommand("systemctl", "stop", cloudflaredService); err != nil {
+		log.Err(err).Msgf("systemctl stop %s error", cloudflaredService)
 		return err
 	}
 	if err := runCommand("systemctl", "stop", "cloudflared-update.timer"); err != nil {
@@ -351,11 +345,18 @@ func uninstallSystemd(log *zerolog.Logger) error {
 			return err
 		}
 	}
-	log.Info().Msgf("Successfully uninstalled cloudflared service from systemd")
+	if err := runCommand("systemctl", "daemon-reload"); err != nil {
+		log.Err(err).Msg("systemctl daemon-reload error")
+		return err
+	}
 	return nil
 }
 
 func uninstallSysv(log *zerolog.Logger) error {
+	if err := runCommand("service", "cloudflared", "stop"); err != nil {
+		log.Err(err).Msg("service cloudflared stop error")
+		return err
+	}
 	if err := sysvTemplate.Remove(); err != nil {
 		log.Err(err).Msg("error removing service template")
 		return err
@@ -370,6 +371,5 @@ func uninstallSysv(log *zerolog.Logger) error {
 			continue
 		}
 	}
-	log.Info().Msgf("Successfully uninstalled cloudflared service from sysv")
 	return nil
 }

cmd/cloudflared/macos_service.go

@@ -21,16 +21,16 @@ const (
 func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
-		Usage: "Manages the Cloudflare Tunnel launch agent",
+		Usage: "Manages the cloudflared launch agent",
 		Subcommands: []*cli.Command{
 			{
 				Name:   "install",
-				Usage:  "Install Cloudflare Tunnel as an user launch agent",
+				Usage:  "Install cloudflared as an user launch agent",
 				Action: cliutil.ConfiguredAction(installLaunchd),
 			},
 			{
 				Name:   "uninstall",
-				Usage:  "Uninstall the Cloudflare Tunnel launch agent",
+				Usage:  "Uninstall the cloudflared launch agent",
 				Action: cliutil.ConfiguredAction(uninstallLaunchd),
 			},
 		},
@@ -50,6 +50,9 @@ func newLaunchdTemplate(installPath, stdoutPath, stderrPath string) *ServiceTemp
 		<key>ProgramArguments</key>
 		<array>
 			<string>{{ .Path }}</string>
+			{{- range $i, $item := .ExtraArgs}}
+			<string>{{ $item }}</string>
+			{{- end}}
 		</array>
 		<key>RunAtLoad</key>
 		<true/>
@@ -111,12 +114,12 @@ func installLaunchd(c *cli.Context) error {
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	if isRootUser() {
-		log.Info().Msg("Installing Cloudflare Tunnel client as a system launch daemon. " +
-			"Cloudflare Tunnel client will run at boot")
+		log.Info().Msg("Installing cloudflared client as a system launch daemon. " +
+			"cloudflared client will run at boot")
 	} else {
-		log.Info().Msg("Installing Cloudflare Tunnel client as an user launch agent. " +
-			"Note that Cloudflare Tunnel client will only run when the user is logged in. " +
-			"If you want to run Cloudflare Tunnel client at boot, install with root permission. " +
+		log.Info().Msg("Installing cloudflared client as an user launch agent. " +
+			"Note that cloudflared client will only run when the user is logged in. " +
+			"If you want to run cloudflared client at boot, install with root permission. " +
 			"For more information, visit https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/run-tunnel/run-as-service")
 	}
 	etPath, err := os.Executable()
@@ -129,6 +132,13 @@ func installLaunchd(c *cli.Context) error {
 		log.Err(err).Msg("Error determining install path")
 		return errors.Wrap(err, "Error determining install path")
 	}
+	extraArgs, err := getServiceExtraArgsFromCliArgs(c, log)
+	if err != nil {
+		errMsg := "Unable to determine extra arguments for launch daemon"
+		log.Err(err).Msg(errMsg)
+		return errors.Wrap(err, errMsg)
+	}
+
 	stdoutPath, err := stdoutPath()
 	if err != nil {
 		log.Err(err).Msg("error determining stdout path")
@@ -140,7 +150,7 @@ func installLaunchd(c *cli.Context) error {
 		return errors.Wrap(err, "error determining stderr path")
 	}
 	launchdTemplate := newLaunchdTemplate(installPath, stdoutPath, stderrPath)
-	templateArgs := ServiceTemplateArgs{Path: etPath}
+	templateArgs := ServiceTemplateArgs{Path: etPath, ExtraArgs: extraArgs}
 	err = launchdTemplate.Generate(&templateArgs)
 	if err != nil {
 		log.Err(err).Msg("error generating launchd template")
@@ -153,16 +163,20 @@ func installLaunchd(c *cli.Context) error {
 	}
 
 	log.Info().Msgf("Outputs are logged to %s and %s", stderrPath, stdoutPath)
-	return runCommand("launchctl", "load", plistPath)
+	err = runCommand("launchctl", "load", plistPath)
+	if err == nil {
+		log.Info().Msg("MacOS service for cloudflared installed successfully")
+	}
+	return err
 }
 
 func uninstallLaunchd(c *cli.Context) error {
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	if isRootUser() {
-		log.Info().Msg("Uninstalling Cloudflare Tunnel as a system launch daemon")
+		log.Info().Msg("Uninstalling cloudflared as a system launch daemon")
 	} else {
-		log.Info().Msg("Uninstalling Cloudflare Tunnel as an user launch agent")
+		log.Info().Msg("Uninstalling cloudflared as a user launch agent")
 	}
 	installPath, err := installPath()
 	if err != nil {
@@ -184,10 +198,13 @@ func uninstallLaunchd(c *cli.Context) error {
 	}
 	err = runCommand("launchctl", "unload", plistPath)
 	if err != nil {
-		log.Err(err).Msg("error unloading")
+		log.Err(err).Msg("error unloading launchd")
 		return err
 	}
 
-	log.Info().Msgf("Outputs are logged to %s and %s", stderrPath, stdoutPath)
-	return launchdTemplate.Remove()
+	err = launchdTemplate.Remove()
+	if err == nil {
+		log.Info().Msg("Launchd for cloudflared was uninstalled successfully")
+	}
+	return err
 }

cmd/cloudflared/main.go

@@ -21,6 +21,8 @@ import (
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/overwatch"
+	"github.com/cloudflare/cloudflared/token"
+	"github.com/cloudflare/cloudflared/tracing"
 	"github.com/cloudflare/cloudflared/watcher"
 )
 
@@ -69,7 +71,7 @@ func main() {
 	app.Copyright = fmt.Sprintf(
 		`(c) %d Cloudflare Inc.
    Your installation of cloudflared software constitutes a symbol of your signature indicating that you accept
-   the terms of the Cloudflare License (https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/license),
+   the terms of the Apache License Version 2.0 (https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/license),
    Terms (https://www.cloudflare.com/terms/) and Privacy Policy (https://www.cloudflare.com/privacypolicy/).`,
 		time.Now().Year(),
 	)
@@ -84,8 +86,10 @@ func main() {
 	app.Commands = commands(cli.ShowVersion)
 
 	tunnel.Init(bInfo, graceShutdownC) // we need this to support the tunnel sub command...
-	access.Init(graceShutdownC)
+	access.Init(graceShutdownC, Version)
 	updater.Init(Version)
+	tracing.Init(Version)
+	token.Init(Version)
 	runApp(app, graceShutdownC)
 }
 

cmd/cloudflared/proxydns/cmd.go

@@ -1,6 +1,7 @@
 package proxydns
 
 import (
+	"context"
 	"net"
 	"os"
 	"os/signal"
@@ -73,7 +74,7 @@ func Run(c *cli.Context) error {
 		log.Fatal().Err(err).Msg("Failed to open the metrics listener")
 	}
 
-	go metrics.ServeMetrics(metricsListener, nil, nil, "", log)
+	go metrics.ServeMetrics(metricsListener, context.Background(), metrics.Config{}, log)
 
 	listener, err := tunneldns.CreateListener(
 		c.String("address"),

cmd/cloudflared/service_template.go

@@ -8,6 +8,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path"
 	"text/template"
 
 	homedir "github.com/mitchellh/go-homedir"
@@ -43,15 +44,26 @@ func (st *ServiceTemplate) Generate(args *ServiceTemplateArgs) error {
 	if err != nil {
 		return err
 	}
+	if _, err = os.Stat(resolvedPath); err == nil {
+		return fmt.Errorf(serviceAlreadyExistsWarn(resolvedPath))
+	}
+
 	var buffer bytes.Buffer
 	err = tmpl.Execute(&buffer, args)
 	if err != nil {
 		return fmt.Errorf("error generating %s: %v", st.Path, err)
 	}
-	fileMode := os.FileMode(0644)
+	fileMode := os.FileMode(0o644)
 	if st.FileMode != 0 {
 		fileMode = st.FileMode
 	}
+
+	plistFolder := path.Dir(resolvedPath)
+	err = os.MkdirAll(plistFolder, 0o755)
+	if err != nil {
+		return fmt.Errorf("error creating %s: %v", plistFolder, err)
+	}
+
 	err = ioutil.WriteFile(resolvedPath, buffer.Bytes(), fileMode)
 	if err != nil {
 		return fmt.Errorf("error writing %s: %v", resolvedPath, err)
@@ -71,6 +83,15 @@ func (st *ServiceTemplate) Remove() error {
 	return nil
 }
 
+func serviceAlreadyExistsWarn(service string) string {
+	return fmt.Sprintf("cloudflared service is already installed at %s; if you are running a cloudflared tunnel, you "+
+		"can point it to multiple origins, avoiding the need to run more than one cloudflared service in the "+
+		"same machine; otherwise if you are really sure, you can do `cloudflared service uninstall` to clean "+
+		"up the existing service and then try again this command",
+		service,
+	)
+}
+
 func runCommand(command string, args ...string) error {
 	cmd := exec.Command(command, args...)
 	stderr, err := cmd.StderrPipe()
@@ -82,10 +103,10 @@ func runCommand(command string, args ...string) error {
 		return fmt.Errorf("error starting %s: %v", command, err)
 	}
 
-	_, _ = ioutil.ReadAll(stderr)
+	output, _ := ioutil.ReadAll(stderr)
 	err = cmd.Wait()
 	if err != nil {
-		return fmt.Errorf("%s returned with error: %v", command, err)
+		return fmt.Errorf("%s %v returned with error code %v due to: %v", command, args, err, string(output))
 	}
 	return nil
 }

cmd/cloudflared/tunnel/cmd.go

@@ -15,6 +15,7 @@ import (
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/getsentry/raven-go"
+	"github.com/google/uuid"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -24,7 +25,6 @@ import (
 	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
@@ -82,6 +82,15 @@ const (
 	LogFieldPIDPathname         = "pidPathname"
 	LogFieldTmpTraceFilename    = "tmpTraceFilename"
 	LogFieldTraceOutputFilepath = "traceOutputFilepath"
+
+	tunnelCmdErrorMessage = `You did not specify any valid additional argument to the cloudflared tunnel command. 
+
+If you are trying to run a Quick Tunnel then you need to explicitly pass the --url flag. 
+Eg. cloudflared tunnel --url localhost:8080/. 
+
+Please note that Quick Tunnels are meant to be ephemeral and should only be used for testing purposes. 
+For production usage, we recommend creating Named Tunnels. (https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/tunnel-guide/)
+`
 )
 
 var (
@@ -109,6 +118,7 @@ func Commands() []*cli.Command {
 		buildIngressSubcommand(),
 		buildDeleteCommand(),
 		buildCleanupCommand(),
+		buildTokenCommand(),
 		// for compatibility, allow following as tunnel subcommands
 		proxydns.Command(true),
 		cliutil.RemovedCommand("db-connect"),
@@ -127,26 +137,34 @@ func buildTunnelCommand(subcommands []*cli.Command) *cli.Command {
 		Name:      "tunnel",
 		Action:    cliutil.ConfiguredAction(TunnelCommand),
 		Category:  "Tunnel",
-		Usage:     "Make a locally-running web service accessible over the internet using Cloudflare Tunnel.",
+		Usage:     "Use Cloudflare Tunnel to expose private services to the Internet or to Cloudflare connected private users.",
 		ArgsUsage: " ",
-		Description: `Cloudflare Tunnel asks you to specify a hostname on a Cloudflare-powered
-		domain you control and a local address. Traffic from that hostname is routed
-		(optionally via a Cloudflare Load Balancer) to this machine and appears on the
-		specified port where it can be served.
+		Description: `    Cloudflare Tunnel allows to expose private services without opening any ingress port on this machine. It can expose:
+  A) Locally reachable HTTP-based private services to the Internet on DNS with Cloudflare as authority (which you can
+then protect with Cloudflare Access).
+  B) Locally reachable TCP/UDP-based private services to Cloudflare connected private users in the same account, e.g.,
+those enrolled to a Zero Trust WARP Client.
 
-		This feature requires your Cloudflare account be subscribed to the Cloudflare Smart Routing feature.
+You can manage your Tunnels via dash.teams.cloudflare.com. This approach will only require you to run a single command
+later in each machine where you wish to run a Tunnel.
 
-		To use, begin by calling login to download a certificate:
+Alternatively, you can manage your Tunnels via the command line. Begin by obtaining a certificate to be able to do so:
 
-			$ cloudflared tunnel login
+	$ cloudflared tunnel login
 
-		With your certificate installed you can then launch your first tunnel,
-		replacing my.site.com with a subdomain of your site:
+With your certificate installed you can then get started with Tunnels:
 
-			$ cloudflared tunnel --hostname my.site.com --url http://localhost:8080
+	$ cloudflared tunnel create my-first-tunnel
+	$ cloudflared tunnel route dns my-first-tunnel my-first-tunnel.mydomain.com
+	$ cloudflared tunnel run --hello-world my-first-tunnel
 
-		If you have a web server running on port 8080 (in this example), it will be available on
-		the internet!`,
+You can now access my-first-tunnel.mydomain.com and be served an example page by your local cloudflared process.
+
+For exposing local TCP/UDP services by IP to your privately connected users, check out:
+
+	$ cloudflared tunnel route ip --help
+
+See https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/tunnel-guide/ for more info.`,
 		Subcommands: subcommands,
 		Flags:       tunnelFlags(false),
 	}
@@ -157,21 +175,27 @@ func TunnelCommand(c *cli.Context) error {
 	if err != nil {
 		return err
 	}
+
 	if name := c.String("name"); name != "" { // Start a named tunnel
 		return runAdhocNamedTunnel(sc, name, c.String(CredFileFlag))
 	}
+
+	// Unauthenticated named tunnel on <random>.<quick-tunnels-service>.com
+	shouldRunQuickTunnel := c.IsSet("url") || c.IsSet("hello-world")
+	if !dnsProxyStandAlone(c, nil) && c.String("hostname") == "" && c.String("quick-service") != "" && shouldRunQuickTunnel {
+		return RunQuickTunnel(sc)
+	}
+
 	if ref := config.GetConfiguration().TunnelID; ref != "" {
 		return fmt.Errorf("Use `cloudflared tunnel run` to start tunnel %s", ref)
 	}
 
-	// Unauthenticated named tunnel on <random>.<quick-tunnels-service>.com
-	// For now, default to legacy setup unless quick-service is specified
-	if !dnsProxyStandAlone(c, nil) && c.String("hostname") == "" && c.String("quick-service") != "" {
-		return RunQuickTunnel(sc)
+	// Start a classic tunnel if hostname is specified.
+	if c.String("hostname") != "" {
+		return runClassicTunnel(sc)
 	}
 
-	// Start a classic tunnel
-	return runClassicTunnel(sc)
+	return errors.New(tunnelCmdErrorMessage)
 }
 
 func Init(info *cliutil.BuildInfo, gracefulShutdown chan struct{}) {
@@ -208,7 +232,7 @@ func runAdhocNamedTunnel(sc *subcommandContext, name, credentialsOutputPath stri
 
 // runClassicTunnel creates a "classic" non-named tunnel
 func runClassicTunnel(sc *subcommandContext) error {
-	return StartServer(sc.c, buildInfo, nil, sc.log, sc.isUIEnabled)
+	return StartServer(sc.c, buildInfo, nil, sc.log)
 }
 
 func routeFromFlag(c *cli.Context) (route cfapi.HostnameRoute, ok bool) {
@@ -226,14 +250,14 @@ func StartServer(
 	info *cliutil.BuildInfo,
 	namedTunnel *connection.NamedTunnelProperties,
 	log *zerolog.Logger,
-	isUIEnabled bool,
 ) error {
 	_ = raven.SetDSN(sentryDSN)
 	var wg sync.WaitGroup
 	listeners := gracenet.Net{}
 	errC := make(chan error)
 
-	if config.GetConfiguration().Source() == "" {
+	// Only log for locally configured tunnels (Token is blank).
+	if config.GetConfiguration().Source() == "" && c.String(TunnelTokenFlag) == "" {
 		log.Info().Msg(config.ErrNoConfigFile.Error())
 	}
 
@@ -321,9 +345,9 @@ func StartServer(
 		return fmt.Errorf(errText)
 	}
 
-	logTransport := logger.CreateTransportLoggerFromContext(c, isUIEnabled)
+	logTransport := logger.CreateTransportLoggerFromContext(c, logger.EnableTerminalLog)
 
-	observer := connection.NewObserver(log, logTransport, isUIEnabled)
+	observer := connection.NewObserver(log, logTransport)
 
 	// Send Quick Tunnel URL to UI if applicable
 	var quickTunnelURL string
@@ -334,11 +358,24 @@ func StartServer(
 		observer.SendURL(quickTunnelURL)
 	}
 
-	tunnelConfig, dynamicConfig, err := prepareTunnelConfig(c, info, log, logTransport, observer, namedTunnel)
+	tunnelConfig, orchestratorConfig, err := prepareTunnelConfig(c, info, log, logTransport, observer, namedTunnel)
 	if err != nil {
 		log.Err(err).Msg("Couldn't start tunnel")
 		return err
 	}
+	var clientID uuid.UUID
+	if tunnelConfig.NamedTunnel != nil {
+		clientID, err = uuid.FromBytes(tunnelConfig.NamedTunnel.Client.ClientID)
+		if err != nil {
+			// set to nil for classic tunnels
+			clientID = uuid.Nil
+		}
+	}
+
+	orchestrator, err := orchestration.NewOrchestrator(ctx, orchestratorConfig, tunnelConfig.Tags, tunnelConfig.Log)
+	if err != nil {
+		return err
+	}
 
 	metricsListener, err := listeners.Listen("tcp", c.String("metrics"))
 	if err != nil {
@@ -349,17 +386,17 @@ func StartServer(
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		readinessServer := metrics.NewReadyServer(log)
+		readinessServer := metrics.NewReadyServer(log, clientID)
 		observer.RegisterSink(readinessServer)
-		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, quickTunnelURL, log)
+		metricsConfig := metrics.Config{
+			ReadyServer:         readinessServer,
+			QuickTunnelHostname: quickTunnelURL,
+			Orchestrator:        orchestrator,
+		}
+		errC <- metrics.ServeMetrics(metricsListener, ctx, metricsConfig, log)
 	}()
 
-	orchestrator, err := orchestration.NewOrchestrator(ctx, dynamicConfig, tunnelConfig.Tags, tunnelConfig.Log)
-	if err != nil {
-		return err
-	}
-
-	reconnectCh := make(chan supervisor.ReconnectSignal, 1)
+	reconnectCh := make(chan supervisor.ReconnectSignal, c.Int("ha-connections"))
 	if c.IsSet("stdin-control") {
 		log.Info().Msg("Enabling control through stdin")
 		go stdinControl(reconnectCh, log)
@@ -374,18 +411,6 @@ func StartServer(
 		errC <- supervisor.StartTunnelDaemon(ctx, tunnelConfig, orchestrator, connectedSignal, reconnectCh, graceShutdownC)
 	}()
 
-	if isUIEnabled {
-		tunnelUI := ui.NewUIModel(
-			info.Version(),
-			hostname,
-			metricsListener.Addr().String(),
-			dynamicConfig.Ingress,
-			tunnelConfig.HAConnections,
-		)
-		app := tunnelUI.Launch(ctx, log, logTransport)
-		observer.RegisterSink(app)
-	}
-
 	gracePeriod, err := gracePeriod(c)
 	if err != nil {
 		return err
@@ -505,6 +530,13 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Usage:   "Cloudflare Edge region to connect to. Omit or set to empty to connect to the global region.",
 			EnvVars: []string{"TUNNEL_REGION"},
 		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    "edge-ip-version",
+			Usage:   "Cloudflare Edge ip address version to connect with. {4, 6, auto}",
+			EnvVars: []string{"TUNNEL_EDGE_IP_VERSION"},
+			Value:   "4",
+			Hidden:  false,
+		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    tlsconfig.CaCertFlag,
 			Usage:   "Certificate Authority authenticating connections with Cloudflare's edge network.",
@@ -580,6 +612,12 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  5,
 			Hidden: true,
 		}),
+		altsrc.NewIntFlag(&cli.IntFlag{
+			Name:   "max-edge-addr-retries",
+			Usage:  "Maximum number of times to retry on edge addrs before falling back to a lower protocol",
+			Value:  8,
+			Hidden: true,
+		}),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:    "retries",
@@ -638,9 +676,9 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   uiFlag,
-			Usage:  "Launch tunnel UI. Tunnel logs are scrollable via 'j', 'k', or arrow keys.",
+			Usage:  "(depreciated) Launch tunnel UI. Tunnel logs are scrollable via 'j', 'k', or arrow keys.",
 			Value:  false,
-			Hidden: shouldHide,
+			Hidden: true,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:   "quick-service",
@@ -654,6 +692,13 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_MAX_FETCH_SIZE"},
 			Hidden:  true,
 		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "post-quantum",
+			Usage:   "When given creates an experimental post-quantum secure tunnel",
+			Aliases: []string{"pq"},
+			EnvVars: []string{"TUNNEL_POST_QUANTUM"},
+			Hidden:  FipsEnabled,
+		}),
 		selectProtocolFlag,
 		overwriteDNSFlag,
 	}...)
@@ -724,43 +769,43 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:    ingress.Socks5Flag,
-			Usage:   "specify if this tunnel is running as a SOCK5 Server",
+			Usage:   legacyTunnelFlag("specify if this tunnel is running as a SOCK5 Server"),
 			EnvVars: []string{"TUNNEL_SOCKS"},
 			Value:   false,
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   ingress.ProxyConnectTimeoutFlag,
-			Usage:  "HTTP proxy timeout for establishing a new connection",
+			Usage:  legacyTunnelFlag("HTTP proxy timeout for establishing a new connection"),
 			Value:  time.Second * 30,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   ingress.ProxyTLSTimeoutFlag,
-			Usage:  "HTTP proxy timeout for completing a TLS handshake",
+			Usage:  legacyTunnelFlag("HTTP proxy timeout for completing a TLS handshake"),
 			Value:  time.Second * 10,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   ingress.ProxyTCPKeepAliveFlag,
-			Usage:  "HTTP proxy TCP keepalive duration",
+			Usage:  legacyTunnelFlag("HTTP proxy TCP keepalive duration"),
 			Value:  time.Second * 30,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   ingress.ProxyNoHappyEyeballsFlag,
-			Usage:  "HTTP proxy should disable \"happy eyeballs\" for IPv4/v6 fallback",
+			Usage:  legacyTunnelFlag("HTTP proxy should disable \"happy eyeballs\" for IPv4/v6 fallback"),
 			Hidden: shouldHide,
 		}),
 		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:   ingress.ProxyKeepAliveConnectionsFlag,
-			Usage:  "HTTP proxy maximum keepalive connection pool size",
+			Usage:  legacyTunnelFlag("HTTP proxy maximum keepalive connection pool size"),
 			Value:  100,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   ingress.ProxyKeepAliveTimeoutFlag,
-			Usage:  "HTTP proxy timeout for closing an idle connection",
+			Usage:  legacyTunnelFlag("HTTP proxy timeout for closing an idle connection"),
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
@@ -778,13 +823,13 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    ingress.HTTPHostHeaderFlag,
-			Usage:   "Sets the HTTP Host header for the local webserver.",
+			Usage:   legacyTunnelFlag("Sets the HTTP Host header for the local webserver."),
 			EnvVars: []string{"TUNNEL_HTTP_HOST_HEADER"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    ingress.OriginServerNameFlag,
-			Usage:   "Hostname on the origin server certificate.",
+			Usage:   legacyTunnelFlag("Hostname on the origin server certificate."),
 			EnvVars: []string{"TUNNEL_ORIGIN_SERVER_NAME"},
 			Hidden:  shouldHide,
 		}),
@@ -796,26 +841,42 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    tlsconfig.OriginCAPoolFlag,
-			Usage:   "Path to the CA for the certificate of your origin. This option should be used only if your certificate is not signed by Cloudflare.",
+			Usage:   legacyTunnelFlag("Path to the CA for the certificate of your origin. This option should be used only if your certificate is not signed by Cloudflare."),
 			EnvVars: []string{"TUNNEL_ORIGIN_CA_POOL"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:    ingress.NoTLSVerifyFlag,
-			Usage:   "Disables TLS verification of the certificate presented by your origin. Will allow any certificate from the origin to be accepted. Note: The connection from your machine to Cloudflare's Edge is still encrypted.",
+			Usage:   legacyTunnelFlag("Disables TLS verification of the certificate presented by your origin. Will allow any certificate from the origin to be accepted. Note: The connection from your machine to Cloudflare's Edge is still encrypted."),
 			EnvVars: []string{"NO_TLS_VERIFY"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:    ingress.NoChunkedEncodingFlag,
-			Usage:   "Disables chunked transfer encoding; useful if you are running a WSGI server.",
+			Usage:   legacyTunnelFlag("Disables chunked transfer encoding; useful if you are running a WSGI server."),
 			EnvVars: []string{"TUNNEL_NO_CHUNKED_ENCODING"},
 			Hidden:  shouldHide,
 		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    ingress.Http2OriginFlag,
+			Usage:   "Enables HTTP/2 origin servers.",
+			EnvVars: []string{"TUNNEL_ORIGIN_ENABLE_HTTP2"},
+			Hidden:  shouldHide,
+			Value:   false,
+		}),
 	}
 	return append(flags, sshFlags(shouldHide)...)
 }
 
+func legacyTunnelFlag(msg string) string {
+	return fmt.Sprintf(
+		"%s This flag only takes effect if you define your origin with `--url` and if you do not use ingress rules."+
+			" The recommended way is to rely on ingress rules and define this property under `originRequest` as per"+
+			" https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/configuration-file/ingress",
+		msg,
+	)
+}
+
 func sshFlags(shouldHide bool) []cli.Flag {
 	return []cli.Flag{
 		altsrc.NewStringFlag(&cli.StringFlag{

cmd/cloudflared/tunnel/configuration.go

@@ -4,6 +4,9 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
+	mathRand "math/rand"
+	"net"
+	"net/netip"
 	"os"
 	"path/filepath"
 	"strings"
@@ -14,9 +17,11 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2/altsrc"
 	"golang.org/x/crypto/ssh/terminal"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
@@ -31,14 +36,19 @@ import (
 )
 
 const LogFieldOriginCertPath = "originCertPath"
+const secretValue = "*****"
 
 var (
 	developerPortal = "https://developers.cloudflare.com/argo-tunnel"
-	quickStartUrl   = developerPortal + "/quickstart/quickstart/"
 	serviceUrl      = developerPortal + "/reference/service/"
 	argumentsUrl    = developerPortal + "/reference/arguments/"
 
 	LogFieldHostname = "hostname"
+
+	secretFlags     = [2]*altsrc.StringFlag{credentialsContentsFlag, tunnelTokenFlag}
+	defaultFeatures = []string{supervisor.FeatureAllowRemoteConfig, supervisor.FeatureSerializedHeaders, supervisor.FeatureDatagramV2, supervisor.FeatureQUICSupportEOF}
+
+	configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq", "edge-ip-version"}
 )
 
 // returns the first path that contains a cert.pem file. If none of the DefaultConfigSearchDirectories
@@ -65,7 +75,11 @@ func generateRandomClientID(log *zerolog.Logger) (string, error) {
 func logClientOptions(c *cli.Context, log *zerolog.Logger) {
 	flags := make(map[string]interface{})
 	for _, flag := range c.FlagNames() {
-		flags[flag] = c.Generic(flag)
+		if isSecretFlag(flag) {
+			flags[flag] = secretValue
+		} else {
+			flags[flag] = c.Generic(flag)
+		}
 	}
 
 	if len(flags) > 0 {
@@ -79,7 +93,11 @@ func logClientOptions(c *cli.Context, log *zerolog.Logger) {
 		if strings.Contains(env, "TUNNEL_") {
 			vars := strings.Split(env, "=")
 			if len(vars) == 2 {
-				envs[vars[0]] = vars[1]
+				if isSecretEnvVar(vars[0]) {
+					envs[vars[0]] = secretValue
+				} else {
+					envs[vars[0]] = vars[1]
+				}
 			}
 		}
 	}
@@ -88,6 +106,26 @@ func logClientOptions(c *cli.Context, log *zerolog.Logger) {
 	}
 }
 
+func isSecretFlag(key string) bool {
+	for _, flag := range secretFlags {
+		if flag.Name == key {
+			return true
+		}
+	}
+	return false
+}
+
+func isSecretEnvVar(key string) bool {
+	for _, flag := range secretFlags {
+		for _, secretEnvVar := range flag.EnvVars {
+			if secretEnvVar == key {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelProperties) bool {
 	return c.IsSet("proxy-dns") && (!c.IsSet("hostname") && !c.IsSet("tag") && !c.IsSet("hello-world") && namedTunnel == nil)
 }
@@ -183,6 +221,23 @@ func prepareTunnelConfig(
 		ingressRules  ingress.Ingress
 		classicTunnel *connection.ClassicTunnelProperties
 	)
+
+	transportProtocol := c.String("protocol")
+
+	needPQ := c.Bool("post-quantum")
+	if needPQ {
+		if FipsEnabled {
+			return nil, nil, fmt.Errorf("post-quantum not supported in FIPS mode")
+		}
+		// Error if the user tries to force a non-quic transport protocol
+		if transportProtocol != connection.AutoSelectFlag && transportProtocol != connection.QUIC.String() {
+			return nil, nil, fmt.Errorf("post-quantum is only supported with the quic transport")
+		}
+		transportProtocol = connection.QUIC.String()
+	}
+
+	protocolFetcher := edgediscovery.ProtocolPercentage
+
 	cfg := config.GetConfiguration()
 	if isNamedTunnel {
 		clientUUID, err := uuid.NewRandom()
@@ -190,7 +245,29 @@ func prepareTunnelConfig(
 			return nil, nil, errors.Wrap(err, "can't generate connector UUID")
 		}
 		log.Info().Msgf("Generated Connector ID: %s", clientUUID)
-		features := append(c.StringSlice("features"), supervisor.FeatureSerializedHeaders)
+		features := append(c.StringSlice("features"), defaultFeatures...)
+		if needPQ {
+			features = append(features, supervisor.FeaturePostQuantum)
+		}
+		if c.IsSet(TunnelTokenFlag) {
+			if transportProtocol == connection.AutoSelectFlag {
+				protocolFetcher = func() (edgediscovery.ProtocolPercents, error) {
+					// If the Tunnel is remotely managed and no protocol is set, we prefer QUIC, but still allow fall-back.
+					preferQuic := []edgediscovery.ProtocolPercent{
+						{
+							Protocol:   connection.QUIC.String(),
+							Percentage: 100,
+						},
+						{
+							Protocol:   connection.HTTP2.String(),
+							Percentage: 100,
+						},
+					}
+					return preferQuic, nil
+				}
+			}
+			log.Info().Msg("Will be fetching remotely managed configuration from Cloudflare API. Defaulting to protocol: quic")
+		}
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],
 			Features: dedup(features),
@@ -233,7 +310,7 @@ func prepareTunnelConfig(
 	}
 
 	warpRoutingEnabled := isWarpRoutingEnabled(cfg.WarpRouting, isNamedTunnel)
-	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), warpRoutingEnabled, namedTunnel, edgediscovery.ProtocolPercentage, supervisor.ResolveTTL, log)
+	protocolSelector, err := connection.NewProtocolSelector(transportProtocol, warpRoutingEnabled, namedTunnel, protocolFetcher, supervisor.ResolveTTL, log, c.Bool("post-quantum"))
 	if err != nil {
 		return nil, nil, err
 	}
@@ -267,6 +344,19 @@ func prepareTunnelConfig(
 		CompressionSetting: h2mux.CompressionSetting(uint64(c.Int("compression-quality"))),
 		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
 	}
+	edgeIPVersion, err := parseConfigIPVersion(c.String("edge-ip-version"))
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var pqKexIdx int
+	if needPQ {
+		pqKexIdx = mathRand.Intn(len(supervisor.PQKexes))
+		log.Info().Msgf(
+			"Using experimental hybrid post-quantum key agreement %s",
+			supervisor.PQKexNames[supervisor.PQKexes[pqKexIdx]],
+		)
+	}
 
 	tunnelConfig := &supervisor.TunnelConfig{
 		GracePeriod:     gracePeriod,
@@ -275,6 +365,7 @@ func prepareTunnelConfig(
 		ClientID:        clientID,
 		EdgeAddrs:       c.StringSlice("edge"),
 		Region:          c.String("region"),
+		EdgeIPVersion:   edgeIPVersion,
 		HAConnections:   c.Int("ha-connections"),
 		IncidentLookup:  supervisor.NewIncidentLookup(),
 		IsAutoupdated:   c.Bool("is-autoupdated"),
@@ -285,19 +376,41 @@ func prepareTunnelConfig(
 		Observer:        observer,
 		ReportedVersion: info.Version(),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
-		Retries:          uint(c.Int("retries")),
-		RunFromTerminal:  isRunningFromTerminal(),
-		NamedTunnel:      namedTunnel,
-		ClassicTunnel:    classicTunnel,
-		MuxerConfig:      muxerConfig,
-		ProtocolSelector: protocolSelector,
-		EdgeTLSConfigs:   edgeTLSConfigs,
+		Retries:            uint(c.Int("retries")),
+		RunFromTerminal:    isRunningFromTerminal(),
+		NamedTunnel:        namedTunnel,
+		ClassicTunnel:      classicTunnel,
+		MuxerConfig:        muxerConfig,
+		ProtocolSelector:   protocolSelector,
+		EdgeTLSConfigs:     edgeTLSConfigs,
+		NeedPQ:             needPQ,
+		PQKexIdx:           pqKexIdx,
+		MaxEdgeAddrRetries: uint8(c.Int("max-edge-addr-retries")),
 	}
-	dynamicConfig := &orchestration.Config{
+	packetConfig, err := newPacketConfig(c, log)
+	if err != nil {
+		log.Warn().Err(err).Msg("ICMP proxy feature is disabled")
+	} else {
+		tunnelConfig.PacketConfig = packetConfig
+	}
+	orchestratorConfig := &orchestration.Config{
 		Ingress:            &ingressRules,
-		WarpRoutingEnabled: warpRoutingEnabled,
+		WarpRouting:        ingress.NewWarpRoutingConfig(&cfg.WarpRouting),
+		ConfigurationFlags: parseConfigFlags(c),
 	}
-	return tunnelConfig, dynamicConfig, nil
+	return tunnelConfig, orchestratorConfig, nil
+}
+
+func parseConfigFlags(c *cli.Context) map[string]string {
+	result := make(map[string]string)
+
+	for _, flag := range configFlags {
+		if v := c.String(flag); c.IsSet(flag) && v != "" {
+			result[flag] = v
+		}
+	}
+
+	return result
 }
 
 func gracePeriod(c *cli.Context) (time.Duration, error) {
@@ -334,3 +447,162 @@ func dedup(slice []string) []string {
 	}
 	return keys
 }
+
+// ParseConfigIPVersion returns the IP version from possible expected values from config
+func parseConfigIPVersion(version string) (v allregions.ConfigIPVersion, err error) {
+	switch version {
+	case "4":
+		v = allregions.IPv4Only
+	case "6":
+		v = allregions.IPv6Only
+	case "auto":
+		v = allregions.Auto
+	default: // unspecified or invalid
+		err = fmt.Errorf("invalid value for edge-ip-version: %s", version)
+	}
+	return
+}
+
+func newPacketConfig(c *cli.Context, logger *zerolog.Logger) (*ingress.GlobalRouterConfig, error) {
+	ipv4Src, err := determineICMPv4Src(c.String("icmpv4-src"), logger)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to determine IPv4 source address for ICMP proxy")
+	}
+	logger.Info().Msgf("ICMP proxy will use %s as source for IPv4", ipv4Src)
+
+	ipv6Src, zone, err := determineICMPv6Src(c.String("icmpv6-src"), logger, ipv4Src)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to determine IPv6 source address for ICMP proxy")
+	}
+	if zone != "" {
+		logger.Info().Msgf("ICMP proxy will use %s in zone %s as source for IPv6", ipv6Src, zone)
+	} else {
+		logger.Info().Msgf("ICMP proxy will use %s as source for IPv6", ipv6Src)
+	}
+
+	icmpRouter, err := ingress.NewICMPRouter(ipv4Src, ipv6Src, zone, logger)
+	if err != nil {
+		return nil, err
+	}
+	return &ingress.GlobalRouterConfig{
+		ICMPRouter: icmpRouter,
+		IPv4Src:    ipv4Src,
+		IPv6Src:    ipv6Src,
+		Zone:       zone,
+	}, nil
+}
+
+func determineICMPv4Src(userDefinedSrc string, logger *zerolog.Logger) (netip.Addr, error) {
+	if userDefinedSrc != "" {
+		addr, err := netip.ParseAddr(userDefinedSrc)
+		if err != nil {
+			return netip.Addr{}, err
+		}
+		if addr.Is4() {
+			return addr, nil
+		}
+		return netip.Addr{}, fmt.Errorf("expect IPv4, but %s is IPv6", userDefinedSrc)
+	}
+
+	addr, err := findLocalAddr(net.ParseIP("192.168.0.1"), 53)
+	if err != nil {
+		addr = netip.IPv4Unspecified()
+		logger.Debug().Err(err).Msgf("Failed to determine the IPv4 for this machine. It will use %s to send/listen for ICMPv4 echo", addr)
+	}
+	return addr, nil
+}
+
+type interfaceIP struct {
+	name string
+	ip   net.IP
+}
+
+func determineICMPv6Src(userDefinedSrc string, logger *zerolog.Logger, ipv4Src netip.Addr) (addr netip.Addr, zone string, err error) {
+	if userDefinedSrc != "" {
+		userDefinedIP, zone, _ := strings.Cut(userDefinedSrc, "%")
+		addr, err := netip.ParseAddr(userDefinedIP)
+		if err != nil {
+			return netip.Addr{}, "", err
+		}
+		if addr.Is6() {
+			return addr, zone, nil
+		}
+		return netip.Addr{}, "", fmt.Errorf("expect IPv6, but %s is IPv4", userDefinedSrc)
+	}
+
+	// Loop through all the interfaces, the preference is
+	// 1. The interface where ipv4Src is in
+	// 2. Interface with IPv6 address
+	// 3. Unspecified interface
+
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		return netip.IPv6Unspecified(), "", nil
+	}
+
+	interfacesWithIPv6 := make([]interfaceIP, 0)
+	for _, interf := range interfaces {
+		interfaceAddrs, err := interf.Addrs()
+		if err != nil {
+			continue
+		}
+
+		foundIPv4SrcInterface := false
+		for _, interfaceAddr := range interfaceAddrs {
+			if ipnet, ok := interfaceAddr.(*net.IPNet); ok {
+				ip := ipnet.IP
+				if ip.Equal(ipv4Src.AsSlice()) {
+					foundIPv4SrcInterface = true
+				}
+				if ip.To4() == nil {
+					interfacesWithIPv6 = append(interfacesWithIPv6, interfaceIP{
+						name: interf.Name,
+						ip:   ip,
+					})
+				}
+			}
+		}
+		// Found the interface of ipv4Src. Loop through the addresses to see if there is an IPv6
+		if foundIPv4SrcInterface {
+			for _, interfaceAddr := range interfaceAddrs {
+				if ipnet, ok := interfaceAddr.(*net.IPNet); ok {
+					ip := ipnet.IP
+					if ip.To4() == nil {
+						addr, err := netip.ParseAddr(ip.String())
+						if err == nil {
+							return addr, interf.Name, nil
+						}
+					}
+				}
+			}
+		}
+	}
+
+	for _, interf := range interfacesWithIPv6 {
+		addr, err := netip.ParseAddr(interf.ip.String())
+		if err == nil {
+			return addr, interf.name, nil
+		}
+	}
+	logger.Debug().Err(err).Msgf("Failed to determine the IPv6 for this machine. It will use %s to send/listen for ICMPv6 echo", netip.IPv6Unspecified())
+
+	return netip.IPv6Unspecified(), "", nil
+}
+
+// FindLocalAddr tries to dial UDP and returns the local address picked by the OS
+func findLocalAddr(dst net.IP, port int) (netip.Addr, error) {
+	udpConn, err := net.DialUDP("udp", nil, &net.UDPAddr{
+		IP:   dst,
+		Port: port,
+	})
+	if err != nil {
+		return netip.Addr{}, err
+	}
+	defer udpConn.Close()
+	localAddrPort, err := netip.ParseAddrPort(udpConn.LocalAddr().String())
+	if err != nil {
+		return netip.Addr{}, err
+	}
+	localAddr := localAddrPort.Addr()
+	return localAddr, nil
+}

cmd/cloudflared/tunnel/fips.go

@@ -0,0 +1,3 @@
+package tunnel
+
+var FipsEnabled bool

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -55,7 +55,6 @@ func RunQuickTunnel(sc *subcommandContext) error {
 		AccountTag:   data.Result.AccountTag,
 		TunnelSecret: data.Result.Secret,
 		TunnelID:     tunnelID,
-		TunnelName:   data.Result.Name,
 	}
 
 	url := data.Result.Hostname
@@ -79,7 +78,6 @@ func RunQuickTunnel(sc *subcommandContext) error {
 		buildInfo,
 		&connection.NamedTunnelProperties{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
 		sc.log,
-		sc.isUIEnabled,
 	)
 }
 

cmd/cloudflared/tunnel/subcommand_context.go

@@ -31,10 +31,9 @@ func (e errInvalidJSONCredential) Error() string {
 // subcommandContext carries structs shared between subcommands, to reduce number of arguments needed to
 // pass between subcommands, and make sure they are only initialized once
 type subcommandContext struct {
-	c           *cli.Context
-	log         *zerolog.Logger
-	isUIEnabled bool
-	fs          fileSystem
+	c   *cli.Context
+	log *zerolog.Logger
+	fs  fileSystem
 
 	// These fields should be accessed using their respective Getter
 	tunnelstoreClient cfapi.Client
@@ -42,16 +41,10 @@ type subcommandContext struct {
 }
 
 func newSubcommandContext(c *cli.Context) (*subcommandContext, error) {
-	isUIEnabled := c.IsSet(uiFlag) && c.String("name") != ""
-
-	// If UI is enabled, terminal log output should be disabled -- log should be written into a UI log window instead
-	log := logger.CreateLoggerFromContext(c, isUIEnabled)
-
 	return &subcommandContext{
-		c:           c,
-		log:         log,
-		isUIEnabled: isUIEnabled,
-		fs:          realFileSystem{},
+		c:   c,
+		log: logger.CreateLoggerFromContext(c, logger.EnableTerminalLog),
+		fs:  realFileSystem{},
 	}, nil
 }
 
@@ -81,7 +74,7 @@ func (sc *subcommandContext) client() (cfapi.Client, error) {
 		sc.c.String("api-url"),
 		credential.cert.AccountID,
 		credential.cert.ZoneID,
-		credential.cert.ServiceKey,
+		credential.cert.APIToken,
 		userAgent,
 		sc.log,
 	)
@@ -185,7 +178,6 @@ func (sc *subcommandContext) create(name string, credentialsFilePath string, sec
 		AccountTag:   credential.cert.AccountID,
 		TunnelSecret: tunnelSecret,
 		TunnelID:     tunnel.ID,
-		TunnelName:   name,
 	}
 	usedCertPath := false
 	if credentialsFilePath == "" {
@@ -221,7 +213,8 @@ func (sc *subcommandContext) create(name string, credentialsFilePath string, sec
 	}
 	fmt.Println(" Keep this file secret. To revoke these credentials, delete the tunnel.")
 	fmt.Printf("\nCreated tunnel %s with id %s\n", tunnel.Name, tunnel.ID)
-	return tunnel, nil
+
+	return &tunnel.Tunnel, nil
 }
 
 func (sc *subcommandContext) list(filter *cfapi.TunnelFilter) ([]*cfapi.Tunnel, error) {
@@ -301,12 +294,17 @@ func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 		return err
 	}
 
+	return sc.runWithCredentials(credentials)
+}
+
+func (sc *subcommandContext) runWithCredentials(credentials connection.Credentials) error {
+	sc.log.Info().Str(LogFieldTunnelID, credentials.TunnelID.String()).Msg("Starting tunnel")
+
 	return StartServer(
 		sc.c,
 		buildInfo,
 		&connection.NamedTunnelProperties{Credentials: credentials},
 		sc.log,
-		sc.isUIEnabled,
 	)
 }
 
@@ -335,6 +333,21 @@ func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
 	return nil
 }
 
+func (sc *subcommandContext) getTunnelTokenCredentials(tunnelID uuid.UUID) (*connection.TunnelToken, error) {
+	client, err := sc.client()
+	if err != nil {
+		return nil, err
+	}
+
+	token, err := client.GetTunnelToken(tunnelID)
+	if err != nil {
+		sc.log.Err(err).Msgf("Could not get the Token for the given Tunnel %v", tunnelID)
+		return nil, err
+	}
+
+	return ParseToken(token)
+}
+
 func (sc *subcommandContext) route(tunnelID uuid.UUID, r cfapi.HostnameRoute) (cfapi.HostnameRouteResult, error) {
 	client, err := sc.client()
 	if err != nil {
@@ -370,7 +383,7 @@ func (sc *subcommandContext) findID(input string) (uuid.UUID, error) {
 	// Look up name in the credentials file.
 	credFinder := newStaticPath(sc.c.String(CredFileFlag), sc.fs)
 	if credentials, err := sc.readTunnelCredentials(credFinder); err == nil {
-		if credentials.TunnelID != uuid.Nil && input == credentials.TunnelName {
+		if credentials.TunnelID != uuid.Nil {
 			return credentials.TunnelID, nil
 		}
 	}

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -35,7 +35,6 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 	type fields struct {
 		c                 *cli.Context
 		log               *zerolog.Logger
-		isUIEnabled       bool
 		fs                fileSystem
 		tunnelstoreClient cfapi.Client
 		userCredential    *userCredential
@@ -116,7 +115,6 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 				AccountTag:   accountTag,
 				TunnelID:     tunnelID,
 				TunnelSecret: secret,
-				TunnelName:   name,
 			},
 		},
 		{
@@ -161,7 +159,6 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 				AccountTag:   accountTag,
 				TunnelID:     tunnelID,
 				TunnelSecret: secret,
-				TunnelName:   name,
 			},
 		},
 	}
@@ -170,7 +167,6 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 			sc := &subcommandContext{
 				c:                 tt.fields.c,
 				log:               tt.fields.log,
-				isUIEnabled:       tt.fields.isUIEnabled,
 				fs:                tt.fields.fs,
 				tunnelstoreClient: tt.fields.tunnelstoreClient,
 				userCredential:    tt.fields.userCredential,
@@ -218,6 +214,10 @@ func (d *deleteMockTunnelStore) GetTunnel(tunnelID uuid.UUID) (*cfapi.Tunnel, er
 	return &tunnel.tunnel, nil
 }
 
+func (d *deleteMockTunnelStore) GetTunnelToken(tunnelID uuid.UUID) (string, error) {
+	return "token", nil
+}
+
 func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
 	tunnel, ok := d.mockTunnels[tunnelID]
 	if !ok {
@@ -305,7 +305,6 @@ func Test_subcommandContext_Delete(t *testing.T) {
 			sc := &subcommandContext{
 				c:                 tt.fields.c,
 				log:               tt.fields.log,
-				isUIEnabled:       tt.fields.isUIEnabled,
 				fs:                tt.fields.fs,
 				tunnelstoreClient: tt.fields.tunnelstoreClient,
 				userCredential:    tt.fields.userCredential,

cmd/cloudflared/tunnel/subcommands.go

@@ -2,6 +2,7 @@ package tunnel
 
 import (
 	"crypto/rand"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -19,7 +20,7 @@ import (
 	"github.com/urfave/cli/v2"
 	"github.com/urfave/cli/v2/altsrc"
 	"golang.org/x/net/idna"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
@@ -34,6 +35,7 @@ const (
 	CredFileFlagAlias    = "cred-file"
 	CredFileFlag         = "credentials-file"
 	CredContentsFlag     = "credentials-contents"
+	TunnelTokenFlag      = "token"
 	overwriteDNSFlagName = "overwrite-dns"
 
 	LogFieldTunnelID = "tunnelID"
@@ -118,6 +120,11 @@ var (
 		Usage:   "Contents of the tunnel credentials JSON file to use. When provided along with credentials-file, this will take precedence.",
 		EnvVars: []string{"TUNNEL_CRED_CONTENTS"},
 	})
+	tunnelTokenFlag = altsrc.NewStringFlag(&cli.StringFlag{
+		Name:    TunnelTokenFlag,
+		Usage:   "The Tunnel token. When provided along with credentials, this will take precedence.",
+		EnvVars: []string{"TUNNEL_TOKEN"},
+	})
 	forceDeleteFlag = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
@@ -127,12 +134,19 @@ var (
 	}
 	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{
 		Name:    "protocol",
-		Value:   "auto",
+		Value:   connection.AutoSelectFlag,
 		Aliases: []string{"p"},
 		Usage:   fmt.Sprintf("Protocol implementation to connect with Cloudflare's edge network. %s", connection.AvailableProtocolFlagMessage),
 		EnvVars: []string{"TUNNEL_TRANSPORT_PROTOCOL"},
 		Hidden:  true,
 	})
+	postQuantumFlag = altsrc.NewBoolFlag(&cli.BoolFlag{
+		Name:    "post-quantum",
+		Usage:   "When given creates an experimental post-quantum secure tunnel",
+		Aliases: []string{"pq"},
+		EnvVars: []string{"TUNNEL_POST_QUANTUM"},
+		Hidden:  FipsEnabled,
+	})
 	sortInfoByFlag = &cli.StringFlag{
 		Name:    "sort-by",
 		Value:   "createdAt",
@@ -162,6 +176,16 @@ var (
 		Usage:   "Base64 encoded secret to set for the tunnel. The decoded secret must be at least 32 bytes long. If not specified, a random 32-byte secret will be generated.",
 		EnvVars: []string{"TUNNEL_CREATE_SECRET"},
 	}
+	icmpv4SrcFlag = &cli.StringFlag{
+		Name:    "icmpv4-src",
+		Usage:   "Source address to send/receive ICMPv4 messages. If not provided cloudflared will dial a local address to determine the source IP or fallback to 0.0.0.0.",
+		EnvVars: []string{"TUNNEL_ICMPV4_SRC"},
+	}
+	icmpv6SrcFlag = &cli.StringFlag{
+		Name:    "icmpv6-src",
+		Usage:   "Source address and the interface name to send/receive ICMPv6 messages. If not provided cloudflared will dial a local address to determine the source IP or fallback to ::.",
+		EnvVars: []string{"TUNNEL_ICMPV6_SRC"},
+	}
 )
 
 func buildCreateCommand() *cli.Command {
@@ -595,8 +619,12 @@ func buildRunCommand() *cli.Command {
 		forceFlag,
 		credentialsFileFlag,
 		credentialsContentsFlag,
+		postQuantumFlag,
 		selectProtocolFlag,
 		featuresFlag,
+		tunnelTokenFlag,
+		icmpv4SrcFlag,
+		icmpv6SrcFlag,
 	}
 	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{
@@ -627,14 +655,6 @@ func runCommand(c *cli.Context) error {
 	if c.NArg() > 1 {
 		return cliutil.UsageError(`"cloudflared tunnel run" accepts only one argument, the ID or name of the tunnel to run.`)
 	}
-	tunnelRef := c.Args().First()
-	if tunnelRef == "" {
-		// see if tunnel id was in the config file
-		tunnelRef = config.GetConfiguration().TunnelID
-		if tunnelRef == "" {
-			return cliutil.UsageError(`"cloudflared tunnel run" requires the ID or name of the tunnel to run as the last command line argument or in the configuration file.`)
-		}
-	}
 
 	if c.String("hostname") != "" {
 		sc.log.Warn().Msg("The property `hostname` in your configuration is ignored because you configured a Named Tunnel " +
@@ -642,7 +662,38 @@ func runCommand(c *cli.Context) error {
 			"your origin will not be reachable. You should remove the `hostname` property to avoid this warning.")
 	}
 
-	return runNamedTunnel(sc, tunnelRef)
+	// Check if token is provided and if not use default tunnelID flag method
+	if tokenStr := c.String(TunnelTokenFlag); tokenStr != "" {
+		if token, err := ParseToken(tokenStr); err == nil {
+			return sc.runWithCredentials(token.Credentials())
+		}
+
+		return cliutil.UsageError("Provided Tunnel token is not valid.")
+	} else {
+		tunnelRef := c.Args().First()
+		if tunnelRef == "" {
+			// see if tunnel id was in the config file
+			tunnelRef = config.GetConfiguration().TunnelID
+			if tunnelRef == "" {
+				return cliutil.UsageError(`"cloudflared tunnel run" requires the ID or name of the tunnel to run as the last command line argument or in the configuration file.`)
+			}
+		}
+
+		return runNamedTunnel(sc, tunnelRef)
+	}
+}
+
+func ParseToken(tokenStr string) (*connection.TunnelToken, error) {
+	content, err := base64.StdEncoding.DecodeString(tokenStr)
+	if err != nil {
+		return nil, err
+	}
+
+	var token connection.TunnelToken
+	if err := json.Unmarshal(content, &token); err != nil {
+		return nil, err
+	}
+	return &token, nil
 }
 
 func runNamedTunnel(sc *subcommandContext, tunnelRef string) error {
@@ -650,9 +701,6 @@ func runNamedTunnel(sc *subcommandContext, tunnelRef string) error {
 	if err != nil {
 		return errors.Wrap(err, "error parsing tunnel ID")
 	}
-
-	sc.log.Info().Str(LogFieldTunnelID, tunnelID.String()).Msg("Starting tunnel")
-
 	return sc.run(tunnelID)
 }
 
@@ -686,6 +734,59 @@ func cleanupCommand(c *cli.Context) error {
 	return sc.cleanupConnections(tunnelIDs)
 }
 
+func buildTokenCommand() *cli.Command {
+	return &cli.Command{
+		Name:               "token",
+		Action:             cliutil.ConfiguredAction(tokenCommand),
+		Usage:              "Fetch the credentials token for an existing tunnel (by name or UUID) that allows to run it",
+		UsageText:          "cloudflared tunnel [tunnel command options] token [subcommand options] TUNNEL",
+		Description:        "cloudflared tunnel token will fetch the credentials token for a given tunnel (by its name or UUID), which is then used to run the tunnel. This command fails if the tunnel does not exist or has been deleted. Use the flag `cloudflared tunnel token --cred-file /my/path/file.json TUNNEL` to output the token to the credentials JSON file. Note: this command only works for Tunnels created since cloudflared version 2022.3.0",
+		Flags:              []cli.Flag{credentialsFileFlagCLIOnly},
+		CustomHelpTemplate: commandHelpTemplate(),
+	}
+}
+
+func tokenCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return errors.Wrap(err, "error setting up logger")
+	}
+
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
+	if c.NArg() != 1 {
+		return cliutil.UsageError(`"cloudflared tunnel token" requires exactly 1 argument, the name or UUID of tunnel to fetch the credentials token for.`)
+	}
+	tunnelID, err := sc.findID(c.Args().First())
+	if err != nil {
+		return errors.Wrap(err, "error parsing tunnel ID")
+	}
+
+	token, err := sc.getTunnelTokenCredentials(tunnelID)
+	if err != nil {
+		return err
+	}
+
+	if path := c.String(CredFileFlag); path != "" {
+		credentials := token.Credentials()
+		err := writeTunnelCredentials(path, &credentials)
+		if err != nil {
+			return errors.Wrapf(err, "error writing token credentials to JSON file in path %s", path)
+		}
+
+		return nil
+	}
+
+	encodedToken, err := token.Encode()
+	if err != nil {
+		return err
+	}
+
+	fmt.Println(encodedToken)
+	return nil
+}
+
 func buildRouteCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "route",
@@ -720,7 +821,7 @@ Further information about managing Cloudflare WARP traffic to your tunnel is ava
 				Name:        "lb",
 				Action:      cliutil.ConfiguredAction(routeLbCommand),
 				Usage:       "Use this tunnel as a load balancer origin, creating pool and load balancer if necessary",
-				UsageText:   "cloudflared tunnel route dns [TUNNEL] [HOSTNAME] [LB-POOL]",
+				UsageText:   "cloudflared tunnel route lb [TUNNEL] [HOSTNAME] [LB-POOL]",
 				Description: `Creates Load Balancer with an origin pool that points to the tunnel.`,
 			},
 			buildRouteIPSubcommand(),

cmd/cloudflared/tunnel/subcommands_test.go

@@ -1,14 +1,18 @@
 package tunnel
 
 import (
+	"encoding/base64"
+	"encoding/json"
 	"path/filepath"
 	"testing"
 
 	"github.com/google/uuid"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/cloudflare/cloudflared/cfapi"
+	"github.com/cloudflare/cloudflared/connection"
 )
 
 func Test_fmtConnections(t *testing.T) {
@@ -177,3 +181,24 @@ func Test_validateHostname(t *testing.T) {
 		})
 	}
 }
+
+func Test_TunnelToken(t *testing.T) {
+	token, err := ParseToken("aabc")
+	require.Error(t, err)
+	require.Nil(t, token)
+
+	expectedToken := &connection.TunnelToken{
+		AccountTag:   "abc",
+		TunnelSecret: []byte("secret"),
+		TunnelID:     uuid.New(),
+	}
+
+	tokenJsonStr, err := json.Marshal(expectedToken)
+	require.NoError(t, err)
+
+	token64 := base64.StdEncoding.EncodeToString(tokenJsonStr)
+
+	token, err = ParseToken(token64)
+	require.NoError(t, err)
+	require.Equal(t, token, expectedToken)
+}

cmd/cloudflared/ui/launch_ui.go

@@ -1,223 +0,0 @@
-package ui
-
-import (
-	"context"
-	"fmt"
-	"strings"
-
-	"github.com/gdamore/tcell"
-	"github.com/rivo/tview"
-	"github.com/rs/zerolog"
-
-	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/ingress"
-)
-
-type connState struct {
-	location string
-}
-
-type uiModel struct {
-	version       string
-	edgeURL       string
-	metricsURL    string
-	localServices []string
-	connections   []connState
-}
-
-type palette struct {
-	url          string
-	connected    string
-	defaultText  string
-	disconnected string
-	reconnecting string
-	unregistered string
-}
-
-func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haConnections int) *uiModel {
-	localServices := make([]string, len(ing.Rules))
-	for i, rule := range ing.Rules {
-		localServices[i] = rule.Service.String()
-	}
-	return &uiModel{
-		version:       version,
-		edgeURL:       hostname,
-		metricsURL:    metricsURL,
-		localServices: localServices,
-		connections:   make([]connState, haConnections),
-	}
-}
-
-func (data *uiModel) Launch(
-	ctx context.Context,
-	log, transportLog *zerolog.Logger,
-) connection.EventSink {
-	// Configure the logger to stream logs into the textview
-
-	// Add TextView as a group to write output to
-	logTextView := NewDynamicColorTextView()
-	// TODO: Format log for UI
-	//log.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
-	//transportLog.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
-
-	// Construct the UI
-	palette := palette{
-		url:          "lightblue",
-		connected:    "lime",
-		defaultText:  "white",
-		disconnected: "red",
-		reconnecting: "orange",
-		unregistered: "orange",
-	}
-
-	app := tview.NewApplication()
-
-	grid := tview.NewGrid().SetGap(1, 0)
-	frame := tview.NewFrame(grid)
-	header := fmt.Sprintf("cloudflared [::b]%s", data.version)
-
-	frame.AddText(header, true, tview.AlignLeft, tcell.ColorWhite)
-
-	// Create table to store connection info and status
-	connTable := tview.NewTable()
-	// SetColumns takes a value for each column, representing the size of the column
-	// Numbers <= 0 represent proportional widths and positive numbers represent absolute widths
-	grid.SetColumns(20, 0)
-
-	// SetRows takes a value for each row, representing the size of the row
-	grid.SetRows(1, 1, len(data.connections), 1, 0)
-
-	// AddItem takes a primitive tview type, row, column, rowSpan, columnSpan, minGridHeight, minGridWidth, and focus
-	grid.AddItem(tview.NewTextView().SetText("Tunnel:"), 0, 0, 1, 1, 0, 0, false)
-	grid.AddItem(tview.NewTextView().SetText("Status:"), 1, 0, 1, 1, 0, 0, false)
-	grid.AddItem(tview.NewTextView().SetText("Connections:"), 2, 0, 1, 1, 0, 0, false)
-
-	grid.AddItem(tview.NewTextView().SetText("Metrics:"), 3, 0, 1, 1, 0, 0, false)
-
-	tunnelHostText := tview.NewTextView().SetText(data.edgeURL)
-
-	grid.AddItem(tunnelHostText, 0, 1, 1, 1, 0, 0, false)
-	status := fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, strings.Join(data.localServices, ", "))
-	grid.AddItem(NewDynamicColorTextView().SetText(status), 1, 1, 1, 1, 0, 0, false)
-
-	grid.AddItem(connTable, 2, 1, 1, 1, 0, 0, false)
-
-	grid.AddItem(NewDynamicColorTextView().SetText(fmt.Sprintf("Metrics at [%s::b]http://%s/metrics", palette.url, data.metricsURL)), 3, 1, 1, 1, 0, 0, false)
-
-	// Add TextView to stream logs
-	// Logs are displayed in a new grid so a border can be set around them
-	logGrid := tview.NewGrid().SetBorders(true).AddItem(logTextView.SetChangedFunc(handleNewText(app, logTextView)), 0, 0, 5, 2, 0, 0, false)
-	// LogFrame holds the Logs header as well as the grid with the textView for streamed logs
-	logFrame := tview.NewFrame(logGrid).AddText("[::b]Logs:[::-]", true, tview.AlignLeft, tcell.ColorWhite).SetBorders(0, 0, 0, 0, 0, 0)
-	// Footer for log frame
-	logFrame.AddText("[::d]Use Ctrl+C to exit[::-]", false, tview.AlignRight, tcell.ColorWhite)
-	grid.AddItem(logFrame, 4, 0, 5, 2, 0, 0, false)
-
-	go func() {
-		<-ctx.Done()
-		app.Stop()
-		return
-	}()
-
-	go func() {
-		if err := app.SetRoot(frame, true).Run(); err != nil {
-			log.Error().Msgf("Error launching UI: %s", err)
-		}
-	}()
-
-	return connection.EventSinkFunc(func(event connection.Event) {
-		switch event.EventType {
-		case connection.Connected:
-			data.setConnTableCell(event, connTable, palette)
-		case connection.Disconnected, connection.Reconnecting, connection.Unregistering:
-			data.changeConnStatus(event, connTable, log, palette)
-		case connection.SetURL:
-			tunnelHostText.SetText(event.URL)
-			data.edgeURL = event.URL
-		case connection.RegisteringTunnel:
-			if data.edgeURL == "" {
-				tunnelHostText.SetText(fmt.Sprintf("Registering tunnel connection %d...", event.Index))
-			}
-		}
-		app.Draw()
-	})
-}
-
-func NewDynamicColorTextView() *tview.TextView {
-	return tview.NewTextView().SetDynamicColors(true)
-}
-
-// Re-draws application when new logs are streamed to UI
-func handleNewText(app *tview.Application, logTextView *tview.TextView) func() {
-	return func() {
-		app.Draw()
-		// SetFocus to enable scrolling in textview
-		app.SetFocus(logTextView)
-	}
-}
-
-func (data *uiModel) changeConnStatus(event connection.Event, table *tview.Table, log *zerolog.Logger, palette palette) {
-	index := int(event.Index)
-	// Get connection location and state
-	connState := data.getConnState(index)
-	// Check if connection is already displayed in UI
-	if connState == nil {
-		log.Info().Msg("Connection is not in the UI table")
-		return
-	}
-
-	locationState := event.Location
-
-	if event.EventType == connection.Reconnecting {
-		locationState = "Reconnecting..."
-	}
-
-	connectionNum := index + 1
-	// Get table cell
-	cell := table.GetCell(index, 0)
-	// Change dot color in front of text as well as location state
-	text := newCellText(palette, connectionNum, locationState, event.EventType)
-	cell.SetText(text)
-}
-
-// Return connection location and row in UI table
-func (data *uiModel) getConnState(connID int) *connState {
-	if connID < len(data.connections) {
-		return &data.connections[connID]
-	}
-
-	return nil
-}
-
-func (data *uiModel) setConnTableCell(event connection.Event, table *tview.Table, palette palette) {
-	index := int(event.Index)
-	connectionNum := index + 1
-
-	// Update slice to keep track of connection location and state in UI table
-	data.connections[index].location = event.Location
-
-	// Update text in table cell to show disconnected state
-	text := newCellText(palette, connectionNum, event.Location, event.EventType)
-	cell := tview.NewTableCell(text)
-	table.SetCell(index, 0, cell)
-}
-
-func newCellText(palette palette, connectionNum int, location string, connectedStatus connection.Status) string {
-	// HA connection indicator formatted as: "• #<CONNECTION_INDEX>: <COLO>",
-	//  where the left middle dot's color depends on the status of the connection
-	const connFmtString = "[%s]\u2022[%s] #%d: %s"
-
-	var dotColor string
-	switch connectedStatus {
-	case connection.Connected:
-		dotColor = palette.connected
-	case connection.Disconnected:
-		dotColor = palette.disconnected
-	case connection.Reconnecting:
-		dotColor = palette.reconnecting
-	case connection.Unregistering:
-		dotColor = palette.unregistered
-	}
-
-	return fmt.Sprintf(connFmtString, dotColor, palette.defaultText, connectionNum, location)
-}

cmd/cloudflared/updater/update.go

@@ -19,7 +19,7 @@ import (
 
 const (
 	DefaultCheckUpdateFreq        = time.Hour * 24
-	noUpdateInShellMessage        = "cloudflared will not automatically update when run from the shell. To enable auto-updates, run cloudflared as a service: https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/run-tunnel/run-as-service"
+	noUpdateInShellMessage        = "cloudflared will not automatically update when run from the shell. To enable auto-updates, run cloudflared as a service: https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/run-tunnel/as-a-service/"
 	noUpdateOnWindowsMessage      = "cloudflared will not automatically update on Windows systems."
 	noUpdateManagedPackageMessage = "cloudflared will not automatically update if installed by a package manager."
 	isManagedInstallFile          = ".installedFromPackageManager"
@@ -30,7 +30,8 @@ const (
 )
 
 var (
-	version string
+	version                string
+	BuiltForPackageManager = ""
 )
 
 // BinaryUpdated implements ExitCoder interface, the app will exit with status code 11
@@ -118,7 +119,11 @@ func Update(c *cli.Context) error {
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	if wasInstalledFromPackageManager() {
-		log.Error().Msg("cloudflared was installed by a package manager. Please update using the same method.")
+		packageManagerName := "a package manager"
+		if BuiltForPackageManager != "" {
+			packageManagerName = BuiltForPackageManager
+		}
+		log.Error().Msg(fmt.Sprintf("cloudflared was installed by %s. Please update using the same method.", packageManagerName))
 		return nil
 	}
 
@@ -281,7 +286,7 @@ func supportAutoUpdate(log *zerolog.Logger) bool {
 
 func wasInstalledFromPackageManager() bool {
 	ok, _ := config.FileExists(filepath.Join(config.DefaultUnixConfigLocation, isManagedInstallFile))
-	return ok
+	return len(BuiltForPackageManager) != 0 || ok
 }
 
 func isRunningFromTerminal() bool {

cmd/cloudflared/windows_service.go

@@ -26,8 +26,8 @@ import (
 
 const (
 	windowsServiceName        = "Cloudflared"
-	windowsServiceDescription = "Cloudflare Tunnel agent"
-	windowsServiceUrl         = "https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/run-tunnel/run-as-service#windows"
+	windowsServiceDescription = "Cloudflared agent"
+	windowsServiceUrl         = "https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/run-tunnel/as-a-service/windows/"
 
 	recoverActionDelay      = time.Second * 20
 	failureCountResetPeriod = time.Hour * 24
@@ -46,16 +46,16 @@ const (
 func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
-		Usage: "Manages the Cloudflare Tunnel Windows service",
+		Usage: "Manages the cloudflared Windows service",
 		Subcommands: []*cli.Command{
 			{
 				Name:   "install",
-				Usage:  "Install Cloudflare Tunnel as a Windows service",
+				Usage:  "Install cloudflared as a Windows service",
 				Action: cliutil.ConfiguredAction(installWindowsService),
 			},
 			{
 				Name:   "uninstall",
-				Usage:  "Uninstall the Cloudflare Tunnel service",
+				Usage:  "Uninstall the cloudflared service",
 				Action: cliutil.ConfiguredAction(uninstallWindowsService),
 			},
 		},
@@ -177,7 +177,7 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 func installWindowsService(c *cli.Context) error {
 	zeroLogger := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
-	zeroLogger.Info().Msg("Installing Cloudflare Tunnel Windows service")
+	zeroLogger.Info().Msg("Installing cloudflared Windows service")
 	exepath, err := os.Executable()
 	if err != nil {
 		return errors.Wrap(err, "Cannot find path name that start the process")
@@ -191,26 +191,39 @@ func installWindowsService(c *cli.Context) error {
 	log := zeroLogger.With().Str(LogFieldWindowsServiceName, windowsServiceName).Logger()
 	if err == nil {
 		s.Close()
-		return fmt.Errorf("Service %s already exists", windowsServiceName)
+		return fmt.Errorf(serviceAlreadyExistsWarn(windowsServiceName))
 	}
+	extraArgs, err := getServiceExtraArgsFromCliArgs(c, &log)
+	if err != nil {
+		errMsg := "Unable to determine extra arguments for windows service"
+		log.Err(err).Msg(errMsg)
+		return errors.Wrap(err, errMsg)
+	}
+
 	config := mgr.Config{StartType: mgr.StartAutomatic, DisplayName: windowsServiceDescription}
-	s, err = m.CreateService(windowsServiceName, exepath, config)
+	s, err = m.CreateService(windowsServiceName, exepath, config, extraArgs...)
 	if err != nil {
 		return errors.Wrap(err, "Cannot install service")
 	}
 	defer s.Close()
-	log.Info().Msg("Cloudflare Tunnel agent service is installed")
+	log.Info().Msg("cloudflared agent service is installed")
 	err = eventlog.InstallAsEventCreate(windowsServiceName, eventlog.Error|eventlog.Warning|eventlog.Info)
 	if err != nil {
 		s.Delete()
 		return errors.Wrap(err, "Cannot install event logger")
 	}
+
 	err = configRecoveryOption(s.Handle)
 	if err != nil {
 		log.Err(err).Msg("Cannot set service recovery actions")
 		log.Info().Msgf("See %s to manually configure service recovery actions", windowsServiceUrl)
 	}
-	return nil
+
+	err = s.Start()
+	if err == nil {
+		log.Info().Msg("Agent service for cloudflared installed successfully")
+	}
+	return err
 }
 
 func uninstallWindowsService(c *cli.Context) error {
@@ -218,7 +231,7 @@ func uninstallWindowsService(c *cli.Context) error {
 		With().
 		Str(LogFieldWindowsServiceName, windowsServiceName).Logger()
 
-	log.Info().Msg("Uninstalling Cloudflare Tunnel Windows Service")
+	log.Info().Msg("Uninstalling cloudflared agent service")
 	m, err := mgr.Connect()
 	if err != nil {
 		return errors.Wrap(err, "Cannot establish a connection to the service control manager")
@@ -226,14 +239,22 @@ func uninstallWindowsService(c *cli.Context) error {
 	defer m.Disconnect()
 	s, err := m.OpenService(windowsServiceName)
 	if err != nil {
-		return fmt.Errorf("Service %s is not installed", windowsServiceName)
+		return fmt.Errorf("Agent service %s is not installed, so it could not be uninstalled", windowsServiceName)
 	}
 	defer s.Close()
+
+	if status, err := s.Query(); err == nil && status.State == svc.Running {
+		log.Info().Msg("Stopping cloudflared agent service")
+		if _, err := s.Control(svc.Stop); err != nil {
+			log.Info().Err(err).Msg("Failed to stop cloudflared agent service, you may need to stop it manually to complete uninstall.")
+		}
+	}
+
 	err = s.Delete()
 	if err != nil {
-		return errors.Wrap(err, "Cannot delete service")
+		return errors.Wrap(err, "Cannot delete agent service")
 	}
-	log.Info().Msg("Cloudflare Tunnel agent service is uninstalled")
+	log.Info().Msg("Agent service for cloudflared was uninstalled successfully")
 	err = eventlog.Remove(windowsServiceName)
 	if err != nil {
 		return errors.Wrap(err, "Cannot remove event logger")

component-tests/README.md

@@ -14,7 +14,7 @@ classic_hostname: "classic-tunnel-component-tests.example.com"
 origincert: "/Users/tunnel/.cloudflared/cert.pem"
 ingress:
 - hostname: named-tunnel-component-tests.example.com
-  service: http_status:200
+  service: hello_world
 - service: http_status:404
 ```
 

component-tests/cli.py

@@ -0,0 +1,95 @@
+import json
+import subprocess
+from time import sleep
+
+from setup import get_config_from_file
+
+SINGLE_CASE_TIMEOUT = 600
+
+class CloudflaredCli:
+    def __init__(self, config, config_path, logger):
+        self.basecmd = [config.cloudflared_binary, "tunnel"]
+        if config_path is not None:
+            self.basecmd += ["--config", str(config_path)]
+        origincert = get_config_from_file()["origincert"]
+        if origincert:
+            self.basecmd += ["--origincert", origincert]
+        self.logger = logger
+
+    def _run_command(self, subcmd, subcmd_name, needs_to_pass=True):
+        cmd = self.basecmd + subcmd
+        # timeout limits the time a subprocess can run. This is useful to guard against running a tunnel when
+        # command/args are in wrong order.
+        result = run_subprocess(cmd, subcmd_name, self.logger, check=needs_to_pass, capture_output=True, timeout=15)
+        return result
+
+    def list_tunnels(self):
+        cmd_args = ["list", "--output", "json"]
+        listed = self._run_command(cmd_args, "list")
+        return json.loads(listed.stdout)
+
+    def get_tunnel_info(self, tunnel_id):
+        info = self._run_command(["info", "--output", "json", tunnel_id], "info")
+        return json.loads(info.stdout)
+
+    def __enter__(self):
+        self.basecmd += ["run"]
+        self.process = subprocess.Popen(self.basecmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        self.logger.info(f"Run cmd {self.basecmd}")
+        return self.process
+
+    def __exit__(self, exc_type, exc_value, exc_traceback):
+        terminate_gracefully(self.process, self.logger, self.basecmd)
+        self.logger.debug(f"{self.basecmd} logs: {self.process.stderr.read()}")
+
+
+def terminate_gracefully(process, logger, cmd):
+    process.terminate()
+    process_terminated = wait_for_terminate(process)
+    if not process_terminated:
+        process.kill()
+        logger.warning(f"{cmd}: cloudflared did not terminate within wait period. Killing process. logs: \
+                stdout: {process.stdout.read()}, stderr: {process.stderr.read()}")
+
+
+def wait_for_terminate(opened_subprocess, attempts=10, poll_interval=1):
+    """
+        wait_for_terminate polls the opened_subprocess every x seconds for a given number of attempts.
+        It returns true if the subprocess was terminated and false if it didn't.
+    """
+    for _ in range(attempts):
+        if _is_process_stopped(opened_subprocess):
+            return True
+        sleep(poll_interval)
+    return False
+
+
+def _is_process_stopped(process):
+    return process.poll() is not None
+
+
+def cert_path():
+    return get_config_from_file()["origincert"]
+
+
+class SubprocessError(Exception):
+    def __init__(self, program, exit_code, cause):
+        self.program = program
+        self.exit_code = exit_code
+        self.cause = cause
+
+
+def run_subprocess(cmd, cmd_name, logger, timeout=SINGLE_CASE_TIMEOUT, **kargs):
+    kargs["timeout"] = timeout
+    try:
+        result = subprocess.run(cmd, **kargs)
+        logger.debug(f"{cmd} log: {result.stdout}", extra={"cmd": cmd_name})
+        return result
+    except subprocess.CalledProcessError as e:
+        err = f"{cmd} return exit code {e.returncode}, stderr" + e.stderr.decode("utf-8")
+        logger.error(err, extra={"cmd": cmd_name, "return_code": e.returncode})
+        raise SubprocessError(cmd[0], e.returncode, e)
+    except subprocess.TimeoutExpired as e:
+        err = f"{cmd} timeout after {e.timeout} seconds, stdout: {e.stdout}, stderr: {e.stderr}"
+        logger.error(err, extra={"cmd": cmd_name, "return_code": "timeout"})
+        raise e

component-tests/config.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 import copy
+import json
+import base64
 
 from dataclasses import dataclass, InitVar
 
@@ -61,6 +63,28 @@ class NamedTunnelConfig(NamedTunnelBaseConfig):
     def get_url(self):
         return "https://" + self.ingress[0]['hostname']
 
+    def base_config(self):
+        config = self.full_config.copy()
+
+        # removes the tunnel reference
+        del(config["tunnel"])
+        del(config["credentials-file"])
+
+        return config
+
+    def get_tunnel_id(self):
+        return self.full_config["tunnel"]
+
+    def get_token(self):
+        creds = self.get_credentials_json()
+        token_dict = {"a": creds["AccountTag"], "t": creds["TunnelID"], "s": creds["TunnelSecret"]}
+        token_json_str = json.dumps(token_dict)
+        return base64.b64encode(token_json_str.encode('utf-8'))
+
+    def get_credentials_json(self):
+        with open(self.credentials_file) as json_file:
+            return json.load(json_file)
+
 
 @dataclass(frozen=True)
 class ClassicTunnelBaseConfig(BaseConfig):

component-tests/constants.py

@@ -1,6 +1,7 @@
 METRICS_PORT = 51000
 MAX_RETRIES = 5
 BACKOFF_SECS = 7
+MAX_LOG_LINES = 50
 
 PROXY_DNS_PORT = 9053
 

component-tests/test_edge_discovery.py

@@ -0,0 +1,165 @@
+import ipaddress
+import socket
+
+import pytest
+
+from constants import protocols
+from cli import CloudflaredCli
+from util import get_tunnel_connector_id, LOGGER, wait_tunnel_ready, write_config
+
+
+class TestEdgeDiscovery:
+    def _extra_config(self, protocol, edge_ip_version):
+        config = {
+            "protocol": protocol,
+        }
+        if edge_ip_version:
+            config["edge-ip-version"] = edge_ip_version
+        return config
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_default_only(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via IPv4-only edge addresses (default is unset "--edge-ip-version 4")
+        """
+        if self.has_ipv6_only():
+            pytest.skip("Host has IPv6 only support and current default is IPv4 only")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, None, self.expect_ipv4_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_ipv4_only(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via IPv4-only edge addresses
+        """
+        if self.has_ipv6_only():
+            pytest.skip("Host has IPv6 only support")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "4", self.expect_ipv4_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_ipv6_only(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via IPv6-only edge addresses
+        """
+        if self.has_ipv4_only():
+            pytest.skip("Host has IPv4 only support")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "6", self.expect_ipv6_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_auto_ip64(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via auto with a preference of IPv6 then IPv4 addresses for a dual stack host
+
+        This test also assumes that the host has IPv6 preference.
+        """
+        if not self.has_dual_stack(address_family_preference=socket.AddressFamily.AF_INET6):
+            pytest.skip("Host does not support dual stack with IPv6 preference")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "auto", self.expect_ipv6_address)
+
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_auto_ip46(self, tmp_path, component_tests_config, protocol):
+        """
+        This test runs a tunnel to connect via auto with a preference of IPv4 then IPv6 addresses for a dual stack host
+
+        This test also assumes that the host has IPv4 preference.
+        """
+        if not self.has_dual_stack(address_family_preference=socket.AddressFamily.AF_INET):
+            pytest.skip("Host does not support dual stack with IPv4 preference")
+        self.expect_address_connections(
+            tmp_path, component_tests_config, protocol, "auto", self.expect_ipv4_address)
+
+    def expect_address_connections(self, tmp_path, component_tests_config, protocol, edge_ip_version, assert_address_type):
+        config = component_tests_config(
+            self._extra_config(protocol, edge_ip_version))
+        config_path = write_config(tmp_path, config.full_config)
+        LOGGER.debug(config)
+        with CloudflaredCli(config, config_path, LOGGER):
+            wait_tunnel_ready(tunnel_url=config.get_url(),
+                              require_min_connections=4)
+            cfd_cli = CloudflaredCli(config, config_path, LOGGER)
+            tunnel_id = config.get_tunnel_id()
+            info = cfd_cli.get_tunnel_info(tunnel_id)
+            connector_id = get_tunnel_connector_id()
+            connector = next(
+                (c for c in info["conns"] if c["id"] == connector_id), None)
+            assert connector, f"Expected connection info from get tunnel info for the connected instance: {info}"
+            conns = connector["conns"]
+            assert conns == None or len(
+                conns) == 4, f"There should be 4 connections registered: {conns}"
+            for conn in conns:
+                origin_ip = conn["origin_ip"]
+                assert origin_ip, f"No available origin_ip for this connection: {conn}"
+                assert_address_type(origin_ip)
+
+    def expect_ipv4_address(self, address):
+        assert type(ipaddress.ip_address(
+            address)) is ipaddress.IPv4Address, f"Expected connection from origin to be a valid IPv4 address: {address}"
+
+    def expect_ipv6_address(self, address):
+        assert type(ipaddress.ip_address(
+            address)) is ipaddress.IPv6Address, f"Expected connection from origin to be a valid IPv6 address: {address}"
+
+    def get_addresses(self):
+        """
+        Returns a list of addresses for the host.
+        """
+        host_addresses = socket.getaddrinfo(
+            "region1.v2.argotunnel.com", 7844, socket.AF_UNSPEC, socket.SOCK_STREAM)
+        assert len(
+            host_addresses) > 0, "No addresses returned from getaddrinfo"
+        return host_addresses
+
+    def has_dual_stack(self, address_family_preference=None):
+        """
+        Returns true if the host has dual stack support and can optionally check 
+        the provided IP family preference.
+        """
+        dual_stack = not self.has_ipv6_only() and not self.has_ipv4_only()
+        if address_family_preference:
+            address = self.get_addresses()[0]
+            return dual_stack and address[0] == address_family_preference
+
+        return dual_stack
+
+    def has_ipv6_only(self):
+        """
+        Returns True if the host has only IPv6 address support.
+        """
+        return self.attempt_connection(socket.AddressFamily.AF_INET6) and not self.attempt_connection(socket.AddressFamily.AF_INET)
+
+    def has_ipv4_only(self):
+        """
+        Returns True if the host has only IPv4 address support.
+        """
+        return self.attempt_connection(socket.AddressFamily.AF_INET) and not self.attempt_connection(socket.AddressFamily.AF_INET6)
+
+    def attempt_connection(self, address_family):
+        """
+        Returns True if a successful socket connection can be made to the 
+        remote host with the provided address family to validate host support 
+        for the provided address family.
+        """
+        address = None
+        for a in self.get_addresses():
+            if a[0] == address_family:
+                address = a
+                break
+        if address is None:
+            # Couldn't even lookup the address family so we can't connect
+            return False
+        af, socktype, proto, canonname, sockaddr = address
+        s = None
+        try:
+            s = socket.socket(af, socktype, proto)
+        except OSError:
+            return False
+        try:
+            s.connect(sockaddr)
+        except OSError:
+            s.close()
+            return False
+        s.close()
+        return True

component-tests/test_logging.py

@@ -2,6 +2,7 @@
 import json
 import os
 
+from constants import MAX_LOG_LINES
 from util import start_cloudflared, wait_tunnel_ready, send_requests
 
 # Rolling logger rotate log files after 1 MB
@@ -11,14 +12,24 @@ expect_message = "Starting Hello"
 
 
 def assert_log_to_terminal(cloudflared):
-    stderr = cloudflared.stderr.read(1500)
-    assert expect_message.encode() in stderr, f"{stderr} doesn't contain {expect_message}"
+    for _ in range(0, MAX_LOG_LINES):
+        line = cloudflared.stderr.readline()
+        if not line:
+            break
+        if expect_message.encode() in line:
+            return
+    raise Exception(f"terminal log doesn't contain {expect_message}")
 
 
 def assert_log_in_file(file):
     with open(file, "r") as f:
-        log = f.read(2000)
-        assert expect_message in log, f"{log} doesn't contain {expect_message}"
+        for _ in range(0, MAX_LOG_LINES):
+            line = f.readline()
+            if not line:
+                break
+            if expect_message in line:
+                return
+    raise Exception(f"log file doesn't contain {expect_message}")
 
 
 def assert_json_log(file):

component-tests/test_pq.py

@@ -0,0 +1,17 @@
+from util import LOGGER, nofips, start_cloudflared, wait_tunnel_ready
+
+
+@nofips
+class TestPostQuantum:
+    def _extra_config(self):
+        config = {
+            "protocol": "quic",
+        }
+        return config
+
+    def test_post_quantum(self, tmp_path, component_tests_config):
+        config = component_tests_config(self._extra_config())
+        LOGGER.debug(config)
+        with start_cloudflared(tmp_path, config, cfd_args=["run", "--post-quantum"], new_process=True):
+            wait_tunnel_ready(tunnel_url=config.get_url(),
+                              require_min_connections=4)

component-tests/test_reconnect.py

@@ -47,17 +47,12 @@ class TestReconnect:
         cloudflared.stdin.flush()
 
     def assert_reconnect(self, config, cloudflared, repeat):
-        wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=self.default_ha_conns)
+        wait_tunnel_ready(tunnel_url=config.get_url(),
+                          require_min_connections=self.default_ha_conns)
         for _ in range(repeat):
-            for i in range(self.default_ha_conns):
+            for _ in range(self.default_ha_conns):
                 self.send_reconnect(cloudflared, self.default_reconnect_secs)
-                expect_connections = self.default_ha_conns-i-1
-                if expect_connections > 0:
-                    # Don't check if tunnel returns 200 here because there is a race condition between wait_tunnel_ready
-                    # retrying to get 200 response and reconnecting
-                    wait_tunnel_ready(require_min_connections=expect_connections)
-                else:
-                    check_tunnel_not_connected()
-
+            check_tunnel_not_connected()
             sleep(self.default_reconnect_secs * 2)
-            wait_tunnel_ready(tunnel_url=config.get_url(), require_min_connections=self.default_ha_conns)
+            wait_tunnel_ready(tunnel_url=config.get_url(),
+                              require_min_connections=self.default_ha_conns)

component-tests/test_service.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 import os
-import platform
+import pathlib
 import subprocess
 from contextlib import contextmanager
 from pathlib import Path
@@ -9,12 +9,7 @@ import pytest
 
 import test_logging
 from conftest import CfdModes
-from util import start_cloudflared, wait_tunnel_ready
-
-
-def select_platform(plat):
-    return pytest.mark.skipif(
-        platform.system() != plat, reason=f"Only runs on {plat}")
+from util import select_platform, start_cloudflared, wait_tunnel_ready, write_config
 
 
 def default_config_dir():
@@ -43,6 +38,21 @@ class TestServiceMode:
 
         self.launchd_service_scenario(config, assert_log_file)
 
+    @select_platform("Darwin")
+    @pytest.mark.skipif(os.path.exists(default_config_file()), reason=f"There is already a config file in default path")
+    def test_launchd_service_with_token(self, tmp_path, component_tests_config):
+        log_file = tmp_path / test_logging.default_log_file
+        additional_config = {
+            "logfile": str(log_file),
+        }
+        config = component_tests_config(additional_config=additional_config)
+
+        # service install doesn't install the config file but in this case we want to use some default settings
+        # so we write the base config without the tunnel credentials and ID
+        write_config(pathlib.Path(default_config_dir()), config.base_config())
+
+        self.launchd_service_scenario(config, use_token=True)
+
     @select_platform("Darwin")
     @pytest.mark.skipif(os.path.exists(default_config_file()), reason=f"There is already a config file in default path")
     def test_launchd_service_rotating_log(self, tmp_path, component_tests_config):
@@ -60,12 +70,13 @@ class TestServiceMode:
 
         self.launchd_service_scenario(config, assert_rotating_log)
 
-    def launchd_service_scenario(self, config, extra_assertions):
-        with self.run_service(Path(default_config_dir()), config):
+    def launchd_service_scenario(self, config, extra_assertions=None, use_token=False):
+        with self.run_service(Path(default_config_dir()), config, use_token=use_token):
             self.launchctl_cmd("list")
             self.launchctl_cmd("start")
             wait_tunnel_ready(tunnel_url=config.get_url())
-            extra_assertions()
+            if extra_assertions is not None:
+                extra_assertions()
             self.launchctl_cmd("stop")
 
         os.remove(default_config_file())
@@ -105,27 +116,48 @@ class TestServiceMode:
 
         self.sysv_service_scenario(config, tmp_path, assert_rotating_log)
 
-    def sysv_service_scenario(self, config, tmp_path, extra_assertions):
-        with self.run_service(tmp_path, config, root=True):
-            self.sysv_cmd("start")
+    @select_platform("Linux")
+    @pytest.mark.skipif(os.path.exists("/etc/cloudflared/config.yml"),
+                        reason=f"There is already a config file in default path")
+    def test_sysv_service_with_token(self, tmp_path, component_tests_config):
+        additional_config = {
+            "loglevel": "debug",
+        }
+
+        config = component_tests_config(additional_config=additional_config)
+
+        # service install doesn't install the config file but in this case we want to use some default settings
+        # so we write the base config without the tunnel credentials and ID
+        config_path = write_config(tmp_path, config.base_config())
+        subprocess.run(["sudo", "cp", config_path, "/etc/cloudflared/config.yml"], check=True)
+
+        self.sysv_service_scenario(config, tmp_path, use_token=True)
+
+    def sysv_service_scenario(self, config, tmp_path, extra_assertions=None, use_token=False):
+        with self.run_service(tmp_path, config, root=True, use_token=use_token):
             self.sysv_cmd("status")
             wait_tunnel_ready(tunnel_url=config.get_url())
-            extra_assertions()
-            self.sysv_cmd("stop")
+            if extra_assertions is not None:
+                extra_assertions()
 
         # Service install copies config file to /etc/cloudflared/config.yml
         subprocess.run(["sudo", "rm", "/etc/cloudflared/config.yml"])
         self.sysv_cmd("status", success=False)
 
     @contextmanager
-    def run_service(self, tmp_path, config, root=False):
+    def run_service(self, tmp_path, config, root=False, use_token=False):
+        args = ["service", "install"]
+
+        if use_token:
+            args.append(config.get_token())
+
         try:
             service = start_cloudflared(
-                tmp_path, config, cfd_args=["service", "install"], cfd_pre_args=[], capture_output=False, root=root)
+                tmp_path, config, cfd_args=args, cfd_pre_args=[], capture_output=False, root=root, skip_config_flag=use_token)
             yield service
         finally:
             start_cloudflared(
-                tmp_path, config, cfd_args=["service", "uninstall"], cfd_pre_args=[], capture_output=False, root=root)
+                tmp_path, config, cfd_args=["service", "uninstall"], cfd_pre_args=[], capture_output=False, root=root, skip_config_flag=use_token)
 
     def launchctl_cmd(self, action, success=True):
         cmd = subprocess.run(

component-tests/test_token.py

@@ -0,0 +1,35 @@
+import base64
+import json
+
+from setup import get_config_from_file, persist_origin_cert
+from util import start_cloudflared
+
+
+class TestToken:
+    def test_get_token(self, tmp_path, component_tests_config):
+        config = component_tests_config()
+        tunnel_id = config.get_tunnel_id()
+
+        token_args = ["--origincert", cert_path(), "token", tunnel_id]
+        output = start_cloudflared(tmp_path, config, token_args)
+
+        assert parse_token(config.get_token()) == parse_token(output.stdout)
+
+    def test_get_credentials_file(self, tmp_path, component_tests_config):
+        config = component_tests_config()
+        tunnel_id = config.get_tunnel_id()
+
+        cred_file = tmp_path / "test_get_credentials_file.json"
+        token_args = ["--origincert", cert_path(), "token", "--cred-file", cred_file, tunnel_id]
+        start_cloudflared(tmp_path, config, token_args)
+
+        with open(cred_file) as json_file:
+            assert config.get_credentials_json() == json.load(json_file)
+
+
+def cert_path():
+    return get_config_from_file()["origincert"]
+
+
+def parse_token(token):
+    return json.loads(base64.b64decode(token))

component-tests/util.py

@@ -1,9 +1,12 @@
 import logging
 import os
+import platform
 import subprocess
 from contextlib import contextmanager
 from time import sleep
 
+import pytest
+
 import requests
 import yaml
 from retrying import retry
@@ -13,6 +16,17 @@ from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
 LOGGER = logging.getLogger(__name__)
 
 
+def select_platform(plat):
+    return pytest.mark.skipif(
+        platform.system() != plat, reason=f"Only runs on {plat}")
+
+def fips_enabled():
+    env_fips = os.getenv("COMPONENT_TESTS_FIPS")
+    return env_fips is not None and env_fips != "0"
+
+nofips = pytest.mark.skipif(
+        fips_enabled(), reason=f"Only runs without FIPS (COMPONENT_TESTS_FIPS=0)")
+
 def write_config(directory, config):
     config_path = directory / "config.yml"
     with open(config_path, 'w') as outfile:
@@ -21,9 +35,14 @@ def write_config(directory, config):
 
 
 def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False,
-                      allow_input=False, capture_output=True, root=False):
-    config_path = write_config(directory, config.full_config)
+                      allow_input=False, capture_output=True, root=False, skip_config_flag=False):
+
+    config_path = None
+    if not skip_config_flag:
+        config_path = write_config(directory, config.full_config)
+
     cmd = cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root)
+
     if new_process:
         return run_cloudflared_background(cmd, allow_input, capture_output)
     # By setting check=True, it will raise an exception if the process exits with non-zero exit code
@@ -36,7 +55,10 @@ def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
         cmd += ["sudo"]
     cmd += [config.cloudflared_binary]
     cmd += cfd_pre_args
-    cmd += ["--config", str(config_path)]
+
+    if config_path is not None:
+        cmd += ["--config", str(config_path)]
+
     cmd += cfd_args
     LOGGER.info(f"Run cmd {cmd} with config {config}")
     return cmd
@@ -46,13 +68,15 @@ def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
 def run_cloudflared_background(cmd, allow_input, capture_output):
     output = subprocess.PIPE if capture_output else subprocess.DEVNULL
     stdin = subprocess.PIPE if allow_input else None
+    cfd = None
     try:
         cfd = subprocess.Popen(cmd, stdin=stdin, stdout=output, stderr=output)
         yield cfd
     finally:
-        cfd.terminate()
-        if capture_output:
-            LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
+        if cfd:
+            cfd.terminate()
+            if capture_output:
+                LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
 
 
 def wait_tunnel_ready(tunnel_url=None, require_min_connections=1, cfd_logs=None):
@@ -91,18 +115,31 @@ def _log_cloudflared_logs(cfd_logs):
             LOGGER.warning(line)
 
 
-@retry(stop_max_attempt_number=MAX_RETRIES * BACKOFF_SECS, wait_fixed=1000)
+@retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
 def check_tunnel_not_connected():
     url = f'http://localhost:{METRICS_PORT}/ready'
 
     try:
-        resp = requests.get(url, timeout=1)
+        resp = requests.get(url, timeout=BACKOFF_SECS)
         assert resp.status_code == 503, f"Expect {url} returns 503, got {resp.status_code}"
+        assert resp.json()[
+            "readyConnections"] == 0, "Expected all connections to be terminated (pending reconnect)"
     # cloudflared might already terminate
     except requests.exceptions.ConnectionError as e:
         LOGGER.warning(f"Failed to connect to {url}, error: {e}")
 
 
+def get_tunnel_connector_id():
+    url = f'http://localhost:{METRICS_PORT}/ready'
+
+    try:
+        resp = requests.get(url, timeout=1)
+        return resp.json()["connectorId"]
+    # cloudflared might already terminated
+    except requests.exceptions.ConnectionError as e:
+        LOGGER.warning(f"Failed to connect to {url}, error: {e}")
+
+
 # In some cases we don't need to check response status, such as when sending batch requests to generate logs
 def send_requests(url, count, require_ok=True):
     errors = 0

config/configuration.go

@@ -1,19 +1,21 @@
 package config
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/url"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"time"
 
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/cloudflare/cloudflared/validation"
 )
@@ -49,7 +51,7 @@ func DefaultConfigDirectory() string {
 		path := os.Getenv("CFDPATH")
 		if path == "" {
 			path = filepath.Join(os.Getenv("ProgramFiles(x86)"), "cloudflared")
-			if _, err := os.Stat(path); os.IsNotExist(err) { //doesn't exist, so return an empty failure string
+			if _, err := os.Stat(path); os.IsNotExist(err) { // doesn't exist, so return an empty failure string
 				return ""
 			}
 		}
@@ -138,7 +140,7 @@ func FindOrCreateConfigPath() string {
 		defer file.Close()
 
 		logDir := DefaultLogDirectory()
-		_ = os.MkdirAll(logDir, os.ModePerm) //try and create it. Doesn't matter if it succeed or not, only byproduct will be no logs
+		_ = os.MkdirAll(logDir, os.ModePerm) // try and create it. Doesn't matter if it succeed or not, only byproduct will be no logs
 
 		c := Root{
 			LogDirectory: logDir,
@@ -175,9 +177,9 @@ func ValidateUrl(c *cli.Context, allowURLFromArgs bool) (*url.URL, error) {
 }
 
 type UnvalidatedIngressRule struct {
-	Hostname      string              `json:"hostname"`
-	Path          string              `json:"path"`
-	Service       string              `json:"service"`
+	Hostname      string              `json:"hostname,omitempty"`
+	Path          string              `json:"path,omitempty"`
+	Service       string              `json:"service,omitempty"`
 	OriginRequest OriginRequestConfig `yaml:"originRequest" json:"originRequest"`
 }
 
@@ -190,41 +192,56 @@ type UnvalidatedIngressRule struct {
 // - To specify a time.Duration in json, use int64 of the nanoseconds
 type OriginRequestConfig struct {
 	// HTTP proxy timeout for establishing a new connection
-	ConnectTimeout *time.Duration `yaml:"connectTimeout" json:"connectTimeout"`
+	ConnectTimeout *CustomDuration `yaml:"connectTimeout" json:"connectTimeout,omitempty"`
 	// HTTP proxy timeout for completing a TLS handshake
-	TLSTimeout *time.Duration `yaml:"tlsTimeout" json:"tlsTimeout"`
+	TLSTimeout *CustomDuration `yaml:"tlsTimeout" json:"tlsTimeout,omitempty"`
 	// HTTP proxy TCP keepalive duration
-	TCPKeepAlive *time.Duration `yaml:"tcpKeepAlive" json:"tcpKeepAlive"`
+	TCPKeepAlive *CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive,omitempty"`
 	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
-	NoHappyEyeballs *bool `yaml:"noHappyEyeballs" json:"noHappyEyeballs"`
+	NoHappyEyeballs *bool `yaml:"noHappyEyeballs" json:"noHappyEyeballs,omitempty"`
 	// HTTP proxy maximum keepalive connection pool size
-	KeepAliveConnections *int `yaml:"keepAliveConnections" json:"keepAliveConnections"`
+	KeepAliveConnections *int `yaml:"keepAliveConnections" json:"keepAliveConnections,omitempty"`
 	// HTTP proxy timeout for closing an idle connection
-	KeepAliveTimeout *time.Duration `yaml:"keepAliveTimeout" json:"keepAliveTimeout"`
+	KeepAliveTimeout *CustomDuration `yaml:"keepAliveTimeout" json:"keepAliveTimeout,omitempty"`
 	// Sets the HTTP Host header for the local webserver.
-	HTTPHostHeader *string `yaml:"httpHostHeader" json:"httpHostHeader"`
+	HTTPHostHeader *string `yaml:"httpHostHeader" json:"httpHostHeader,omitempty"`
 	// Hostname on the origin server certificate.
-	OriginServerName *string `yaml:"originServerName" json:"originServerName"`
+	OriginServerName *string `yaml:"originServerName" json:"originServerName,omitempty"`
 	// Path to the CA for the certificate of your origin.
 	// This option should be used only if your certificate is not signed by Cloudflare.
-	CAPool *string `yaml:"caPool" json:"caPool"`
+	CAPool *string `yaml:"caPool" json:"caPool,omitempty"`
 	// Disables TLS verification of the certificate presented by your origin.
 	// Will allow any certificate from the origin to be accepted.
 	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
-	NoTLSVerify *bool `yaml:"noTLSVerify" json:"noTLSVerify"`
+	NoTLSVerify *bool `yaml:"noTLSVerify" json:"noTLSVerify,omitempty"`
 	// Disables chunked transfer encoding.
 	// Useful if you are running a WSGI server.
-	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding" json:"disableChunkedEncoding"`
+	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding" json:"disableChunkedEncoding,omitempty"`
 	// Runs as jump host
-	BastionMode *bool `yaml:"bastionMode" json:"bastionMode"`
+	BastionMode *bool `yaml:"bastionMode" json:"bastionMode,omitempty"`
 	// Listen address for the proxy.
-	ProxyAddress *string `yaml:"proxyAddress" json:"proxyAddress"`
+	ProxyAddress *string `yaml:"proxyAddress" json:"proxyAddress,omitempty"`
 	// Listen port for the proxy.
-	ProxyPort *uint `yaml:"proxyPort" json:"proxyPort"`
+	ProxyPort *uint `yaml:"proxyPort" json:"proxyPort,omitempty"`
 	// Valid options are 'socks' or empty.
-	ProxyType *string `yaml:"proxyType" json:"proxyType"`
+	ProxyType *string `yaml:"proxyType" json:"proxyType,omitempty"`
 	// IP rules for the proxy service
-	IPRules []IngressIPRule `yaml:"ipRules" json:"ipRules"`
+	IPRules []IngressIPRule `yaml:"ipRules" json:"ipRules,omitempty"`
+	// Attempt to connect to origin with HTTP/2
+	Http2Origin *bool `yaml:"http2Origin" json:"http2Origin,omitempty"`
+	// Access holds all access related configs
+	Access *AccessConfig `yaml:"access" json:"access,omitempty"`
+}
+
+type AccessConfig struct {
+	// Required when set to true will fail every request that does not arrive through an access authenticated endpoint.
+	Required bool `yaml:"required" json:"required,omitempty"`
+
+	// TeamName is the organization team name to get the public key certificates for.
+	TeamName string `yaml:"teamName" json:"teamName"`
+
+	// AudTag is the AudTag to verify access JWT against.
+	AudTag []string `yaml:"audTag" json:"audTag"`
 }
 
 type IngressIPRule struct {
@@ -242,7 +259,9 @@ type Configuration struct {
 }
 
 type WarpRoutingConfig struct {
-	Enabled bool `yaml:"enabled" json:"enabled"`
+	Enabled        bool            `yaml:"enabled" json:"enabled"`
+	ConnectTimeout *CustomDuration `yaml:"connectTimeout" json:"connectTimeout,omitempty"`
+	TCPKeepAlive   *CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive,omitempty"`
 }
 
 type configFileSettings struct {
@@ -372,7 +391,8 @@ func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (settings *configFileSe
 	log.Debug().Msgf("Loading configuration from %s", configFile)
 	file, err := os.Open(configFile)
 	if err != nil {
-		if os.IsNotExist(err) {
+		// If does not exist and config file was not specificly specified then return ErrNoConfigFile found.
+		if os.IsNotExist(err) && !c.IsSet("config") {
 			err = ErrNoConfigFile
 		}
 		return nil, "", err
@@ -390,7 +410,7 @@ func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (settings *configFileSe
 	// Parse it again, with strict mode, to find warnings.
 	if file, err := os.Open(configFile); err == nil {
 		decoder := yaml.NewDecoder(file)
-		decoder.SetStrict(true)
+		decoder.KnownFields(true)
 		var unusedConfig configFileSettings
 		if err := decoder.Decode(&unusedConfig); err != nil {
 			warnings = err.Error()
@@ -399,3 +419,34 @@ func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (settings *configFileSe
 
 	return &configuration, warnings, nil
 }
+
+// A CustomDuration is a Duration that has custom serialization for JSON.
+// JSON in Javascript assumes that int fields are 32 bits and Duration fields are deserialized assuming that numbers
+// are in nanoseconds, which in 32bit integers limits to just 2 seconds.
+// This type assumes that when serializing/deserializing from JSON, that the number is in seconds, while it maintains
+// the YAML serde assumptions.
+type CustomDuration struct {
+	time.Duration
+}
+
+func (s CustomDuration) MarshalJSON() ([]byte, error) {
+	return json.Marshal(s.Duration.Seconds())
+}
+
+func (s *CustomDuration) UnmarshalJSON(data []byte) error {
+	seconds, err := strconv.ParseInt(string(data), 10, 64)
+	if err != nil {
+		return err
+	}
+
+	s.Duration = time.Duration(seconds * int64(time.Second))
+	return nil
+}
+
+func (s *CustomDuration) MarshalYAML() (interface{}, error) {
+	return s.Duration.String(), nil
+}
+
+func (s *CustomDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	return unmarshal(&s.Duration)
+}

config/configuration_test.go

@@ -6,7 +6,8 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
-	yaml "gopkg.in/yaml.v2"
+	"github.com/stretchr/testify/require"
+	yaml "gopkg.in/yaml.v3"
 )
 
 func TestConfigFileSettings(t *testing.T) {
@@ -22,7 +23,9 @@ func TestConfigFileSettings(t *testing.T) {
 			Service:  "https://localhost:8001",
 		}
 		warpRouting = WarpRoutingConfig{
-			Enabled: true,
+			Enabled:        true,
+			ConnectTimeout: &CustomDuration{Duration: 2 * time.Second},
+			TCPKeepAlive:   &CustomDuration{Duration: 10 * time.Second},
 		}
 	)
 	rawYAML := `
@@ -47,6 +50,9 @@ ingress:
    service: https://localhost:8001
 warp-routing: 
   enabled: true
+  connectTimeout: 2s
+  tcpKeepAlive: 10s
+
 retries: 5
 grace-period: 30s
 percentage: 3.14
@@ -110,14 +116,13 @@ counters:
 
 }
 
-func TestUnmarshalOriginRequestConfig(t *testing.T) {
-	raw := []byte(`
+var rawJsonConfig = []byte(`
 {
-	"connectTimeout": 10000000000,
-	"tlsTimeout": 30000000000,
-	"tcpKeepAlive": 30000000000,
+	"connectTimeout": 10,
+	"tlsTimeout": 30,
+	"tcpKeepAlive": 30,
 	"noHappyEyeballs": true,
-	"keepAliveTimeout": 60000000000,
+	"keepAliveTimeout": 60,
 	"keepAliveConnections": 10,
 	"httpHostHeader": "app.tunnel.com",
 	"originServerName": "app.tunnel.com",
@@ -139,16 +144,43 @@ func TestUnmarshalOriginRequestConfig(t *testing.T) {
 			"ports": [443, 4443],
 			"allow": true
 		}
-	]
+	],
+	"http2Origin": true
 }
 `)
+
+func TestMarshalUnmarshalOriginRequest(t *testing.T) {
+	testCases := []struct {
+		name          string
+		marshalFunc   func(in interface{}) (out []byte, err error)
+		unMarshalFunc func(in []byte, out interface{}) (err error)
+	}{
+		{"json", json.Marshal, json.Unmarshal},
+		{"yaml", yaml.Marshal, yaml.Unmarshal},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assertConfig(t, tc.marshalFunc, tc.unMarshalFunc)
+		})
+	}
+}
+
+func assertConfig(
+	t *testing.T,
+	marshalFunc func(in interface{}) (out []byte, err error),
+	unMarshalFunc func(in []byte, out interface{}) (err error),
+) {
 	var config OriginRequestConfig
-	assert.NoError(t, json.Unmarshal(raw, &config))
-	assert.Equal(t, time.Second*10, *config.ConnectTimeout)
-	assert.Equal(t, time.Second*30, *config.TLSTimeout)
-	assert.Equal(t, time.Second*30, *config.TCPKeepAlive)
+	var config2 OriginRequestConfig
+
+	assert.NoError(t, json.Unmarshal(rawJsonConfig, &config))
+
+	assert.Equal(t, time.Second*10, config.ConnectTimeout.Duration)
+	assert.Equal(t, time.Second*30, config.TLSTimeout.Duration)
+	assert.Equal(t, time.Second*30, config.TCPKeepAlive.Duration)
 	assert.Equal(t, true, *config.NoHappyEyeballs)
-	assert.Equal(t, time.Second*60, *config.KeepAliveTimeout)
+	assert.Equal(t, time.Second*60, config.KeepAliveTimeout.Duration)
 	assert.Equal(t, 10, *config.KeepAliveConnections)
 	assert.Equal(t, "app.tunnel.com", *config.HTTPHostHeader)
 	assert.Equal(t, "app.tunnel.com", *config.OriginServerName)
@@ -160,6 +192,7 @@ func TestUnmarshalOriginRequestConfig(t *testing.T) {
 	assert.Equal(t, true, *config.NoTLSVerify)
 	assert.Equal(t, uint(9000), *config.ProxyPort)
 	assert.Equal(t, "socks", *config.ProxyType)
+	assert.Equal(t, true, *config.Http2Origin)
 
 	privateV4 := "10.0.0.0/8"
 	privateV6 := "fc00::/7"
@@ -176,4 +209,12 @@ func TestUnmarshalOriginRequestConfig(t *testing.T) {
 		},
 	}
 	assert.Equal(t, ipRules, config.IPRules)
+
+	// validate that serializing and deserializing again matches the deserialization from raw string
+	result, err := marshalFunc(config)
+	require.NoError(t, err)
+	err = unMarshalFunc(result, &config2)
+	require.NoError(t, err)
+
+	require.Equal(t, config2, config)
 }

config/manager.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/cloudflare/cloudflared/watcher"
 )

connection/connection.go

@@ -2,6 +2,7 @@ package connection
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"io"
 	"math"
@@ -11,7 +12,9 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/pkg/errors"
 
+	"github.com/cloudflare/cloudflared/tracing"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/websocket"
 )
@@ -21,13 +24,22 @@ const (
 	LogFieldConnIndex      = "connIndex"
 	MaxGracePeriod         = time.Minute * 3
 	MaxConcurrentStreams   = math.MaxUint32
+
+	contentTypeHeader = "content-type"
+	sseContentType    = "text/event-stream"
+	grpcContentType   = "application/grpc"
 )
 
-var switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols, http.StatusText(http.StatusSwitchingProtocols))
+var (
+	switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols, http.StatusText(http.StatusSwitchingProtocols))
+	flushableContentTypes = []string{sseContentType, grpcContentType}
+)
 
 type Orchestrator interface {
 	UpdateConfig(version int32, config []byte) *pogs.UpdateConfigurationResponse
+	GetConfigJSON() ([]byte, error)
 	GetOriginProxy() (OriginProxy, error)
+	WarpRoutingEnabled() (enabled bool)
 }
 
 type NamedTunnelProperties struct {
@@ -41,7 +53,6 @@ type Credentials struct {
 	AccountTag   string
 	TunnelSecret []byte
 	TunnelID     uuid.UUID
-	TunnelName   string
 }
 
 func (c *Credentials) Auth() pogs.TunnelAuth {
@@ -51,6 +62,30 @@ func (c *Credentials) Auth() pogs.TunnelAuth {
 	}
 }
 
+// TunnelToken are Credentials but encoded with custom fields namings.
+type TunnelToken struct {
+	AccountTag   string    `json:"a"`
+	TunnelSecret []byte    `json:"s"`
+	TunnelID     uuid.UUID `json:"t"`
+}
+
+func (t TunnelToken) Credentials() Credentials {
+	return Credentials{
+		AccountTag:   t.AccountTag,
+		TunnelSecret: t.TunnelSecret,
+		TunnelID:     t.TunnelID,
+	}
+}
+
+func (t TunnelToken) Encode() (string, error) {
+	val, err := json.Marshal(t)
+	if err != nil {
+		return "", errors.Wrap(err, "could not JSON encode token")
+	}
+
+	return base64.StdEncoding.EncodeToString(val), nil
+}
+
 type ClassicTunnelProperties struct {
 	Hostname   string
 	OriginCert []byte
@@ -66,6 +101,7 @@ const (
 	TypeTCP
 	TypeControlStream
 	TypeHTTP
+	TypeConfiguration
 )
 
 // ShouldFlush returns whether this kind of connection should actively flush data
@@ -95,22 +131,24 @@ func (t Type) String() string {
 
 // OriginProxy is how data flows from cloudflared to the origin services running behind it.
 type OriginProxy interface {
-	ProxyHTTP(w ResponseWriter, req *http.Request, isWebsocket bool) error
+	ProxyHTTP(w ResponseWriter, tr *tracing.TracedHTTPRequest, isWebsocket bool) error
 	ProxyTCP(ctx context.Context, rwa ReadWriteAcker, req *TCPRequest) error
 }
 
 // TCPRequest defines the input format needed to perform a TCP proxy.
 type TCPRequest struct {
-	Dest    string
-	CFRay   string
-	LBProbe bool
+	Dest      string
+	CFRay     string
+	LBProbe   bool
+	FlowID    string
+	CfTraceID string
 }
 
 // ReadWriteAcker is a readwriter with the ability to Acknowledge to the downstream (edge) that the origin has
 // accepted the connection.
 type ReadWriteAcker interface {
 	io.ReadWriter
-	AckConnection() error
+	AckConnection(tracePropagation string) error
 }
 
 // HTTPResponseReadWriteAcker is an HTTP implementation of ReadWriteAcker.
@@ -139,22 +177,28 @@ func (h *HTTPResponseReadWriteAcker) Write(p []byte) (int, error) {
 
 // AckConnection acks an HTTP connection by sending a switch protocols status code that enables the caller to
 // upgrade to streams.
-func (h *HTTPResponseReadWriteAcker) AckConnection() error {
+func (h *HTTPResponseReadWriteAcker) AckConnection(tracePropagation string) error {
 	resp := &http.Response{
 		Status:        switchingProtocolText,
 		StatusCode:    http.StatusSwitchingProtocols,
 		ContentLength: -1,
+		Header:        http.Header{},
 	}
 
 	if secWebsocketKey := h.req.Header.Get("Sec-WebSocket-Key"); secWebsocketKey != "" {
 		resp.Header = websocket.NewResponseHeader(h.req)
 	}
 
+	if tracePropagation != "" {
+		resp.Header.Add(tracing.CanonicalCloudflaredTracingHeader, tracePropagation)
+	}
+
 	return h.w.WriteRespHeaders(resp.StatusCode, resp.Header)
 }
 
 type ResponseWriter interface {
 	WriteRespHeaders(status int, header http.Header) error
+	AddTrailer(trailerName, trailerValue string)
 	io.Writer
 }
 
@@ -163,10 +207,18 @@ type ConnectedFuse interface {
 	IsConnected() bool
 }
 
-func IsServerSentEvent(headers http.Header) bool {
-	if contentType := headers.Get("content-type"); contentType != "" {
-		return strings.HasPrefix(strings.ToLower(contentType), "text/event-stream")
+// Helper method to let the caller know what content-types should require a flush on every
+// write to a ResponseWriter.
+func shouldFlush(headers http.Header) bool {
+	if contentType := headers.Get(contentTypeHeader); contentType != "" {
+		contentType = strings.ToLower(contentType)
+		for _, c := range flushableContentTypes {
+			if strings.HasPrefix(contentType, c) {
+				return true
+			}
+		}
 	}
+
 	return false
 }
 

connection/connection_test.go

@@ -6,12 +6,11 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
-	"testing"
 	"time"
 
 	"github.com/rs/zerolog"
-	"github.com/stretchr/testify/assert"
 
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/websocket"
 )
@@ -29,6 +28,8 @@ var (
 	testLargeResp = make([]byte, largeFileSize)
 )
 
+var _ ReadWriteAcker = (*HTTPResponseReadWriteAcker)(nil)
+
 type testRequest struct {
 	name           string
 	endpoint       string
@@ -41,6 +42,10 @@ type mockOrchestrator struct {
 	originProxy OriginProxy
 }
 
+func (mcr *mockOrchestrator) GetConfigJSON() ([]byte, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
 func (*mockOrchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
 	return &tunnelpogs.UpdateConfigurationResponse{
 		LastAppliedVersion: version,
@@ -51,13 +56,18 @@ func (mcr *mockOrchestrator) GetOriginProxy() (OriginProxy, error) {
 	return mcr.originProxy, nil
 }
 
+func (mcr *mockOrchestrator) WarpRoutingEnabled() (enabled bool) {
+	return true
+}
+
 type mockOriginProxy struct{}
 
 func (moc *mockOriginProxy) ProxyHTTP(
 	w ResponseWriter,
-	req *http.Request,
+	tr *tracing.TracedHTTPRequest,
 	isWebsocket bool,
 ) error {
+	req := tr.Request
 	if isWebsocket {
 		switch req.URL.Path {
 		case "/ws/echo":
@@ -189,40 +199,3 @@ func (mcf mockConnectedFuse) Connected() {}
 func (mcf mockConnectedFuse) IsConnected() bool {
 	return true
 }
-
-func TestIsEventStream(t *testing.T) {
-	tests := []struct {
-		headers       http.Header
-		isEventStream bool
-	}{
-		{
-			headers:       newHeader("Content-Type", "text/event-stream"),
-			isEventStream: true,
-		},
-		{
-			headers:       newHeader("content-type", "text/event-stream"),
-			isEventStream: true,
-		},
-		{
-			headers:       newHeader("Content-Type", "text/event-stream; charset=utf-8"),
-			isEventStream: true,
-		},
-		{
-			headers:       newHeader("Content-Type", "application/json"),
-			isEventStream: false,
-		},
-		{
-			headers:       http.Header{},
-			isEventStream: false,
-		},
-	}
-	for _, test := range tests {
-		assert.Equal(t, test.isEventStream, IsServerSentEvent(test.headers))
-	}
-}
-
-func newHeader(key, value string) http.Header {
-	header := http.Header{}
-	header.Add(key, value)
-	return header
-}

connection/control.go

@@ -2,7 +2,9 @@ package connection
 
 import (
 	"context"
+	"fmt"
 	"io"
+	"net"
 	"time"
 
 	"github.com/rs/zerolog"
@@ -19,6 +21,8 @@ type controlStream struct {
 	connectedFuse         ConnectedFuse
 	namedTunnelProperties *NamedTunnelProperties
 	connIndex             uint8
+	edgeAddress           net.IP
+	protocol              Protocol
 
 	newRPCClientFunc RPCClientFunc
 
@@ -30,20 +34,26 @@ type controlStream struct {
 // ControlStreamHandler registers connections with origintunneld and initiates graceful shutdown.
 type ControlStreamHandler interface {
 	// ServeControlStream handles the control plane of the transport in the current goroutine calling this
-	ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions) error
+	ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions, tunnelConfigGetter TunnelConfigJSONGetter) error
 	// IsStopped tells whether the method above has finished
 	IsStopped() bool
 }
 
+type TunnelConfigJSONGetter interface {
+	GetConfigJSON() ([]byte, error)
+}
+
 // NewControlStream returns a new instance of ControlStreamHandler
 func NewControlStream(
 	observer *Observer,
 	connectedFuse ConnectedFuse,
 	namedTunnelConfig *NamedTunnelProperties,
 	connIndex uint8,
+	edgeAddress net.IP,
 	newRPCClientFunc RPCClientFunc,
 	gracefulShutdownC <-chan struct{},
 	gracePeriod time.Duration,
+	protocol Protocol,
 ) ControlStreamHandler {
 	if newRPCClientFunc == nil {
 		newRPCClientFunc = newRegistrationRPCClient
@@ -54,8 +64,10 @@ func NewControlStream(
 		namedTunnelProperties: namedTunnelConfig,
 		newRPCClientFunc:      newRPCClientFunc,
 		connIndex:             connIndex,
+		edgeAddress:           edgeAddress,
 		gracefulShutdownC:     gracefulShutdownC,
 		gracePeriod:           gracePeriod,
+		protocol:              protocol,
 	}
 }
 
@@ -63,15 +75,31 @@ func (c *controlStream) ServeControlStream(
 	ctx context.Context,
 	rw io.ReadWriteCloser,
 	connOptions *tunnelpogs.ConnectionOptions,
+	tunnelConfigGetter TunnelConfigJSONGetter,
 ) error {
 	rpcClient := c.newRPCClientFunc(ctx, rw, c.observer.log)
 
-	if err := rpcClient.RegisterConnection(ctx, c.namedTunnelProperties, connOptions, c.connIndex, c.observer); err != nil {
+	registrationDetails, err := rpcClient.RegisterConnection(ctx, c.namedTunnelProperties, connOptions, c.connIndex, c.edgeAddress, c.observer)
+	if err != nil {
 		rpcClient.Close()
 		return err
 	}
+
+	c.observer.logServerInfo(c.connIndex, registrationDetails.Location, c.edgeAddress, fmt.Sprintf("Connection %s registered with protocol: %s", registrationDetails.UUID, c.protocol))
+	c.observer.sendConnectedEvent(c.connIndex, c.protocol, registrationDetails.Location)
 	c.connectedFuse.Connected()
 
+	// if conn index is 0 and tunnel is not remotely managed, then send local ingress rules configuration
+	if c.connIndex == 0 && !registrationDetails.TunnelIsRemotelyManaged {
+		if tunnelConfig, err := tunnelConfigGetter.GetConfigJSON(); err == nil {
+			if err := rpcClient.SendLocalConfiguration(ctx, tunnelConfig, c.observer); err != nil {
+				c.observer.log.Err(err).Msg("unable to send local configuration")
+			}
+		} else {
+			c.observer.log.Err(err).Msg("failed to obtain current configuration")
+		}
+	}
+
 	c.waitForUnregister(ctx, rpcClient)
 	return nil
 }

connection/errors.go

@@ -18,6 +18,19 @@ func (e DupConnRegisterTunnelError) Error() string {
 	return "already connected to this server, trying another address"
 }
 
+// Dial to edge server with quic failed
+type EdgeQuicDialError struct {
+	Cause error
+}
+
+func (e *EdgeQuicDialError) Error() string {
+	return "failed to dial to edge with quic: " + e.Cause.Error()
+}
+
+func (e *EdgeQuicDialError) Unwrap() error {
+	return e.Cause
+}
+
 // RegisterTunnel error from server
 type ServerRegisterTunnelError struct {
 	Cause     error

connection/event.go

@@ -5,6 +5,7 @@ type Event struct {
 	Index     uint8
 	EventType Status
 	Location  string
+	Protocol  Protocol
 	URL       string
 }
 

connection/h2mux.go

@@ -12,6 +12,7 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/websocket"
 )
@@ -68,6 +69,7 @@ func NewH2muxConnection(
 	connIndex uint8,
 	observer *Observer,
 	gracefulShutdownC <-chan struct{},
+	log *zerolog.Logger,
 ) (*h2muxConnection, error, bool) {
 	h := &h2muxConnection{
 		orchestrator:      orchestrator,
@@ -78,6 +80,7 @@ func NewH2muxConnection(
 		observer:          observer,
 		gracefulShutdownC: gracefulShutdownC,
 		newRPCClientFunc:  newRegistrationRPCClient,
+		log:               log,
 	}
 
 	// Establish a muxed connection with the edge
@@ -233,7 +236,7 @@ func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
 		return err
 	}
 
-	err = originProxy.ProxyHTTP(respWriter, req, sourceConnectionType == TypeWebsocket)
+	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedHTTPRequest(req, h.log), sourceConnectionType == TypeWebsocket)
 	if err != nil {
 		respWriter.WriteErrorResponse()
 	}
@@ -256,6 +259,10 @@ type h2muxRespWriter struct {
 	*h2mux.MuxedStream
 }
 
+func (rp *h2muxRespWriter) AddTrailer(trailerName, trailerValue string) {
+	// do nothing. we don't support trailers over h2mux
+}
+
 func (rp *h2muxRespWriter) WriteRespHeaders(status int, header http.Header) error {
 	headers := H1ResponseToH2ResponseHeaders(status, header)
 	headers = append(headers, h2mux.Header{Name: ResponseMetaHeader, Value: responseMetaHeaderOrigin})

connection/h2mux_test.go

@@ -47,8 +47,8 @@ func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
 		edgeMuxChan <- edgeMux
 	}()
 	var connIndex = uint8(0)
-	testObserver := NewObserver(&log, &log, false)
-	h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil)
+	testObserver := NewObserver(&log, &log)
+	h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil, &log)
 	require.NoError(t, err)
 	return h2muxConn, <-edgeMuxChan
 }

connection/http2.go

@@ -2,6 +2,7 @@ package connection
 
 import (
 	"context"
+	gojson "encoding/json"
 	"fmt"
 	"io"
 	"net"
@@ -14,6 +15,7 @@ import (
 	"github.com/rs/zerolog"
 	"golang.org/x/net/http2"
 
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
@@ -23,6 +25,7 @@ const (
 	InternalTCPProxySrcHeader = "Cf-Cloudflared-Proxy-Src"
 	WebsocketUpgrade          = "websocket"
 	ControlStreamUpgrade      = "control-stream"
+	ConfigurationUpdate       = "update-configuration"
 )
 
 var errEdgeConnectionClosed = fmt.Errorf("connection with edge closed")
@@ -100,7 +103,7 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	connType := determineHTTP2Type(r)
 	handleMissingRequestParts(connType, r)
 
-	respWriter, err := NewHTTP2RespWriter(r, w, connType)
+	respWriter, err := NewHTTP2RespWriter(r, w, connType, c.log)
 	if err != nil {
 		c.observer.log.Error().Msg(err.Error())
 		return
@@ -114,15 +117,23 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	switch connType {
 	case TypeControlStream:
-		if err := c.controlStreamHandler.ServeControlStream(r.Context(), respWriter, c.connOptions); err != nil {
+		if err := c.controlStreamHandler.ServeControlStream(r.Context(), respWriter, c.connOptions, c.orchestrator); err != nil {
 			c.controlStreamErr = err
 			c.log.Error().Err(err)
 			respWriter.WriteErrorResponse()
 		}
 
+	case TypeConfiguration:
+		if err := c.handleConfigurationUpdate(respWriter, r); err != nil {
+			c.log.Error().Err(err)
+			respWriter.WriteErrorResponse()
+		}
+
 	case TypeWebsocket, TypeHTTP:
 		stripWebsocketUpgradeHeader(r)
-		if err := originProxy.ProxyHTTP(respWriter, r, connType == TypeWebsocket); err != nil {
+		// Check for tracing on request
+		tr := tracing.NewTracedHTTPRequest(r, c.log)
+		if err := originProxy.ProxyHTTP(respWriter, tr, connType == TypeWebsocket); err != nil {
 			err := fmt.Errorf("Failed to proxy HTTP: %w", err)
 			c.log.Error().Err(err)
 			respWriter.WriteErrorResponse()
@@ -138,9 +149,10 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 		rws := NewHTTPResponseReadWriterAcker(respWriter, r)
 		if err := originProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
-			Dest:    host,
-			CFRay:   FindCfRayHeader(r),
-			LBProbe: IsLBProbeRequest(r),
+			Dest:      host,
+			CFRay:     FindCfRayHeader(r),
+			LBProbe:   IsLBProbeRequest(r),
+			CfTraceID: r.Header.Get(tracing.TracerContextName),
 		}); err != nil {
 			respWriter.WriteErrorResponse()
 		}
@@ -152,6 +164,26 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// ConfigurationUpdateBody is the representation followed by the edge to send updates to cloudflared.
+type ConfigurationUpdateBody struct {
+	Version int32             `json:"version"`
+	Config  gojson.RawMessage `json:"config"`
+}
+
+func (c *HTTP2Connection) handleConfigurationUpdate(respWriter *http2RespWriter, r *http.Request) error {
+	var configBody ConfigurationUpdateBody
+	if err := json.NewDecoder(r.Body).Decode(&configBody); err != nil {
+		return err
+	}
+	resp := c.orchestrator.UpdateConfig(configBody.Version, configBody.Config)
+	bdy, err := json.Marshal(resp)
+	if err != nil {
+		return err
+	}
+	_, err = respWriter.Write(bdy)
+	return err
+}
+
 func (c *HTTP2Connection) close() {
 	// Wait for all serve HTTP handlers to return
 	c.activeRequestsWG.Wait()
@@ -159,18 +191,21 @@ func (c *HTTP2Connection) close() {
 }
 
 type http2RespWriter struct {
-	r           io.Reader
-	w           http.ResponseWriter
-	flusher     http.Flusher
-	shouldFlush bool
+	r             io.Reader
+	w             http.ResponseWriter
+	flusher       http.Flusher
+	shouldFlush   bool
+	statusWritten bool
+	log           *zerolog.Logger
 }
 
-func NewHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type) (*http2RespWriter, error) {
+func NewHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type, log *zerolog.Logger) (*http2RespWriter, error) {
 	flusher, isFlusher := w.(http.Flusher)
 	if !isFlusher {
 		respWriter := &http2RespWriter{
-			r: r.Body,
-			w: w,
+			r:   r.Body,
+			w:   w,
+			log: log,
 		}
 		respWriter.WriteErrorResponse()
 		return nil, fmt.Errorf("%T doesn't implement http.Flusher", w)
@@ -181,14 +216,24 @@ func NewHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type) (
 		w:           w,
 		flusher:     flusher,
 		shouldFlush: connType.shouldFlush(),
+		log:         log,
 	}, nil
 }
 
+func (rp *http2RespWriter) AddTrailer(trailerName, trailerValue string) {
+	if !rp.statusWritten {
+		rp.log.Warn().Msg("Tried to add Trailer to response before status written. Ignoring...")
+		return
+	}
+
+	rp.w.Header().Add(http2.TrailerPrefix+trailerName, trailerValue)
+}
+
 func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) error {
 	dest := rp.w.Header()
 	userHeaders := make(http.Header, len(header))
 	for name, values := range header {
-		// Since these are http2 headers, they're required to be lowercase
+		// lowercase headers for simplicity check
 		h2name := strings.ToLower(name)
 
 		if h2name == "content-length" {
@@ -197,6 +242,12 @@ func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) erro
 			dest[name] = values
 		}
 
+		if h2name == tracing.IntCloudflaredTracingHeader {
+			// Add cf-int-cloudflared-tracing header outside of serialized userHeaders
+			dest[tracing.CanonicalCloudflaredTracingHeader] = values
+			continue
+		}
+
 		if !IsControlResponseHeader(h2name) || IsWebsocketClientHeader(h2name) {
 			// User headers, on the other hand, must all be serialized so that
 			// HTTP/2 header validation won't be applied to HTTP/1 header values
@@ -206,18 +257,21 @@ func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) erro
 
 	// Perform user header serialization and set them in the single header
 	dest.Set(CanonicalResponseUserHeaders, SerializeHeaders(userHeaders))
+
 	rp.setResponseMetaHeader(responseMetaHeaderOrigin)
 	// HTTP2 removes support for 101 Switching Protocols https://tools.ietf.org/html/rfc7540#section-8.1.1
 	if status == http.StatusSwitchingProtocols {
 		status = http.StatusOK
 	}
 	rp.w.WriteHeader(status)
-	if IsServerSentEvent(header) {
+	if shouldFlush(header) {
 		rp.shouldFlush = true
 	}
 	if rp.shouldFlush {
 		rp.flusher.Flush()
 	}
+
+	rp.statusWritten = true
 	return nil
 }
 
@@ -239,7 +293,7 @@ func (rp *http2RespWriter) Write(p []byte) (n int, err error) {
 		// Implementer of OriginClient should make sure it doesn't write to the connection after Proxy returns
 		// Register a recover routine just in case.
 		if r := recover(); r != nil {
-			println(fmt.Sprintf("Recover from http2 response writer panic, error %s", debug.Stack()))
+			rp.log.Debug().Msgf("Recover from http2 response writer panic, error %s", debug.Stack())
 		}
 	}()
 	n, err = rp.w.Write(p)
@@ -255,6 +309,8 @@ func (rp *http2RespWriter) Close() error {
 
 func determineHTTP2Type(r *http.Request) Type {
 	switch {
+	case isConfigurationUpdate(r):
+		return TypeConfiguration
 	case isWebsocketUpgrade(r):
 		return TypeWebsocket
 	case IsTCPStream(r):
@@ -288,6 +344,10 @@ func isWebsocketUpgrade(r *http.Request) bool {
 	return r.Header.Get(InternalUpgradeHeader) == WebsocketUpgrade
 }
 
+func isConfigurationUpdate(r *http.Request) bool {
+	return r.Header.Get(InternalUpgradeHeader) == ConfigurationUpdate
+}
+
 // IsTCPStream discerns if the connection request needs a tcp stream proxy.
 func IsTCPStream(r *http.Request) bool {
 	return r.Header.Get(InternalTCPProxySrcHeader) != ""

connection/http2_test.go

@@ -1,6 +1,7 @@
 package connection
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -14,6 +15,7 @@ import (
 	"time"
 
 	"github.com/gobwas/ws/wsutil"
+	"github.com/google/uuid"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -31,7 +33,7 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 	edgeConn, cfdConn := net.Pipe()
 	var connIndex = uint8(0)
 	log := zerolog.Nop()
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
@@ -39,7 +41,9 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 		connIndex,
 		nil,
 		nil,
+		nil,
 		1*time.Second,
+		HTTP2,
 	)
 	return NewHTTP2Connection(
 		cfdConn,
@@ -53,6 +57,41 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 	), edgeConn
 }
 
+func TestHTTP2ConfigurationSet(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	endpoint := fmt.Sprintf("http://localhost:8080/ok")
+	reqBody := []byte(`{
+"version": 2, 
+"config": {"warp-routing": {"enabled": true},  "originRequest" : {"connectTimeout": 10}, "ingress" : [ {"hostname": "test", "service": "https://localhost:8000" } , {"service": "http_status:404"} ]}}
+`)
+	reader := bytes.NewReader(reqBody)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, endpoint, reader)
+	require.NoError(t, err)
+	req.Header.Set(InternalUpgradeHeader, ConfigurationUpdate)
+
+	resp, err := edgeHTTP2Conn.RoundTrip(req)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	bdy, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, `{"lastAppliedVersion":2,"err":null}`, string(bdy))
+	cancel()
+	wg.Wait()
+
+}
+
 func TestServeHTTP(t *testing.T) {
 	tests := []testRequest{
 		{
@@ -130,18 +169,27 @@ type mockNamedTunnelRPCClient struct {
 	unregistered chan struct{}
 }
 
+func (mc mockNamedTunnelRPCClient) SendLocalConfiguration(c context.Context, config []byte, observer *Observer) error {
+	return nil
+}
+
 func (mc mockNamedTunnelRPCClient) RegisterConnection(
 	c context.Context,
 	properties *NamedTunnelProperties,
 	options *tunnelpogs.ConnectionOptions,
 	connIndex uint8,
+	edgeAddress net.IP,
 	observer *Observer,
-) error {
+) (*tunnelpogs.ConnectionDetails, error) {
 	if mc.shouldFail != nil {
-		return mc.shouldFail
+		return nil, mc.shouldFail
 	}
 	close(mc.registered)
-	return nil
+	return &tunnelpogs.ConnectionDetails{
+		Location:                "LIS",
+		UUID:                    uuid.New(),
+		TunnelIsRemotelyManaged: false,
+	}, nil
 }
 
 func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
@@ -309,15 +357,17 @@ func TestServeControlStream(t *testing.T) {
 		unregistered: make(chan struct{}),
 	}
 
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
 		&NamedTunnelProperties{},
 		1,
+		nil,
 		rpcClientFactory.newMockRPCClient,
 		nil,
 		1*time.Second,
+		HTTP2,
 	)
 	http2Conn.controlStreamHandler = controlStream
 
@@ -359,15 +409,17 @@ func TestFailRegistration(t *testing.T) {
 		unregistered: make(chan struct{}),
 	}
 
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
 		&NamedTunnelProperties{},
 		http2Conn.connIndex,
+		nil,
 		rpcClientFactory.newMockRPCClient,
 		nil,
 		1*time.Second,
+		HTTP2,
 	)
 	http2Conn.controlStreamHandler = controlStream
 
@@ -404,16 +456,18 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 	events := &eventCollectorSink{}
 
 	shutdownC := make(chan struct{})
-	obs := NewObserver(&log, &log, false)
+	obs := NewObserver(&log, &log)
 	obs.RegisterSink(events)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
 		&NamedTunnelProperties{},
 		http2Conn.connIndex,
+		nil,
 		rpcClientFactory.newMockRPCClient,
 		shutdownC,
 		1*time.Second,
+		HTTP2,
 	)
 
 	http2Conn.controlStreamHandler = controlStream
@@ -441,7 +495,7 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 
 	select {
 	case <-rpcClientFactory.registered:
-		break //ok
+		break // ok
 	case <-time.Tick(time.Second):
 		t.Fatal("timeout out waiting for registration")
 	}
@@ -451,7 +505,7 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 
 	select {
 	case <-rpcClientFactory.unregistered:
-		break //ok
+		break // ok
 	case <-time.Tick(time.Second):
 		t.Fatal("timeout out waiting for unregistered signal")
 	}

connection/metrics.go

@@ -13,6 +13,7 @@ const (
 	MetricsNamespace = "cloudflared"
 	TunnelSubsystem  = "tunnel"
 	muxerSubsystem   = "muxer"
+	configSubsystem  = "config"
 )
 
 type muxerMetrics struct {
@@ -36,6 +37,11 @@ type muxerMetrics struct {
 	compRateAve      *prometheus.GaugeVec
 }
 
+type localConfigMetrics struct {
+	pushes       prometheus.Counter
+	pushesErrors prometheus.Counter
+}
+
 type tunnelMetrics struct {
 	timerRetries    prometheus.Gauge
 	serverLocations *prometheus.GaugeVec
@@ -51,6 +57,39 @@ type tunnelMetrics struct {
 	muxerMetrics        *muxerMetrics
 	tunnelsHA           tunnelsForHA
 	userHostnamesCounts *prometheus.CounterVec
+
+	localConfigMetrics *localConfigMetrics
+}
+
+func newLocalConfigMetrics() *localConfigMetrics {
+
+	pushesMetric := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: configSubsystem,
+			Name:      "local_config_pushes",
+			Help:      "Number of local configuration pushes to the edge",
+		},
+	)
+
+	pushesErrorsMetric := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: configSubsystem,
+			Name:      "local_config_pushes_errors",
+			Help:      "Number of errors occurred during local configuration pushes",
+		},
+	)
+
+	prometheus.MustRegister(
+		pushesMetric,
+		pushesErrorsMetric,
+	)
+
+	return &localConfigMetrics{
+		pushes:       pushesMetric,
+		pushesErrors: pushesErrorsMetric,
+	}
 }
 
 func newMuxerMetrics() *muxerMetrics {
@@ -328,7 +367,7 @@ func initTunnelMetrics() *tunnelMetrics {
 			Name:      "server_locations",
 			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
 		},
-		[]string{"connection_id", "location"},
+		[]string{"connection_id", "edge_location"},
 	)
 	prometheus.MustRegister(serverLocations)
 
@@ -386,6 +425,7 @@ func initTunnelMetrics() *tunnelMetrics {
 		regFail:             registerFail,
 		rpcFail:             rpcFail,
 		userHostnamesCounts: userHostnamesCounts,
+		localConfigMetrics:  newLocalConfigMetrics(),
 	}
 }
 

connection/observer.go

@@ -1,6 +1,7 @@
 package connection
 
 import (
+	"net"
 	"strings"
 
 	"github.com/rs/zerolog"
@@ -8,6 +9,7 @@ import (
 
 const (
 	LogFieldLocation          = "location"
+	LogFieldIPAddress         = "ip"
 	observerChannelBufferSize = 16
 )
 
@@ -16,7 +18,6 @@ type Observer struct {
 	logTransport    *zerolog.Logger
 	metrics         *tunnelMetrics
 	tunnelEventChan chan Event
-	uiEnabled       bool
 	addSinkChan     chan EventSink
 }
 
@@ -24,12 +25,11 @@ type EventSink interface {
 	OnTunnelEvent(event Event)
 }
 
-func NewObserver(log, logTransport *zerolog.Logger, uiEnabled bool) *Observer {
+func NewObserver(log, logTransport *zerolog.Logger) *Observer {
 	o := &Observer{
 		log:             log,
 		logTransport:    logTransport,
 		metrics:         newTunnelMetrics(),
-		uiEnabled:       uiEnabled,
 		tunnelEventChan: make(chan Event, observerChannelBufferSize),
 		addSinkChan:     make(chan EventSink, observerChannelBufferSize),
 	}
@@ -41,11 +41,12 @@ func (o *Observer) RegisterSink(sink EventSink) {
 	o.addSinkChan <- sink
 }
 
-func (o *Observer) logServerInfo(connIndex uint8, location, msg string) {
+func (o *Observer) logServerInfo(connIndex uint8, location string, address net.IP, msg string) {
 	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
 	o.log.Info().
 		Uint8(LogFieldConnIndex, connIndex).
 		Str(LogFieldLocation, location).
+		IPAddr(LogFieldIPAddress, address).
 		Msg(msg)
 	o.metrics.registerServerLocation(uint8ToString(connIndex), location)
 }
@@ -54,8 +55,8 @@ func (o *Observer) sendRegisteringEvent(connIndex uint8) {
 	o.sendEvent(Event{Index: connIndex, EventType: RegisteringTunnel})
 }
 
-func (o *Observer) sendConnectedEvent(connIndex uint8, location string) {
-	o.sendEvent(Event{Index: connIndex, EventType: Connected, Location: location})
+func (o *Observer) sendConnectedEvent(connIndex uint8, protocol Protocol, location string) {
+	o.sendEvent(Event{Index: connIndex, EventType: Connected, Protocol: protocol, Location: location})
 }
 
 func (o *Observer) SendURL(url string) {

connection/observer_test.go

@@ -12,7 +12,7 @@ import (
 )
 
 func TestSendUrl(t *testing.T) {
-	observer := NewObserver(&log, &log, false)
+	observer := NewObserver(&log, &log)
 
 	observer.SendURL("my-url.com")
 	assert.Equal(t, 1.0, getCounterValue(t, observer.metrics.userHostnamesCounts, "https://my-url.com"))
@@ -63,7 +63,7 @@ func TestRegisterServerLocation(t *testing.T) {
 }
 
 func TestObserverEventsDontBlock(t *testing.T) {
-	observer := NewObserver(&log, &log, false)
+	observer := NewObserver(&log, &log)
 	var mu sync.Mutex
 	observer.RegisterSink(EventSinkFunc(func(_ Event) {
 		// callback will block if lock is already held

connection/protocol.go

@@ -1,6 +1,7 @@
 package connection
 
 import (
+	"errors"
 	"fmt"
 	"hash/fnv"
 	"sync"
@@ -19,7 +20,7 @@ const (
 	edgeH2TLSServerName = "h2.cftunnel.com"
 	// edgeQUICServerName is the server name to establish quic connection with edge.
 	edgeQUICServerName = "quic.cftunnel.com"
-	autoSelectFlag     = "auto"
+	AutoSelectFlag     = "auto"
 )
 
 var (
@@ -130,6 +131,7 @@ type autoProtocolSelector struct {
 	refreshAfter    time.Time
 	ttl             time.Duration
 	log             *zerolog.Logger
+	needPQ          bool
 }
 
 func newAutoProtocolSelector(
@@ -139,6 +141,7 @@ func newAutoProtocolSelector(
 	fetchFunc PercentageFetcher,
 	ttl time.Duration,
 	log *zerolog.Logger,
+	needPQ bool,
 ) *autoProtocolSelector {
 	return &autoProtocolSelector{
 		current:         current,
@@ -148,6 +151,7 @@ func newAutoProtocolSelector(
 		refreshAfter:    time.Now().Add(ttl),
 		ttl:             ttl,
 		log:             log,
+		needPQ:          needPQ,
 	}
 }
 
@@ -187,6 +191,9 @@ func getProtocol(protocolPool []Protocol, fetchFunc PercentageFetcher, switchThr
 func (s *autoProtocolSelector) Fallback() (Protocol, bool) {
 	s.lock.RLock()
 	defer s.lock.RUnlock()
+	if s.needPQ {
+		return 0, false
+	}
 	return s.current.fallback()
 }
 
@@ -199,9 +206,14 @@ func NewProtocolSelector(
 	fetchFunc PercentageFetcher,
 	ttl time.Duration,
 	log *zerolog.Logger,
+	needPQ bool,
 ) (ProtocolSelector, error) {
 	// Classic tunnel is only supported with h2mux
 	if namedTunnel == nil {
+		if needPQ {
+			return nil, errors.New("Classic tunnel does not support post-quantum")
+		}
+
 		return &staticProtocolSelector{
 			current: H2mux,
 		}, nil
@@ -209,8 +221,11 @@ func NewProtocolSelector(
 
 	threshold := switchThreshold(namedTunnel.Credentials.AccountTag)
 	fetchedProtocol, err := getProtocol([]Protocol{QUIC, HTTP2}, fetchFunc, threshold)
-	if err != nil {
-		log.Err(err).Msg("Unable to lookup protocol. Defaulting to `http2`. If this fails, you can set `--protocol h2mux` in your cloudflared command.")
+	if err != nil && protocolFlag == "auto" {
+		log.Err(err).Msg("Unable to lookup protocol. Defaulting to `http2`. If this fails, you can attempt `--protocol quic` instead.")
+		if needPQ {
+			return nil, errors.New("http2 does not support post-quantum")
+		}
 		return &staticProtocolSelector{
 			current: HTTP2,
 		}, nil
@@ -221,10 +236,10 @@ func NewProtocolSelector(
 			protocolFlag = HTTP2.String()
 			fetchedProtocol = HTTP2Warp
 		}
-		return selectWarpRoutingProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol)
+		return selectWarpRoutingProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol, needPQ)
 	}
 
-	return selectNamedTunnelProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol)
+	return selectNamedTunnelProtocols(protocolFlag, fetchFunc, ttl, log, threshold, fetchedProtocol, needPQ)
 }
 
 func selectNamedTunnelProtocols(
@@ -234,6 +249,7 @@ func selectNamedTunnelProtocols(
 	log *zerolog.Logger,
 	threshold int32,
 	protocol Protocol,
+	needPQ bool,
 ) (ProtocolSelector, error) {
 	// If the user picks a protocol, then we stick to it no matter what.
 	switch protocolFlag {
@@ -247,8 +263,8 @@ func selectNamedTunnelProtocols(
 
 	// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
 	// fallback on failures.
-	if protocolFlag == autoSelectFlag {
-		return newAutoProtocolSelector(protocol, []Protocol{QUIC, HTTP2, H2mux}, threshold, fetchFunc, ttl, log), nil
+	if protocolFlag == AutoSelectFlag {
+		return newAutoProtocolSelector(protocol, []Protocol{QUIC, HTTP2, H2mux}, threshold, fetchFunc, ttl, log, needPQ), nil
 	}
 
 	return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
@@ -261,6 +277,7 @@ func selectWarpRoutingProtocols(
 	log *zerolog.Logger,
 	threshold int32,
 	protocol Protocol,
+	needPQ bool,
 ) (ProtocolSelector, error) {
 	// If the user picks a protocol, then we stick to it no matter what.
 	switch protocolFlag {
@@ -272,8 +289,8 @@ func selectWarpRoutingProtocols(
 
 	// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
 	// fallback on failures.
-	if protocolFlag == autoSelectFlag {
-		return newAutoProtocolSelector(protocol, []Protocol{QUICWarp, HTTP2Warp}, threshold, fetchFunc, ttl, log), nil
+	if protocolFlag == AutoSelectFlag {
+		return newAutoProtocolSelector(protocol, []Protocol{QUICWarp, HTTP2Warp}, threshold, fetchFunc, ttl, log, needPQ), nil
 	}
 
 	return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)

connection/protocol_test.go

@@ -91,14 +91,14 @@ func TestNewProtocolSelector(t *testing.T) {
 		},
 		{
 			name:              "named tunnel quic and http2 disabled",
-			protocol:          "auto",
+			protocol:          AutoSelectFlag,
 			expectedProtocol:  H2mux,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
 			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:             "named tunnel quic disabled",
-			protocol:         "auto",
+			protocol:         AutoSelectFlag,
 			expectedProtocol: HTTP2,
 			// Hasfallback true is because if http2 fails, then we further fallback to h2mux.
 			hasFallback:       true,
@@ -108,21 +108,21 @@ func TestNewProtocolSelector(t *testing.T) {
 		},
 		{
 			name:              "named tunnel auto all http2 disabled",
-			protocol:          "auto",
+			protocol:          AutoSelectFlag,
 			expectedProtocol:  H2mux,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
 			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel auto to h2mux",
-			protocol:          "auto",
+			protocol:          AutoSelectFlag,
 			expectedProtocol:  H2mux,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}),
 			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel auto to http2",
-			protocol:          "auto",
+			protocol:          AutoSelectFlag,
 			expectedProtocol:  HTTP2,
 			hasFallback:       true,
 			expectedFallback:  H2mux,
@@ -131,7 +131,7 @@ func TestNewProtocolSelector(t *testing.T) {
 		},
 		{
 			name:              "named tunnel auto to quic",
-			protocol:          "auto",
+			protocol:          AutoSelectFlag,
 			expectedProtocol:  QUIC,
 			hasFallback:       true,
 			expectedFallback:  HTTP2,
@@ -167,7 +167,7 @@ func TestNewProtocolSelector(t *testing.T) {
 		},
 		{
 			name:               "warp routing quic",
-			protocol:           "auto",
+			protocol:           AutoSelectFlag,
 			expectedProtocol:   QUICWarp,
 			hasFallback:        true,
 			expectedFallback:   HTTP2Warp,
@@ -177,7 +177,7 @@ func TestNewProtocolSelector(t *testing.T) {
 		},
 		{
 			name:               "warp routing auto",
-			protocol:           "auto",
+			protocol:           AutoSelectFlag,
 			expectedProtocol:   HTTP2Warp,
 			hasFallback:        false,
 			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
@@ -186,7 +186,7 @@ func TestNewProtocolSelector(t *testing.T) {
 		},
 		{
 			name:               "warp routing auto- quic",
-			protocol:           "auto",
+			protocol:           AutoSelectFlag,
 			expectedProtocol:   QUICWarp,
 			hasFallback:        true,
 			expectedFallback:   HTTP2Warp,
@@ -209,7 +209,7 @@ func TestNewProtocolSelector(t *testing.T) {
 		},
 		{
 			name:              "named tunnel fetch error",
-			protocol:          "auto",
+			protocol:          AutoSelectFlag,
 			fetchFunc:         mockFetcher(true),
 			namedTunnelConfig: testNamedTunnelProperties,
 			expectedProtocol:  HTTP2,
@@ -219,7 +219,7 @@ func TestNewProtocolSelector(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			selector, err := NewProtocolSelector(test.protocol, test.warpRoutingEnabled, test.namedTunnelConfig, test.fetchFunc, testNoTTL, &log)
+			selector, err := NewProtocolSelector(test.protocol, test.warpRoutingEnabled, test.namedTunnelConfig, test.fetchFunc, testNoTTL, &log, false)
 			if test.wantErr {
 				assert.Error(t, err, fmt.Sprintf("test %s failed", test.name))
 			} else {
@@ -237,7 +237,7 @@ func TestNewProtocolSelector(t *testing.T) {
 
 func TestAutoProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
-	selector, err := NewProtocolSelector("auto", noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log)
+	selector, err := NewProtocolSelector(AutoSelectFlag, noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log, false)
 	assert.NoError(t, err)
 	assert.Equal(t, H2mux, selector.Current())
 
@@ -267,7 +267,7 @@ func TestAutoProtocolSelectorRefresh(t *testing.T) {
 func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
 	// Since the user chooses http2 on purpose, we always stick to it.
-	selector, err := NewProtocolSelector("http2", noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log)
+	selector, err := NewProtocolSelector("http2", noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log, false)
 	assert.NoError(t, err)
 	assert.Equal(t, HTTP2, selector.Current())
 
@@ -297,7 +297,7 @@ func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 func TestProtocolSelectorRefreshTTL(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}}
-	selector, err := NewProtocolSelector("auto", noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), time.Hour, &log)
+	selector, err := NewProtocolSelector(AutoSelectFlag, noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), time.Hour, &log, false)
 	assert.NoError(t, err)
 	assert.Equal(t, QUIC, selector.Current())
 

connection/quic.go

@@ -7,19 +7,26 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"net/netip"
 	"strconv"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/lucas-clemente/quic-go"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/cloudflare/cloudflared/datagramsession"
 	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/packet"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
@@ -30,14 +37,27 @@ const (
 	HTTPMethodKey = "HttpMethod"
 	// HTTPHostKey is used to get or set http Method in QUIC ALPN if the underlying proxy connection type is HTTP.
 	HTTPHostKey = "HttpHost"
+
+	QUICMetadataFlowID = "FlowID"
+	// emperically this capacity has been working well
+	demuxChanCapacity = 16
+)
+
+var (
+	portForConnIndex = make(map[uint8]int, 0)
+	portMapMutex     sync.Mutex
 )
 
 // QUICConnection represents the type that facilitates Proxying via QUIC streams.
 type QUICConnection struct {
-	session              quic.Session
-	logger               *zerolog.Logger
-	orchestrator         Orchestrator
-	sessionManager       datagramsession.Manager
+	session      quic.Connection
+	logger       *zerolog.Logger
+	orchestrator Orchestrator
+	// sessionManager tracks active sessions. It receives datagrams from quic connection via datagramMuxer
+	sessionManager datagramsession.Manager
+	// datagramMuxer mux/demux datagrams from quic connection
+	datagramMuxer        *quicpogs.DatagramMuxerV2
+	packetRouter         *ingress.PacketRouter
 	controlStreamHandler ControlStreamHandler
 	connOptions          *tunnelpogs.ConnectionOptions
 }
@@ -46,29 +66,44 @@ type QUICConnection struct {
 func NewQUICConnection(
 	quicConfig *quic.Config,
 	edgeAddr net.Addr,
+	connIndex uint8,
 	tlsConfig *tls.Config,
 	orchestrator Orchestrator,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler ControlStreamHandler,
 	logger *zerolog.Logger,
+	packetRouterConfig *ingress.GlobalRouterConfig,
 ) (*QUICConnection, error) {
-	session, err := quic.DialAddr(edgeAddr.String(), tlsConfig, quicConfig)
-	if err != nil {
-		return nil, fmt.Errorf("failed to dial to edge: %w", err)
-	}
-
-	datagramMuxer, err := quicpogs.NewDatagramMuxer(session)
+	udpConn, err := createUDPConnForConnIndex(connIndex, logger)
 	if err != nil {
 		return nil, err
 	}
 
-	sessionManager := datagramsession.NewManager(datagramMuxer, logger)
+	session, err := quic.Dial(udpConn, edgeAddr, edgeAddr.String(), tlsConfig, quicConfig)
+	if err != nil {
+		// close the udp server socket in case of error connecting to the edge
+		udpConn.Close()
+		return nil, &EdgeQuicDialError{Cause: err}
+	}
+
+	// wrap the session, so that the UDPConn is closed after session is closed.
+	session = &wrapCloseableConnQuicConnection{
+		session,
+		udpConn,
+	}
+
+	sessionDemuxChan := make(chan *packet.Session, demuxChanCapacity)
+	datagramMuxer := quicpogs.NewDatagramMuxerV2(session, logger, sessionDemuxChan)
+	sessionManager := datagramsession.NewManager(logger, datagramMuxer.SendToSession, sessionDemuxChan)
+	packetRouter := ingress.NewPacketRouter(packetRouterConfig, datagramMuxer, logger, orchestrator.WarpRoutingEnabled)
 
 	return &QUICConnection{
 		session:              session,
 		orchestrator:         orchestrator,
 		logger:               logger,
 		sessionManager:       sessionManager,
+		datagramMuxer:        datagramMuxer,
+		packetRouter:         packetRouter,
 		controlStreamHandler: controlStreamHandler,
 		connOptions:          connOptions,
 	}, nil
@@ -104,13 +139,21 @@ func (q *QUICConnection) Serve(ctx context.Context) error {
 		defer cancel()
 		return q.sessionManager.Serve(ctx)
 	})
+	errGroup.Go(func() error {
+		defer cancel()
+		return q.datagramMuxer.ServeReceive(ctx)
+	})
+	errGroup.Go(func() error {
+		defer cancel()
+		return q.packetRouter.Serve(ctx)
+	})
 
 	return errGroup.Wait()
 }
 
 func (q *QUICConnection) serveControlStream(ctx context.Context, controlStream quic.Stream) error {
 	// This blocks until the control plane is done.
-	err := q.controlStreamHandler.ServeControlStream(ctx, controlStream, q.connOptions)
+	err := q.controlStreamHandler.ServeControlStream(ctx, controlStream, q.connOptions, q.orchestrator)
 	if err != nil {
 		// Not wrapping error here to be consistent with the http2 message.
 		return err
@@ -119,6 +162,11 @@ func (q *QUICConnection) serveControlStream(ctx context.Context, controlStream q
 	return nil
 }
 
+// Close closes the session with no errors specified.
+func (q *QUICConnection) Close() {
+	q.session.CloseWithError(0, "")
+}
+
 func (q *QUICConnection) acceptStream(ctx context.Context) error {
 	defer q.Close()
 	for {
@@ -130,23 +178,26 @@ func (q *QUICConnection) acceptStream(ctx context.Context) error {
 			}
 			return fmt.Errorf("failed to accept QUIC stream: %w", err)
 		}
-		go func() {
-			stream := quicpogs.NewSafeStreamCloser(quicStream)
-			defer stream.Close()
-
-			if err = q.handleStream(stream); err != nil {
-				q.logger.Err(err).Msg("Failed to handle QUIC stream")
-			}
-		}()
+		go q.runStream(quicStream)
 	}
 }
 
-// Close closes the session with no errors specified.
-func (q *QUICConnection) Close() {
-	q.session.CloseWithError(0, "")
+func (q *QUICConnection) runStream(quicStream quic.Stream) {
+	ctx := quicStream.Context()
+	stream := quicpogs.NewSafeStreamCloser(quicStream)
+	defer stream.Close()
+
+	// we are going to fuse readers/writers from stream <- cloudflared -> origin, and we want to guarantee that
+	// code executed in the code path of handleStream don't trigger an earlier close to the downstream write stream.
+	// So, we wrap the stream with a no-op write closer and only this method can actually close write side of the stream.
+	// A call to close will simulate a close to the read-side, which will fail subsequent reads.
+	noCloseStream := &nopCloserReadWriter{ReadWriteCloser: stream}
+	if err := q.handleStream(ctx, noCloseStream); err != nil {
+		q.logger.Err(err).Msg("Failed to handle QUIC stream")
+	}
 }
 
-func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
+func (q *QUICConnection) handleStream(ctx context.Context, stream io.ReadWriteCloser) error {
 	signature, err := quicpogs.DetermineProtocol(stream)
 	if err != nil {
 		return err
@@ -155,9 +206,9 @@ func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
 	case quicpogs.DataStreamProtocolSignature:
 		reqServerStream, err := quicpogs.NewRequestServerStream(stream, signature)
 		if err != nil {
-			return nil
+			return err
 		}
-		return q.handleDataStream(reqServerStream)
+		return q.handleDataStream(ctx, reqServerStream)
 	case quicpogs.RPCStreamProtocolSignature:
 		rpcStream, err := quicpogs.NewRPCServerStream(stream, signature)
 		if err != nil {
@@ -169,28 +220,43 @@ func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
 	}
 }
 
-func (q *QUICConnection) handleDataStream(stream *quicpogs.RequestServerStream) error {
-	connectRequest, err := stream.ReadConnectRequestData()
+func (q *QUICConnection) handleDataStream(ctx context.Context, stream *quicpogs.RequestServerStream) error {
+	request, err := stream.ReadConnectRequestData()
 	if err != nil {
 		return err
 	}
 
+	if err := q.dispatchRequest(ctx, stream, err, request); err != nil {
+		_ = stream.WriteConnectResponseData(err)
+		q.logger.Err(err).Str("type", request.Type.String()).Str("dest", request.Dest).Msg("Request failed")
+	}
+
+	return nil
+}
+
+func (q *QUICConnection) dispatchRequest(ctx context.Context, stream *quicpogs.RequestServerStream, err error, request *quicpogs.ConnectRequest) error {
 	originProxy, err := q.orchestrator.GetOriginProxy()
 	if err != nil {
 		return err
 	}
-	switch connectRequest.Type {
+
+	switch request.Type {
 	case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
-		req, err := buildHTTPRequest(connectRequest, stream)
+		tracedReq, err := buildHTTPRequest(ctx, request, stream, q.logger)
 		if err != nil {
 			return err
 		}
-
 		w := newHTTPResponseAdapter(stream)
-		return originProxy.ProxyHTTP(w, req, connectRequest.Type == quicpogs.ConnectionTypeWebsocket)
+		return originProxy.ProxyHTTP(w, tracedReq, request.Type == quicpogs.ConnectionTypeWebsocket)
+
 	case quicpogs.ConnectionTypeTCP:
 		rwa := &streamReadWriteAcker{stream}
-		return originProxy.ProxyTCP(context.Background(), rwa, &TCPRequest{Dest: connectRequest.Dest})
+		metadata := request.MetadataMap()
+		return originProxy.ProxyTCP(ctx, rwa, &TCPRequest{
+			Dest:      request.Dest,
+			FlowID:    metadata[QUICMetadataFlowID],
+			CfTraceID: metadata[tracing.TracerContextName],
+		})
 	}
 	return nil
 }
@@ -200,24 +266,42 @@ func (q *QUICConnection) handleRPCStream(rpcStream *quicpogs.RPCServerStream) er
 }
 
 // RegisterUdpSession is the RPC method invoked by edge to register and run a session
-func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration) error {
+func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration, traceContext string) (*tunnelpogs.RegisterUdpSessionResponse, error) {
+	traceCtx := tracing.NewTracedContext(ctx, traceContext, q.logger)
+	ctx, registerSpan := traceCtx.Tracer().Start(traceCtx, "register-session", trace.WithAttributes(
+		attribute.String("session-id", sessionID.String()),
+		attribute.String("dst", fmt.Sprintf("%s:%d", dstIP, dstPort)),
+	))
 	// Each session is a series of datagram from an eyeball to a dstIP:dstPort.
 	// (src port, dst IP, dst port) uniquely identifies a session, so it needs a dedicated connected socket.
 	originProxy, err := ingress.DialUDP(dstIP, dstPort)
 	if err != nil {
 		q.logger.Err(err).Msgf("Failed to create udp proxy to %s:%d", dstIP, dstPort)
-		return err
+		tracing.EndWithErrorStatus(registerSpan, err)
+		return nil, err
 	}
+	registerSpan.SetAttributes(
+		attribute.Bool("socket-bind-success", true),
+		attribute.String("src", originProxy.LocalAddr().String()),
+	)
+
 	session, err := q.sessionManager.RegisterSession(ctx, sessionID, originProxy)
 	if err != nil {
 		q.logger.Err(err).Str("sessionID", sessionID.String()).Msgf("Failed to register udp session")
-		return err
+		tracing.EndWithErrorStatus(registerSpan, err)
+		return nil, err
 	}
 
 	go q.serveUDPSession(session, closeAfterIdleHint)
 
-	q.logger.Debug().Msgf("Registered session %v, %v, %v", sessionID, dstIP, dstPort)
-	return nil
+	q.logger.Debug().Str("sessionID", sessionID.String()).Str("src", originProxy.LocalAddr().String()).Str("dst", fmt.Sprintf("%s:%d", dstIP, dstPort)).Msgf("Registered session")
+	tracing.End(registerSpan)
+
+	resp := tunnelpogs.RegisterUdpSessionResponse{
+		Spans: traceCtx.GetProtoSpans(),
+	}
+
+	return &resp, nil
 }
 
 func (q *QUICConnection) serveUDPSession(session *datagramsession.Session, closeAfterIdleHint time.Duration) {
@@ -276,8 +360,12 @@ type streamReadWriteAcker struct {
 }
 
 // AckConnection acks response back to the proxy.
-func (s *streamReadWriteAcker) AckConnection() error {
-	return s.WriteConnectResponseData(nil)
+func (s *streamReadWriteAcker) AckConnection(tracePropagation string) error {
+	metadata := quicpogs.Metadata{
+		Key: tracing.CanonicalCloudflaredTracingHeader,
+		Val: tracePropagation,
+	}
+	return s.WriteConnectResponseData(nil, metadata)
 }
 
 // httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
@@ -289,6 +377,10 @@ func newHTTPResponseAdapter(s *quicpogs.RequestServerStream) httpResponseAdapter
 	return httpResponseAdapter{s}
 }
 
+func (hrw httpResponseAdapter) AddTrailer(trailerName, trailerValue string) {
+	// we do not support trailers over QUIC
+}
+
 func (hrw httpResponseAdapter) WriteRespHeaders(status int, header http.Header) error {
 	metadata := make([]quicpogs.Metadata, 0)
 	metadata = append(metadata, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(status)})
@@ -305,14 +397,19 @@ func (hrw httpResponseAdapter) WriteErrorResponse(err error) {
 	hrw.WriteConnectResponseData(err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
 }
 
-func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadCloser) (*http.Request, error) {
+func buildHTTPRequest(
+	ctx context.Context,
+	connectRequest *quicpogs.ConnectRequest,
+	body io.ReadCloser,
+	log *zerolog.Logger,
+) (*tracing.TracedHTTPRequest, error) {
 	metadata := connectRequest.MetadataMap()
 	dest := connectRequest.Dest
 	method := metadata[HTTPMethodKey]
 	host := metadata[HTTPHostKey]
 	isWebsocket := connectRequest.Type == quicpogs.ConnectionTypeWebsocket
 
-	req, err := http.NewRequest(method, dest, body)
+	req, err := http.NewRequestWithContext(ctx, method, dest, body)
 	if err != nil {
 		return nil, err
 	}
@@ -342,10 +439,13 @@ func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadClose
 	//   * there is no transfer-encoding=chunked already set.
 	// So, if transfer cannot be chunked and content length is 0, we dont set a request body.
 	if !isWebsocket && !isTransferEncodingChunked(req) && req.ContentLength == 0 {
-		req.Body = nil
+		req.Body = http.NoBody
 	}
 	stripWebsocketUpgradeHeader(req)
-	return req, err
+
+	// Check for tracing on request
+	tracedReq := tracing.NewTracedHTTPRequest(req, log)
+	return tracedReq, err
 }
 
 func setContentLength(req *http.Request) error {
@@ -362,3 +462,107 @@ func isTransferEncodingChunked(req *http.Request) bool {
 	// separated value as well.
 	return strings.Contains(strings.ToLower(transferEncodingVal), "chunked")
 }
+
+// A helper struct that guarantees a call to close only affects read side, but not write side.
+type nopCloserReadWriter struct {
+	io.ReadWriteCloser
+
+	// for use by Read only
+	// we don't need a memory barrier here because there is an implicit assumption that
+	// Read calls can't happen concurrently by different go-routines.
+	sawEOF bool
+	// should be updated and read using atomic primitives.
+	// value is read in Read method and written in Close method, which could be done by different
+	// go-routines.
+	closed uint32
+}
+
+func (np *nopCloserReadWriter) Read(p []byte) (n int, err error) {
+	if np.sawEOF {
+		return 0, io.EOF
+	}
+
+	if atomic.LoadUint32(&np.closed) > 0 {
+		return 0, fmt.Errorf("closed by handler")
+	}
+
+	n, err = np.ReadWriteCloser.Read(p)
+	if err == io.EOF {
+		np.sawEOF = true
+	}
+
+	return
+}
+
+func (np *nopCloserReadWriter) Close() error {
+	atomic.StoreUint32(&np.closed, 1)
+
+	return nil
+}
+
+// muxerWrapper wraps DatagramMuxerV2 to satisfy the packet.FunnelUniPipe interface
+type muxerWrapper struct {
+	muxer *quicpogs.DatagramMuxerV2
+}
+
+func (rp *muxerWrapper) SendPacket(dst netip.Addr, pk packet.RawPacket) error {
+	return rp.muxer.SendPacket(quicpogs.RawPacket(pk))
+}
+
+func (rp *muxerWrapper) ReceivePacket(ctx context.Context) (packet.RawPacket, error) {
+	pk, err := rp.muxer.ReceivePacket(ctx)
+	if err != nil {
+		return packet.RawPacket{}, err
+	}
+	rawPacket, ok := pk.(quicpogs.RawPacket)
+	if ok {
+		return packet.RawPacket(rawPacket), nil
+	}
+	return packet.RawPacket{}, fmt.Errorf("unexpected packet type %+v", pk)
+}
+
+func (rp *muxerWrapper) Close() error {
+	return nil
+}
+
+func createUDPConnForConnIndex(connIndex uint8, logger *zerolog.Logger) (*net.UDPConn, error) {
+	portMapMutex.Lock()
+	defer portMapMutex.Unlock()
+
+	// if port was not set yet, it will be zero, so bind will randomly allocate one.
+	if port, ok := portForConnIndex[connIndex]; ok {
+		udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: port})
+		// if there wasn't an error, or if port was 0 (independently of error or not, just return)
+		if err == nil {
+			return udpConn, nil
+		} else {
+			logger.Debug().Err(err).Msgf("Unable to reuse port %d for connIndex %d. Falling back to random allocation.", port, connIndex)
+		}
+	}
+
+	// if we reached here, then there was an error or port as not been allocated it.
+	udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
+	if err == nil {
+		udpAddr, ok := (udpConn.LocalAddr()).(*net.UDPAddr)
+		if !ok {
+			return nil, fmt.Errorf("unable to cast to udpConn")
+		}
+		portForConnIndex[connIndex] = udpAddr.Port
+	} else {
+		delete(portForConnIndex, connIndex)
+	}
+
+	return udpConn, err
+}
+
+type wrapCloseableConnQuicConnection struct {
+	quic.Connection
+	udpConn *net.UDPConn
+}
+
+func (w *wrapCloseableConnQuicConnection) CloseWithError(errorCode quic.ApplicationErrorCode, reason string) error {
+	err := w.Connection.CloseWithError(errorCode, reason)
+	w.udpConn.Close()
+
+	return err
+}

connection/quic_test.go

@@ -10,7 +10,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
-	"sync"
+	"strings"
 	"testing"
 	"time"
 
@@ -24,17 +24,22 @@ import (
 
 	"github.com/cloudflare/cloudflared/datagramsession"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
+	"github.com/cloudflare/cloudflared/tracing"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
 	testTLSServerConfig = quicpogs.GenerateTLSConfig()
 	testQUICConfig      = &quic.Config{
-		KeepAlive:       true,
-		EnableDatagrams: true,
+		ConnectionIDLength: 16,
+		KeepAlivePeriod:    5 * time.Second,
+		EnableDatagrams:    true,
 	}
 )
 
+var _ ReadWriteAcker = (*streamReadWriteAcker)(nil)
+
 // TestQUICServer tests if a quic server accepts and responds to a quic client with the acceptance protocol.
 // It also serves as a demonstration for communication with the QUIC connection started by a cloudflared.
 func TestQUICServer(t *testing.T) {
@@ -136,24 +141,33 @@ func TestQUICServer(t *testing.T) {
 		},
 	}
 
-	for _, test := range tests {
+	for i, test := range tests {
+		test := test // capture range variable
 		t.Run(test.desc, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
-			var wg sync.WaitGroup
-			wg.Add(1)
+
+			quicListener, err := quic.Listen(udpListener, testTLSServerConfig, testQUICConfig)
+			require.NoError(t, err)
+
+			serverDone := make(chan struct{})
 			go func() {
-				defer wg.Done()
 				quicServer(
-					t, udpListener, testTLSServerConfig, testQUICConfig,
-					test.dest, test.connectionType, test.metadata, test.message, test.expectedResponse,
+					ctx, t, quicListener, test.dest, test.connectionType, test.metadata, test.message, test.expectedResponse,
 				)
+				close(serverDone)
 			}()
 
-			qc := testQUICConnection(udpListener.LocalAddr(), t)
-			go qc.Serve(ctx)
+			qc := testQUICConnection(udpListener.LocalAddr(), t, uint8(i))
 
-			wg.Wait()
+			connDone := make(chan struct{})
+			go func() {
+				qc.Serve(ctx)
+				close(connDone)
+			}()
+
+			<-serverDone
 			cancel()
+			<-connDone
 		})
 	}
 }
@@ -162,7 +176,7 @@ type fakeControlStream struct {
 	ControlStreamHandler
 }
 
-func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions) error {
+func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions, tunnelConfigGetter TunnelConfigJSONGetter) error {
 	<-ctx.Done()
 	return nil
 }
@@ -171,23 +185,16 @@ func (fakeControlStream) IsStopped() bool {
 }
 
 func quicServer(
+	ctx context.Context,
 	t *testing.T,
-	conn net.PacketConn,
-	tlsConf *tls.Config,
-	config *quic.Config,
+	listener quic.Listener,
 	dest string,
 	connectionType quicpogs.ConnectionType,
 	metadata []quicpogs.Metadata,
 	message []byte,
 	expectedResponse []byte,
 ) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	earlyListener, err := quic.Listen(conn, tlsConf, config)
-	require.NoError(t, err)
-
-	session, err := earlyListener.Accept(ctx)
+	session, err := listener.Accept(ctx)
 	require.NoError(t, err)
 
 	quicStream, err := session.OpenStreamSync(context.Background())
@@ -219,9 +226,10 @@ func quicServer(
 
 type mockOriginProxyWithRequest struct{}
 
-func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, r *http.Request, isWebsocket bool) error {
+func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, tr *tracing.TracedHTTPRequest, isWebsocket bool) error {
 	// These are a series of crude tests to ensure the headers and http related data is transferred from
 	// metadata.
+	r := tr.Request
 	if r.Method == "" {
 		return errors.New("method not sent")
 	}
@@ -345,7 +353,7 @@ func TestBuildHTTPRequest(t *testing.T) {
 				},
 				ContentLength: 0,
 				Host:          "cf.host",
-				Body:          nil,
+				Body:          http.NoBody,
 			},
 			body: io.NopCloser(&bytes.Buffer{}),
 		},
@@ -473,18 +481,20 @@ func TestBuildHTTPRequest(t *testing.T) {
 		},
 	}
 
+	log := zerolog.Nop()
 	for _, test := range tests {
+		test := test // capture range variable
 		t.Run(test.name, func(t *testing.T) {
-			req, err := buildHTTPRequest(test.connectRequest, test.body)
+			req, err := buildHTTPRequest(context.Background(), test.connectRequest, test.body, &log)
 			assert.NoError(t, err)
 			test.req = test.req.WithContext(req.Context())
-			assert.Equal(t, test.req, req)
+			assert.Equal(t, test.req, req.Request)
 		})
 	}
 }
 
 func (moc *mockOriginProxyWithRequest) ProxyTCP(ctx context.Context, rwa ReadWriteAcker, tcpRequest *TCPRequest) error {
-	rwa.AckConnection()
+	rwa.AckConnection("")
 	io.Copy(rwa, rwa)
 	return nil
 }
@@ -498,9 +508,10 @@ func TestServeUDPSession(t *testing.T) {
 	defer udpListener.Close()
 
 	ctx, cancel := context.WithCancel(context.Background())
+	val := udpListener.LocalAddr()
 
 	// Establish QUIC connection with edge
-	edgeQUICSessionChan := make(chan quic.Session)
+	edgeQUICSessionChan := make(chan quic.Connection)
 	go func() {
 		earlyListener, err := quic.Listen(udpListener, testTLSServerConfig, testQUICConfig)
 		require.NoError(t, err)
@@ -510,7 +521,8 @@ func TestServeUDPSession(t *testing.T) {
 		edgeQUICSessionChan <- edgeQUICSession
 	}()
 
-	qc := testQUICConnection(udpListener.LocalAddr(), t)
+	// Random index to avoid reusing port
+	qc := testQUICConnection(val, t, 28)
 	go qc.Serve(ctx)
 
 	edgeQUICSession := <-edgeQUICSessionChan
@@ -520,7 +532,76 @@ func TestServeUDPSession(t *testing.T) {
 	cancel()
 }
 
-func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.Session, closeType closeReason, expectedReason string, t *testing.T) {
+func TestNopCloserReadWriterCloseBeforeEOF(t *testing.T) {
+	readerWriter := nopCloserReadWriter{ReadWriteCloser: &mockReaderNoopWriter{Reader: strings.NewReader("123456789")}}
+	buffer := make([]byte, 5)
+
+	n, err := readerWriter.Read(buffer)
+	require.NoError(t, err)
+	require.Equal(t, n, 5)
+
+	// close
+	require.NoError(t, readerWriter.Close())
+
+	// read should get error
+	n, err = readerWriter.Read(buffer)
+	require.Equal(t, n, 0)
+	require.Equal(t, err, fmt.Errorf("closed by handler"))
+}
+
+func TestNopCloserReadWriterCloseAfterEOF(t *testing.T) {
+	readerWriter := nopCloserReadWriter{ReadWriteCloser: &mockReaderNoopWriter{Reader: strings.NewReader("123456789")}}
+	buffer := make([]byte, 20)
+
+	n, err := readerWriter.Read(buffer)
+	require.NoError(t, err)
+	require.Equal(t, n, 9)
+
+	// force another read to read eof
+	_, err = readerWriter.Read(buffer)
+	require.Equal(t, err, io.EOF)
+
+	// close
+	require.NoError(t, readerWriter.Close())
+
+	// read should get EOF still
+	n, err = readerWriter.Read(buffer)
+	require.Equal(t, n, 0)
+	require.Equal(t, err, io.EOF)
+}
+
+func TestCreateUDPConnReuseSourcePort(t *testing.T) {
+	logger := zerolog.Nop()
+	conn, err := createUDPConnForConnIndex(0, &logger)
+	require.NoError(t, err)
+
+	getPortFunc := func(conn *net.UDPConn) int {
+		addr := conn.LocalAddr().(*net.UDPAddr)
+		return addr.Port
+	}
+
+	initialPort := getPortFunc(conn)
+
+	// close conn
+	conn.Close()
+
+	// should get the same port as before.
+	conn, err = createUDPConnForConnIndex(0, &logger)
+	require.NoError(t, err)
+	require.Equal(t, initialPort, getPortFunc(conn))
+
+	// new index, should get a different port
+	conn1, err := createUDPConnForConnIndex(1, &logger)
+	require.NoError(t, err)
+	require.NotEqual(t, initialPort, getPortFunc(conn1))
+
+	// not closing the conn and trying to obtain a new conn for same index should give a different random port
+	conn, err = createUDPConnForConnIndex(0, &logger)
+	require.NoError(t, err)
+	require.NotEqual(t, initialPort, getPortFunc(conn))
+}
+
+func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.Connection, closeType closeReason, expectedReason string, t *testing.T) {
 	var (
 		payload = []byte(t.Name())
 	)
@@ -536,8 +617,12 @@ func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.
 		close(sessionDone)
 	}()
 
-	// Send a message to the quic session on edge side, it should be deumx to this datagram session
-	muxedPayload := append(payload, sessionID[:]...)
+	// Send a message to the quic session on edge side, it should be deumx to this datagram v2 session
+	muxedPayload, err := quicpogs.SuffixSessionID(sessionID, payload)
+	require.NoError(t, err)
+	muxedPayload, err = quicpogs.SuffixType(muxedPayload, quicpogs.DatagramTypeUDP)
+	require.NoError(t, err)
+
 	err = edgeQUICSession.SendMessage(muxedPayload)
 	require.NoError(t, err)
 
@@ -581,7 +666,7 @@ const (
 	closedByTimeout
 )
 
-func runRPCServer(ctx context.Context, session quic.Session, sessionRPCServer tunnelpogs.SessionManager, configRPCServer tunnelpogs.ConfigurationManager, t *testing.T) {
+func runRPCServer(ctx context.Context, session quic.Connection, sessionRPCServer tunnelpogs.SessionManager, configRPCServer tunnelpogs.ConfigurationManager, t *testing.T) {
 	stream, err := session.AcceptStream(ctx)
 	require.NoError(t, err)
 
@@ -606,8 +691,8 @@ type mockSessionRPCServer struct {
 	calledUnregisterChan chan struct{}
 }
 
-func (s mockSessionRPCServer) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfter time.Duration) error {
-	return fmt.Errorf("mockSessionRPCServer doesn't implement RegisterUdpSession")
+func (s mockSessionRPCServer) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfter time.Duration, traceContext string) (*pogs.RegisterUdpSessionResponse, error) {
+	return nil, fmt.Errorf("mockSessionRPCServer doesn't implement RegisterUdpSession")
 }
 
 func (s mockSessionRPCServer) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, reason string) error {
@@ -621,7 +706,7 @@ func (s mockSessionRPCServer) UnregisterUdpSession(ctx context.Context, sessionI
 	return nil
 }
 
-func testQUICConnection(udpListenerAddr net.Addr, t *testing.T) *QUICConnection {
+func testQUICConnection(udpListenerAddr net.Addr, t *testing.T, index uint8) *QUICConnection {
 	tlsClientConfig := &tls.Config{
 		InsecureSkipVerify: true,
 		NextProtos:         []string{"argotunnel"},
@@ -631,12 +716,26 @@ func testQUICConnection(udpListenerAddr net.Addr, t *testing.T) *QUICConnection
 	qc, err := NewQUICConnection(
 		testQUICConfig,
 		udpListenerAddr,
+		index,
 		tlsClientConfig,
 		&mockOrchestrator{originProxy: &mockOriginProxyWithRequest{}},
 		&tunnelpogs.ConnectionOptions{},
 		fakeControlStream{},
 		&log,
+		nil,
 	)
 	require.NoError(t, err)
 	return qc
 }
+
+type mockReaderNoopWriter struct {
+	io.Reader
+}
+
+func (m *mockReaderNoopWriter) Write(p []byte) (n int, err error) {
+	return len(p), nil
+}
+
+func (m *mockReaderNoopWriter) Close() error {
+	return nil
+}

connection/rpc.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net"
 	"time"
 
 	"github.com/rs/zerolog"
@@ -57,6 +58,12 @@ type NamedTunnelRPCClient interface {
 		config *NamedTunnelProperties,
 		options *tunnelpogs.ConnectionOptions,
 		connIndex uint8,
+		edgeAddress net.IP,
+		observer *Observer,
+	) (*tunnelpogs.ConnectionDetails, error)
+	SendLocalConfiguration(
+		c context.Context,
+		config []byte,
 		observer *Observer,
 	) error
 	GracefulShutdown(ctx context.Context, gracePeriod time.Duration)
@@ -89,8 +96,9 @@ func (rsc *registrationServerClient) RegisterConnection(
 	properties *NamedTunnelProperties,
 	options *tunnelpogs.ConnectionOptions,
 	connIndex uint8,
+	edgeAddress net.IP,
 	observer *Observer,
-) error {
+) (*tunnelpogs.ConnectionDetails, error) {
 	conn, err := rsc.client.RegisterConnection(
 		ctx,
 		properties.Credentials.Auth(),
@@ -101,18 +109,26 @@ func (rsc *registrationServerClient) RegisterConnection(
 	if err != nil {
 		if err.Error() == DuplicateConnectionError {
 			observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
-			return errDuplicationConnection
+			return nil, errDuplicationConnection
 		}
 		observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
-		return serverRegistrationErrorFromRPC(err)
+		return nil, serverRegistrationErrorFromRPC(err)
 	}
 
 	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
 
-	observer.logServerInfo(connIndex, conn.Location, fmt.Sprintf("Connection %s registered", conn.UUID))
-	observer.sendConnectedEvent(connIndex, conn.Location)
+	return conn, nil
+}
 
-	return nil
+func (rsc *registrationServerClient) SendLocalConfiguration(ctx context.Context, config []byte, observer *Observer) (err error) {
+	observer.metrics.localConfigMetrics.pushes.Inc()
+	defer func() {
+		if err != nil {
+			observer.metrics.localConfigMetrics.pushesErrors.Inc()
+		}
+	}()
+
+	return rsc.client.SendLocalConfiguration(ctx, config)
 }
 
 func (rsc *registrationServerClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
@@ -258,7 +274,7 @@ func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelSe
 		h.observer.log.Err(err).Msg("Failed to retrieve server information")
 		return err
 	}
-	h.observer.logServerInfo(h.connIndex, serverInfo.LocationName, "Connection established")
+	h.observer.logServerInfo(h.connIndex, serverInfo.LocationName, net.IP{}, "Connection established")
 	return nil
 }
 
@@ -274,9 +290,13 @@ func (h *h2muxConnection) registerNamedTunnel(
 	rpcClient := h.newRPCClientFunc(ctx, stream, h.observer.log)
 	defer rpcClient.Close()
 
-	if err = rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+	registrationDetails, err := rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, nil, h.observer)
+	if err != nil {
 		return err
 	}
+	h.observer.logServerInfo(h.connIndex, registrationDetails.Location, nil, fmt.Sprintf("Connection %s registered", registrationDetails.UUID))
+	h.observer.sendConnectedEvent(h.connIndex, H2mux, registrationDetails.Location)
+
 	return nil
 }
 

datagramsession/event.go

@@ -42,9 +42,3 @@ func (sc *errClosedSession) Error() string {
 		return fmt.Sprintf("session closed by local due to %s", sc.message)
 	}
 }
-
-// newDatagram is an event when transport receives new datagram
-type newDatagram struct {
-	sessionID uuid.UUID
-	payload   []byte
-}

datagramsession/manager.go

@@ -2,16 +2,23 @@ package datagramsession
 
 import (
 	"context"
+	"fmt"
 	"io"
+	"time"
 
 	"github.com/google/uuid"
-	"github.com/lucas-clemente/quic-go"
 	"github.com/rs/zerolog"
-	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/packet"
 )
 
 const (
 	requestChanCapacity = 16
+	defaultReqTimeout   = time.Second * 5
+)
+
+var (
+	errSessionManagerClosed = fmt.Errorf("session manager closed")
 )
 
 // Manager defines the APIs to manage sessions from the same transport.
@@ -22,90 +29,114 @@ type Manager interface {
 	RegisterSession(ctx context.Context, sessionID uuid.UUID, dstConn io.ReadWriteCloser) (*Session, error)
 	// UnregisterSession stops tracking the session and terminates it
 	UnregisterSession(ctx context.Context, sessionID uuid.UUID, message string, byRemote bool) error
+	// UpdateLogger updates the logger used by the Manager
+	UpdateLogger(log *zerolog.Logger)
 }
 
 type manager struct {
 	registrationChan   chan *registerSessionEvent
 	unregistrationChan chan *unregisterSessionEvent
-	datagramChan       chan *newDatagram
-	transport          transport
+	sendFunc           transportSender
+	receiveChan        <-chan *packet.Session
+	closedChan         <-chan struct{}
 	sessions           map[uuid.UUID]*Session
 	log                *zerolog.Logger
+	// timeout waiting for an API to finish. This can be overriden in test
+	timeout time.Duration
 }
 
-func NewManager(transport transport, log *zerolog.Logger) Manager {
+func NewManager(log *zerolog.Logger, sendF transportSender, receiveChan <-chan *packet.Session) *manager {
 	return &manager{
 		registrationChan:   make(chan *registerSessionEvent),
 		unregistrationChan: make(chan *unregisterSessionEvent),
-		// datagramChan is buffered, so it can read more datagrams from transport while the event loop is processing other events
-		datagramChan: make(chan *newDatagram, requestChanCapacity),
-		transport:    transport,
-		sessions:     make(map[uuid.UUID]*Session),
-		log:          log,
+		sendFunc:           sendF,
+		receiveChan:        receiveChan,
+		closedChan:         make(chan struct{}),
+		sessions:           make(map[uuid.UUID]*Session),
+		log:                log,
+		timeout:            defaultReqTimeout,
 	}
 }
 
+func (m *manager) UpdateLogger(log *zerolog.Logger) {
+	// Benign data race, no problem if the old pointer is read or not concurrently.
+	m.log = log
+}
+
 func (m *manager) Serve(ctx context.Context) error {
-	errGroup, ctx := errgroup.WithContext(ctx)
-	errGroup.Go(func() error {
-		for {
-			sessionID, payload, err := m.transport.ReceiveFrom()
-			if err != nil {
-				if aerr, ok := err.(*quic.ApplicationError); ok && uint64(aerr.ErrorCode) == uint64(quic.NoError) {
-					return nil
-				} else {
-					return err
-				}
-			}
-			datagram := &newDatagram{
-				sessionID: sessionID,
-				payload:   payload,
-			}
-			select {
-			case <-ctx.Done():
-				return ctx.Err()
-			// Only the event loop routine can update/lookup the sessions map to avoid concurrent access
-			// Send the datagram to the event loop. It will find the session to send to
-			case m.datagramChan <- datagram:
-			}
+	for {
+		select {
+		case <-ctx.Done():
+			m.shutdownSessions(ctx.Err())
+			return ctx.Err()
+		// receiveChan is buffered, so the transport can read more datagrams from transport while the event loop is
+		// processing other events
+		case datagram := <-m.receiveChan:
+			m.sendToSession(datagram)
+		case registration := <-m.registrationChan:
+			m.registerSession(ctx, registration)
+		case unregistration := <-m.unregistrationChan:
+			m.unregisterSession(unregistration)
 		}
-	})
-	errGroup.Go(func() error {
-		for {
-			select {
-			case <-ctx.Done():
-				return nil
-			case datagram := <-m.datagramChan:
-				m.sendToSession(datagram)
-			case registration := <-m.registrationChan:
-				m.registerSession(ctx, registration)
-			// TODO: TUN-5422: Unregister inactive session upon timeout
-			case unregistration := <-m.unregistrationChan:
-				m.unregisterSession(unregistration)
-			}
-		}
-	})
-	return errGroup.Wait()
+	}
+}
+
+func (m *manager) shutdownSessions(err error) {
+	if err == nil {
+		err = errSessionManagerClosed
+	}
+	closeSessionErr := &errClosedSession{
+		message: err.Error(),
+		// Usually connection with remote has been closed, so set this to true to skip unregistering from remote
+		byRemote: true,
+	}
+	for _, s := range m.sessions {
+		s.close(closeSessionErr)
+	}
 }
 
 func (m *manager) RegisterSession(ctx context.Context, sessionID uuid.UUID, originProxy io.ReadWriteCloser) (*Session, error) {
+	ctx, cancel := context.WithTimeout(ctx, m.timeout)
+	defer cancel()
 	event := newRegisterSessionEvent(sessionID, originProxy)
 	select {
 	case <-ctx.Done():
+		m.log.Error().Msg("Datagram session registration timeout")
 		return nil, ctx.Err()
 	case m.registrationChan <- event:
 		session := <-event.resultChan
 		return session, nil
+	// Once closedChan is closed, manager won't accept more registration because nothing is
+	// reading from registrationChan and it's an unbuffered channel
+	case <-m.closedChan:
+		return nil, errSessionManagerClosed
 	}
 }
 
 func (m *manager) registerSession(ctx context.Context, registration *registerSessionEvent) {
-	session := newSession(registration.sessionID, m.transport, registration.originProxy, m.log)
+	session := m.newSession(registration.sessionID, registration.originProxy)
 	m.sessions[registration.sessionID] = session
 	registration.resultChan <- session
 }
 
+func (m *manager) newSession(id uuid.UUID, dstConn io.ReadWriteCloser) *Session {
+	logger := m.log.With().Str("sessionID", id.String()).Logger()
+	return &Session{
+		ID:       id,
+		sendFunc: m.sendFunc,
+		dstConn:  dstConn,
+		// activeAtChan has low capacity. It can be full when there are many concurrent read/write. markActive() will
+		// drop instead of blocking because last active time only needs to be an approximation
+		activeAtChan: make(chan time.Time, 2),
+		// capacity is 2 because close() and dstToTransport routine in Serve() can write to this channel
+		closeChan: make(chan error, 2),
+		log:       &logger,
+	}
+}
+
 func (m *manager) UnregisterSession(ctx context.Context, sessionID uuid.UUID, message string, byRemote bool) error {
+	ctx, cancel := context.WithTimeout(ctx, m.timeout)
+	defer cancel()
 	event := &unregisterSessionEvent{
 		sessionID: sessionID,
 		err: &errClosedSession{
@@ -115,9 +146,12 @@ func (m *manager) UnregisterSession(ctx context.Context, sessionID uuid.UUID, me
 	}
 	select {
 	case <-ctx.Done():
+		m.log.Error().Msg("Datagram session unregistration timeout")
 		return ctx.Err()
 	case m.unregistrationChan <- event:
 		return nil
+	case <-m.closedChan:
+		return errSessionManagerClosed
 	}
 }
 
@@ -129,16 +163,13 @@ func (m *manager) unregisterSession(unregistration *unregisterSessionEvent) {
 	}
 }
 
-func (m *manager) sendToSession(datagram *newDatagram) {
-	session, ok := m.sessions[datagram.sessionID]
+func (m *manager) sendToSession(datagram *packet.Session) {
+	session, ok := m.sessions[datagram.ID]
 	if !ok {
-		m.log.Error().Str("sessionID", datagram.sessionID.String()).Msg("session not found")
+		m.log.Error().Str("sessionID", datagram.ID.String()).Msg("session not found")
 		return
 	}
 	// session writes to destination over a connected UDP socket, which should not be blocking, so this call doesn't
 	// need to run in another go routine
-	_, err := session.transportToDst(datagram.payload)
-	if err != nil {
-		m.log.Err(err).Str("sessionID", datagram.sessionID.String()).Msg("Failed to write payload to session")
-	}
+	session.transportToDst(datagram.Payload)
 }

datagramsession/manager_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"sync"
 	"testing"
 	"time"
 
@@ -13,26 +14,30 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/packet"
+)
+
+var (
+	nopLogger = zerolog.Nop()
 )
 
 func TestManagerServe(t *testing.T) {
 	const (
-		sessions            = 20
-		msgs                = 50
+		sessions            = 2
+		msgs                = 5
 		remoteUnregisterMsg = "eyeball closed connection"
 	)
-	log := zerolog.Nop()
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(1),
-		respChan: newDatagramChannel(1),
-	}
-	mg := NewManager(transport, &log)
 
-	eyeballTracker := make(map[uuid.UUID]*datagramChannel)
-	for i := 0; i < sessions; i++ {
-		sessionID := uuid.New()
-		eyeballTracker[sessionID] = newDatagramChannel(1)
+	requestChan := make(chan *packet.Session)
+	transport := mockQUICTransport{
+		sessions: make(map[uuid.UUID]chan []byte),
 	}
+	for i := 0; i < sessions; i++ {
+		transport.sessions[uuid.New()] = make(chan []byte)
+	}
+
+	mg := NewManager(&nopLogger, transport.MuxSession, requestChan)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	serveDone := make(chan struct{})
@@ -41,53 +46,42 @@ func TestManagerServe(t *testing.T) {
 		close(serveDone)
 	}(ctx)
 
-	go func(ctx context.Context) {
-		for {
-			sessionID, payload, err := transport.respChan.Receive(ctx)
-			if err != nil {
-				require.Equal(t, context.Canceled, err)
-				return
-			}
-			respChan := eyeballTracker[sessionID]
-			require.NoError(t, respChan.Send(ctx, sessionID, payload))
-		}
-	}(ctx)
-
 	errGroup, ctx := errgroup.WithContext(ctx)
-	for sID, receiver := range eyeballTracker {
+	for sessionID, eyeballRespChan := range transport.sessions {
+		// Assign loop variables to local variables
+		sID := sessionID
+		payload := testPayload(sID)
+		expectResp := testResponse(payload)
+
+		cfdConn, originConn := net.Pipe()
+
+		origin := mockOrigin{
+			expectMsgCount: msgs,
+			expectedMsg:    payload,
+			expectedResp:   expectResp,
+			conn:           originConn,
+		}
+
+		eyeball := mockEyeballSession{
+			id:               sID,
+			expectedMsgCount: msgs,
+			expectedMsg:      payload,
+			expectedResponse: expectResp,
+			respReceiver:     eyeballRespChan,
+		}
+
 		// Assign loop variables to local variables
-		sessionID := sID
-		eyeballRespReceiver := receiver
 		errGroup.Go(func() error {
-			payload := testPayload(sessionID)
-			expectResp := testResponse(payload)
-
-			cfdConn, originConn := net.Pipe()
-
-			origin := mockOrigin{
-				expectMsgCount: msgs,
-				expectedMsg:    payload,
-				expectedResp:   expectResp,
-				conn:           originConn,
-			}
-			eyeball := mockEyeball{
-				expectMsgCount:  msgs,
-				expectedMsg:     expectResp,
-				expectSessionID: sessionID,
-				respReceiver:    eyeballRespReceiver,
-			}
-
+			session, err := mg.RegisterSession(ctx, sID, cfdConn)
+			require.NoError(t, err)
 			reqErrGroup, reqCtx := errgroup.WithContext(ctx)
 			reqErrGroup.Go(func() error {
 				return origin.serve()
 			})
 			reqErrGroup.Go(func() error {
-				return eyeball.serve(reqCtx)
+				return eyeball.serve(reqCtx, requestChan)
 			})
 
-			session, err := mg.RegisterSession(ctx, sessionID, cfdConn)
-			require.NoError(t, err)
-
 			sessionDone := make(chan struct{})
 			go func() {
 				closedByRemote, err := session.Serve(ctx, time.Minute*2)
@@ -100,26 +94,112 @@ func TestManagerServe(t *testing.T) {
 				close(sessionDone)
 			}()
 
-			for i := 0; i < msgs; i++ {
-				require.NoError(t, transport.newRequest(ctx, sessionID, testPayload(sessionID)))
-			}
-
 			// Make sure eyeball and origin have received all messages before unregistering the session
 			require.NoError(t, reqErrGroup.Wait())
 
-			require.NoError(t, mg.UnregisterSession(ctx, sessionID, remoteUnregisterMsg, true))
+			require.NoError(t, mg.UnregisterSession(ctx, sID, remoteUnregisterMsg, true))
 			<-sessionDone
-
 			return nil
 		})
 	}
 
 	require.NoError(t, errGroup.Wait())
 	cancel()
-	transport.close()
 	<-serveDone
 }
 
+func TestTimeout(t *testing.T) {
+	const (
+		testTimeout = time.Millisecond * 50
+	)
+
+	mg := NewManager(&nopLogger, nil, nil)
+	mg.timeout = testTimeout
+	ctx := context.Background()
+	sessionID := uuid.New()
+	// session manager is not running, so event loop is not running and therefore calling the APIs should timeout
+	session, err := mg.RegisterSession(ctx, sessionID, nil)
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+	require.Nil(t, session)
+
+	err = mg.UnregisterSession(ctx, sessionID, "session gone", true)
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+}
+
+func TestUnregisterSessionCloseSession(t *testing.T) {
+	sessionID := uuid.New()
+	payload := []byte(t.Name())
+	sender := newMockTransportSender(sessionID, payload)
+	mg := NewManager(&nopLogger, sender.muxSession, nil)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	managerDone := make(chan struct{})
+	go func() {
+		err := mg.Serve(ctx)
+		require.Error(t, err)
+		close(managerDone)
+	}()
+
+	cfdConn, originConn := net.Pipe()
+	session, err := mg.RegisterSession(ctx, sessionID, cfdConn)
+	require.NoError(t, err)
+	require.NotNil(t, session)
+
+	unregisteredChan := make(chan struct{})
+	go func() {
+		_, err := originConn.Write(payload)
+		require.NoError(t, err)
+
+		err = mg.UnregisterSession(ctx, sessionID, "eyeball closed session", true)
+		require.NoError(t, err)
+
+		close(unregisteredChan)
+	}()
+
+	closedByRemote, err := session.Serve(ctx, time.Minute)
+	require.True(t, closedByRemote)
+	require.Error(t, err)
+
+	<-unregisteredChan
+	cancel()
+	<-managerDone
+}
+
+func TestManagerCtxDoneCloseSessions(t *testing.T) {
+	sessionID := uuid.New()
+	payload := []byte(t.Name())
+	sender := newMockTransportSender(sessionID, payload)
+	mg := NewManager(&nopLogger, sender.muxSession, nil)
+	ctx, cancel := context.WithCancel(context.Background())
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		err := mg.Serve(ctx)
+		require.Error(t, err)
+	}()
+
+	cfdConn, originConn := net.Pipe()
+	session, err := mg.RegisterSession(ctx, sessionID, cfdConn)
+	require.NoError(t, err)
+	require.NotNil(t, session)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, err := originConn.Write(payload)
+		require.NoError(t, err)
+		cancel()
+	}()
+
+	closedByRemote, err := session.Serve(ctx, time.Minute)
+	require.False(t, closedByRemote)
+	require.Error(t, err)
+
+	wg.Wait()
+}
+
 type mockOrigin struct {
 	expectMsgCount int
 	expectedMsg    []byte
@@ -141,7 +221,6 @@ func (mo *mockOrigin) serve() error {
 		if !bytes.Equal(readBuffer[:n], mo.expectedMsg) {
 			return fmt.Errorf("Expect %v, read %v", mo.expectedMsg, readBuffer[:n])
 		}
-
 		_, err = mo.conn.Write(mo.expectedResp)
 		if err != nil {
 			return err
@@ -158,65 +237,35 @@ func testResponse(msg []byte) []byte {
 	return []byte(fmt.Sprintf("Response to %v", msg))
 }
 
-type mockEyeball struct {
-	expectMsgCount  int
-	expectedMsg     []byte
-	expectSessionID uuid.UUID
-	respReceiver    *datagramChannel
+type mockQUICTransport struct {
+	sessions map[uuid.UUID]chan []byte
 }
 
-func (me *mockEyeball) serve(ctx context.Context) error {
-	for i := 0; i < me.expectMsgCount; i++ {
-		sessionID, msg, err := me.respReceiver.Receive(ctx)
-		if err != nil {
-			return err
-		}
-		if sessionID != me.expectSessionID {
-			return fmt.Errorf("Expect session %s, got %s", me.expectSessionID, sessionID)
-		}
-		if !bytes.Equal(msg, me.expectedMsg) {
-			return fmt.Errorf("Expect %v, read %v", me.expectedMsg, msg)
-		}
-	}
+func (me *mockQUICTransport) MuxSession(session *packet.Session) error {
+	s := me.sessions[session.ID]
+	s <- session.Payload
 	return nil
 }
 
-// datagramChannel is a channel for Datagram with wrapper to send/receive with context
-type datagramChannel struct {
-	datagramChan chan *newDatagram
-	closedChan   chan struct{}
+type mockEyeballSession struct {
+	id               uuid.UUID
+	expectedMsgCount int
+	expectedMsg      []byte
+	expectedResponse []byte
+	respReceiver     <-chan []byte
 }
 
-func newDatagramChannel(capacity uint) *datagramChannel {
-	return &datagramChannel{
-		datagramChan: make(chan *newDatagram, capacity),
-		closedChan:   make(chan struct{}),
+func (me *mockEyeballSession) serve(ctx context.Context, requestChan chan *packet.Session) error {
+	for i := 0; i < me.expectedMsgCount; i++ {
+		requestChan <- &packet.Session{
+			ID:      me.id,
+			Payload: me.expectedMsg,
+		}
+		resp := <-me.respReceiver
+		if !bytes.Equal(resp, me.expectedResponse) {
+			return fmt.Errorf("Expect %v, read %v", me.expectedResponse, resp)
+		}
+		fmt.Println("Resp", resp)
 	}
-}
-
-func (rc *datagramChannel) Send(ctx context.Context, sessionID uuid.UUID, payload []byte) error {
-	select {
-	case <-ctx.Done():
-		return ctx.Err()
-	case <-rc.closedChan:
-		return fmt.Errorf("datagram channel closed")
-	case rc.datagramChan <- &newDatagram{sessionID: sessionID, payload: payload}:
-		return nil
-	}
-}
-
-func (rc *datagramChannel) Receive(ctx context.Context) (uuid.UUID, []byte, error) {
-	select {
-	case <-ctx.Done():
-		return uuid.Nil, nil, ctx.Err()
-	case <-rc.closedChan:
-		return uuid.Nil, nil, fmt.Errorf("datagram channel closed")
-	case msg := <-rc.datagramChan:
-		return msg.sessionID, msg.payload, nil
-	}
-}
-
-func (rc *datagramChannel) Close() {
-	// No need to close msgChan, it will be garbage collect once there is no reference to it
-	close(rc.closedChan)
+	return nil
 }

datagramsession/session.go

@@ -2,12 +2,16 @@ package datagramsession
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
+	"net"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/packet"
 )
 
 const (
@@ -18,41 +22,33 @@ func SessionIdleErr(timeout time.Duration) error {
 	return fmt.Errorf("session idle for %v", timeout)
 }
 
+type transportSender func(session *packet.Session) error
+
+// ErrVithVariableSeverity are errors that have variable severity
+type ErrVithVariableSeverity interface {
+	error
+	// LogLevel return the severity of this error
+	LogLevel() zerolog.Level
+}
+
 // Session is a bidirectional pipe of datagrams between transport and dstConn
-// Currently the only implementation of transport is quic DatagramMuxer
 // Destination can be a connection with origin or with eyeball
 // When the destination is origin:
-// - Datagrams from edge are read by Manager from the transport. Manager finds the corresponding Session and calls the
-//   write method of the Session to send to origin
-// - Datagrams from origin are read from conn and SentTo transport. Transport will return them to eyeball
+// - Manager receives datagrams from receiveChan and calls the transportToDst method of the Session to send to origin
+// - Datagrams from origin are read from conn and Send to transport using the transportSender callback. Transport will return them to eyeball
 // When the destination is eyeball:
-// - Datagrams from eyeball are read from conn and SentTo transport. Transport will send them to cloudflared
-// - Datagrams from cloudflared are read by Manager from the transport. Manager finds the corresponding Session and calls the
-//   write method of the Session to send to eyeball
+// - Datagrams from eyeball are read from conn and Send to transport. Transport will send them to cloudflared using the transportSender callback.
+// - Manager receives datagrams from receiveChan and calls the transportToDst method of the Session to send to the eyeball
 type Session struct {
-	ID        uuid.UUID
-	transport transport
-	dstConn   io.ReadWriteCloser
+	ID       uuid.UUID
+	sendFunc transportSender
+	dstConn  io.ReadWriteCloser
 	// activeAtChan is used to communicate the last read/write time
 	activeAtChan chan time.Time
 	closeChan    chan error
 	log          *zerolog.Logger
 }
 
-func newSession(id uuid.UUID, transport transport, dstConn io.ReadWriteCloser, log *zerolog.Logger) *Session {
-	return &Session{
-		ID:        id,
-		transport: transport,
-		dstConn:   dstConn,
-		// activeAtChan has low capacity. It can be full when there are many concurrent read/write. markActive() will
-		// drop instead of blocking because last active time only needs to be an approximation
-		activeAtChan: make(chan time.Time, 2),
-		// capacity is 2 because close() and dstToTransport routine in Serve() can write to this channel
-		closeChan: make(chan error, 2),
-		log:       log,
-	}
-}
-
 func (s *Session) Serve(ctx context.Context, closeAfterIdle time.Duration) (closedByRemote bool, err error) {
 	go func() {
 		// QUIC implementation copies data to another buffer before returning https://github.com/lucas-clemente/quic-go/blob/v0.24.0/session.go#L1967-L1975
@@ -60,9 +56,20 @@ func (s *Session) Serve(ctx context.Context, closeAfterIdle time.Duration) (clos
 		const maxPacketSize = 1500
 		readBuffer := make([]byte, maxPacketSize)
 		for {
-			if err := s.dstToTransport(readBuffer); err != nil {
-				s.closeChan <- err
-				return
+			if closeSession, err := s.dstToTransport(readBuffer); err != nil {
+				if errors.Is(err, net.ErrClosed) {
+					s.log.Debug().Msg("Destination connection closed")
+				} else {
+					level := zerolog.ErrorLevel
+					if variableErr, ok := err.(ErrVithVariableSeverity); ok {
+						level = variableErr.LogLevel()
+					}
+					s.log.WithLevel(level).Err(err).Msg("Failed to send session payload from destination to transport")
+				}
+				if closeSession {
+					s.closeChan <- err
+					return
+				}
 			}
 		}
 	}()
@@ -103,32 +110,29 @@ func (s *Session) waitForCloseCondition(ctx context.Context, closeAfterIdle time
 	}
 }
 
-func (s *Session) dstToTransport(buffer []byte) error {
+func (s *Session) dstToTransport(buffer []byte) (closeSession bool, err error) {
 	n, err := s.dstConn.Read(buffer)
 	s.markActive()
 	// https://pkg.go.dev/io#Reader suggests caller should always process n > 0 bytes
-	if n > 0 {
-		if n <= int(s.transport.MTU()) {
-			err = s.transport.SendTo(s.ID, buffer[:n])
-		} else {
-			// drop packet for now, eventually reply with ICMP for PMTUD
-			s.log.Debug().
-				Str("session", s.ID.String()).
-				Int("len", n).
-				Int("mtu", s.transport.MTU()).
-				Msg("dropped packet exceeding MTU")
+	if n > 0 || err == nil {
+		session := packet.Session{
+			ID:      s.ID,
+			Payload: buffer[:n],
+		}
+		if sendErr := s.sendFunc(&session); sendErr != nil {
+			return false, sendErr
 		}
 	}
-	// Some UDP application might send 0-size payload.
-	if err == nil && n == 0 {
-		err = s.transport.SendTo(s.ID, []byte{})
-	}
-	return err
+	return err != nil, err
 }
 
 func (s *Session) transportToDst(payload []byte) (int, error) {
 	s.markActive()
-	return s.dstConn.Write(payload)
+	n, err := s.dstConn.Write(payload)
+	if err != nil {
+		s.log.Err(err).Msg("Failed to write payload to session")
+	}
+	return n, err
 }
 
 // Sends the last active time to the idle checker loop without blocking. activeAtChan will only be full when there

datagramsession/session_test.go

@@ -14,6 +14,8 @@ import (
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/packet"
 )
 
 // TestCloseSession makes sure a session will stop after context is done
@@ -41,12 +43,10 @@ func testSessionReturns(t *testing.T, closeBy closeMethod, closeAfterIdle time.D
 	sessionID := uuid.New()
 	cfdConn, originConn := net.Pipe()
 	payload := testPayload(sessionID)
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(1),
-		respChan: newDatagramChannel(1),
-	}
+
 	log := zerolog.Nop()
-	session := newSession(sessionID, transport, cfdConn, &log)
+	mg := NewManager(&log, nil, nil)
+	session := mg.newSession(sessionID, cfdConn)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	sessionDone := make(chan struct{})
@@ -117,12 +117,11 @@ func testActiveSessionNotClosed(t *testing.T, readFromDst bool, writeToDst bool)
 	sessionID := uuid.New()
 	cfdConn, originConn := net.Pipe()
 	payload := testPayload(sessionID)
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(100),
-		respChan: newDatagramChannel(100),
-	}
-	log := zerolog.Nop()
-	session := newSession(sessionID, transport, cfdConn, &log)
+
+	respChan := make(chan *packet.Session)
+	sender := newMockTransportSender(sessionID, payload)
+	mg := NewManager(&nopLogger, sender.muxSession, respChan)
+	session := mg.newSession(sessionID, cfdConn)
 
 	startTime := time.Now()
 	activeUntil := startTime.Add(activeTime)
@@ -184,7 +183,8 @@ func testActiveSessionNotClosed(t *testing.T, readFromDst bool, writeToDst bool)
 
 func TestMarkActiveNotBlocking(t *testing.T) {
 	const concurrentCalls = 50
-	session := newSession(uuid.New(), nil, nil, nil)
+	mg := NewManager(&nopLogger, nil, nil)
+	session := mg.newSession(uuid.New(), nil)
 	var wg sync.WaitGroup
 	wg.Add(concurrentCalls)
 	for i := 0; i < concurrentCalls; i++ {
@@ -196,15 +196,17 @@ func TestMarkActiveNotBlocking(t *testing.T) {
 	wg.Wait()
 }
 
+// Some UDP application might send 0-size payload.
 func TestZeroBytePayload(t *testing.T) {
 	sessionID := uuid.New()
 	cfdConn, originConn := net.Pipe()
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(1),
-		respChan: newDatagramChannel(1),
+
+	sender := sendOnceTransportSender{
+		baseSender: newMockTransportSender(sessionID, make([]byte, 0)),
+		sentChan:   make(chan struct{}),
 	}
-	log := zerolog.Nop()
-	session := newSession(sessionID, transport, cfdConn, &log)
+	mg := NewManager(&nopLogger, sender.muxSession, nil)
+	session := mg.newSession(sessionID, cfdConn)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	errGroup, ctx := errgroup.WithContext(ctx)
@@ -224,11 +226,39 @@ func TestZeroBytePayload(t *testing.T) {
 		return nil
 	})
 
-	receivedSessionID, payload, err := transport.respChan.Receive(ctx)
-	require.NoError(t, err)
-	require.Len(t, payload, 0)
-	require.Equal(t, sessionID, receivedSessionID)
-
+	<-sender.sentChan
 	cancel()
 	require.NoError(t, errGroup.Wait())
 }
+
+type mockTransportSender struct {
+	expectedSessionID uuid.UUID
+	expectedPayload   []byte
+}
+
+func newMockTransportSender(expectedSessionID uuid.UUID, expectedPayload []byte) *mockTransportSender {
+	return &mockTransportSender{
+		expectedSessionID: expectedSessionID,
+		expectedPayload:   expectedPayload,
+	}
+}
+
+func (mts *mockTransportSender) muxSession(session *packet.Session) error {
+	if session.ID != mts.expectedSessionID {
+		return fmt.Errorf("Expect session %s, got %s", mts.expectedSessionID, session.ID)
+	}
+	if !bytes.Equal(session.Payload, mts.expectedPayload) {
+		return fmt.Errorf("Expect %v, read %v", mts.expectedPayload, session.Payload)
+	}
+	return nil
+}
+
+type sendOnceTransportSender struct {
+	baseSender *mockTransportSender
+	sentChan   chan struct{}
+}
+
+func (sots *sendOnceTransportSender) muxSession(session *packet.Session) error {
+	defer close(sots.sentChan)
+	return sots.baseSender.muxSession(session)
+}

datagramsession/transport.go

@@ -1,13 +0,0 @@
-package datagramsession
-
-import "github.com/google/uuid"
-
-// Transport is a connection between cloudflared and edge that can multiplex datagrams from multiple sessions
-type transport interface {
-	// SendTo writes payload for a session to the transport
-	SendTo(sessionID uuid.UUID, payload []byte) error
-	// ReceiveFrom reads the next datagram from the transport
-	ReceiveFrom() (uuid.UUID, []byte, error)
-	// Max transmission unit to receive from the transport
-	MTU() int
-}

datagramsession/transport_test.go

@@ -1,36 +0,0 @@
-package datagramsession
-
-import (
-	"context"
-
-	"github.com/google/uuid"
-)
-
-type mockQUICTransport struct {
-	reqChan  *datagramChannel
-	respChan *datagramChannel
-}
-
-func (mt *mockQUICTransport) SendTo(sessionID uuid.UUID, payload []byte) error {
-	buf := make([]byte, len(payload))
-	// The QUIC implementation copies data to another buffer before returning https://github.com/lucas-clemente/quic-go/blob/v0.24.0/session.go#L1967-L1975
-	copy(buf, payload)
-	return mt.respChan.Send(context.Background(), sessionID, buf)
-}
-
-func (mt *mockQUICTransport) ReceiveFrom() (uuid.UUID, []byte, error) {
-	return mt.reqChan.Receive(context.Background())
-}
-
-func (mt *mockQUICTransport) MTU() int {
-	return 1280
-}
-
-func (mt *mockQUICTransport) newRequest(ctx context.Context, sessionID uuid.UUID, payload []byte) error {
-	return mt.reqChan.Send(ctx, sessionID, payload)
-}
-
-func (mt *mockQUICTransport) close() {
-	mt.reqChan.Close()
-	mt.respChan.Close()
-}

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17.5 as builder
+FROM golang:1.19 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
@@ -6,4 +6,4 @@ RUN apt-get update
 COPY . .
 # compile cloudflared
 RUN make cloudflared
-RUN cp /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+RUN cp /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/

edgediscovery/allregions/address.go

@@ -0,0 +1,64 @@
+package allregions
+
+// Region contains cloudflared edge addresses. The edge is partitioned into several regions for
+// redundancy purposes.
+type AddrSet map[*EdgeAddr]UsedBy
+
+// AddrUsedBy finds the address used by the given connection in this region.
+// Returns nil if the connection isn't using any IP.
+func (a AddrSet) AddrUsedBy(connID int) *EdgeAddr {
+	for addr, used := range a {
+		if used.Used && used.ConnID == connID {
+			return addr
+		}
+	}
+	return nil
+}
+
+// AvailableAddrs counts how many unused addresses this region contains.
+func (a AddrSet) AvailableAddrs() int {
+	n := 0
+	for _, usedby := range a {
+		if !usedby.Used {
+			n++
+		}
+	}
+	return n
+}
+
+// GetUnusedIP returns a random unused address in this region.
+// Returns nil if all addresses are in use.
+func (a AddrSet) GetUnusedIP(excluding *EdgeAddr) *EdgeAddr {
+	for addr, usedby := range a {
+		if !usedby.Used && addr != excluding {
+			return addr
+		}
+	}
+	return nil
+}
+
+// Use the address, assigning it to a proxy connection.
+func (a AddrSet) Use(addr *EdgeAddr, connID int) {
+	if addr == nil {
+		return
+	}
+	a[addr] = InUse(connID)
+}
+
+// GetAnyAddress returns an arbitrary address from the region.
+func (a AddrSet) GetAnyAddress() *EdgeAddr {
+	for addr := range a {
+		return addr
+	}
+	return nil
+}
+
+// GiveBack the address, ensuring it is no longer assigned to an IP.
+// Returns true if the address is in this region.
+func (a AddrSet) GiveBack(addr *EdgeAddr) (ok bool) {
+	if _, ok := a[addr]; !ok {
+		return false
+	}
+	a[addr] = Unused()
+	return true
+}

edgediscovery/allregions/address_test.go

@@ -0,0 +1,247 @@
+package allregions
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestAddrSet_AddrUsedBy(t *testing.T) {
+	type args struct {
+		connID int
+	}
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		args    args
+		want    *EdgeAddr
+	}{
+		{
+			name: "happy trivial test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+			},
+			args: args{connID: 0},
+			want: &addr0,
+		},
+		{
+			name: "sad trivial test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+			},
+			args: args{connID: 1},
+			want: nil,
+		},
+		{
+			name: "sad test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{connID: 3},
+			want: nil,
+		},
+		{
+			name: "happy test",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{connID: 1},
+			want: &addr1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.AddrUsedBy(tt.args.connID); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Region.AddrUsedBy() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAddrSet_AvailableAddrs(t *testing.T) {
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		want    int
+	}{
+		{
+			name: "contains addresses",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			},
+			want: 1,
+		},
+		{
+			name: "all free",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+				&addr1: Unused(),
+				&addr2: Unused(),
+			},
+			want: 3,
+		},
+		{
+			name: "all used",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			want: 0,
+		},
+		{
+			name:    "empty",
+			addrSet: AddrSet{},
+			want:    0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.AvailableAddrs(); got != tt.want {
+				t.Errorf("Region.AvailableAddrs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAddrSet_GetUnusedIP(t *testing.T) {
+	type args struct {
+		excluding *EdgeAddr
+	}
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		args    args
+		want    *EdgeAddr
+	}{
+		{
+			name: "happy test with excluding set",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: &addr0},
+			want: &addr1,
+		},
+		{
+			name: "happy test with no excluding",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: nil},
+			want: &addr1,
+		},
+		{
+			name: "sad test with no excluding",
+			addrSet: AddrSet{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: nil},
+			want: nil,
+		},
+		{
+			name: "sad test with excluding",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			},
+			args: args{excluding: &addr0},
+			want: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.GetUnusedIP(tt.args.excluding); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Region.GetUnusedIP() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestAddrSet_GiveBack(t *testing.T) {
+	type args struct {
+		addr *EdgeAddr
+	}
+	tests := []struct {
+		name           string
+		addrSet        AddrSet
+		args           args
+		wantOk         bool
+		availableAfter int
+	}{
+		{
+			name: "sad test with excluding",
+			addrSet: AddrSet{
+				&addr1: InUse(1),
+			},
+			args:           args{addr: &addr1},
+			wantOk:         true,
+			availableAfter: 1,
+		},
+		{
+			name: "sad test with excluding",
+			addrSet: AddrSet{
+				&addr1: InUse(1),
+			},
+			args:           args{addr: &addr2},
+			wantOk:         false,
+			availableAfter: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if gotOk := tt.addrSet.GiveBack(tt.args.addr); gotOk != tt.wantOk {
+				t.Errorf("Region.GiveBack() = %v, want %v", gotOk, tt.wantOk)
+			}
+			if tt.availableAfter != tt.addrSet.AvailableAddrs() {
+				t.Errorf("Region.AvailableAddrs() = %v, want %v", tt.addrSet.AvailableAddrs(), tt.availableAfter)
+			}
+		})
+	}
+}
+
+func TestAddrSet_GetAnyAddress(t *testing.T) {
+	tests := []struct {
+		name    string
+		addrSet AddrSet
+		wantNil bool
+	}{
+		{
+			name:    "Sad test -- GetAnyAddress should only fail if the region is empty",
+			addrSet: AddrSet{},
+			wantNil: true,
+		},
+		{
+			name: "Happy test (all addresses unused)",
+			addrSet: AddrSet{
+				&addr0: Unused(),
+			},
+			wantNil: false,
+		},
+		{
+			name: "Happy test (GetAnyAddress can still return addresses used by proxy conns)",
+			addrSet: AddrSet{
+				&addr0: InUse(2),
+			},
+			wantNil: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := tt.addrSet.GetAnyAddress(); tt.wantNil != (got == nil) {
+				t.Errorf("Region.GetAnyAddress() = %v, but should it return nil? %v", got, tt.wantNil)
+			}
+		})
+	}
+}

edgediscovery/allregions/discovery.go

@@ -13,7 +13,7 @@ import (
 
 const (
 	// Used to discover HA origintunneld servers
-	srvService = "origintunneld"
+	srvService = "v2-origintunneld"
 	srvProto   = "tcp"
 	srvName    = "argotunnel.com"
 
@@ -32,20 +32,53 @@ var (
 	netLookupIP  = net.LookupIP
 )
 
+// ConfigIPVersion is the selection of IP versions from config
+type ConfigIPVersion int8
+
+const (
+	Auto     ConfigIPVersion = 2
+	IPv4Only ConfigIPVersion = 4
+	IPv6Only ConfigIPVersion = 6
+)
+
+// IPVersion is the IP version of an EdgeAddr
+type EdgeIPVersion int8
+
+const (
+	V4 EdgeIPVersion = 4
+	V6 EdgeIPVersion = 6
+)
+
+// String returns the enum's constant name.
+func (c EdgeIPVersion) String() string {
+	switch c {
+	case V4:
+		return "4"
+	case V6:
+		return "6"
+	default:
+		return ""
+	}
+}
+
 // EdgeAddr is a representation of possible ways to refer an edge location.
 type EdgeAddr struct {
-	TCP *net.TCPAddr
-	UDP *net.UDPAddr
+	TCP       *net.TCPAddr
+	UDP       *net.UDPAddr
+	IPVersion EdgeIPVersion
 }
 
 // If the call to net.LookupSRV fails, try to fall back to DoT from Cloudflare directly.
 //
 // Note: Instead of DoT, we could also have used DoH. Either of these:
-//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
-//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
+//   - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
+//   - indirectly via `tunneldns.NewUpstreamHTTPS()`
+//
 // But both of these cases miss out on a key feature from the stdlib:
-//     "The returned records are sorted by priority and randomized by weight within a priority."
-//     (https://golang.org/pkg/net/#Resolver.LookupSRV)
+//
+//	"The returned records are sorted by priority and randomized by weight within a priority."
+//	(https://golang.org/pkg/net/#Resolver.LookupSRV)
+//
 // Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it.
 // See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552
 var fallbackLookupSRV = lookupSRVWithDOT
@@ -85,6 +118,9 @@ func edgeDiscovery(log *zerolog.Logger, srvService string) ([][]*EdgeAddr, error
 		if err != nil {
 			return nil, err
 		}
+		for _, e := range edgeAddrs {
+			log.Debug().Msgf("Edge Address: %+v", *e)
+		}
 		resolvedAddrPerCNAME = append(resolvedAddrPerCNAME, edgeAddrs)
 	}
 
@@ -120,9 +156,14 @@ func resolveSRV(srv *net.SRV) ([]*EdgeAddr, error) {
 	}
 	addrs := make([]*EdgeAddr, len(ips))
 	for i, ip := range ips {
+		version := V6
+		if ip.To4() != nil {
+			version = V4
+		}
 		addrs[i] = &EdgeAddr{
-			TCP: &net.TCPAddr{IP: ip, Port: int(srv.Port)},
-			UDP: &net.UDPAddr{IP: ip, Port: int(srv.Port)},
+			TCP:       &net.TCPAddr{IP: ip, Port: int(srv.Port)},
+			UDP:       &net.UDPAddr{IP: ip, Port: int(srv.Port)},
+			IPVersion: version,
 		}
 	}
 	return addrs, nil
@@ -143,11 +184,15 @@ func ResolveAddrs(addrs []string, log *zerolog.Logger) (resolved []*EdgeAddr) {
 			log.Error().Str(logFieldAddress, addr).Err(err).Msg("failed to resolve to UDP address")
 			continue
 		}
+		version := V6
+		if udpAddr.IP.To4() != nil {
+			version = V4
+		}
 		resolved = append(resolved, &EdgeAddr{
-			TCP: tcpAddr,
-			UDP: udpAddr,
+			TCP:       tcpAddr,
+			UDP:       udpAddr,
+			IPVersion: version,
 		})
-
 	}
 	return
 }

edgediscovery/allregions/mocks_for_test.go

@@ -9,6 +9,115 @@ import (
 	"testing/quick"
 )
 
+var (
+	v4Addrs = []*EdgeAddr{&addr0, &addr1, &addr2, &addr3}
+	v6Addrs = []*EdgeAddr{&addr4, &addr5, &addr6, &addr7}
+	addr0   = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr1 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.1"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.1"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr2 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.2"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.2"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr3 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.3"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.3"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	addr4 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+	addr5 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+	addr6 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+	addr7 = EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V6,
+	}
+)
+
 type mockAddrs struct {
 	// a set of synthetic SRV records
 	addrMap map[net.SRV][]*EdgeAddr
@@ -36,7 +145,7 @@ func newMockAddrs(port uint16, numRegions uint8, numAddrsPerRegion uint8) mockAd
 				IP:   net.ParseIP(fmt.Sprintf("10.0.%v.%v", r, a)),
 				Port: int(port),
 			}
-			addrs = append(addrs, &EdgeAddr{tcpAddr, udpAddr})
+			addrs = append(addrs, &EdgeAddr{tcpAddr, udpAddr, V4})
 		}
 		addrMap[srv] = addrs
 		numAddrs += len(addrs)

edgediscovery/allregions/region.go

@@ -1,79 +1,155 @@
 package allregions
 
+import "time"
+
+const (
+	timeoutDuration = 10 * time.Minute
+)
+
 // Region contains cloudflared edge addresses. The edge is partitioned into several regions for
 // redundancy purposes.
 type Region struct {
-	connFor map[*EdgeAddr]UsedBy
+	primaryIsActive bool
+	active          AddrSet
+	primary         AddrSet
+	secondary       AddrSet
+	primaryTimeout  time.Time
+	timeoutDuration time.Duration
 }
 
 // NewRegion creates a region with the given addresses, which are all unused.
-func NewRegion(addrs []*EdgeAddr) Region {
+func NewRegion(addrs []*EdgeAddr, overrideIPVersion ConfigIPVersion) Region {
 	// The zero value of UsedBy is Unused(), so we can just initialize the map's values with their
 	// zero values.
-	connFor := make(map[*EdgeAddr]UsedBy)
-	for _, addr := range addrs {
-		connFor[addr] = Unused()
+	connForv4 := make(AddrSet)
+	connForv6 := make(AddrSet)
+	systemPreference := V6
+	for i, addr := range addrs {
+		if i == 0 {
+			// First family of IPs returned is system preference of IP
+			systemPreference = addr.IPVersion
+		}
+		switch addr.IPVersion {
+		case V4:
+			connForv4[addr] = Unused()
+		case V6:
+			connForv6[addr] = Unused()
+		}
 	}
+
+	// Process as system preference
+	var primary AddrSet
+	var secondary AddrSet
+	switch systemPreference {
+	case V4:
+		primary = connForv4
+		secondary = connForv6
+	case V6:
+		primary = connForv6
+		secondary = connForv4
+	}
+
+	// Override with provided preference
+	switch overrideIPVersion {
+	case IPv4Only:
+		primary = connForv4
+		secondary = make(AddrSet) // empty
+	case IPv6Only:
+		primary = connForv6
+		secondary = make(AddrSet) // empty
+	case Auto:
+		// no change
+	default:
+		// no change
+	}
+
 	return Region{
-		connFor: connFor,
+		primaryIsActive: true,
+		active:          primary,
+		primary:         primary,
+		secondary:       secondary,
+		timeoutDuration: timeoutDuration,
 	}
 }
 
 // AddrUsedBy finds the address used by the given connection in this region.
 // Returns nil if the connection isn't using any IP.
 func (r *Region) AddrUsedBy(connID int) *EdgeAddr {
-	for addr, used := range r.connFor {
-		if used.Used && used.ConnID == connID {
-			return addr
-		}
+	edgeAddr := r.primary.AddrUsedBy(connID)
+	if edgeAddr == nil {
+		edgeAddr = r.secondary.AddrUsedBy(connID)
 	}
-	return nil
+	return edgeAddr
 }
 
 // AvailableAddrs counts how many unused addresses this region contains.
 func (r Region) AvailableAddrs() int {
-	n := 0
-	for _, usedby := range r.connFor {
-		if !usedby.Used {
-			n++
-		}
-	}
-	return n
+	return r.active.AvailableAddrs()
 }
 
-// GetUnusedIP returns a random unused address in this region.
-// Returns nil if all addresses are in use.
-func (r Region) GetUnusedIP(excluding *EdgeAddr) *EdgeAddr {
-	for addr, usedby := range r.connFor {
-		if !usedby.Used && addr != excluding {
-			return addr
-		}
-	}
-	return nil
-}
-
-// Use the address, assigning it to a proxy connection.
-func (r Region) Use(addr *EdgeAddr, connID int) {
-	if addr == nil {
-		return
-	}
-	r.connFor[addr] = InUse(connID)
-}
-
-// GetAnyAddress returns an arbitrary address from the region.
-func (r Region) GetAnyAddress() *EdgeAddr {
-	for addr := range r.connFor {
+// AssignAnyAddress returns a random unused address in this region now
+// assigned to the connID excluding the provided EdgeAddr.
+// Returns nil if all addresses are in use for the region.
+func (r Region) AssignAnyAddress(connID int, excluding *EdgeAddr) *EdgeAddr {
+	if addr := r.active.GetUnusedIP(excluding); addr != nil {
+		r.active.Use(addr, connID)
 		return addr
 	}
 	return nil
 }
 
+// GetAnyAddress returns an arbitrary address from the region.
+func (r Region) GetAnyAddress() *EdgeAddr {
+	return r.active.GetAnyAddress()
+}
+
 // GiveBack the address, ensuring it is no longer assigned to an IP.
 // Returns true if the address is in this region.
-func (r Region) GiveBack(addr *EdgeAddr) (ok bool) {
-	if _, ok := r.connFor[addr]; !ok {
-		return false
+func (r *Region) GiveBack(addr *EdgeAddr, hasConnectivityError bool) (ok bool) {
+	if ok = r.primary.GiveBack(addr); !ok {
+		// Attempt to give back the address in the secondary set
+		if ok = r.secondary.GiveBack(addr); !ok {
+			// Address is not in this region
+			return
+		}
 	}
-	r.connFor[addr] = Unused()
-	return true
+
+	// No connectivity error: no worry
+	if !hasConnectivityError {
+		return
+	}
+
+	// If using primary and returned address is IPv6 and secondary is available
+	if r.primaryIsActive && addr.IPVersion == V6 && len(r.secondary) > 0 {
+		r.active = r.secondary
+		r.primaryIsActive = false
+		r.primaryTimeout = time.Now().Add(r.timeoutDuration)
+		return
+	}
+
+	// Do nothing for IPv4 or if secondary is empty
+	if r.primaryIsActive {
+		return
+	}
+
+	// Immediately return to primary pool, regardless of current primary timeout
+	if addr.IPVersion == V4 {
+		activatePrimary(r)
+		return
+	}
+
+	// Timeout exceeded and can be reset to primary pool
+	if r.primaryTimeout.Before(time.Now()) {
+		activatePrimary(r)
+		return
+	}
+
+	return
+}
+
+// activatePrimary sets the primary set to the active set and resets the timeout.
+func activatePrimary(r *Region) {
+	r.active = r.primary
+	r.primaryIsActive = true
+	r.primaryTimeout = time.Now() // reset timeout
 }

edgediscovery/allregions/region_test.go

@@ -1,284 +1,357 @@
 package allregions
 
 import (
-	"reflect"
+	"net"
 	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
 )
 
+func makeAddrSet(addrs []*EdgeAddr) AddrSet {
+	addrSet := make(AddrSet, len(addrs))
+	for _, addr := range addrs {
+		addrSet[addr] = Unused()
+	}
+	return addrSet
+}
+
 func TestRegion_New(t *testing.T) {
-	r := NewRegion([]*EdgeAddr{&addr0, &addr1, &addr2})
-	if r.AvailableAddrs() != 3 {
-		t.Errorf("r.AvailableAddrs() == %v but want 3", r.AvailableAddrs())
-	}
-}
-
-func TestRegion_AddrUsedBy(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	type args struct {
-		connID int
-	}
 	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   *EdgeAddr
+		name          string
+		addrs         []*EdgeAddr
+		mode          ConfigIPVersion
+		expectedAddrs int
+		primary       AddrSet
+		secondary     AddrSet
 	}{
 		{
-			name: "happy trivial test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-			}},
-			args: args{connID: 0},
-			want: &addr0,
+			name:          "IPv4 addresses with IPv4Only",
+			addrs:         v4Addrs,
+			mode:          IPv4Only,
+			expectedAddrs: len(v4Addrs),
+			primary:       makeAddrSet(v4Addrs),
+			secondary:     AddrSet{},
 		},
 		{
-			name: "sad trivial test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-			}},
-			args: args{connID: 1},
-			want: nil,
+			name:          "IPv6 addresses with IPv4Only",
+			addrs:         v6Addrs,
+			mode:          IPv4Only,
+			expectedAddrs: 0,
+			primary:       AddrSet{},
+			secondary:     AddrSet{},
 		},
 		{
-			name: "sad test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{connID: 3},
-			want: nil,
+			name:          "IPv6 addresses with IPv6Only",
+			addrs:         v6Addrs,
+			mode:          IPv6Only,
+			expectedAddrs: len(v6Addrs),
+			primary:       makeAddrSet(v6Addrs),
+			secondary:     AddrSet{},
 		},
 		{
-			name: "happy test",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{connID: 1},
-			want: &addr1,
+			name:          "IPv6 addresses with IPv4Only",
+			addrs:         v6Addrs,
+			mode:          IPv4Only,
+			expectedAddrs: 0,
+			primary:       AddrSet{},
+			secondary:     AddrSet{},
+		},
+		{
+			name:          "IPv4 (first) and IPv6 addresses with Auto",
+			addrs:         append(v4Addrs, v6Addrs...),
+			mode:          Auto,
+			expectedAddrs: len(v4Addrs),
+			primary:       makeAddrSet(v4Addrs),
+			secondary:     makeAddrSet(v6Addrs),
+		},
+		{
+			name:          "IPv6 (first) and IPv4 addresses with Auto",
+			addrs:         append(v6Addrs, v4Addrs...),
+			mode:          Auto,
+			expectedAddrs: len(v6Addrs),
+			primary:       makeAddrSet(v6Addrs),
+			secondary:     makeAddrSet(v4Addrs),
+		},
+		{
+			name:          "IPv4 addresses with Auto",
+			addrs:         v4Addrs,
+			mode:          Auto,
+			expectedAddrs: len(v4Addrs),
+			primary:       makeAddrSet(v4Addrs),
+			secondary:     AddrSet{},
+		},
+		{
+			name:          "IPv6 addresses with Auto",
+			addrs:         v6Addrs,
+			mode:          Auto,
+			expectedAddrs: len(v6Addrs),
+			primary:       makeAddrSet(v6Addrs),
+			secondary:     AddrSet{},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			r := &Region{
-				connFor: tt.fields.connFor,
+			r := NewRegion(tt.addrs, tt.mode)
+			assert.Equal(t, tt.expectedAddrs, r.AvailableAddrs())
+			assert.Equal(t, tt.primary, r.primary)
+			assert.Equal(t, tt.secondary, r.secondary)
+		})
+	}
+}
+
+func TestRegion_AnyAddress_EmptyActiveSet(t *testing.T) {
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv6 addresses with IPv4Only",
+			addrs: v6Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv4 addresses with IPv6Only",
+			addrs: v4Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			addr := r.GetAnyAddress()
+			assert.Nil(t, addr)
+			addr = r.AssignAnyAddress(0, nil)
+			assert.Nil(t, addr)
+		})
+	}
+}
+
+func TestRegion_AssignAnyAddress_FullyUsedActiveSet(t *testing.T) {
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			total := r.active.AvailableAddrs()
+			for i := 0; i < total; i++ {
+				addr := r.AssignAnyAddress(i, nil)
+				assert.NotNil(t, addr)
 			}
-			if got := r.AddrUsedBy(tt.args.connID); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("Region.AddrUsedBy() = %v, want %v", got, tt.want)
+			addr := r.AssignAnyAddress(9, nil)
+			assert.Nil(t, addr)
+		})
+	}
+}
+
+var giveBackTests = []struct {
+	name          string
+	addrs         []*EdgeAddr
+	mode          ConfigIPVersion
+	expectedAddrs int
+	primary       AddrSet
+	secondary     AddrSet
+	primarySwap   bool
+}{
+	{
+		name:          "IPv4 addresses with IPv4Only",
+		addrs:         v4Addrs,
+		mode:          IPv4Only,
+		expectedAddrs: len(v4Addrs),
+		primary:       makeAddrSet(v4Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv6 addresses with IPv6Only",
+		addrs:         v6Addrs,
+		mode:          IPv6Only,
+		expectedAddrs: len(v6Addrs),
+		primary:       makeAddrSet(v6Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv4 (first) and IPv6 addresses with Auto",
+		addrs:         append(v4Addrs, v6Addrs...),
+		mode:          Auto,
+		expectedAddrs: len(v4Addrs),
+		primary:       makeAddrSet(v4Addrs),
+		secondary:     makeAddrSet(v6Addrs),
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv6 (first) and IPv4 addresses with Auto",
+		addrs:         append(v6Addrs, v4Addrs...),
+		mode:          Auto,
+		expectedAddrs: len(v6Addrs),
+		primary:       makeAddrSet(v6Addrs),
+		secondary:     makeAddrSet(v4Addrs),
+		primarySwap:   true,
+	},
+	{
+		name:          "IPv4 addresses with Auto",
+		addrs:         v4Addrs,
+		mode:          Auto,
+		expectedAddrs: len(v4Addrs),
+		primary:       makeAddrSet(v4Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+	{
+		name:          "IPv6 addresses with Auto",
+		addrs:         v6Addrs,
+		mode:          Auto,
+		expectedAddrs: len(v6Addrs),
+		primary:       makeAddrSet(v6Addrs),
+		secondary:     AddrSet{},
+		primarySwap:   false,
+	},
+}
+
+func TestRegion_GiveBack_NoConnectivityError(t *testing.T) {
+	for _, tt := range giveBackTests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			addr := r.AssignAnyAddress(0, nil)
+			assert.NotNil(t, addr)
+			assert.True(t, r.GiveBack(addr, false))
+		})
+	}
+}
+
+func TestRegion_GiveBack_ForeignAddr(t *testing.T) {
+	invalid := EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("123.4.5.0"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: V4,
+	}
+	for _, tt := range giveBackTests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			assert.False(t, r.GiveBack(&invalid, false))
+			assert.False(t, r.GiveBack(&invalid, true))
+		})
+	}
+}
+
+func TestRegion_GiveBack_SwapPrimary(t *testing.T) {
+	for _, tt := range giveBackTests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := NewRegion(tt.addrs, tt.mode)
+			addr := r.AssignAnyAddress(0, nil)
+			assert.NotNil(t, addr)
+			assert.True(t, r.GiveBack(addr, true))
+			assert.Equal(t, tt.primarySwap, !r.primaryIsActive)
+			if tt.primarySwap {
+				assert.Equal(t, r.secondary, r.active)
+				assert.False(t, r.primaryTimeout.IsZero())
+			} else {
+				assert.Equal(t, r.primary, r.active)
+				assert.True(t, r.primaryTimeout.IsZero())
 			}
 		})
 	}
 }
 
-func TestRegion_AvailableAddrs(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		want   int
-	}{
-		{
-			name: "contains addresses",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: Unused(),
-				&addr2: InUse(2),
-			}},
-			want: 1,
-		},
-		{
-			name: "all free",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-				&addr1: Unused(),
-				&addr2: Unused(),
-			}},
-			want: 3,
-		},
-		{
-			name: "all used",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			want: 0,
-		},
-		{
-			name:   "empty",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{}},
-			want:   0,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if got := r.AvailableAddrs(); got != tt.want {
-				t.Errorf("Region.AvailableAddrs() = %v, want %v", got, tt.want)
-			}
-		})
-	}
+func TestRegion_GiveBack_IPv4_ResetPrimary(t *testing.T) {
+	r := NewRegion(append(v6Addrs, v4Addrs...), Auto)
+	// Exhaust all IPv6 addresses
+	a0 := r.AssignAnyAddress(0, nil)
+	a1 := r.AssignAnyAddress(1, nil)
+	a2 := r.AssignAnyAddress(2, nil)
+	a3 := r.AssignAnyAddress(3, nil)
+	assert.NotNil(t, a0)
+	assert.NotNil(t, a1)
+	assert.NotNil(t, a2)
+	assert.NotNil(t, a3)
+	// Give back the first IPv6 address to fallback to secondary IPv4 address set
+	assert.True(t, r.GiveBack(a0, true))
+	assert.False(t, r.primaryIsActive)
+	// Give back another IPv6 address
+	assert.True(t, r.GiveBack(a1, true))
+	// Primary shouldn't change
+	assert.False(t, r.primaryIsActive)
+	// Request an address (should be IPv4 from secondary)
+	a4_v4 := r.AssignAnyAddress(4, nil)
+	assert.NotNil(t, a4_v4)
+	assert.Equal(t, V4, a4_v4.IPVersion)
+	a5_v4 := r.AssignAnyAddress(5, nil)
+	assert.NotNil(t, a5_v4)
+	assert.Equal(t, V4, a5_v4.IPVersion)
+	a6_v4 := r.AssignAnyAddress(6, nil)
+	assert.NotNil(t, a6_v4)
+	assert.Equal(t, V4, a6_v4.IPVersion)
+	// Return IPv4 address (without failure)
+	// Primary shouldn't change because it is not a connectivity failure
+	assert.True(t, r.GiveBack(a4_v4, false))
+	assert.False(t, r.primaryIsActive)
+	// Return IPv4 address (with failure)
+	// Primary should change because it is a connectivity failure
+	assert.True(t, r.GiveBack(a5_v4, true))
+	assert.True(t, r.primaryIsActive)
+	// Return IPv4 address (with failure)
+	// Primary shouldn't change because the address is returned to the inactive
+	// secondary address set
+	assert.True(t, r.GiveBack(a6_v4, true))
+	assert.True(t, r.primaryIsActive)
+	// Return IPv6 address (without failure)
+	// Primary shoudn't change because it is not a connectivity failure
+	assert.True(t, r.GiveBack(a2, false))
+	assert.True(t, r.primaryIsActive)
 }
 
-func TestRegion_GetUnusedIP(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	type args struct {
-		excluding *EdgeAddr
-	}
-	tests := []struct {
-		name   string
-		fields fields
-		args   args
-		want   *EdgeAddr
-	}{
-		{
-			name: "happy test with excluding set",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-				&addr1: Unused(),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: &addr0},
-			want: &addr1,
-		},
-		{
-			name: "happy test with no excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: Unused(),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: nil},
-			want: &addr1,
-		},
-		{
-			name: "sad test with no excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(0),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: nil},
-			want: nil,
-		},
-		{
-			name: "sad test with excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-				&addr1: InUse(1),
-				&addr2: InUse(2),
-			}},
-			args: args{excluding: &addr0},
-			want: nil,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if got := r.GetUnusedIP(tt.args.excluding); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("Region.GetUnusedIP() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func TestRegion_GiveBack(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	type args struct {
-		addr *EdgeAddr
-	}
-	tests := []struct {
-		name           string
-		fields         fields
-		args           args
-		wantOk         bool
-		availableAfter int
-	}{
-		{
-			name: "sad test with excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr1: InUse(1),
-			}},
-			args:           args{addr: &addr1},
-			wantOk:         true,
-			availableAfter: 1,
-		},
-		{
-			name: "sad test with excluding",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr1: InUse(1),
-			}},
-			args:           args{addr: &addr2},
-			wantOk:         false,
-			availableAfter: 0,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if gotOk := r.GiveBack(tt.args.addr); gotOk != tt.wantOk {
-				t.Errorf("Region.GiveBack() = %v, want %v", gotOk, tt.wantOk)
-			}
-			if tt.availableAfter != r.AvailableAddrs() {
-				t.Errorf("Region.AvailableAddrs() = %v, want %v", r.AvailableAddrs(), tt.availableAfter)
-			}
-		})
-	}
-}
-
-func TestRegion_GetAnyAddress(t *testing.T) {
-	type fields struct {
-		connFor map[*EdgeAddr]UsedBy
-	}
-	tests := []struct {
-		name    string
-		fields  fields
-		wantNil bool
-	}{
-		{
-			name:    "Sad test -- GetAnyAddress should only fail if the region is empty",
-			fields:  fields{connFor: map[*EdgeAddr]UsedBy{}},
-			wantNil: true,
-		},
-		{
-			name: "Happy test (all addresses unused)",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: Unused(),
-			}},
-			wantNil: false,
-		},
-		{
-			name: "Happy test (GetAnyAddress can still return addresses used by proxy conns)",
-			fields: fields{connFor: map[*EdgeAddr]UsedBy{
-				&addr0: InUse(2),
-			}},
-			wantNil: false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			r := Region{
-				connFor: tt.fields.connFor,
-			}
-			if got := r.GetAnyAddress(); tt.wantNil != (got == nil) {
-				t.Errorf("Region.GetAnyAddress() = %v, but should it return nil? %v", got, tt.wantNil)
-			}
-		})
-	}
+func TestRegion_GiveBack_Timeout(t *testing.T) {
+	r := NewRegion(append(v6Addrs, v4Addrs...), Auto)
+	a0 := r.AssignAnyAddress(0, nil)
+	a1 := r.AssignAnyAddress(1, nil)
+	a2 := r.AssignAnyAddress(2, nil)
+	assert.NotNil(t, a0)
+	assert.NotNil(t, a1)
+	assert.NotNil(t, a2)
+	// Give back IPv6 address to set timeout
+	assert.True(t, r.GiveBack(a0, true))
+	assert.False(t, r.primaryIsActive)
+	assert.False(t, r.primaryTimeout.IsZero())
+	// Request an address (should be IPv4 from secondary)
+	a3_v4 := r.AssignAnyAddress(3, nil)
+	assert.NotNil(t, a3_v4)
+	assert.Equal(t, V4, a3_v4.IPVersion)
+	assert.False(t, r.primaryIsActive)
+	// Give back IPv6 address inside timeout (no change)
+	assert.True(t, r.GiveBack(a2, true))
+	assert.False(t, r.primaryIsActive)
+	assert.False(t, r.primaryTimeout.IsZero())
+	// Accelerate timeout
+	r.primaryTimeout = time.Now().Add(-time.Minute)
+	// Return IPv6 address
+	assert.True(t, r.GiveBack(a1, true))
+	assert.True(t, r.primaryIsActive)
+	// Returning an IPv4 address after primary is active shouldn't change primary
+	// even with a connectivity error
+	assert.True(t, r.GiveBack(a3_v4, true))
+	assert.True(t, r.primaryIsActive)
 }

edgediscovery/allregions/regions.go

@@ -2,6 +2,7 @@ package allregions
 
 import (
 	"fmt"
+	"math/rand"
 
 	"github.com/rs/zerolog"
 )
@@ -18,7 +19,7 @@ type Regions struct {
 // ------------------------------------
 
 // ResolveEdge resolves the Cloudflare edge, returning all regions discovered.
-func ResolveEdge(log *zerolog.Logger, region string) (*Regions, error) {
+func ResolveEdge(log *zerolog.Logger, region string, overrideIPVersion ConfigIPVersion) (*Regions, error) {
 	edgeAddrs, err := edgeDiscovery(log, getRegionalServiceName(region))
 	if err != nil {
 		return nil, err
@@ -27,8 +28,8 @@ func ResolveEdge(log *zerolog.Logger, region string) (*Regions, error) {
 		return nil, fmt.Errorf("expected at least 2 Cloudflare Regions regions, but SRV only returned %v", len(edgeAddrs))
 	}
 	return &Regions{
-		region1: NewRegion(edgeAddrs[0]),
-		region2: NewRegion(edgeAddrs[1]),
+		region1: NewRegion(edgeAddrs[0], overrideIPVersion),
+		region2: NewRegion(edgeAddrs[1], overrideIPVersion),
 	}, nil
 }
 
@@ -56,8 +57,8 @@ func NewNoResolve(addrs []*EdgeAddr) *Regions {
 	}
 
 	return &Regions{
-		region1: NewRegion(region1),
-		region2: NewRegion(region2),
+		region1: NewRegion(region1, Auto),
+		region2: NewRegion(region2, Auto),
 	}
 }
 
@@ -85,6 +86,15 @@ func (rs *Regions) AddrUsedBy(connID int) *EdgeAddr {
 // GetUnusedAddr gets an unused addr from the edge, excluding the given addr. Prefer to use addresses
 // evenly across both regions.
 func (rs *Regions) GetUnusedAddr(excluding *EdgeAddr, connID int) *EdgeAddr {
+	// If both regions have the same number of available addrs, lets randomise which one
+	// we pick. The rest of this algorithm will continue to make sure we always use addresses
+	// evenly across both regions.
+	if rs.region1.AvailableAddrs() == rs.region2.AvailableAddrs() {
+		regions := []Region{rs.region1, rs.region2}
+		firstChoice := rand.Intn(2)
+		return getAddrs(excluding, connID, &regions[firstChoice], &regions[1-firstChoice])
+	}
+
 	if rs.region1.AvailableAddrs() > rs.region2.AvailableAddrs() {
 		return getAddrs(excluding, connID, &rs.region1, &rs.region2)
 	}
@@ -95,14 +105,12 @@ func (rs *Regions) GetUnusedAddr(excluding *EdgeAddr, connID int) *EdgeAddr {
 // getAddrs tries to grab address form `first` region, then `second` region
 // this is an unrolled loop over 2 element array
 func getAddrs(excluding *EdgeAddr, connID int, first *Region, second *Region) *EdgeAddr {
-	addr := first.GetUnusedIP(excluding)
+	addr := first.AssignAnyAddress(connID, excluding)
 	if addr != nil {
-		first.Use(addr, connID)
 		return addr
 	}
-	addr = second.GetUnusedIP(excluding)
+	addr = second.AssignAnyAddress(connID, excluding)
 	if addr != nil {
-		second.Use(addr, connID)
 		return addr
 	}
 
@@ -116,18 +124,18 @@ func (rs *Regions) AvailableAddrs() int {
 
 // GiveBack the address so that other connections can use it.
 // Returns true if the address is in this edge.
-func (rs *Regions) GiveBack(addr *EdgeAddr) bool {
-	if found := rs.region1.GiveBack(addr); found {
+func (rs *Regions) GiveBack(addr *EdgeAddr, hasConnectivityError bool) bool {
+	if found := rs.region1.GiveBack(addr, hasConnectivityError); found {
 		return found
 	}
-	return rs.region2.GiveBack(addr)
+	return rs.region2.GiveBack(addr, hasConnectivityError)
 }
 
 // Return regionalized service name if `region` isn't empty, otherwise return the global service name for origintunneld
 func getRegionalServiceName(region string) string {
 	if region != "" {
-		return region + "-" + srvService // Example: `us-origintunneld`
+		return region + "-" + srvService // Example: `us-v2-origintunneld`
 	}
 
-	return srvService // Global service is just `origintunneld`
+	return srvService // Global service is just `v2-origintunneld`
 }

edgediscovery/allregions/regions_test.go

@@ -1,134 +1,215 @@
 package allregions
 
 import (
-	"net"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-var (
-	addr0 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.0"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.0"),
-			Port: 8000,
-			Zone: "",
-		},
+func makeRegions(addrs []*EdgeAddr, mode ConfigIPVersion) Regions {
+	r1addrs := make([]*EdgeAddr, 0)
+	r2addrs := make([]*EdgeAddr, 0)
+	for i, addr := range addrs {
+		if i%2 == 0 {
+			r1addrs = append(r1addrs, addr)
+		} else {
+			r2addrs = append(r2addrs, addr)
+		}
 	}
-	addr1 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.1"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.1"),
-			Port: 8000,
-			Zone: "",
-		},
-	}
-	addr2 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.2"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.2"),
-			Port: 8000,
-			Zone: "",
-		},
-	}
-	addr3 = EdgeAddr{
-		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.4.5.3"),
-			Port: 8000,
-			Zone: "",
-		},
-		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.4.5.3"),
-			Port: 8000,
-			Zone: "",
-		},
-	}
-)
-
-func makeRegions() Regions {
-	r1 := NewRegion([]*EdgeAddr{&addr0, &addr1})
-	r2 := NewRegion([]*EdgeAddr{&addr2, &addr3})
+	r1 := NewRegion(r1addrs, mode)
+	r2 := NewRegion(r2addrs, mode)
 	return Regions{region1: r1, region2: r2}
 }
 
 func TestRegions_AddrUsedBy(t *testing.T) {
-	rs := makeRegions()
-	addr1 := rs.GetUnusedAddr(nil, 1)
-	assert.Equal(t, addr1, rs.AddrUsedBy(1))
-	addr2 := rs.GetUnusedAddr(nil, 2)
-	assert.Equal(t, addr2, rs.AddrUsedBy(2))
-	addr3 := rs.GetUnusedAddr(nil, 3)
-	assert.Equal(t, addr3, rs.AddrUsedBy(3))
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			addr1 := rs.GetUnusedAddr(nil, 1)
+			assert.Equal(t, addr1, rs.AddrUsedBy(1))
+			addr2 := rs.GetUnusedAddr(nil, 2)
+			assert.Equal(t, addr2, rs.AddrUsedBy(2))
+			addr3 := rs.GetUnusedAddr(nil, 3)
+			assert.Equal(t, addr3, rs.AddrUsedBy(3))
+		})
+	}
 }
 
 func TestRegions_Giveback_Region1(t *testing.T) {
-	rs := makeRegions()
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
-	rs.region2.Use(&addr2, 2)
-	rs.region2.Use(&addr3, 3)
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			addr := rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			rs.region2.AssignAnyAddress(2, nil)
+			rs.region2.AssignAnyAddress(3, nil)
 
-	assert.Equal(t, 0, rs.AvailableAddrs())
+			assert.Equal(t, 0, rs.AvailableAddrs())
 
-	rs.GiveBack(&addr0)
-	assert.Equal(t, &addr0, rs.GetUnusedAddr(nil, 3))
+			rs.GiveBack(addr, false)
+			assert.Equal(t, addr, rs.GetUnusedAddr(nil, 0))
+		})
+	}
 }
 
 func TestRegions_Giveback_Region2(t *testing.T) {
-	rs := makeRegions()
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
-	rs.region2.Use(&addr2, 2)
-	rs.region2.Use(&addr3, 3)
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			addr := rs.region2.AssignAnyAddress(2, nil)
+			rs.region2.AssignAnyAddress(3, nil)
 
-	assert.Equal(t, 0, rs.AvailableAddrs())
+			assert.Equal(t, 0, rs.AvailableAddrs())
 
-	rs.GiveBack(&addr2)
-	assert.Equal(t, &addr2, rs.GetUnusedAddr(nil, 2))
+			rs.GiveBack(addr, false)
+			assert.Equal(t, addr, rs.GetUnusedAddr(nil, 2))
+		})
+	}
 }
 
 func TestRegions_GetUnusedAddr_OneAddrLeft(t *testing.T) {
-	rs := makeRegions()
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
+			rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			rs.region2.AssignAnyAddress(2, nil)
+			addr := rs.region2.active.GetUnusedIP(nil)
 
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
-	rs.region2.Use(&addr2, 2)
-
-	assert.Equal(t, 1, rs.AvailableAddrs())
-	assert.Equal(t, &addr3, rs.GetUnusedAddr(nil, 3))
+			assert.Equal(t, 1, rs.AvailableAddrs())
+			assert.Equal(t, addr, rs.GetUnusedAddr(nil, 3))
+		})
+	}
 }
 
 func TestRegions_GetUnusedAddr_Excluding_Region1(t *testing.T) {
-	rs := makeRegions()
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
 
-	rs.region1.Use(&addr0, 0)
-	rs.region1.Use(&addr1, 1)
+			rs.region1.AssignAnyAddress(0, nil)
+			rs.region1.AssignAnyAddress(1, nil)
+			addr := rs.region2.active.GetUnusedIP(nil)
+			a2 := rs.region2.active.GetUnusedIP(addr)
 
-	assert.Equal(t, 2, rs.AvailableAddrs())
-	assert.Equal(t, &addr3, rs.GetUnusedAddr(&addr2, 3))
+			assert.Equal(t, 2, rs.AvailableAddrs())
+			assert.Equal(t, addr, rs.GetUnusedAddr(a2, 3))
+		})
+	}
 }
 
 func TestRegions_GetUnusedAddr_Excluding_Region2(t *testing.T) {
-	rs := makeRegions()
+	tests := []struct {
+		name  string
+		addrs []*EdgeAddr
+		mode  ConfigIPVersion
+	}{
+		{
+			name:  "IPv4 addresses with IPv4Only",
+			addrs: v4Addrs,
+			mode:  IPv4Only,
+		},
+		{
+			name:  "IPv6 addresses with IPv6Only",
+			addrs: v6Addrs,
+			mode:  IPv6Only,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			rs := makeRegions(tt.addrs, tt.mode)
 
-	rs.region2.Use(&addr2, 0)
-	rs.region2.Use(&addr3, 1)
+			rs.region2.AssignAnyAddress(0, nil)
+			rs.region2.AssignAnyAddress(1, nil)
+			addr := rs.region1.active.GetUnusedIP(nil)
+			a2 := rs.region1.active.GetUnusedIP(addr)
 
-	assert.Equal(t, 2, rs.AvailableAddrs())
-	assert.Equal(t, &addr1, rs.GetUnusedAddr(&addr0, 1))
+			assert.Equal(t, 2, rs.AvailableAddrs())
+			assert.Equal(t, addr, rs.GetUnusedAddr(a2, 1))
+		})
+	}
 }
 
 func TestNewNoResolveBalancesRegions(t *testing.T) {

edgediscovery/edgediscovery.go

@@ -1,7 +1,6 @@
 package edgediscovery
 
 import (
-	"fmt"
 	"sync"
 
 	"github.com/rs/zerolog"
@@ -11,9 +10,16 @@ import (
 
 const (
 	LogFieldConnIndex = "connIndex"
+	LogFieldIPAddress = "ip"
 )
 
-var errNoAddressesLeft = fmt.Errorf("there are no free edge addresses left")
+var errNoAddressesLeft = ErrNoAddressesLeft{}
+
+type ErrNoAddressesLeft struct{}
+
+func (e ErrNoAddressesLeft) Error() string {
+	return "there are no free edge addresses left to resolve to"
+}
 
 // Edge finds addresses on the Cloudflare edge and hands them out to connections.
 type Edge struct {
@@ -28,8 +34,8 @@ type Edge struct {
 
 // ResolveEdge runs the initial discovery of the Cloudflare edge, finding Addrs that can be allocated
 // to connections.
-func ResolveEdge(log *zerolog.Logger, region string) (*Edge, error) {
-	regions, err := allregions.ResolveEdge(log, region)
+func ResolveEdge(log *zerolog.Logger, region string, edgeIpVersion allregions.ConfigIPVersion) (*Edge, error) {
+	regions, err := allregions.ResolveEdge(log, region, edgeIpVersion)
 	if err != nil {
 		return new(Edge), err
 	}
@@ -51,15 +57,6 @@ func StaticEdge(log *zerolog.Logger, hostnames []string) (*Edge, error) {
 	}, nil
 }
 
-// MockEdge creates a Cloudflare Edge from arbitrary TCP addresses. Used for testing.
-func MockEdge(log *zerolog.Logger, addrs []*allregions.EdgeAddr) *Edge {
-	regions := allregions.NewNoResolve(addrs)
-	return &Edge{
-		log:     log,
-		regions: regions,
-	}
-}
-
 // ------------------------------------
 // Methods
 // ------------------------------------
@@ -93,12 +90,15 @@ func (ed *Edge) GetAddr(connIndex int) (*allregions.EdgeAddr, error) {
 		log.Debug().Msg("edgediscovery - GetAddr: No addresses left to give proxy connection")
 		return nil, errNoAddressesLeft
 	}
-	log.Debug().Msg("edgediscovery - GetAddr: Giving connection its new address")
+	log = ed.log.With().
+		Int(LogFieldConnIndex, connIndex).
+		IPAddr(LogFieldIPAddress, addr.UDP.IP).Logger()
+	log.Debug().Msgf("edgediscovery - GetAddr: Giving connection its new address")
 	return addr, nil
 }
 
 // GetDifferentAddr gives back the proxy connection's edge Addr and uses a new one.
-func (ed *Edge) GetDifferentAddr(connIndex int) (*allregions.EdgeAddr, error) {
+func (ed *Edge) GetDifferentAddr(connIndex int, hasConnectivityError bool) (*allregions.EdgeAddr, error) {
 	log := ed.log.With().Int(LogFieldConnIndex, connIndex).Logger()
 
 	ed.Lock()
@@ -106,7 +106,7 @@ func (ed *Edge) GetDifferentAddr(connIndex int) (*allregions.EdgeAddr, error) {
 
 	oldAddr := ed.regions.AddrUsedBy(connIndex)
 	if oldAddr != nil {
-		ed.regions.GiveBack(oldAddr)
+		ed.regions.GiveBack(oldAddr, hasConnectivityError)
 	}
 	addr := ed.regions.GetUnusedAddr(oldAddr, connIndex)
 	if addr == nil {
@@ -114,7 +114,10 @@ func (ed *Edge) GetDifferentAddr(connIndex int) (*allregions.EdgeAddr, error) {
 		// note: if oldAddr were not nil, it will become available on the next iteration
 		return nil, errNoAddressesLeft
 	}
-	log.Debug().Msg("edgediscovery - GetDifferentAddr: Giving connection its new address")
+	log = ed.log.With().
+		Int(LogFieldConnIndex, connIndex).
+		IPAddr(LogFieldIPAddress, addr.UDP.IP).Logger()
+	log.Debug().Msgf("edgediscovery - GetDifferentAddr: Giving connection its new address from the address list: %v", ed.regions.AvailableAddrs())
 	return addr, nil
 }
 
@@ -127,9 +130,11 @@ func (ed *Edge) AvailableAddrs() int {
 
 // GiveBack the address so that other connections can use it.
 // Returns true if the address is in this edge.
-func (ed *Edge) GiveBack(addr *allregions.EdgeAddr) bool {
+func (ed *Edge) GiveBack(addr *allregions.EdgeAddr, hasConnectivityError bool) bool {
 	ed.Lock()
 	defer ed.Unlock()
-	ed.log.Debug().Msg("edgediscovery - GiveBack: Address now unused")
-	return ed.regions.GiveBack(addr)
+	log := ed.log.With().
+		IPAddr(LogFieldIPAddress, addr.UDP.IP).Logger()
+	log.Debug().Msgf("edgediscovery - GiveBack: Address now unused")
+	return ed.regions.GiveBack(addr, hasConnectivityError)
 }

edgediscovery/edgediscovery_test.go

@@ -11,56 +11,113 @@ import (
 )
 
 var (
-	addr0 = allregions.EdgeAddr{
+	testLogger = zerolog.Nop()
+	v4Addrs    = []*allregions.EdgeAddr{&addr0, &addr1, &addr2, &addr3}
+	v6Addrs    = []*allregions.EdgeAddr{&addr4, &addr5, &addr6, &addr7}
+	addr0      = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.0"),
+			IP:   net.ParseIP("123.4.5.0"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.0"),
+			IP:   net.ParseIP("123.4.5.0"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
 	}
 	addr1 = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.1"),
+			IP:   net.ParseIP("123.4.5.1"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.1"),
+			IP:   net.ParseIP("123.4.5.1"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
 	}
 	addr2 = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.2"),
+			IP:   net.ParseIP("123.4.5.2"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.2"),
+			IP:   net.ParseIP("123.4.5.2"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
 	}
 	addr3 = allregions.EdgeAddr{
 		TCP: &net.TCPAddr{
-			IP:   net.ParseIP("123.0.0.3"),
+			IP:   net.ParseIP("123.4.5.3"),
 			Port: 8000,
 			Zone: "",
 		},
 		UDP: &net.UDPAddr{
-			IP:   net.ParseIP("123.0.0.3"),
+			IP:   net.ParseIP("123.4.5.3"),
 			Port: 8000,
 			Zone: "",
 		},
+		IPVersion: allregions.V4,
+	}
+	addr4 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::1"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
+	}
+	addr5 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::2"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
+	}
+	addr6 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::3"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
+	}
+	addr7 = allregions.EdgeAddr{
+		TCP: &net.TCPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		UDP: &net.UDPAddr{
+			IP:   net.ParseIP("2606:4700:a0::4"),
+			Port: 8000,
+			Zone: "",
+		},
+		IPVersion: allregions.V6,
 	}
-
-	testLogger = zerolog.Nop()
 )
 
 func TestGiveBack(t *testing.T) {
@@ -75,7 +132,7 @@ func TestGiveBack(t *testing.T) {
 	assert.Equal(t, 3, edge.AvailableAddrs())
 
 	// Get it back
-	edge.GiveBack(addr)
+	edge.GiveBack(addr, false)
 	assert.Equal(t, 4, edge.AvailableAddrs())
 }
 
@@ -107,7 +164,7 @@ func TestGetAddrForRPC(t *testing.T) {
 	assert.Equal(t, 4, edge.AvailableAddrs())
 
 	// Get it back
-	edge.GiveBack(addr)
+	edge.GiveBack(addr, false)
 	assert.Equal(t, 4, edge.AvailableAddrs())
 }
 
@@ -122,13 +179,13 @@ func TestOnePerRegion(t *testing.T) {
 	assert.NotNil(t, a1)
 
 	// if the first address is bad, get the second one
-	a2, err := edge.GetDifferentAddr(connID)
+	a2, err := edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.NotNil(t, a2)
 	assert.NotEqual(t, a1, a2)
 
 	// now that second one is bad, get the first one again
-	a3, err := edge.GetDifferentAddr(connID)
+	a3, err := edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.Equal(t, a1, a3)
 }
@@ -144,11 +201,11 @@ func TestOnlyOneAddrLeft(t *testing.T) {
 	assert.NotNil(t, addr)
 
 	// If that edge address is "bad", there's no alternative address.
-	_, err = edge.GetDifferentAddr(connID)
+	_, err = edge.GetDifferentAddr(connID, false)
 	assert.Error(t, err)
 
 	// previously bad address should become available again on next iteration.
-	addr, err = edge.GetDifferentAddr(connID)
+	addr, err = edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.NotNil(t, addr)
 }
@@ -190,8 +247,17 @@ func TestGetDifferentAddr(t *testing.T) {
 	assert.Equal(t, 3, edge.AvailableAddrs())
 
 	// If the same connection requests another address, it should get the same one.
-	addr2, err := edge.GetDifferentAddr(connID)
+	addr2, err := edge.GetDifferentAddr(connID, false)
 	assert.NoError(t, err)
 	assert.NotEqual(t, addr, addr2)
 	assert.Equal(t, 3, edge.AvailableAddrs())
 }
+
+// MockEdge creates a Cloudflare Edge from arbitrary TCP addresses. Used for testing.
+func MockEdge(log *zerolog.Logger, addrs []*allregions.EdgeAddr) *Edge {
+	regions := allregions.NewNoResolve(addrs)
+	return &Edge{
+		log:     log,
+		regions: regions,
+	}
+}