Release 2022.5.2

Allow for cloudflared to be built with debug mode to be used with dlv exec

.docker-images

@@ -2,6 +2,8 @@ images:
   - name: cloudflared
     dockerfile: Dockerfile
     context: .
+    versions: 
+      - latest
     registries:
     - name: docker.io/cloudflare
       user: env:DOCKER_USER

.github/ISSUE_TEMPLATE/bug-report---.md

@@ -0,0 +1,31 @@
+---
+name: "Bug report \U0001F41B"
+about: Create a report to help us improve cloudflared
+title: ''
+labels: awaiting reply, bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Configure '...'
+2. Run '....'
+3. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Environment and versions**
+ - OS: [e.g. MacOS]
+ - Architecture: [e.g. AMD, ARM]
+ - Version: [e.g. 2022.02.0]
+
+**Logs and errors**
+If applicable, add logs or errors to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature-request---.md

@@ -0,0 +1,17 @@
+---
+name: "Feature request \U0001F4A1"
+about: Suggest a feature or enhancement for cloudflared
+title: ''
+labels: awaiting reply, feature-request
+assignees: ''
+
+---
+
+**Describe the feature you'd like**
+A clear and concise description of the feature. What problem does it solve for you?
+
+**Describe alternatives you've considered**
+Are there any alternatives to solving this problem? If so, what was your experience with them?
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.gitignore

@@ -10,6 +10,7 @@ cscope.*
 /cloudflared.exe
 /cloudflared.msi
 /cloudflared-x86-64*
+/cloudflared.1
 /packaging
 .DS_Store
 *-session.log

.teamcity/update-homebrew-core.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if ! VERSION="$(git describe --tags --exact-match 2>/dev/null)" ; then
+    echo "Skipping public release for an untagged commit."
+    echo "##teamcity[buildStatus status='SUCCESS' text='Skipped due to lack of tag']"
+    exit 0
+fi
+
+if [[ "${HOMEBREW_GITHUB_API_TOKEN:-}" == "" ]] ; then
+    echo "Missing GITHUB_API_TOKEN"
+    exit 1
+fi
+
+# "install" Homebrew
+git clone https://github.com/Homebrew/brew tmp/homebrew
+eval "$(tmp/homebrew/bin/brew shellenv)"
+brew update --force --quiet
+chmod -R go-w "$(brew --prefix)/share/zsh"
+
+git config --global user.name "cloudflare-warp-bot"
+git config --global user.email "warp-bot@cloudflare.com"
+
+# bump formula pr
+brew bump-formula-pr cloudflared --version="$VERSION" --no-browse

CHANGES.md

@@ -1,4 +1,59 @@
-**Experimental**: This is a new format for release notes. The format and availability is subject to change.
+## 2022.4.0
+### Bug Fixes
+- `cloudflared tunnel run` no longer logs the Tunnel token or JSON credentials in clear text as those are the secret
+that allows to run the Tunnel.
+
+## 2022.3.4
+### New Features
+- It is now possible to retrieve the credentials that allow to run a Tunnel in case you forgot/lost them. This is
+achievable with: `cloudflared tunnel token --cred-file /path/to/file.json TUNNEL`. This new feature only works for
+Tunnels created with cloudflared version 2022.3.0 or more recent.
+
+### Bug Fixes
+- `cloudflared service install` now starts the underlying agent service on Linux operating system (similarly to the
+behaviour in Windows and MacOS).
+
+## 2022.3.3
+### Bug Fixes
+- `cloudflared service install` now starts the underlying agent service on Windows operating system (similarly to the
+behaviour in MacOS).
+
+## 2022.3.1
+### Bug Fixes
+- Various fixes to the reliability of `quic` protocol, including an edge case that could lead to cloudflared crashing.
+
+## 2022.3.0
+### New Features
+- It is now possible to configure Ingress Rules to point to an origin served by unix socket with either HTTP or HTTPS.
+If the origin starts with `unix:/` then we assume HTTP (existing behavior). Otherwise, the origin can start with
+`unix+tls:/` for HTTPS.
+
+## 2022.2.1
+### New Features
+- This project now has a new LICENSE that is more compliant with open source purposes.
+
+### Bug Fixes
+- Various fixes to the reliability of `quic` protocol.
+
+## 2022.1.3
+### New Features
+- New `cloudflared tunnel vnet` commands to allow for private routing to be virtualized. This means that the same CIDR
+can now be used to point to two different Tunnels with `cloudflared tunnel route ip` command. More information will be
+made available on blog.cloudflare.com and developers.cloudflare.com/cloudflare-one once the feature is globally available.
+
+### Bug Fixes
+- Correctly handle proxying UDP datagrams with no payload.
+- Bug fix for origins that use Server-Sent Events (SSE).
+
+## 2022.1.0
+### Improvements
+- If a specific `protocol` property is defined (e.g. for `quic`), cloudflared no longer falls back to an older protocol
+(such as `http2`) in face of connectivity errors. This is important because some features are only supported in a specific
+protocol (e.g. UDP proxying only works for `quic`). Hence, if a user chooses a protocol, cloudflared now adheres to it
+no matter what.
+
+### Bug Fixes
+- Stopping cloudflared running with `quic` protocol now respects graceful shutdown.
 
 ## 2021.12.2
 ### Bug Fixes

LICENSE

@@ -1,155 +1,202 @@
-SERVICES AGREEMENT
 
-Your installation of this software is symbol of your signature indicating that
-you accept the terms of this Services Agreement (this "Agreement").  This
-Agreement is a legal agreement between you (either an individual or a single
-entity) and CloudFlare, Inc. for the services being provided to you by
-CloudFlare or its authorized representative (the "Services"), including any
-computer software and any associated media, printed materials, and "online" or
-electronic documentation provided in connection with the Services (the
-"Software" and together with the Services are hereinafter collectively referred
-to as the "Solution").  If the user is not an individual, then "you" means your
-company, its officers, members, employees, agents, representatives, successors
-and assigns.  BY USING THE SOLUTION, YOU ARE INDICATING THAT YOU HAVE READ, AND
-AGREE TO BE BOUND BY, THE POLICIES, TERMS, AND CONDITIONS SET FORTH BELOW IN
-THEIR ENTIRETY WITHOUT LIMITATION OR QUALIFICATION, AS WELL AS BY ALL APPLICABLE
-LAWS AND REGULATIONS, AS IF YOU HAD HANDWRITTEN YOUR NAME ON A CONTRACT.  IF YOU
-DO NOT AGREE TO THESE TERMS AND CONDITIONS, YOU MAY NOT USE THE SOLUTION.
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-1.    GRANT OF RIGHTS
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-1.1	Grant of License.  The Solution is licensed by CloudFlare and its
-licensors, not sold.  Subject to the terms and conditions of this Agreement,
-CloudFlare hereby grants you a nonexclusive, nonsublicensable, nontransferable
-license to use the Solution.  You may examine source code, if provided to you,
-solely for the limited purpose of evaluating the Software for security flaws.
-You may also use the Service to create derivative works which are exclusively
-compatible with any CloudFlare product serviceand no other product or service.
-This license applies to the parts of the Solution developed by CloudFlare.  The
-Solution may also incorporate externally maintained libraries and other open software.
-These resources may be governed by other licenses.
+   1. Definitions.
 
-1.2	Restrictions.  The license granted herein is granted solely to you and
-not, by implication or otherwise, to any of your parents, subsidiaries or
-affiliates.  No right is granted hereunder to use the Solution to perform
-services for third parties. All rights not expressly granted hereunder are
-reserved to CloudFlare.  You may not use the Solution except as explicitly
-permitted under this Agreement.  You are expressly prohibited from modifying,
-adapting, translating, preparing derivative works from, decompiling, reverse
-engineering, disassembling or otherwise attempting to derive source code from
-the Software used to provide the Services or any internal data files generated
-by the Solution.  You are also prohibited from removing, obscuring or altering
-any copyright notice, trademarks, or other proprietary rights notices affixed to
-or associated with the Solution.
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
 
-1.3	Ownership.  As between the parties, CloudFlare and/or its licensors own
-and shall retain all right, title, and interest in and to the Solution,
-including any and all technology embodied therein, including all copyrights,
-patents, trade secrets, trade dress and other proprietary rights associated
-therewith, and any derivative works created there from.
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
 
-2.	LIMITATION OF LIABILITY
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
 
-YOU EXPRESSLY ACKNOWLEDGE AND AGREE THAT DOWNLOADING THE SOFTWARE IS AT YOUR
-SOLE RISK.  THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTY OF ANY KIND
-AND CLOUDFLARE, ITS LICENSORS AND ITS AUTHORIZED REPRESENTATIVES (TOGETHER FOR
-PURPOSES HEREOF, "CLOUDFLARE") EXPRESSLY DISCLAIM ALL WARRANTIES, EXPRESS OR
-IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  CLOUDFLARE DOES NOT
-WARRANT THAT THE FUNCTIONS CONTAINED IN THE SOFTWARE WILL MEET YOUR
-REQUIREMENTS, OR THAT THE OPERATION OF THE SOFTWARE WILL BE UNINTERRUPTED OR
-ERROR-FREE, OR THAT DEFECTS IN THE SOFTWARE WILL BE CORRECTED.  FURTHERMORE,
-CLOUDFLARE DOES NOT WARRANT OR MAKE ANY REPRESENTATIONS REGARDING THE SOFTWARE
-OR RELATED DOCUMENTATION IN TERMS OF THEIR CORRECTNESS, ACCURACY, RELIABILITY,
-OR OTHERWISE. NO ORAL OR WRITTEN INFORMATION OR ADVICE GIVEN BY CLOUDFLARE SHALL
-CREATE A WARRANTY OR IN ANY WAY INCREASE THE SCOPE OF THIS WARRANTY.
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
 
-3.	CONFIDENTIALITY
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
 
-It may be necessary during the set up and performance of the Solution for the
-parties to exchange Confidential Information. "Confidential Information" means
-any information whether oral, or written, of a private, secret, proprietary or
-confidential nature, concerning either party or its business operations,
-including without limitation: (a) your data and (b) CloudFlare's access control
-systems, specialized network equipment and techniques related to the Solution,
-use policies, which include trade secrets of CloudFlare and its licensors. Each
-party agrees to use the same degree of care to protect the confidentiality of
-the Confidential Information of the other party and to prevent its unauthorized
-use or dissemination as it uses to protect its own Confidential Information of a
-similar nature, but in no event shall exercise less than due diligence and
-reasonable care. Each party agrees to use the Confidential Information of the
-other party only for purposes related to the performance of this Agreement. All
-Confidential Information remains the property of the party disclosing the
-information and no license or other rights to Confidential Information is
-granted or implied hereby.
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
 
-4.	TERM AND TERMINATION
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
 
-4.1	Term.  This Agreement shall be effective upon download or install of the
-Software.
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
 
-4.2	Termination. This Agreement may be terminated by CloudFlare or its
-authorized representative by written notice to you if any of the following
-events occur:  (i) you fail to pay any amounts due for the Services and the
-Solution when due and after written notice of such nonpayment has been given to
-you; (ii) you are in material breach of any term, condition, or provision of
-this Agreement or any other agreement executed by you with CloudFlare or its
-authorized representative in connection with the provision of the Solution and
-Services (a "Related Agreement"); or (iii) you terminate or suspend your
-business, becomes subject to any bankruptcy or insolvency proceeding under
-federal or state statutes, or become insolvent or subject to direct control by a
-trustee, receiver or similar authority.
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
 
-4.3	Effect of Termination.  Upon the termination of this Agreement for any
-reason: (1) all license rights granted hereunder shall terminate and (2) all
-Confidential Information shall be returned to the disclosing party or destroyed.
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
 
-5.	MISCELLANEOUS
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
 
-5.1	Assignment.  You may not assign any of your rights or delegate any of
-your obligations under this Agreement, whether by operation of law or otherwise,
-without the prior express written consent of CloudFlare or its authorized
-representative.  Any such assignment without the prior express written consent
-of CloudFlare or its authorized representative shall be void.  Subject to the
-foregoing, this Agreement will bind and inure to the benefit of the parties,
-their respective successors and permitted assigns.
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
 
-5.2	Waiver and Amendment.  No modification, amendment or waiver of any
-provision of this Agreement shall be effective unless in writing and signed by
-the party to be charged.  No failure or delay by either party in exercising any
-right, power, or remedy under this Agreement, except as specifically provided
-herein, shall operate as a waiver of any such right, power or remedy.  Without
-limiting the foregoing, terms and conditions on any purchase orders or similar
-materials submitted by you to CloudFlare or its authorized representative shall
-be of no force or effect.
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
 
-5.3	Governing Law.  This Agreement shall be governed by the laws of the State
-of California, USA, excluding conflict of laws and provisions, and excluding the
-United Nations Convention on Contracts for the International Sale of Goods.
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
 
-5.4	Notices.  All notices, demands or consents required or permitted under
-this Agreement shall be in writing.  Notice shall be sent to you at the e-mail
-address provided by you to CloudFlare or its authorized representative in
-connection with the Solution.
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
 
-5.5	Independent Contractors. The parties are independent contractors.
-Neither party shall be deemed to be an employee, agent, partner or legal
-representative of the other for any purpose and neither shall have any right,
-power or authority to create any obligation or responsibility on behalf of the
-other.
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
 
-5.6	Severability.  If any provision of this Agreement is held by a court of
-competent jurisdiction to be contrary to law, such provision shall be changed
-and interpreted so as to best accomplish the objectives of the original
-provision to the fullest extent allowed by law and the remaining provisions of
-this Agreement shall remain in full force and effect.
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
 
-5.7	Force Majeure.  CloudFlare shall not be liable to the other party for any
-failure or delay in performance caused by reasons beyond its reasonable control.
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
 
-5.8	Complete Understanding.  This Agreement and the Related Agreement
-constitute the final, complete and exclusive agreement between the parties with
-respect to the subject matter hereof, and supersedes all previous written and
-oral agreements and communications related to the subject matter of this
-Agreement.  To the extent this Agreement and the Related Agreement conflict,
-this Agreement shall control.
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -23,6 +23,9 @@ endif
 
 DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
 VERSION_FLAGS := -X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"
+ifdef PACKAGE_MANAGER
+	VERSION_FLAGS := $(VERSION_FLAGS) -X "github.com/cloudflare/cloudflared/cmd/cloudflared/updater.BuiltForPackageManager=$(PACKAGE_MANAGER)"
+endif
 
 LINK_FLAGS :=
 ifeq ($(FIPS), true)
@@ -37,10 +40,15 @@ ifneq ($(GO_BUILD_TAGS),)
 	GO_BUILD_TAGS := -tags "$(GO_BUILD_TAGS)"
 endif
 
-IMPORT_PATH   := github.com/cloudflare/cloudflared
-PACKAGE_DIR   := $(CURDIR)/packaging
-INSTALL_BINDIR := /usr/bin/
-MAN_DIR := /usr/share/man/man1/
+ifeq ($(debug), 1)
+	GO_BUILD_TAGS += -gcflags="all=-N -l"
+endif
+
+IMPORT_PATH    := github.com/cloudflare/cloudflared
+PACKAGE_DIR    := $(CURDIR)/packaging
+PREFIX         := /usr
+INSTALL_BINDIR := $(PREFIX)/bin/
+INSTALL_MANDIR := $(PREFIX)/share/man/man1/
 
 LOCAL_ARCH ?= $(shell uname -m)
 ifneq ($(GOARCH),)
@@ -59,6 +67,8 @@ else ifeq ($(LOCAL_ARCH),arm64)
     TARGET_ARCH ?= arm64
 else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 4),armv)
     TARGET_ARCH ?= arm
+else ifeq ($(LOCAL_ARCH),s390x)
+    TARGET_ARCH ?= s390x
 else
     $(error This system's architecture $(LOCAL_ARCH) isn't supported)
 endif
@@ -141,32 +151,40 @@ publish-deb: cloudflared-deb
 publish-rpm: cloudflared-rpm
 	$(call publish_package,rpm,yum)
 
+cloudflared.1: cloudflared_man_template
+	cat cloudflared_man_template | sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' > cloudflared.1
+
+install: cloudflared cloudflared.1
+	mkdir -p $(DESTDIR)$(INSTALL_BINDIR) $(DESTDIR)$(INSTALL_MANDIR)
+	install -m755 cloudflared $(DESTDIR)$(INSTALL_BINDIR)/cloudflared
+	install -m644 cloudflared.1 $(DESTDIR)$(INSTALL_MANDIR)/cloudflared.1
+
 # When we build packages, the package name will be FIPS-aware.
 # But we keep the binary installed by it to be named "cloudflared" regardless.
 define build_package
 	mkdir -p $(PACKAGE_DIR)
 	cp cloudflared $(PACKAGE_DIR)/cloudflared
-	cat cloudflared_man_template | sed -e 's/\$${VERSION}/$(VERSION)/; s/\$${DATE}/$(DATE)/' > $(PACKAGE_DIR)/cloudflared.1
+	cp cloudflared.1 $(PACKAGE_DIR)/cloudflared.1
 	fakeroot fpm -C $(PACKAGE_DIR) -s dir -t $(1) \
 		--description 'Cloudflare Tunnel daemon' \
 		--vendor 'Cloudflare' \
-		--license 'Cloudflare Service Agreement' \
+		--license 'Apache License Version 2.0' \
 		--url 'https://github.com/cloudflare/cloudflared' \
 		-m 'Cloudflare <support@cloudflare.com>' \
 		-a $(TARGET_ARCH) -v $(VERSION) -n $(DEB_PACKAGE_NAME) $(NIGHTLY_FLAGS) --after-install postinst.sh --after-remove postrm.sh \
-		cloudflared=$(INSTALL_BINDIR) cloudflared.1=$(MAN_DIR)
+		cloudflared=$(INSTALL_BINDIR) cloudflared.1=$(INSTALL_MANDIR)
 endef
 
 .PHONY: cloudflared-deb
-cloudflared-deb: cloudflared
+cloudflared-deb: cloudflared cloudflared.1
 	$(call build_package,deb)
 
 .PHONY: cloudflared-rpm
-cloudflared-rpm: cloudflared
+cloudflared-rpm: cloudflared cloudflared.1
 	$(call build_package,rpm)
 
 .PHONY: cloudflared-pkg
-cloudflared-pkg: cloudflared
+cloudflared-pkg: cloudflared cloudflared.1
 	$(call build_package,osxpkg)
 
 .PHONY: cloudflared-msi
@@ -247,6 +265,10 @@ github-release: cloudflared
 github-release-built-pkgs:
 	python3 github_release.py --path $(PWD)/built_artifacts --release-version $(VERSION)
 
+.PHONY: release-pkgs-linux
+release-pkgs-linux:
+	python3 ./release_pkgs.py
+
 .PHONY: github-message
 github-message:
 	python3 github_message.py --release-version $(VERSION)
@@ -279,4 +301,4 @@ vet:
 
 .PHONY: goimports
 goimports:
-	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done
+	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done

RELEASE_NOTES

@@ -1,3 +1,119 @@
+2022.5.2
+- 2022-05-23 TUN-6270: Import gpg keys from environment variables
+- 2022-05-24 TUN-6209: Improve feedback process if release_pkgs to deb and rpm fail
+- 2022-05-24 TUN-6280: Don't wrap qlog connection tracer for gatethering QUIC metrics since we're not writing qlog files.
+- 2022-05-25 TUN-6209: Sign RPM packages
+- 2022-05-25 TUN-6285: Upload pkg assets to repos when cloudflared is released.
+- 2022-05-24 TUN-6282: Upgrade golang to 1.17.10, go-boring to 1.17.9
+- 2022-05-26 TUN-6292: Debug builds for cloudflared
+- 2022-05-28 TUN-6304: Fixed some file permission issues
+- 2022-05-11 TUN-6197: Publish to brew core should not try to open the browser
+- 2022-05-12 TUN-5943: Add RPM support
+- 2022-05-18 TUN-6248: Fix panic in cloudflared during tracing when origin doesn't provide header map
+- 2022-05-18 TUN-6250: Add upstream response status code to tracing span attributes
+
+2022.5.1
+- 2022-05-06 TUN-6146: Release_pkgs is now a generic command line script
+- 2022-05-06 TUN-6185: Fix tcpOverWSOriginService not using original scheme for String representation
+- 2022-05-05 TUN-6175: Simply debian packaging by structural upload
+- 2022-05-05 TUN-5945: Added support for Ubuntu releases
+- 2022-05-04 TUN-6054: Create and upload deb packages to R2
+- 2022-05-03 TUN-6161: Set git user/email for brew core release
+- 2022-05-03 TUN-6166: Fix mocked QUIC transport for UDP proxy manager to return expected error
+- 2022-04-27 TUN-6016: Push local managed tunnels configuration to the edge
+2022.5.0
+- 2022-05-02 TUN-6158: Update golang.org/x/crypto
+- 2022-04-20 VULN-8383 Bump yaml.v2 to yaml.v3
+- 2022-04-21 TUN-6123: For a given connection with edge, close all datagram sessions through this connection when it's closed
+- 2022-04-20 TUN-6015: Add RPC method for pushing local config
+- 2022-04-21 TUN-6130: Fix vendoring due to case sensitive typo in package
+- 2022-04-27 TUN-6142: Add tunnel details support to RPC
+- 2022-04-28 TUN-6014: Add remote config flag as default feature
+- 2022-04-12 TUN-6000: Another fix for publishing to brew core
+- 2022-04-11 TUN-5990: Add otlp span export to response header
+- 2022-04-19 TUN-6070: First connection retries other edge IPs if the error is quic timeout(likely due to firewall blocking UDP)
+- 2022-04-11 TUN-6030: Add ttfb span for origin http request
+
+2022.4.1
+- 2022-04-11 TUN-6035: Reduce buffer size when proxying data
+- 2022-04-11 TUN-6038: Reduce buffer size used for proxying data
+- 2022-04-11 TUN-6043: Allow UI-managed Tunnels to fallback from QUIC but warn about that
+- 2022-04-07 TUN-6000 add version argument to bump-formula-pr
+- 2022-04-06 TUN-5989: Add in-memory otlp exporter
+
+2022.4.0
+- 2022-04-01 TUN-5973: Add backoff for non-recoverable errors as well
+- 2022-04-05 TUN-5992: Use QUIC protocol for remotely managed tunnels when protocol is unspecified
+- 2022-04-06 Update Makefile
+- 2022-04-06 TUN-5995: Update prometheus to 1.12.1 to avoid vulnerabilities
+- 2022-04-07 TUN-5995: Force prometheus v1.12.1 usage
+- 2022-04-07 TUN-4130: cloudflared docker images now have a latest tag
+- 2022-03-30 TUN-5842: Fix flaky TestConcurrentUpdateAndRead by making sure resources are released
+- 2022-03-30 carrier: fix dropped errors
+- 2022-03-25 TUN-5959: tidy go.mod
+- 2022-03-25 TUN-5958: Fix release to homebrew core
+- 2022-03-28 TUN-5960: Do not log the tunnel token or json credentials
+- 2022-03-28 TUN-5956: Add timeout to session manager APIs
+
+2022.3.4
+- 2022-03-22 TUN-5918: Clean up text in cloudflared tunnel --help
+- 2022-03-22 TUN-5895 run brew bump-formula-pr on release
+- 2022-03-22 TUN-5915: New cloudflared command to allow to retrieve the token credentials for a Tunnel
+- 2022-03-24 TUN-5933: Better messaging to help user when installing service if it is already installed
+- 2022-03-25 TUN-5954: Start cloudflared service in Linux too similarly to other OSs
+- 2022-03-14 TUN-5869: Add configuration endpoint in metrics server
+
+2022.3.3
+- 2022-03-17 TUN-5893: Start windows service on install, stop on uninstall. Previously user had to manually start the service after running 'cloudflared tunnel install' and stop the service before running uninstall command.
+- 2022-03-17 Revert "CC-796: Remove dependency on unsupported version of go-oidc"
+- 2022-03-18 TUN-5881: Clarify success (or lack thereof) of (un)installing cloudflared service
+- 2022-03-18 CC-796: Remove dependency on unsupported version of go-oidc
+- 2022-03-18 TUN-5907: Change notes for 2022.3.3
+
+2022.3.2
+- 2022-03-10 TUN-5833: Create constant for allow-remote-config
+- 2022-03-15 TUN-5867: Return error if service was already installed
+- 2022-03-16 TUN-5833: Send feature `allow_remote_config` if Tunnel is run with --token
+- 2022-03-08 TUN-5849: Remove configuration debug log
+- 2022-03-08 TUN-5850: Update CHANGES.md with latest releases
+- 2022-03-08 TUN-5851: Update all references to point to Apache License 2.0
+- 2022-03-07 TUN-5853 Add "install" make target and build package manager info into executable
+- 2022-03-08 TUN-5801: Add custom wrapper for OriginConfig for JSON serde
+- 2022-03-09 TUN-5703: Add prometheus metric for current configuration version
+- 2022-02-05 CC-796: Remove dependency on unsupported version of go-oidc
+
+2022.3.1
+- 2022-03-04 TUN-5837: Log panic recovery in http2 logic with debug level log
+- 2022-03-04  TUN-5696: HTTP/2 Configuration Update
+- 2022-03-04 TUN-5836: Avoid websocket#Stream function from crashing cloudflared with unexpected memory access
+- 2022-03-05 TUN-5836: QUIC transport no longer sets body to nil in any condition
+
+2022.3.0
+- 2022-03-02 TUN-5680: Adapt component tests for new service install based on token
+- 2022-02-21 TUN-5682: Remove name field from credentials
+- 2022-02-21 TUN-5681: Add support for running tunnel using Token
+- 2022-02-28 TUN-5824: Update updater no-update-in-shell link
+- 2022-02-28 TUN-5823: Warn about legacy flags that are ignored when ingress rules are used
+- 2022-02-28 TUN-5737: Support https protocol over unix socket origin
+- 2022-02-23 TUN-5679: Add support for service install using Tunnel Token
+
+2022.2.2
+- 2022-02-22 TUN-5754: Allow ingress validate to take plaintext option
+- 2022-02-17 TUN-5678: Cloudflared uses typed tunnel API
+
+2022.2.1
+- 2022-02-10 TUN-5184: Handle errors in bidrectional streaming (websocket#Stream) gracefully when 1 side has ended
+- 2022-02-14 Update issue templates
+- 2022-02-14 Update issue templates
+- 2022-02-11 TUN-5768: Update cloudflared license file
+- 2022-02-11 TUN-5698: Make ingress rules and warp routing dynamically configurable
+- 2022-02-14 TUN-5678: Adapt cloudflared to use new typed APIs
+- 2022-02-17 Revert "TUN-5678: Adapt cloudflared to use new typed APIs"
+- 2022-02-11 TUN-5697: Listen for UpdateConfiguration RPC in quic transport
+- 2022-02-04 TUN-5744: Add a test to make sure cloudflared uses scheme defined in ingress rule, not X-Forwarded-Proto header
+- 2022-02-07 TUN-5749: Refactor cloudflared to pave way for reconfigurable ingress - Split origin into supervisor and proxy packages - Create configManager to handle dynamic config
+- 2021-10-19 TUN-5184: Make sure outstanding websocket write is finished, and no more writes after shutdown
+
 2022.2.0
 - 2022-02-02 TUN-4947: Use http when talking to Unix sockets origins
 - 2022-02-02 TUN-5695: Define RPC method to update configuration

carrier/websocket.go

@@ -55,6 +55,9 @@ func createWebsocketStream(options *StartOptions, log *zerolog.Logger) (*cfwebso
 	}
 
 	dump, err := httputil.DumpRequest(req, false)
+	if err != nil {
+		return nil, err
+	}
 	log.Debug().Msgf("Websocket request: %s", string(dump))
 
 	dialer := &websocket.Dialer{
@@ -182,6 +185,9 @@ func createAccessWebSocketStream(options *StartOptions, log *zerolog.Logger) (*w
 	}
 
 	dump, err := httputil.DumpRequest(req, false)
+	if err != nil {
+		return nil, nil, err
+	}
 	log.Debug().Msgf("Access Websocket request: %s", string(dump))
 
 	conn, resp, err := clientConnect(req, nil)

cfapi/base_client.go

@@ -48,7 +48,7 @@ func NewRESTClient(baseURL, accountTag, zoneTag, authToken, userAgent string, lo
 	if strings.HasSuffix(baseURL, "/") {
 		baseURL = baseURL[:len(baseURL)-1]
 	}
-	accountLevelEndpoint, err := url.Parse(fmt.Sprintf("%s/accounts/%s/tunnels", baseURL, accountTag))
+	accountLevelEndpoint, err := url.Parse(fmt.Sprintf("%s/accounts/%s/cfd_tunnel", baseURL, accountTag))
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create account level endpoint")
 	}

cfapi/client.go

@@ -5,8 +5,9 @@ import (
 )
 
 type TunnelClient interface {
-	CreateTunnel(name string, tunnelSecret []byte) (*Tunnel, error)
+	CreateTunnel(name string, tunnelSecret []byte) (*TunnelWithToken, error)
 	GetTunnel(tunnelID uuid.UUID) (*Tunnel, error)
+	GetTunnelToken(tunnelID uuid.UUID) (string, error)
 	DeleteTunnel(tunnelID uuid.UUID) error
 	ListTunnels(filter *TunnelFilter) ([]*Tunnel, error)
 	ListActiveClients(tunnelID uuid.UUID) ([]*ActiveClient, error)

cfapi/tunnel.go

@@ -23,6 +23,11 @@ type Tunnel struct {
 	Connections []Connection `json:"connections"`
 }
 
+type TunnelWithToken struct {
+	Tunnel
+	Token string `json:"token"`
+}
+
 type Connection struct {
 	ColoName           string    `json:"colo_name"`
 	ID                 uuid.UUID `json:"id"`
@@ -63,7 +68,7 @@ func (cp CleanupParams) encode() string {
 	return cp.queryParams.Encode()
 }
 
-func (r *RESTClient) CreateTunnel(name string, tunnelSecret []byte) (*Tunnel, error) {
+func (r *RESTClient) CreateTunnel(name string, tunnelSecret []byte) (*TunnelWithToken, error) {
 	if name == "" {
 		return nil, errors.New("tunnel name required")
 	}
@@ -83,7 +88,11 @@ func (r *RESTClient) CreateTunnel(name string, tunnelSecret []byte) (*Tunnel, er
 
 	switch resp.StatusCode {
 	case http.StatusOK:
-		return unmarshalTunnel(resp.Body)
+		var tunnel TunnelWithToken
+		if serdeErr := parseResponse(resp.Body, &tunnel); err != nil {
+			return nil, serdeErr
+		}
+		return &tunnel, nil
 	case http.StatusConflict:
 		return nil, ErrTunnelNameConflict
 	}
@@ -107,6 +116,23 @@ func (r *RESTClient) GetTunnel(tunnelID uuid.UUID) (*Tunnel, error) {
 	return nil, r.statusCodeToError("get tunnel", resp)
 }
 
+func (r *RESTClient) GetTunnelToken(tunnelID uuid.UUID) (token string, err error) {
+	endpoint := r.baseEndpoints.accountLevel
+	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/token", tunnelID))
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return "", errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		err = parseResponse(resp.Body, &token)
+		return token, err
+	}
+
+	return "", r.statusCodeToError("get tunnel token", resp)
+}
+
 func (r *RESTClient) DeleteTunnel(tunnelID uuid.UUID) error {
 	endpoint := r.baseEndpoints.accountLevel
 	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v", tunnelID))

cfio/copy.go

@@ -0,0 +1,27 @@
+package cfio
+
+import (
+	"io"
+	"sync"
+)
+
+const defaultBufferSize = 16 * 1024
+
+var bufferPool = sync.Pool{
+	New: func() interface{} {
+		return make([]byte, defaultBufferSize)
+	},
+}
+
+func Copy(dst io.Writer, src io.Reader) (written int64, err error) {
+	_, okWriteTo := src.(io.WriterTo)
+	_, okReadFrom := dst.(io.ReaderFrom)
+	var buffer []byte = nil
+
+	if !(okWriteTo || okReadFrom) {
+		buffer = bufferPool.Get().([]byte)
+		defer bufferPool.Put(buffer)
+	}
+
+	return io.CopyBuffer(dst, src, buffer)
+}

cfsetup.yaml

@@ -1,5 +1,5 @@
-pinned_go: &pinned_go go=1.17.5-1
-pinned_go_fips: &pinned_go_fips go-boring=1.17.5-1
+pinned_go: &pinned_go go=1.17.10-1
+pinned_go_fips: &pinned_go_fips go-boring=1.17.9-1
 
 build_dir: &build_dir /cfsetup_build
 default-flavor: buster
@@ -40,16 +40,22 @@ stretch: &stretch
       - libffi-dev
       - python3-setuptools
       - python3-pip
+      - reprepro
+      - createrepo
     pre-cache: &github_release_pkgs_pre_cache
       - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
       - chmod a+x /usr/local/bin/wixl
       - pip3 install pynacl==1.4.0
       - pip3 install pygithub==1.55
+      - pip3 install boto3==1.22.9
+      - pip3 install python-gnupg==0.4.9
     post-cache:
       # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts
       - ./build-packages.sh
       # release the packages built and moved to /cfsetup/built_artifacts
       - make github-release-built-pkgs
+      # publish packages to linux repos
+      - make release-pkgs-linux
   # handle FIPS separately so that we built with gofips compiler
   github-fips-release-pkgs:
     build_dir: *build_dir
@@ -192,8 +198,11 @@ stretch: &stretch
     builddeps:
       - openssh-client
       - s3cmd
+      - jq
+      - build-essential
     post-cache:
       - .teamcity/update-homebrew.sh
+      - .teamcity/update-homebrew-core.sh
   github-message-release:
     build_dir: *build_dir
     builddeps:
@@ -246,10 +255,10 @@ centos-7:
     pre-cache:
       - yum install -y fakeroot
       - yum upgrade -y binutils-2.27-44.base.el7.x86_64
-      - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.17.5.linux-amd64.tar.gz
+      - wget https://go.dev/dl/go1.17.10.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.17.10.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
       - export GOARCH=amd64
-      - make publish-rpm
+      - make publish-rpm

cmd/cloudflared/common_service.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"github.com/rs/zerolog"
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
+)
+
+func buildArgsForToken(c *cli.Context, log *zerolog.Logger) ([]string, error) {
+	token := c.Args().First()
+	if _, err := tunnel.ParseToken(token); err != nil {
+		return nil, cliutil.UsageError("Provided tunnel token is not valid (%s).", err)
+	}
+
+	return []string{
+		"tunnel", "run", "--token", token,
+	}, nil
+}
+
+func getServiceExtraArgsFromCliArgs(c *cli.Context, log *zerolog.Logger) ([]string, error) {
+	if c.NArg() > 0 {
+		// currently, we only support extra args for token
+		return buildArgsForToken(c, log)
+	} else {
+		// empty extra args
+		return make([]string, 0), nil
+	}
+}

cmd/cloudflared/linux_service.go

@@ -6,7 +6,6 @@ package main
 import (
 	"fmt"
 	"os"
-	"path/filepath"
 
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
@@ -20,22 +19,16 @@ import (
 func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
-		Usage: "Manages the Cloudflare Tunnel system service",
+		Usage: "Manages the cloudflared system service",
 		Subcommands: []*cli.Command{
 			{
 				Name:   "install",
-				Usage:  "Install Cloudflare Tunnel as a system service",
+				Usage:  "Install cloudflared as a system service",
 				Action: cliutil.ConfiguredAction(installLinuxService),
-				Flags: []cli.Flag{
-					&cli.BoolFlag{
-						Name:  "legacy",
-						Usage: "Generate service file for non-named tunnels",
-					},
-				},
 			},
 			{
 				Name:   "uninstall",
-				Usage:  "Uninstall the Cloudflare Tunnel service",
+				Usage:  "Uninstall the cloudflared service",
 				Action: cliutil.ConfiguredAction(uninstallLinuxService),
 			},
 		},
@@ -50,19 +43,20 @@ const (
 	serviceConfigFile     = "config.yml"
 	serviceCredentialFile = "cert.pem"
 	serviceConfigPath     = serviceConfigDir + "/" + serviceConfigFile
+	cloudflaredService    = "cloudflared.service"
 )
 
 var systemdTemplates = []ServiceTemplate{
 	{
-		Path: "/etc/systemd/system/cloudflared.service",
+		Path: fmt.Sprintf("/etc/systemd/system/%s", cloudflaredService),
 		Content: `[Unit]
-Description=Cloudflare Tunnel
+Description=cloudflared
 After=network.target
 
 [Service]
 TimeoutStartSec=0
 Type=notify
-ExecStart={{ .Path }} --config /etc/cloudflared/config.yml --no-autoupdate{{ range .ExtraArgs }} {{ . }}{{ end }}
+ExecStart={{ .Path }} --no-autoupdate{{ range .ExtraArgs }} {{ . }}{{ end }}
 Restart=on-failure
 RestartSec=5s
 
@@ -73,7 +67,7 @@ WantedBy=multi-user.target
 	{
 		Path: "/etc/systemd/system/cloudflared-update.service",
 		Content: `[Unit]
-Description=Update Cloudflare Tunnel
+Description=Update cloudflared
 After=network.target
 
 [Service]
@@ -83,7 +77,7 @@ ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 11 ]; then s
 	{
 		Path: "/etc/systemd/system/cloudflared-update.timer",
 		Content: `[Unit]
-Description=Update Cloudflare Tunnel
+Description=Update cloudflared
 
 [Timer]
 OnCalendar=daily
@@ -100,7 +94,7 @@ var sysvTemplate = ServiceTemplate{
 	Content: `#!/bin/sh
 # For RedHat and cousins:
 # chkconfig: 2345 99 01
-# description: Cloudflare Tunnel agent
+# description: cloudflared
 # processname: {{.Path}}
 ### BEGIN INIT INFO
 # Provides:          {{.Path}}
@@ -108,11 +102,11 @@ var sysvTemplate = ServiceTemplate{
 # Required-Stop:
 # Default-Start:     2 3 4 5
 # Default-Stop:      0 1 6
-# Short-Description: Cloudflare Tunnel
-# Description:       Cloudflare Tunnel agent
+# Short-Description: cloudflared
+# Description:       cloudflared agent
 ### END INIT INFO
 name=$(basename $(readlink -f $0))
-cmd="{{.Path}} --config /etc/cloudflared/config.yml --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s{{ range .ExtraArgs }} {{ . }}{{ end }}"
+cmd="{{.Path}} --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s{{ range .ExtraArgs }} {{ . }}{{ end }}"
 pid_file="/var/run/$name.pid"
 stdout_log="/var/log/$name.log"
 stderr_log="/var/log/$name.err"
@@ -191,27 +185,6 @@ func isSystemd() bool {
 	return false
 }
 
-func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile string, log *zerolog.Logger) error {
-	srcCredentialPath := filepath.Join(userConfigDir, userCredentialFile)
-	destCredentialPath := filepath.Join(serviceConfigDir, serviceCredentialFile)
-	if srcCredentialPath != destCredentialPath {
-		if err := copyCredential(srcCredentialPath, destCredentialPath); err != nil {
-			return err
-		}
-	}
-
-	srcConfigPath := filepath.Join(userConfigDir, userConfigFile)
-	destConfigPath := filepath.Join(serviceConfigDir, serviceConfigFile)
-	if srcConfigPath != destConfigPath {
-		if err := copyConfig(srcConfigPath, destConfigPath); err != nil {
-			return err
-		}
-		log.Info().Msgf("Copied %s to %s", srcConfigPath, destConfigPath)
-	}
-
-	return nil
-}
-
 func installLinuxService(c *cli.Context) error {
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
@@ -223,61 +196,69 @@ func installLinuxService(c *cli.Context) error {
 		Path: etPath,
 	}
 
-	if err := ensureConfigDirExists(serviceConfigDir); err != nil {
+	var extraArgsFunc func(c *cli.Context, log *zerolog.Logger) ([]string, error)
+	if c.NArg() == 0 {
+		extraArgsFunc = buildArgsForConfig
+	} else {
+		extraArgsFunc = buildArgsForToken
+	}
+
+	extraArgs, err := extraArgsFunc(c, log)
+	if err != nil {
 		return err
 	}
-	if c.Bool("legacy") {
-		userConfigDir := filepath.Dir(c.String("config"))
-		userConfigFile := filepath.Base(c.String("config"))
-		userCredentialFile := config.DefaultCredentialFile
-		if err = copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile, log); err != nil {
-			log.Err(err).Msgf("Failed to copy user configuration. Before running the service, ensure that %s contains two files, %s and %s",
-				serviceConfigDir, serviceCredentialFile, serviceConfigFile)
-			return err
-		}
-		templateArgs.ExtraArgs = []string{
-			"--origincert", serviceConfigDir + "/" + serviceCredentialFile,
-		}
-	} else {
-		src, _, err := config.ReadConfigFile(c, log)
-		if err != nil {
-			return err
-		}
 
-		// can't use context because this command doesn't define "credentials-file" flag
-		configPresent := func(s string) bool {
-			val, err := src.String(s)
-			return err == nil && val != ""
-		}
-		if src.TunnelID == "" || !configPresent(tunnel.CredFileFlag) {
-			return fmt.Errorf(`Configuration file %s must contain entries for the tunnel to run and its associated credentials:
-tunnel: TUNNEL-UUID
-credentials-file: CREDENTIALS-FILE
-`, src.Source())
-		}
-		if src.Source() != serviceConfigPath {
-			if exists, err := config.FileExists(serviceConfigPath); err != nil || exists {
-				return fmt.Errorf("Possible conflicting configuration in %[1]s and %[2]s. Either remove %[2]s or run `cloudflared --config %[2]s service install`", src.Source(), serviceConfigPath)
-			}
-
-			if err := copyFile(src.Source(), serviceConfigPath); err != nil {
-				return fmt.Errorf("failed to copy %s to %s: %w", src.Source(), serviceConfigPath, err)
-			}
-		}
-
-		templateArgs.ExtraArgs = []string{
-			"tunnel", "run",
-		}
-	}
+	templateArgs.ExtraArgs = extraArgs
 
 	switch {
 	case isSystemd():
 		log.Info().Msgf("Using Systemd")
-		return installSystemd(&templateArgs, log)
+		err = installSystemd(&templateArgs, log)
 	default:
 		log.Info().Msgf("Using SysV")
-		return installSysv(&templateArgs, log)
+		err = installSysv(&templateArgs, log)
 	}
+
+	if err == nil {
+		log.Info().Msg("Linux service for cloudflared installed successfully")
+	}
+	return err
+}
+
+func buildArgsForConfig(c *cli.Context, log *zerolog.Logger) ([]string, error) {
+	if err := ensureConfigDirExists(serviceConfigDir); err != nil {
+		return nil, err
+	}
+
+	src, _, err := config.ReadConfigFile(c, log)
+	if err != nil {
+		return nil, err
+	}
+
+	// can't use context because this command doesn't define "credentials-file" flag
+	configPresent := func(s string) bool {
+		val, err := src.String(s)
+		return err == nil && val != ""
+	}
+	if src.TunnelID == "" || !configPresent(tunnel.CredFileFlag) {
+		return nil, fmt.Errorf(`Configuration file %s must contain entries for the tunnel to run and its associated credentials:
+tunnel: TUNNEL-UUID
+credentials-file: CREDENTIALS-FILE
+`, src.Source())
+	}
+	if src.Source() != serviceConfigPath {
+		if exists, err := config.FileExists(serviceConfigPath); err != nil || exists {
+			return nil, fmt.Errorf("Possible conflicting configuration in %[1]s and %[2]s. Either remove %[2]s or run `cloudflared --config %[2]s service install`", src.Source(), serviceConfigPath)
+		}
+
+		if err := copyFile(src.Source(), serviceConfigPath); err != nil {
+			return nil, fmt.Errorf("failed to copy %s to %s: %w", src.Source(), serviceConfigPath, err)
+		}
+	}
+
+	return []string{
+		"--config", "/etc/cloudflared/config.yml", "tunnel", "run",
+	}, nil
 }
 
 func installSystemd(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
@@ -288,16 +269,19 @@ func installSystemd(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) erro
 			return err
 		}
 	}
-	if err := runCommand("systemctl", "enable", "cloudflared.service"); err != nil {
-		log.Err(err).Msg("systemctl enable cloudflared.service error")
+	if err := runCommand("systemctl", "enable", cloudflaredService); err != nil {
+		log.Err(err).Msgf("systemctl enable %s error", cloudflaredService)
 		return err
 	}
 	if err := runCommand("systemctl", "start", "cloudflared-update.timer"); err != nil {
 		log.Err(err).Msg("systemctl start cloudflared-update.timer error")
 		return err
 	}
-	log.Info().Msg("systemctl daemon-reload")
-	return runCommand("systemctl", "daemon-reload")
+	if err := runCommand("systemctl", "daemon-reload"); err != nil {
+		log.Err(err).Msg("systemctl daemon-reload error")
+		return err
+	}
+	return runCommand("systemctl", "start", cloudflaredService)
 }
 
 func installSysv(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
@@ -320,25 +304,35 @@ func installSysv(templateArgs *ServiceTemplateArgs, log *zerolog.Logger) error {
 			continue
 		}
 	}
-	return nil
+	return runCommand("service", "cloudflared", "start")
 }
 
 func uninstallLinuxService(c *cli.Context) error {
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
+	var err error
 	switch {
 	case isSystemd():
 		log.Info().Msg("Using Systemd")
-		return uninstallSystemd(log)
+		err = uninstallSystemd(log)
 	default:
 		log.Info().Msg("Using SysV")
-		return uninstallSysv(log)
+		err = uninstallSysv(log)
 	}
+
+	if err == nil {
+		log.Info().Msg("Linux service for cloudflared uninstalled successfully")
+	}
+	return err
 }
 
 func uninstallSystemd(log *zerolog.Logger) error {
-	if err := runCommand("systemctl", "disable", "cloudflared.service"); err != nil {
-		log.Err(err).Msg("systemctl disable cloudflared.service error")
+	if err := runCommand("systemctl", "disable", cloudflaredService); err != nil {
+		log.Err(err).Msgf("systemctl disable %s error", cloudflaredService)
+		return err
+	}
+	if err := runCommand("systemctl", "stop", cloudflaredService); err != nil {
+		log.Err(err).Msgf("systemctl stop %s error", cloudflaredService)
 		return err
 	}
 	if err := runCommand("systemctl", "stop", "cloudflared-update.timer"); err != nil {
@@ -351,11 +345,18 @@ func uninstallSystemd(log *zerolog.Logger) error {
 			return err
 		}
 	}
-	log.Info().Msgf("Successfully uninstalled cloudflared service from systemd")
+	if err := runCommand("systemctl", "daemon-reload"); err != nil {
+		log.Err(err).Msg("systemctl daemon-reload error")
+		return err
+	}
 	return nil
 }
 
 func uninstallSysv(log *zerolog.Logger) error {
+	if err := runCommand("service", "cloudflared", "stop"); err != nil {
+		log.Err(err).Msg("service cloudflared stop error")
+		return err
+	}
 	if err := sysvTemplate.Remove(); err != nil {
 		log.Err(err).Msg("error removing service template")
 		return err
@@ -370,6 +371,5 @@ func uninstallSysv(log *zerolog.Logger) error {
 			continue
 		}
 	}
-	log.Info().Msgf("Successfully uninstalled cloudflared service from sysv")
 	return nil
 }

cmd/cloudflared/macos_service.go

@@ -21,16 +21,16 @@ const (
 func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
-		Usage: "Manages the Cloudflare Tunnel launch agent",
+		Usage: "Manages the cloudflared launch agent",
 		Subcommands: []*cli.Command{
 			{
 				Name:   "install",
-				Usage:  "Install Cloudflare Tunnel as an user launch agent",
+				Usage:  "Install cloudflared as an user launch agent",
 				Action: cliutil.ConfiguredAction(installLaunchd),
 			},
 			{
 				Name:   "uninstall",
-				Usage:  "Uninstall the Cloudflare Tunnel launch agent",
+				Usage:  "Uninstall the cloudflared launch agent",
 				Action: cliutil.ConfiguredAction(uninstallLaunchd),
 			},
 		},
@@ -50,6 +50,9 @@ func newLaunchdTemplate(installPath, stdoutPath, stderrPath string) *ServiceTemp
 		<key>ProgramArguments</key>
 		<array>
 			<string>{{ .Path }}</string>
+			{{- range $i, $item := .ExtraArgs}}
+			<string>{{ $item }}</string>
+			{{- end}}
 		</array>
 		<key>RunAtLoad</key>
 		<true/>
@@ -111,12 +114,12 @@ func installLaunchd(c *cli.Context) error {
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	if isRootUser() {
-		log.Info().Msg("Installing Cloudflare Tunnel client as a system launch daemon. " +
-			"Cloudflare Tunnel client will run at boot")
+		log.Info().Msg("Installing cloudflared client as a system launch daemon. " +
+			"cloudflared client will run at boot")
 	} else {
-		log.Info().Msg("Installing Cloudflare Tunnel client as an user launch agent. " +
-			"Note that Cloudflare Tunnel client will only run when the user is logged in. " +
-			"If you want to run Cloudflare Tunnel client at boot, install with root permission. " +
+		log.Info().Msg("Installing cloudflared client as an user launch agent. " +
+			"Note that cloudflared client will only run when the user is logged in. " +
+			"If you want to run cloudflared client at boot, install with root permission. " +
 			"For more information, visit https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/run-tunnel/run-as-service")
 	}
 	etPath, err := os.Executable()
@@ -129,6 +132,13 @@ func installLaunchd(c *cli.Context) error {
 		log.Err(err).Msg("Error determining install path")
 		return errors.Wrap(err, "Error determining install path")
 	}
+	extraArgs, err := getServiceExtraArgsFromCliArgs(c, log)
+	if err != nil {
+		errMsg := "Unable to determine extra arguments for launch daemon"
+		log.Err(err).Msg(errMsg)
+		return errors.Wrap(err, errMsg)
+	}
+
 	stdoutPath, err := stdoutPath()
 	if err != nil {
 		log.Err(err).Msg("error determining stdout path")
@@ -140,7 +150,7 @@ func installLaunchd(c *cli.Context) error {
 		return errors.Wrap(err, "error determining stderr path")
 	}
 	launchdTemplate := newLaunchdTemplate(installPath, stdoutPath, stderrPath)
-	templateArgs := ServiceTemplateArgs{Path: etPath}
+	templateArgs := ServiceTemplateArgs{Path: etPath, ExtraArgs: extraArgs}
 	err = launchdTemplate.Generate(&templateArgs)
 	if err != nil {
 		log.Err(err).Msg("error generating launchd template")
@@ -153,16 +163,20 @@ func installLaunchd(c *cli.Context) error {
 	}
 
 	log.Info().Msgf("Outputs are logged to %s and %s", stderrPath, stdoutPath)
-	return runCommand("launchctl", "load", plistPath)
+	err = runCommand("launchctl", "load", plistPath)
+	if err == nil {
+		log.Info().Msg("MacOS service for cloudflared installed successfully")
+	}
+	return err
 }
 
 func uninstallLaunchd(c *cli.Context) error {
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	if isRootUser() {
-		log.Info().Msg("Uninstalling Cloudflare Tunnel as a system launch daemon")
+		log.Info().Msg("Uninstalling cloudflared as a system launch daemon")
 	} else {
-		log.Info().Msg("Uninstalling Cloudflare Tunnel as an user launch agent")
+		log.Info().Msg("Uninstalling cloudflared as a user launch agent")
 	}
 	installPath, err := installPath()
 	if err != nil {
@@ -184,10 +198,13 @@ func uninstallLaunchd(c *cli.Context) error {
 	}
 	err = runCommand("launchctl", "unload", plistPath)
 	if err != nil {
-		log.Err(err).Msg("error unloading")
+		log.Err(err).Msg("error unloading launchd")
 		return err
 	}
 
-	log.Info().Msgf("Outputs are logged to %s and %s", stderrPath, stdoutPath)
-	return launchdTemplate.Remove()
+	err = launchdTemplate.Remove()
+	if err == nil {
+		log.Info().Msg("Launchd for cloudflared was uninstalled successfully")
+	}
+	return err
 }

cmd/cloudflared/main.go

@@ -21,6 +21,7 @@ import (
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/overwatch"
+	"github.com/cloudflare/cloudflared/tracing"
 	"github.com/cloudflare/cloudflared/watcher"
 )
 
@@ -69,7 +70,7 @@ func main() {
 	app.Copyright = fmt.Sprintf(
 		`(c) %d Cloudflare Inc.
    Your installation of cloudflared software constitutes a symbol of your signature indicating that you accept
-   the terms of the Cloudflare License (https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/license),
+   the terms of the Apache License Version 2.0 (https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/license),
    Terms (https://www.cloudflare.com/terms/) and Privacy Policy (https://www.cloudflare.com/privacypolicy/).`,
 		time.Now().Year(),
 	)
@@ -86,6 +87,7 @@ func main() {
 	tunnel.Init(bInfo, graceShutdownC) // we need this to support the tunnel sub command...
 	access.Init(graceShutdownC)
 	updater.Init(Version)
+	tracing.Init(Version)
 	runApp(app, graceShutdownC)
 }
 

cmd/cloudflared/proxydns/cmd.go

@@ -73,7 +73,7 @@ func Run(c *cli.Context) error {
 		log.Fatal().Err(err).Msg("Failed to open the metrics listener")
 	}
 
-	go metrics.ServeMetrics(metricsListener, nil, nil, "", log)
+	go metrics.ServeMetrics(metricsListener, nil, nil, "", nil, log)
 
 	listener, err := tunneldns.CreateListener(
 		c.String("address"),

cmd/cloudflared/service_template.go

@@ -43,6 +43,10 @@ func (st *ServiceTemplate) Generate(args *ServiceTemplateArgs) error {
 	if err != nil {
 		return err
 	}
+	if _, err = os.Stat(resolvedPath); err == nil {
+		return fmt.Errorf(serviceAlreadyExistsWarn(resolvedPath))
+	}
+
 	var buffer bytes.Buffer
 	err = tmpl.Execute(&buffer, args)
 	if err != nil {
@@ -71,6 +75,15 @@ func (st *ServiceTemplate) Remove() error {
 	return nil
 }
 
+func serviceAlreadyExistsWarn(service string) string {
+	return fmt.Sprintf("cloudflared service is already installed at %s; if you are running a cloudflared tunnel, you "+
+		"can point it to multiple origins, avoiding the need to run more than one cloudflared service in the "+
+		"same machine; otherwise if you are really sure, you can do `cloudflared service uninstall` to clean "+
+		"up the existing service and then try again this command",
+		service,
+	)
+}
+
 func runCommand(command string, args ...string) error {
 	cmd := exec.Command(command, args...)
 	stderr, err := cmd.StderrPipe()
@@ -82,10 +95,10 @@ func runCommand(command string, args ...string) error {
 		return fmt.Errorf("error starting %s: %v", command, err)
 	}
 
-	_, _ = ioutil.ReadAll(stderr)
+	output, _ := ioutil.ReadAll(stderr)
 	err = cmd.Wait()
 	if err != nil {
-		return fmt.Errorf("%s returned with error: %v", command, err)
+		return fmt.Errorf("%s %v returned with error code %v due to: %v", command, args, err, string(output))
 	}
 	return nil
 }

cmd/cloudflared/tunnel/cmd.go

@@ -31,8 +31,9 @@ import (
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
-	"github.com/cloudflare/cloudflared/origin"
+	"github.com/cloudflare/cloudflared/orchestration"
 	"github.com/cloudflare/cloudflared/signal"
+	"github.com/cloudflare/cloudflared/supervisor"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunneldns"
 )
@@ -108,6 +109,7 @@ func Commands() []*cli.Command {
 		buildIngressSubcommand(),
 		buildDeleteCommand(),
 		buildCleanupCommand(),
+		buildTokenCommand(),
 		// for compatibility, allow following as tunnel subcommands
 		proxydns.Command(true),
 		cliutil.RemovedCommand("db-connect"),
@@ -126,26 +128,34 @@ func buildTunnelCommand(subcommands []*cli.Command) *cli.Command {
 		Name:      "tunnel",
 		Action:    cliutil.ConfiguredAction(TunnelCommand),
 		Category:  "Tunnel",
-		Usage:     "Make a locally-running web service accessible over the internet using Cloudflare Tunnel.",
+		Usage:     "Use Cloudflare Tunnel to expose private services to the Internet or to Cloudflare connected private users.",
 		ArgsUsage: " ",
-		Description: `Cloudflare Tunnel asks you to specify a hostname on a Cloudflare-powered
-		domain you control and a local address. Traffic from that hostname is routed
-		(optionally via a Cloudflare Load Balancer) to this machine and appears on the
-		specified port where it can be served.
+		Description: `    Cloudflare Tunnel allows to expose private services without opening any ingress port on this machine. It can expose:
+  A) Locally reachable HTTP-based private services to the Internet on DNS with Cloudflare as authority (which you can
+then protect with Cloudflare Access).
+  B) Locally reachable TCP/UDP-based private services to Cloudflare connected private users in the same account, e.g.,
+those enrolled to a Zero Trust WARP Client.
 
-		This feature requires your Cloudflare account be subscribed to the Cloudflare Smart Routing feature.
+You can manage your Tunnels via dash.teams.cloudflare.com. This approach will only require you to run a single command
+later in each machine where you wish to run a Tunnel.
 
-		To use, begin by calling login to download a certificate:
+Alternatively, you can manage your Tunnels via the command line. Begin by obtaining a certificate to be able to do so:
 
-			$ cloudflared tunnel login
+	$ cloudflared tunnel login
 
-		With your certificate installed you can then launch your first tunnel,
-		replacing my.site.com with a subdomain of your site:
+With your certificate installed you can then get started with Tunnels:
 
-			$ cloudflared tunnel --hostname my.site.com --url http://localhost:8080
+	$ cloudflared tunnel create my-first-tunnel
+	$ cloudflared tunnel route dns my-first-tunnel my-first-tunnel.mydomain.com
+	$ cloudflared tunnel run --hello-world my-first-tunnel
 
-		If you have a web server running on port 8080 (in this example), it will be available on
-		the internet!`,
+You can now access my-first-tunnel.mydomain.com and be served an example page by your local cloudflared process.
+
+For exposing local TCP/UDP services by IP to your privately connected users, check out:
+
+	$ cloudflared tunnel route ip --help
+
+See https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/install-and-setup/tunnel-guide/ for more info.`,
 		Subcommands: subcommands,
 		Flags:       tunnelFlags(false),
 	}
@@ -223,7 +233,7 @@ func routeFromFlag(c *cli.Context) (route cfapi.HostnameRoute, ok bool) {
 func StartServer(
 	c *cli.Context,
 	info *cliutil.BuildInfo,
-	namedTunnel *connection.NamedTunnelConfig,
+	namedTunnel *connection.NamedTunnelProperties,
 	log *zerolog.Logger,
 	isUIEnabled bool,
 ) error {
@@ -333,12 +343,17 @@ func StartServer(
 		observer.SendURL(quickTunnelURL)
 	}
 
-	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, info, log, logTransport, observer, namedTunnel)
+	tunnelConfig, orchestratorConfig, err := prepareTunnelConfig(c, info, log, logTransport, observer, namedTunnel)
 	if err != nil {
 		log.Err(err).Msg("Couldn't start tunnel")
 		return err
 	}
 
+	orchestrator, err := orchestration.NewOrchestrator(ctx, orchestratorConfig, tunnelConfig.Tags, tunnelConfig.Log)
+	if err != nil {
+		return err
+	}
+
 	metricsListener, err := listeners.Listen("tcp", c.String("metrics"))
 	if err != nil {
 		log.Err(err).Msg("Error opening metrics server listener")
@@ -350,14 +365,10 @@ func StartServer(
 		defer wg.Done()
 		readinessServer := metrics.NewReadyServer(log)
 		observer.RegisterSink(readinessServer)
-		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, quickTunnelURL, log)
+		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, quickTunnelURL, orchestrator, log)
 	}()
 
-	if err := ingressRules.StartOrigins(&wg, log, ctx.Done(), errC); err != nil {
-		return err
-	}
-
-	reconnectCh := make(chan origin.ReconnectSignal, 1)
+	reconnectCh := make(chan supervisor.ReconnectSignal, 1)
 	if c.IsSet("stdin-control") {
 		log.Info().Msg("Enabling control through stdin")
 		go stdinControl(reconnectCh, log)
@@ -369,7 +380,7 @@ func StartServer(
 			wg.Done()
 			log.Info().Msg("Tunnel server stopped")
 		}()
-		errC <- origin.StartTunnelDaemon(ctx, tunnelConfig, connectedSignal, reconnectCh, graceShutdownC)
+		errC <- supervisor.StartTunnelDaemon(ctx, tunnelConfig, orchestrator, connectedSignal, reconnectCh, graceShutdownC)
 	}()
 
 	if isUIEnabled {
@@ -377,7 +388,7 @@ func StartServer(
 			info.Version(),
 			hostname,
 			metricsListener.Addr().String(),
-			&ingressRules,
+			orchestratorConfig.Ingress,
 			tunnelConfig.HAConnections,
 		)
 		app := tunnelUI.Launch(ctx, log, logTransport)
@@ -722,43 +733,43 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:    ingress.Socks5Flag,
-			Usage:   "specify if this tunnel is running as a SOCK5 Server",
+			Usage:   legacyTunnelFlag("specify if this tunnel is running as a SOCK5 Server"),
 			EnvVars: []string{"TUNNEL_SOCKS"},
 			Value:   false,
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   ingress.ProxyConnectTimeoutFlag,
-			Usage:  "HTTP proxy timeout for establishing a new connection",
+			Usage:  legacyTunnelFlag("HTTP proxy timeout for establishing a new connection"),
 			Value:  time.Second * 30,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   ingress.ProxyTLSTimeoutFlag,
-			Usage:  "HTTP proxy timeout for completing a TLS handshake",
+			Usage:  legacyTunnelFlag("HTTP proxy timeout for completing a TLS handshake"),
 			Value:  time.Second * 10,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   ingress.ProxyTCPKeepAliveFlag,
-			Usage:  "HTTP proxy TCP keepalive duration",
+			Usage:  legacyTunnelFlag("HTTP proxy TCP keepalive duration"),
 			Value:  time.Second * 30,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   ingress.ProxyNoHappyEyeballsFlag,
-			Usage:  "HTTP proxy should disable \"happy eyeballs\" for IPv4/v6 fallback",
+			Usage:  legacyTunnelFlag("HTTP proxy should disable \"happy eyeballs\" for IPv4/v6 fallback"),
 			Hidden: shouldHide,
 		}),
 		altsrc.NewIntFlag(&cli.IntFlag{
 			Name:   ingress.ProxyKeepAliveConnectionsFlag,
-			Usage:  "HTTP proxy maximum keepalive connection pool size",
+			Usage:  legacyTunnelFlag("HTTP proxy maximum keepalive connection pool size"),
 			Value:  100,
 			Hidden: shouldHide,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:   ingress.ProxyKeepAliveTimeoutFlag,
-			Usage:  "HTTP proxy timeout for closing an idle connection",
+			Usage:  legacyTunnelFlag("HTTP proxy timeout for closing an idle connection"),
 			Value:  time.Second * 90,
 			Hidden: shouldHide,
 		}),
@@ -776,13 +787,13 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    ingress.HTTPHostHeaderFlag,
-			Usage:   "Sets the HTTP Host header for the local webserver.",
+			Usage:   legacyTunnelFlag("Sets the HTTP Host header for the local webserver."),
 			EnvVars: []string{"TUNNEL_HTTP_HOST_HEADER"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    ingress.OriginServerNameFlag,
-			Usage:   "Hostname on the origin server certificate.",
+			Usage:   legacyTunnelFlag("Hostname on the origin server certificate."),
 			EnvVars: []string{"TUNNEL_ORIGIN_SERVER_NAME"},
 			Hidden:  shouldHide,
 		}),
@@ -794,19 +805,19 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    tlsconfig.OriginCAPoolFlag,
-			Usage:   "Path to the CA for the certificate of your origin. This option should be used only if your certificate is not signed by Cloudflare.",
+			Usage:   legacyTunnelFlag("Path to the CA for the certificate of your origin. This option should be used only if your certificate is not signed by Cloudflare."),
 			EnvVars: []string{"TUNNEL_ORIGIN_CA_POOL"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:    ingress.NoTLSVerifyFlag,
-			Usage:   "Disables TLS verification of the certificate presented by your origin. Will allow any certificate from the origin to be accepted. Note: The connection from your machine to Cloudflare's Edge is still encrypted.",
+			Usage:   legacyTunnelFlag("Disables TLS verification of the certificate presented by your origin. Will allow any certificate from the origin to be accepted. Note: The connection from your machine to Cloudflare's Edge is still encrypted."),
 			EnvVars: []string{"NO_TLS_VERIFY"},
 			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:    ingress.NoChunkedEncodingFlag,
-			Usage:   "Disables chunked transfer encoding; useful if you are running a WSGI server.",
+			Usage:   legacyTunnelFlag("Disables chunked transfer encoding; useful if you are running a WSGI server."),
 			EnvVars: []string{"TUNNEL_NO_CHUNKED_ENCODING"},
 			Hidden:  shouldHide,
 		}),
@@ -814,6 +825,15 @@ func configureProxyFlags(shouldHide bool) []cli.Flag {
 	return append(flags, sshFlags(shouldHide)...)
 }
 
+func legacyTunnelFlag(msg string) string {
+	return fmt.Sprintf(
+		"%s This flag only takes effect if you define your origin with `--url` and if you do not use ingress rules."+
+			" The recommended way is to rely on ingress rules and define this property under `originRequest` as per"+
+			" https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/configuration-file/ingress",
+		msg,
+	)
+}
+
 func sshFlags(shouldHide bool) []cli.Flag {
 	return []cli.Flag{
 		altsrc.NewStringFlag(&cli.StringFlag{
@@ -998,7 +1018,7 @@ func configureProxyDNSFlags(shouldHide bool) []cli.Flag {
 	}
 }
 
-func stdinControl(reconnectCh chan origin.ReconnectSignal, log *zerolog.Logger) {
+func stdinControl(reconnectCh chan supervisor.ReconnectSignal, log *zerolog.Logger) {
 	for {
 		scanner := bufio.NewScanner(os.Stdin)
 		for scanner.Scan() {
@@ -1009,7 +1029,7 @@ func stdinControl(reconnectCh chan origin.ReconnectSignal, log *zerolog.Logger)
 			case "":
 				break
 			case "reconnect":
-				var reconnect origin.ReconnectSignal
+				var reconnect supervisor.ReconnectSignal
 				if len(parts) > 1 {
 					var err error
 					if reconnect.Delay, err = time.ParseDuration(parts[1]); err != nil {

cmd/cloudflared/tunnel/configuration.go

@@ -14,6 +14,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2/altsrc"
 	"golang.org/x/crypto/ssh/terminal"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
@@ -23,21 +24,27 @@ import (
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/ingress"
-	"github.com/cloudflare/cloudflared/origin"
+	"github.com/cloudflare/cloudflared/orchestration"
+	"github.com/cloudflare/cloudflared/supervisor"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/validation"
 )
 
 const LogFieldOriginCertPath = "originCertPath"
+const secretValue = "*****"
 
 var (
 	developerPortal = "https://developers.cloudflare.com/argo-tunnel"
-	quickStartUrl   = developerPortal + "/quickstart/quickstart/"
 	serviceUrl      = developerPortal + "/reference/service/"
 	argumentsUrl    = developerPortal + "/reference/arguments/"
 
 	LogFieldHostname = "hostname"
+
+	secretFlags     = [2]*altsrc.StringFlag{credentialsContentsFlag, tunnelTokenFlag}
+	defaultFeatures = []string{supervisor.FeatureAllowRemoteConfig, supervisor.FeatureSerializedHeaders}
+
+	configFlags = []string{"autoupdate-freq", "no-autoupdate", "retries", "protocol", "loglevel", "transport-loglevel", "origincert", "metrics", "metrics-update-freq"}
 )
 
 // returns the first path that contains a cert.pem file. If none of the DefaultConfigSearchDirectories
@@ -64,7 +71,11 @@ func generateRandomClientID(log *zerolog.Logger) (string, error) {
 func logClientOptions(c *cli.Context, log *zerolog.Logger) {
 	flags := make(map[string]interface{})
 	for _, flag := range c.FlagNames() {
-		flags[flag] = c.Generic(flag)
+		if isSecretFlag(flag) {
+			flags[flag] = secretValue
+		} else {
+			flags[flag] = c.Generic(flag)
+		}
 	}
 
 	if len(flags) > 0 {
@@ -78,7 +89,11 @@ func logClientOptions(c *cli.Context, log *zerolog.Logger) {
 		if strings.Contains(env, "TUNNEL_") {
 			vars := strings.Split(env, "=")
 			if len(vars) == 2 {
-				envs[vars[0]] = vars[1]
+				if isSecretEnvVar(vars[0]) {
+					envs[vars[0]] = secretValue
+				} else {
+					envs[vars[0]] = vars[1]
+				}
 			}
 		}
 	}
@@ -87,7 +102,27 @@ func logClientOptions(c *cli.Context, log *zerolog.Logger) {
 	}
 }
 
-func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelConfig) bool {
+func isSecretFlag(key string) bool {
+	for _, flag := range secretFlags {
+		if flag.Name == key {
+			return true
+		}
+	}
+	return false
+}
+
+func isSecretEnvVar(key string) bool {
+	for _, flag := range secretFlags {
+		for _, secretEnvVar := range flag.EnvVars {
+			if secretEnvVar == key {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelProperties) bool {
 	return c.IsSet("proxy-dns") && (!c.IsSet("hostname") && !c.IsSet("tag") && !c.IsSet("hello-world") && namedTunnel == nil)
 }
 
@@ -152,44 +187,67 @@ func prepareTunnelConfig(
 	info *cliutil.BuildInfo,
 	log, logTransport *zerolog.Logger,
 	observer *connection.Observer,
-	namedTunnel *connection.NamedTunnelConfig,
-) (*origin.TunnelConfig, ingress.Ingress, error) {
+	namedTunnel *connection.NamedTunnelProperties,
+) (*supervisor.TunnelConfig, *orchestration.Config, error) {
 	isNamedTunnel := namedTunnel != nil
 
 	configHostname := c.String("hostname")
 	hostname, err := validation.ValidateHostname(configHostname)
 	if err != nil {
 		log.Err(err).Str(LogFieldHostname, configHostname).Msg("Invalid hostname")
-		return nil, ingress.Ingress{}, errors.Wrap(err, "Invalid hostname")
+		return nil, nil, errors.Wrap(err, "Invalid hostname")
 	}
 	clientID := c.String("id")
 	if !c.IsSet("id") {
 		clientID, err = generateRandomClientID(log)
 		if err != nil {
-			return nil, ingress.Ingress{}, err
+			return nil, nil, err
 		}
 	}
 
 	tags, err := NewTagSliceFromCLI(c.StringSlice("tag"))
 	if err != nil {
 		log.Err(err).Msg("Tag parse failure")
-		return nil, ingress.Ingress{}, errors.Wrap(err, "Tag parse failure")
+		return nil, nil, errors.Wrap(err, "Tag parse failure")
 	}
 
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
 
 	var (
 		ingressRules  ingress.Ingress
-		classicTunnel *connection.ClassicTunnelConfig
+		classicTunnel *connection.ClassicTunnelProperties
 	)
+
+	transportProtocol := c.String("protocol")
+	protocolFetcher := edgediscovery.ProtocolPercentage
+
 	cfg := config.GetConfiguration()
 	if isNamedTunnel {
 		clientUUID, err := uuid.NewRandom()
 		if err != nil {
-			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate connector UUID")
+			return nil, nil, errors.Wrap(err, "can't generate connector UUID")
 		}
 		log.Info().Msgf("Generated Connector ID: %s", clientUUID)
-		features := append(c.StringSlice("features"), origin.FeatureSerializedHeaders)
+		features := append(c.StringSlice("features"), defaultFeatures...)
+		if c.IsSet(TunnelTokenFlag) {
+			if transportProtocol == connection.AutoSelectFlag {
+				protocolFetcher = func() (edgediscovery.ProtocolPercents, error) {
+					// If the Tunnel is remotely managed and no protocol is set, we prefer QUIC, but still allow fall-back.
+					preferQuic := []edgediscovery.ProtocolPercent{
+						{
+							Protocol:   connection.QUIC.String(),
+							Percentage: 100,
+						},
+						{
+							Protocol:   connection.HTTP2.String(),
+							Percentage: 100,
+						},
+					}
+					return preferQuic, nil
+				}
+			}
+			log.Info().Msg("Will be fetching remotely managed configuration from Cloudflare API. Defaulting to protocol: quic")
+		}
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],
 			Features: dedup(features),
@@ -198,10 +256,10 @@ func prepareTunnelConfig(
 		}
 		ingressRules, err = ingress.ParseIngress(cfg)
 		if err != nil && err != ingress.ErrNoIngressRules {
-			return nil, ingress.Ingress{}, err
+			return nil, nil, err
 		}
 		if !ingressRules.IsEmpty() && c.IsSet("url") {
-			return nil, ingress.Ingress{}, ingress.ErrURLIncompatibleWithIngress
+			return nil, nil, ingress.ErrURLIncompatibleWithIngress
 		}
 	} else {
 
@@ -212,10 +270,10 @@ func prepareTunnelConfig(
 
 		originCert, err := getOriginCert(originCertPath, &originCertLog)
 		if err != nil {
-			return nil, ingress.Ingress{}, errors.Wrap(err, "Error getting origin cert")
+			return nil, nil, errors.Wrap(err, "Error getting origin cert")
 		}
 
-		classicTunnel = &connection.ClassicTunnelConfig{
+		classicTunnel = &connection.ClassicTunnelProperties{
 			Hostname:   hostname,
 			OriginCert: originCert,
 			// turn off use of reconnect token and auth refresh when using named tunnels
@@ -227,20 +285,14 @@ func prepareTunnelConfig(
 	if ingressRules.IsEmpty() {
 		ingressRules, err = ingress.NewSingleOrigin(c, !isNamedTunnel)
 		if err != nil {
-			return nil, ingress.Ingress{}, err
+			return nil, nil, err
 		}
 	}
 
-	var warpRoutingService *ingress.WarpRoutingService
 	warpRoutingEnabled := isWarpRoutingEnabled(cfg.WarpRouting, isNamedTunnel)
-	if warpRoutingEnabled {
-		warpRoutingService = ingress.NewWarpRoutingService()
-		log.Info().Msgf("Warp-routing is enabled")
-	}
-
-	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), warpRoutingEnabled, namedTunnel, edgediscovery.ProtocolPercentage, origin.ResolveTTL, log)
+	protocolSelector, err := connection.NewProtocolSelector(transportProtocol, warpRoutingEnabled, namedTunnel, protocolFetcher, supervisor.ResolveTTL, log)
 	if err != nil {
-		return nil, ingress.Ingress{}, err
+		return nil, nil, err
 	}
 	log.Info().Msgf("Initial protocol %s", protocolSelector.Current())
 
@@ -248,11 +300,11 @@ func prepareTunnelConfig(
 	for _, p := range connection.ProtocolList {
 		tlsSettings := p.TLSSettings()
 		if tlsSettings == nil {
-			return nil, ingress.Ingress{}, fmt.Errorf("%s has unknown TLS settings", p)
+			return nil, nil, fmt.Errorf("%s has unknown TLS settings", p)
 		}
 		edgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c, tlsSettings.ServerName)
 		if err != nil {
-			return nil, ingress.Ingress{}, errors.Wrap(err, "unable to create TLS config to connect with edge")
+			return nil, nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
 		}
 		if len(tlsSettings.NextProtos) > 0 {
 			edgeTLSConfig.NextProtos = tlsSettings.NextProtos
@@ -260,15 +312,9 @@ func prepareTunnelConfig(
 		edgeTLSConfigs[p] = edgeTLSConfig
 	}
 
-	originProxy := origin.NewOriginProxy(ingressRules, warpRoutingService, tags, log)
 	gracePeriod, err := gracePeriod(c)
 	if err != nil {
-		return nil, ingress.Ingress{}, err
-	}
-	connectionConfig := &connection.Config{
-		OriginProxy:     originProxy,
-		GracePeriod:     gracePeriod,
-		ReplaceExisting: c.Bool("force"),
+		return nil, nil, err
 	}
 	muxerConfig := &connection.MuxerConfig{
 		HeartbeatInterval: c.Duration("heartbeat-interval"),
@@ -279,21 +325,22 @@ func prepareTunnelConfig(
 		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
 	}
 
-	return &origin.TunnelConfig{
-		ConnectionConfig: connectionConfig,
-		OSArch:           info.OSArch(),
-		ClientID:         clientID,
-		EdgeAddrs:        c.StringSlice("edge"),
-		Region:           c.String("region"),
-		HAConnections:    c.Int("ha-connections"),
-		IncidentLookup:   origin.NewIncidentLookup(),
-		IsAutoupdated:    c.Bool("is-autoupdated"),
-		LBPool:           c.String("lb-pool"),
-		Tags:             tags,
-		Log:              log,
-		LogTransport:     logTransport,
-		Observer:         observer,
-		ReportedVersion:  info.Version(),
+	tunnelConfig := &supervisor.TunnelConfig{
+		GracePeriod:     gracePeriod,
+		ReplaceExisting: c.Bool("force"),
+		OSArch:          info.OSArch(),
+		ClientID:        clientID,
+		EdgeAddrs:       c.StringSlice("edge"),
+		Region:          c.String("region"),
+		HAConnections:   c.Int("ha-connections"),
+		IncidentLookup:  supervisor.NewIncidentLookup(),
+		IsAutoupdated:   c.Bool("is-autoupdated"),
+		LBPool:          c.String("lb-pool"),
+		Tags:            tags,
+		Log:             log,
+		LogTransport:    logTransport,
+		Observer:        observer,
+		ReportedVersion: info.Version(),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		Retries:          uint(c.Int("retries")),
 		RunFromTerminal:  isRunningFromTerminal(),
@@ -302,7 +349,25 @@ func prepareTunnelConfig(
 		MuxerConfig:      muxerConfig,
 		ProtocolSelector: protocolSelector,
 		EdgeTLSConfigs:   edgeTLSConfigs,
-	}, ingressRules, nil
+	}
+	orchestratorConfig := &orchestration.Config{
+		Ingress:            &ingressRules,
+		WarpRoutingEnabled: warpRoutingEnabled,
+		ConfigurationFlags: parseConfigFlags(c),
+	}
+	return tunnelConfig, orchestratorConfig, nil
+}
+
+func parseConfigFlags(c *cli.Context) map[string]string {
+	result := make(map[string]string)
+
+	for _, flag := range configFlags {
+		if v := c.String(flag); c.IsSet(flag) && v != "" {
+			result[flag] = v
+		}
+	}
+
+	return result
 }
 
 func gracePeriod(c *cli.Context) (time.Duration, error) {

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -1,6 +1,7 @@
 package tunnel
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/url"
 
@@ -12,6 +13,15 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
+const ingressDataJSONFlagName = "json"
+
+var ingressDataJSON = &cli.StringFlag{
+	Name:    ingressDataJSONFlagName,
+	Aliases: []string{"j"},
+	Usage:   `Accepts data in the form of json as an input rather than read from a file`,
+	EnvVars: []string{"TUNNEL_INGRESS_VALIDATE_JSON"},
+}
+
 func buildIngressSubcommand() *cli.Command {
 	return &cli.Command{
 		Name:      "ingress",
@@ -49,6 +59,7 @@ func buildValidateIngressCommand() *cli.Command {
 		Usage:       "Validate the ingress configuration ",
 		UsageText:   "cloudflared tunnel [--config FILEPATH] ingress validate",
 		Description: "Validates the configuration file, ensuring your ingress rules are OK.",
+		Flags:       []cli.Flag{ingressDataJSON},
 	}
 }
 
@@ -69,12 +80,11 @@ func buildTestURLCommand() *cli.Command {
 
 // validateIngressCommand check the syntax of the ingress rules in the cloudflared config file
 func validateIngressCommand(c *cli.Context, warnings string) error {
-	conf := config.GetConfiguration()
-	if conf.Source() == "" {
-		fmt.Println("No configuration file was found. Please create one, or use the --config flag to specify its filepath. You can use the help command to learn more about configuration files")
-		return nil
+	conf, err := getConfiguration(c)
+	if err != nil {
+		return err
 	}
-	fmt.Println("Validating rules from", conf.Source())
+
 	if _, err := ingress.ParseIngress(conf); err != nil {
 		return errors.Wrap(err, "Validation failed")
 	}
@@ -90,6 +100,22 @@ func validateIngressCommand(c *cli.Context, warnings string) error {
 	return nil
 }
 
+func getConfiguration(c *cli.Context) (*config.Configuration, error) {
+	var conf *config.Configuration
+	if c.IsSet(ingressDataJSONFlagName) {
+		ingressJSON := c.String(ingressDataJSONFlagName)
+		fmt.Println("Validating rules from cmdline flag --json")
+		err := json.Unmarshal([]byte(ingressJSON), &conf)
+		return conf, err
+	}
+	conf = config.GetConfiguration()
+	if conf.Source() == "" {
+		return nil, errors.New("No configuration file was found. Please create one, or use the --config flag to specify its filepath. You can use the help command to learn more about configuration files")
+	}
+	fmt.Println("Validating rules from", conf.Source())
+	return conf, nil
+}
+
 // testURLCommand checks which ingress rule matches the given URL.
 func testURLCommand(c *cli.Context) error {
 	requestArg := c.Args().First()

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -55,7 +55,6 @@ func RunQuickTunnel(sc *subcommandContext) error {
 		AccountTag:   data.Result.AccountTag,
 		TunnelSecret: data.Result.Secret,
 		TunnelID:     tunnelID,
-		TunnelName:   data.Result.Name,
 	}
 
 	url := data.Result.Hostname
@@ -77,7 +76,7 @@ func RunQuickTunnel(sc *subcommandContext) error {
 	return StartServer(
 		sc.c,
 		buildInfo,
-		&connection.NamedTunnelConfig{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
+		&connection.NamedTunnelProperties{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
 		sc.log,
 		sc.isUIEnabled,
 	)

cmd/cloudflared/tunnel/subcommand_context.go

@@ -185,7 +185,6 @@ func (sc *subcommandContext) create(name string, credentialsFilePath string, sec
 		AccountTag:   credential.cert.AccountID,
 		TunnelSecret: tunnelSecret,
 		TunnelID:     tunnel.ID,
-		TunnelName:   name,
 	}
 	usedCertPath := false
 	if credentialsFilePath == "" {
@@ -221,7 +220,8 @@ func (sc *subcommandContext) create(name string, credentialsFilePath string, sec
 	}
 	fmt.Println(" Keep this file secret. To revoke these credentials, delete the tunnel.")
 	fmt.Printf("\nCreated tunnel %s with id %s\n", tunnel.Name, tunnel.ID)
-	return tunnel, nil
+
+	return &tunnel.Tunnel, nil
 }
 
 func (sc *subcommandContext) list(filter *cfapi.TunnelFilter) ([]*cfapi.Tunnel, error) {
@@ -301,10 +301,16 @@ func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 		return err
 	}
 
+	return sc.runWithCredentials(credentials)
+}
+
+func (sc *subcommandContext) runWithCredentials(credentials connection.Credentials) error {
+	sc.log.Info().Str(LogFieldTunnelID, credentials.TunnelID.String()).Msg("Starting tunnel")
+
 	return StartServer(
 		sc.c,
 		buildInfo,
-		&connection.NamedTunnelConfig{Credentials: credentials},
+		&connection.NamedTunnelProperties{Credentials: credentials},
 		sc.log,
 		sc.isUIEnabled,
 	)
@@ -335,6 +341,21 @@ func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
 	return nil
 }
 
+func (sc *subcommandContext) getTunnelTokenCredentials(tunnelID uuid.UUID) (*connection.TunnelToken, error) {
+	client, err := sc.client()
+	if err != nil {
+		return nil, err
+	}
+
+	token, err := client.GetTunnelToken(tunnelID)
+	if err != nil {
+		sc.log.Err(err).Msgf("Could not get the Token for the given Tunnel %v", tunnelID)
+		return nil, err
+	}
+
+	return ParseToken(token)
+}
+
 func (sc *subcommandContext) route(tunnelID uuid.UUID, r cfapi.HostnameRoute) (cfapi.HostnameRouteResult, error) {
 	client, err := sc.client()
 	if err != nil {
@@ -370,7 +391,7 @@ func (sc *subcommandContext) findID(input string) (uuid.UUID, error) {
 	// Look up name in the credentials file.
 	credFinder := newStaticPath(sc.c.String(CredFileFlag), sc.fs)
 	if credentials, err := sc.readTunnelCredentials(credFinder); err == nil {
-		if credentials.TunnelID != uuid.Nil && input == credentials.TunnelName {
+		if credentials.TunnelID != uuid.Nil {
 			return credentials.TunnelID, nil
 		}
 	}

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
 	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/cfapi"
@@ -115,7 +116,6 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 				AccountTag:   accountTag,
 				TunnelID:     tunnelID,
 				TunnelSecret: secret,
-				TunnelName:   name,
 			},
 		},
 		{
@@ -160,7 +160,6 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 				AccountTag:   accountTag,
 				TunnelID:     tunnelID,
 				TunnelSecret: secret,
-				TunnelName:   name,
 			},
 		},
 	}
@@ -217,6 +216,10 @@ func (d *deleteMockTunnelStore) GetTunnel(tunnelID uuid.UUID) (*cfapi.Tunnel, er
 	return &tunnel.tunnel, nil
 }
 
+func (d *deleteMockTunnelStore) GetTunnelToken(tunnelID uuid.UUID) (string, error) {
+	return "token", nil
+}
+
 func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
 	tunnel, ok := d.mockTunnels[tunnelID]
 	if !ok {
@@ -322,3 +325,48 @@ func Test_subcommandContext_Delete(t *testing.T) {
 		})
 	}
 }
+
+func Test_subcommandContext_ValidateIngressCommand(t *testing.T) {
+	var tests = []struct {
+		name        string
+		c           *cli.Context
+		wantErr     bool
+		expectedErr error
+	}{
+		{
+			name: "read a valid configuration from data",
+			c: func() *cli.Context {
+				data := `{ "warp-routing": {"enabled": true},  "originRequest" : {"connectTimeout": 10}, "ingress" : [ {"hostname": "test", "service": "https://localhost:8000" } , {"service": "http_status:404"} ]}`
+				flagSet := flag.NewFlagSet("json", flag.PanicOnError)
+				flagSet.String(ingressDataJSONFlagName, data, "")
+				c := cli.NewContext(cli.NewApp(), flagSet, nil)
+				_ = c.Set(ingressDataJSONFlagName, data)
+				return c
+			}(),
+		},
+		{
+			name: "read an invalid configuration with multiple mistakes",
+			c: func() *cli.Context {
+				data := `{ "ingress" : [ {"hostname": "test", "service": "localhost:8000" } , {"service": "http_status:invalid_status"} ]}`
+				flagSet := flag.NewFlagSet("json", flag.PanicOnError)
+				flagSet.String(ingressDataJSONFlagName, data, "")
+				c := cli.NewContext(cli.NewApp(), flagSet, nil)
+				_ = c.Set(ingressDataJSONFlagName, data)
+				return c
+			}(),
+			wantErr:     true,
+			expectedErr: errors.New("Validation failed: localhost:8000 is an invalid address, please make sure it has a scheme and a hostname"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateIngressCommand(tt.c, "")
+			if tt.wantErr {
+				assert.Equal(t, tt.expectedErr.Error(), err.Error())
+			} else {
+				assert.Nil(t, err)
+			}
+		})
+	}
+}

cmd/cloudflared/tunnel/subcommands.go

@@ -2,6 +2,7 @@ package tunnel
 
 import (
 	"crypto/rand"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -34,6 +35,7 @@ const (
 	CredFileFlagAlias    = "cred-file"
 	CredFileFlag         = "credentials-file"
 	CredContentsFlag     = "credentials-contents"
+	TunnelTokenFlag      = "token"
 	overwriteDNSFlagName = "overwrite-dns"
 
 	LogFieldTunnelID = "tunnelID"
@@ -118,6 +120,11 @@ var (
 		Usage:   "Contents of the tunnel credentials JSON file to use. When provided along with credentials-file, this will take precedence.",
 		EnvVars: []string{"TUNNEL_CRED_CONTENTS"},
 	})
+	tunnelTokenFlag = altsrc.NewStringFlag(&cli.StringFlag{
+		Name:    TunnelTokenFlag,
+		Usage:   "The Tunnel token. When provided along with credentials, this will take precedence.",
+		EnvVars: []string{"TUNNEL_TOKEN"},
+	})
 	forceDeleteFlag = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
@@ -127,7 +134,7 @@ var (
 	}
 	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{
 		Name:    "protocol",
-		Value:   "auto",
+		Value:   connection.AutoSelectFlag,
 		Aliases: []string{"p"},
 		Usage:   fmt.Sprintf("Protocol implementation to connect with Cloudflare's edge network. %s", connection.AvailableProtocolFlagMessage),
 		EnvVars: []string{"TUNNEL_TRANSPORT_PROTOCOL"},
@@ -597,6 +604,7 @@ func buildRunCommand() *cli.Command {
 		credentialsContentsFlag,
 		selectProtocolFlag,
 		featuresFlag,
+		tunnelTokenFlag,
 	}
 	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{
@@ -627,14 +635,6 @@ func runCommand(c *cli.Context) error {
 	if c.NArg() > 1 {
 		return cliutil.UsageError(`"cloudflared tunnel run" accepts only one argument, the ID or name of the tunnel to run.`)
 	}
-	tunnelRef := c.Args().First()
-	if tunnelRef == "" {
-		// see if tunnel id was in the config file
-		tunnelRef = config.GetConfiguration().TunnelID
-		if tunnelRef == "" {
-			return cliutil.UsageError(`"cloudflared tunnel run" requires the ID or name of the tunnel to run as the last command line argument or in the configuration file.`)
-		}
-	}
 
 	if c.String("hostname") != "" {
 		sc.log.Warn().Msg("The property `hostname` in your configuration is ignored because you configured a Named Tunnel " +
@@ -642,7 +642,38 @@ func runCommand(c *cli.Context) error {
 			"your origin will not be reachable. You should remove the `hostname` property to avoid this warning.")
 	}
 
-	return runNamedTunnel(sc, tunnelRef)
+	// Check if token is provided and if not use default tunnelID flag method
+	if tokenStr := c.String(TunnelTokenFlag); tokenStr != "" {
+		if token, err := ParseToken(tokenStr); err == nil {
+			return sc.runWithCredentials(token.Credentials())
+		}
+
+		return cliutil.UsageError("Provided Tunnel token is not valid.")
+	} else {
+		tunnelRef := c.Args().First()
+		if tunnelRef == "" {
+			// see if tunnel id was in the config file
+			tunnelRef = config.GetConfiguration().TunnelID
+			if tunnelRef == "" {
+				return cliutil.UsageError(`"cloudflared tunnel run" requires the ID or name of the tunnel to run as the last command line argument or in the configuration file.`)
+			}
+		}
+
+		return runNamedTunnel(sc, tunnelRef)
+	}
+}
+
+func ParseToken(tokenStr string) (*connection.TunnelToken, error) {
+	content, err := base64.StdEncoding.DecodeString(tokenStr)
+	if err != nil {
+		return nil, err
+	}
+
+	var token connection.TunnelToken
+	if err := json.Unmarshal(content, &token); err != nil {
+		return nil, err
+	}
+	return &token, nil
 }
 
 func runNamedTunnel(sc *subcommandContext, tunnelRef string) error {
@@ -650,9 +681,6 @@ func runNamedTunnel(sc *subcommandContext, tunnelRef string) error {
 	if err != nil {
 		return errors.Wrap(err, "error parsing tunnel ID")
 	}
-
-	sc.log.Info().Str(LogFieldTunnelID, tunnelID.String()).Msg("Starting tunnel")
-
 	return sc.run(tunnelID)
 }
 
@@ -686,6 +714,59 @@ func cleanupCommand(c *cli.Context) error {
 	return sc.cleanupConnections(tunnelIDs)
 }
 
+func buildTokenCommand() *cli.Command {
+	return &cli.Command{
+		Name:               "token",
+		Action:             cliutil.ConfiguredAction(tokenCommand),
+		Usage:              "Fetch the credentials token for an existing tunnel (by name or UUID) that allows to run it",
+		UsageText:          "cloudflared tunnel [tunnel command options] token [subcommand options] TUNNEL",
+		Description:        "cloudflared tunnel token will fetch the credentials token for a given tunnel (by its name or UUID), which is then used to run the tunnel. This command fails if the tunnel does not exist or has been deleted. Use the flag `cloudflared tunnel token --cred-file /my/path/file.json TUNNEL` to output the token to the credentials JSON file. Note: this command only works for Tunnels created since cloudflared version 2022.3.0",
+		Flags:              []cli.Flag{credentialsFileFlagCLIOnly},
+		CustomHelpTemplate: commandHelpTemplate(),
+	}
+}
+
+func tokenCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return errors.Wrap(err, "error setting up logger")
+	}
+
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
+	if c.NArg() != 1 {
+		return cliutil.UsageError(`"cloudflared tunnel token" requires exactly 1 argument, the name or UUID of tunnel to fetch the credentials token for.`)
+	}
+	tunnelID, err := sc.findID(c.Args().First())
+	if err != nil {
+		return errors.Wrap(err, "error parsing tunnel ID")
+	}
+
+	token, err := sc.getTunnelTokenCredentials(tunnelID)
+	if err != nil {
+		return err
+	}
+
+	if path := c.String(CredFileFlag); path != "" {
+		credentials := token.Credentials()
+		err := writeTunnelCredentials(path, &credentials)
+		if err != nil {
+			return errors.Wrapf(err, "error writing token credentials to JSON file in path %s", path)
+		}
+
+		return nil
+	}
+
+	encodedToken, err := token.Encode()
+	if err != nil {
+		return err
+	}
+
+	fmt.Printf("%s", encodedToken)
+	return nil
+}
+
 func buildRouteCommand() *cli.Command {
 	return &cli.Command{
 		Name:      "route",

cmd/cloudflared/tunnel/subcommands_test.go

@@ -1,14 +1,18 @@
 package tunnel
 
 import (
+	"encoding/base64"
+	"encoding/json"
 	"path/filepath"
 	"testing"
 
 	"github.com/google/uuid"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/cloudflare/cloudflared/cfapi"
+	"github.com/cloudflare/cloudflared/connection"
 )
 
 func Test_fmtConnections(t *testing.T) {
@@ -177,3 +181,24 @@ func Test_validateHostname(t *testing.T) {
 		})
 	}
 }
+
+func Test_TunnelToken(t *testing.T) {
+	token, err := ParseToken("aabc")
+	require.Error(t, err)
+	require.Nil(t, token)
+
+	expectedToken := &connection.TunnelToken{
+		AccountTag:   "abc",
+		TunnelSecret: []byte("secret"),
+		TunnelID:     uuid.New(),
+	}
+
+	tokenJsonStr, err := json.Marshal(expectedToken)
+	require.NoError(t, err)
+
+	token64 := base64.StdEncoding.EncodeToString(tokenJsonStr)
+
+	token, err = ParseToken(token64)
+	require.NoError(t, err)
+	require.Equal(t, token, expectedToken)
+}

cmd/cloudflared/updater/update.go

@@ -19,7 +19,7 @@ import (
 
 const (
 	DefaultCheckUpdateFreq        = time.Hour * 24
-	noUpdateInShellMessage        = "cloudflared will not automatically update when run from the shell. To enable auto-updates, run cloudflared as a service: https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/run-tunnel/run-as-service"
+	noUpdateInShellMessage        = "cloudflared will not automatically update when run from the shell. To enable auto-updates, run cloudflared as a service: https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/run-tunnel/as-a-service/"
 	noUpdateOnWindowsMessage      = "cloudflared will not automatically update on Windows systems."
 	noUpdateManagedPackageMessage = "cloudflared will not automatically update if installed by a package manager."
 	isManagedInstallFile          = ".installedFromPackageManager"
@@ -30,7 +30,8 @@ const (
 )
 
 var (
-	version string
+	version                string
+	BuiltForPackageManager = ""
 )
 
 // BinaryUpdated implements ExitCoder interface, the app will exit with status code 11
@@ -118,7 +119,11 @@ func Update(c *cli.Context) error {
 	log := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
 	if wasInstalledFromPackageManager() {
-		log.Error().Msg("cloudflared was installed by a package manager. Please update using the same method.")
+		packageManagerName := "a package manager"
+		if BuiltForPackageManager != "" {
+			packageManagerName = BuiltForPackageManager
+		}
+		log.Error().Msg(fmt.Sprintf("cloudflared was installed by %s. Please update using the same method.", packageManagerName))
 		return nil
 	}
 
@@ -281,7 +286,7 @@ func supportAutoUpdate(log *zerolog.Logger) bool {
 
 func wasInstalledFromPackageManager() bool {
 	ok, _ := config.FileExists(filepath.Join(config.DefaultUnixConfigLocation, isManagedInstallFile))
-	return ok
+	return len(BuiltForPackageManager) != 0 || ok
 }
 
 func isRunningFromTerminal() bool {

cmd/cloudflared/windows_service.go

@@ -26,8 +26,8 @@ import (
 
 const (
 	windowsServiceName        = "Cloudflared"
-	windowsServiceDescription = "Cloudflare Tunnel agent"
-	windowsServiceUrl         = "https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/run-tunnel/run-as-service#windows"
+	windowsServiceDescription = "Cloudflared agent"
+	windowsServiceUrl         = "https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/run-tunnel/as-a-service/windows/"
 
 	recoverActionDelay      = time.Second * 20
 	failureCountResetPeriod = time.Hour * 24
@@ -46,16 +46,16 @@ const (
 func runApp(app *cli.App, graceShutdownC chan struct{}) {
 	app.Commands = append(app.Commands, &cli.Command{
 		Name:  "service",
-		Usage: "Manages the Cloudflare Tunnel Windows service",
+		Usage: "Manages the cloudflared Windows service",
 		Subcommands: []*cli.Command{
 			{
 				Name:   "install",
-				Usage:  "Install Cloudflare Tunnel as a Windows service",
+				Usage:  "Install cloudflared as a Windows service",
 				Action: cliutil.ConfiguredAction(installWindowsService),
 			},
 			{
 				Name:   "uninstall",
-				Usage:  "Uninstall the Cloudflare Tunnel service",
+				Usage:  "Uninstall the cloudflared service",
 				Action: cliutil.ConfiguredAction(uninstallWindowsService),
 			},
 		},
@@ -177,7 +177,7 @@ func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeReques
 func installWindowsService(c *cli.Context) error {
 	zeroLogger := logger.CreateLoggerFromContext(c, logger.EnableTerminalLog)
 
-	zeroLogger.Info().Msg("Installing Cloudflare Tunnel Windows service")
+	zeroLogger.Info().Msg("Installing cloudflared Windows service")
 	exepath, err := os.Executable()
 	if err != nil {
 		return errors.Wrap(err, "Cannot find path name that start the process")
@@ -191,26 +191,39 @@ func installWindowsService(c *cli.Context) error {
 	log := zeroLogger.With().Str(LogFieldWindowsServiceName, windowsServiceName).Logger()
 	if err == nil {
 		s.Close()
-		return fmt.Errorf("Service %s already exists", windowsServiceName)
+		return fmt.Errorf(serviceAlreadyExistsWarn(windowsServiceName))
 	}
+	extraArgs, err := getServiceExtraArgsFromCliArgs(c, &log)
+	if err != nil {
+		errMsg := "Unable to determine extra arguments for windows service"
+		log.Err(err).Msg(errMsg)
+		return errors.Wrap(err, errMsg)
+	}
+
 	config := mgr.Config{StartType: mgr.StartAutomatic, DisplayName: windowsServiceDescription}
-	s, err = m.CreateService(windowsServiceName, exepath, config)
+	s, err = m.CreateService(windowsServiceName, exepath, config, extraArgs...)
 	if err != nil {
 		return errors.Wrap(err, "Cannot install service")
 	}
 	defer s.Close()
-	log.Info().Msg("Cloudflare Tunnel agent service is installed")
+	log.Info().Msg("cloudflared agent service is installed")
 	err = eventlog.InstallAsEventCreate(windowsServiceName, eventlog.Error|eventlog.Warning|eventlog.Info)
 	if err != nil {
 		s.Delete()
 		return errors.Wrap(err, "Cannot install event logger")
 	}
+
 	err = configRecoveryOption(s.Handle)
 	if err != nil {
 		log.Err(err).Msg("Cannot set service recovery actions")
 		log.Info().Msgf("See %s to manually configure service recovery actions", windowsServiceUrl)
 	}
-	return nil
+
+	err = s.Start()
+	if err == nil {
+		log.Info().Msg("Agent service for cloudflared installed successfully")
+	}
+	return err
 }
 
 func uninstallWindowsService(c *cli.Context) error {
@@ -218,7 +231,7 @@ func uninstallWindowsService(c *cli.Context) error {
 		With().
 		Str(LogFieldWindowsServiceName, windowsServiceName).Logger()
 
-	log.Info().Msg("Uninstalling Cloudflare Tunnel Windows Service")
+	log.Info().Msg("Uninstalling cloudflared agent service")
 	m, err := mgr.Connect()
 	if err != nil {
 		return errors.Wrap(err, "Cannot establish a connection to the service control manager")
@@ -226,14 +239,22 @@ func uninstallWindowsService(c *cli.Context) error {
 	defer m.Disconnect()
 	s, err := m.OpenService(windowsServiceName)
 	if err != nil {
-		return fmt.Errorf("Service %s is not installed", windowsServiceName)
+		return fmt.Errorf("Agent service %s is not installed, so it could not be uninstalled", windowsServiceName)
 	}
 	defer s.Close()
+
+	if status, err := s.Query(); err == nil && status.State == svc.Running {
+		log.Info().Msg("Stopping cloudflared agent service")
+		if _, err := s.Control(svc.Stop); err != nil {
+			log.Info().Err(err).Msg("Failed to stop cloudflared agent service, you may need to stop it manually to complete uninstall.")
+		}
+	}
+
 	err = s.Delete()
 	if err != nil {
-		return errors.Wrap(err, "Cannot delete service")
+		return errors.Wrap(err, "Cannot delete agent service")
 	}
-	log.Info().Msg("Cloudflare Tunnel agent service is uninstalled")
+	log.Info().Msg("Agent service for cloudflared was uninstalled successfully")
 	err = eventlog.Remove(windowsServiceName)
 	if err != nil {
 		return errors.Wrap(err, "Cannot remove event logger")

component-tests/README.md

@@ -14,7 +14,7 @@ classic_hostname: "classic-tunnel-component-tests.example.com"
 origincert: "/Users/tunnel/.cloudflared/cert.pem"
 ingress:
 - hostname: named-tunnel-component-tests.example.com
-  service: http_status:200
+  service: hello_world
 - service: http_status:404
 ```
 

component-tests/config.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 import copy
+import json
+import base64
 
 from dataclasses import dataclass, InitVar
 
@@ -61,6 +63,28 @@ class NamedTunnelConfig(NamedTunnelBaseConfig):
     def get_url(self):
         return "https://" + self.ingress[0]['hostname']
 
+    def base_config(self):
+        config = self.full_config.copy()
+
+        # removes the tunnel reference
+        del(config["tunnel"])
+        del(config["credentials-file"])
+
+        return config
+
+    def get_tunnel_id(self):
+        return self.full_config["tunnel"]
+
+    def get_token(self):
+        creds = self.get_credentials_json()
+        token_dict = {"a": creds["AccountTag"], "t": creds["TunnelID"], "s": creds["TunnelSecret"]}
+        token_json_str = json.dumps(token_dict)
+        return base64.b64encode(token_json_str.encode('utf-8'))
+
+    def get_credentials_json(self):
+        with open(self.credentials_file) as json_file:
+            return json.load(json_file)
+
 
 @dataclass(frozen=True)
 class ClassicTunnelBaseConfig(BaseConfig):

component-tests/test_service.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 import os
+import pathlib
 import platform
 import subprocess
 from contextlib import contextmanager
@@ -9,7 +10,7 @@ import pytest
 
 import test_logging
 from conftest import CfdModes
-from util import start_cloudflared, wait_tunnel_ready
+from util import start_cloudflared, wait_tunnel_ready, write_config
 
 
 def select_platform(plat):
@@ -43,6 +44,21 @@ class TestServiceMode:
 
         self.launchd_service_scenario(config, assert_log_file)
 
+    @select_platform("Darwin")
+    @pytest.mark.skipif(os.path.exists(default_config_file()), reason=f"There is already a config file in default path")
+    def test_launchd_service_with_token(self, tmp_path, component_tests_config):
+        log_file = tmp_path / test_logging.default_log_file
+        additional_config = {
+            "logfile": str(log_file),
+        }
+        config = component_tests_config(additional_config=additional_config)
+
+        # service install doesn't install the config file but in this case we want to use some default settings
+        # so we write the base config without the tunnel credentials and ID
+        write_config(pathlib.Path(default_config_dir()), config.base_config())
+
+        self.launchd_service_scenario(config, use_token=True)
+
     @select_platform("Darwin")
     @pytest.mark.skipif(os.path.exists(default_config_file()), reason=f"There is already a config file in default path")
     def test_launchd_service_rotating_log(self, tmp_path, component_tests_config):
@@ -60,12 +76,13 @@ class TestServiceMode:
 
         self.launchd_service_scenario(config, assert_rotating_log)
 
-    def launchd_service_scenario(self, config, extra_assertions):
-        with self.run_service(Path(default_config_dir()), config):
+    def launchd_service_scenario(self, config, extra_assertions=None, use_token=False):
+        with self.run_service(Path(default_config_dir()), config, use_token=use_token):
             self.launchctl_cmd("list")
             self.launchctl_cmd("start")
             wait_tunnel_ready(tunnel_url=config.get_url())
-            extra_assertions()
+            if extra_assertions is not None:
+                extra_assertions()
             self.launchctl_cmd("stop")
 
         os.remove(default_config_file())
@@ -105,27 +122,48 @@ class TestServiceMode:
 
         self.sysv_service_scenario(config, tmp_path, assert_rotating_log)
 
-    def sysv_service_scenario(self, config, tmp_path, extra_assertions):
-        with self.run_service(tmp_path, config, root=True):
-            self.sysv_cmd("start")
+    @select_platform("Linux")
+    @pytest.mark.skipif(os.path.exists("/etc/cloudflared/config.yml"),
+                        reason=f"There is already a config file in default path")
+    def test_sysv_service_with_token(self, tmp_path, component_tests_config):
+        additional_config = {
+            "loglevel": "debug",
+        }
+
+        config = component_tests_config(additional_config=additional_config)
+
+        # service install doesn't install the config file but in this case we want to use some default settings
+        # so we write the base config without the tunnel credentials and ID
+        config_path = write_config(tmp_path, config.base_config())
+        subprocess.run(["sudo", "cp", config_path, "/etc/cloudflared/config.yml"], check=True)
+
+        self.sysv_service_scenario(config, tmp_path, use_token=True)
+
+    def sysv_service_scenario(self, config, tmp_path, extra_assertions=None, use_token=False):
+        with self.run_service(tmp_path, config, root=True, use_token=use_token):
             self.sysv_cmd("status")
             wait_tunnel_ready(tunnel_url=config.get_url())
-            extra_assertions()
-            self.sysv_cmd("stop")
+            if extra_assertions is not None:
+                extra_assertions()
 
         # Service install copies config file to /etc/cloudflared/config.yml
         subprocess.run(["sudo", "rm", "/etc/cloudflared/config.yml"])
         self.sysv_cmd("status", success=False)
 
     @contextmanager
-    def run_service(self, tmp_path, config, root=False):
+    def run_service(self, tmp_path, config, root=False, use_token=False):
+        args = ["service", "install"]
+
+        if use_token:
+            args.append(config.get_token())
+
         try:
             service = start_cloudflared(
-                tmp_path, config, cfd_args=["service", "install"], cfd_pre_args=[], capture_output=False, root=root)
+                tmp_path, config, cfd_args=args, cfd_pre_args=[], capture_output=False, root=root, skip_config_flag=use_token)
             yield service
         finally:
             start_cloudflared(
-                tmp_path, config, cfd_args=["service", "uninstall"], cfd_pre_args=[], capture_output=False, root=root)
+                tmp_path, config, cfd_args=["service", "uninstall"], cfd_pre_args=[], capture_output=False, root=root, skip_config_flag=use_token)
 
     def launchctl_cmd(self, action, success=True):
         cmd = subprocess.run(

component-tests/test_token.py

@@ -0,0 +1,35 @@
+import base64
+import json
+
+from setup import get_config_from_file, persist_origin_cert
+from util import start_cloudflared
+
+
+class TestToken:
+    def test_get_token(self, tmp_path, component_tests_config):
+        config = component_tests_config()
+        tunnel_id = config.get_tunnel_id()
+
+        token_args = ["--origincert", cert_path(), "token", tunnel_id]
+        output = start_cloudflared(tmp_path, config, token_args)
+
+        assert parse_token(config.get_token()) == parse_token(output.stdout)
+
+    def test_get_credentials_file(self, tmp_path, component_tests_config):
+        config = component_tests_config()
+        tunnel_id = config.get_tunnel_id()
+
+        cred_file = tmp_path / "test_get_credentials_file.json"
+        token_args = ["--origincert", cert_path(), "token", "--cred-file", cred_file, tunnel_id]
+        start_cloudflared(tmp_path, config, token_args)
+
+        with open(cred_file) as json_file:
+            assert config.get_credentials_json() == json.load(json_file)
+
+
+def cert_path():
+    return get_config_from_file()["origincert"]
+
+
+def parse_token(token):
+    return json.loads(base64.b64decode(token))

component-tests/util.py

@@ -21,9 +21,14 @@ def write_config(directory, config):
 
 
 def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False,
-                      allow_input=False, capture_output=True, root=False):
-    config_path = write_config(directory, config.full_config)
+                      allow_input=False, capture_output=True, root=False, skip_config_flag=False):
+
+    config_path = None
+    if not skip_config_flag:
+        config_path = write_config(directory, config.full_config)
+
     cmd = cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root)
+
     if new_process:
         return run_cloudflared_background(cmd, allow_input, capture_output)
     # By setting check=True, it will raise an exception if the process exits with non-zero exit code
@@ -36,7 +41,10 @@ def cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root):
         cmd += ["sudo"]
     cmd += [config.cloudflared_binary]
     cmd += cfd_pre_args
-    cmd += ["--config", str(config_path)]
+
+    if config_path is not None:
+        cmd += ["--config", str(config_path)]
+
     cmd += cfd_args
     LOGGER.info(f"Run cmd {cmd} with config {config}")
     return cmd

config/configuration.go

@@ -1,19 +1,21 @@
 package config
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/url"
 	"os"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"time"
 
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/urfave/cli/v2"
-	yaml "gopkg.in/yaml.v2"
+	yaml "gopkg.in/yaml.v3"
 
 	"github.com/cloudflare/cloudflared/validation"
 )
@@ -49,7 +51,7 @@ func DefaultConfigDirectory() string {
 		path := os.Getenv("CFDPATH")
 		if path == "" {
 			path = filepath.Join(os.Getenv("ProgramFiles(x86)"), "cloudflared")
-			if _, err := os.Stat(path); os.IsNotExist(err) { //doesn't exist, so return an empty failure string
+			if _, err := os.Stat(path); os.IsNotExist(err) { // doesn't exist, so return an empty failure string
 				return ""
 			}
 		}
@@ -138,7 +140,7 @@ func FindOrCreateConfigPath() string {
 		defer file.Close()
 
 		logDir := DefaultLogDirectory()
-		_ = os.MkdirAll(logDir, os.ModePerm) //try and create it. Doesn't matter if it succeed or not, only byproduct will be no logs
+		_ = os.MkdirAll(logDir, os.ModePerm) // try and create it. Doesn't matter if it succeed or not, only byproduct will be no logs
 
 		c := Root{
 			LogDirectory: logDir,
@@ -175,9 +177,9 @@ func ValidateUrl(c *cli.Context, allowURLFromArgs bool) (*url.URL, error) {
 }
 
 type UnvalidatedIngressRule struct {
-	Hostname      string              `json:"hostname"`
-	Path          string              `json:"path"`
-	Service       string              `json:"service"`
+	Hostname      string              `json:"hostname,omitempty"`
+	Path          string              `json:"path,omitempty"`
+	Service       string              `json:"service,omitempty"`
 	OriginRequest OriginRequestConfig `yaml:"originRequest" json:"originRequest"`
 }
 
@@ -190,41 +192,41 @@ type UnvalidatedIngressRule struct {
 // - To specify a time.Duration in json, use int64 of the nanoseconds
 type OriginRequestConfig struct {
 	// HTTP proxy timeout for establishing a new connection
-	ConnectTimeout *time.Duration `yaml:"connectTimeout" json:"connectTimeout"`
+	ConnectTimeout *CustomDuration `yaml:"connectTimeout" json:"connectTimeout,omitempty"`
 	// HTTP proxy timeout for completing a TLS handshake
-	TLSTimeout *time.Duration `yaml:"tlsTimeout" json:"tlsTimeout"`
+	TLSTimeout *CustomDuration `yaml:"tlsTimeout" json:"tlsTimeout,omitempty"`
 	// HTTP proxy TCP keepalive duration
-	TCPKeepAlive *time.Duration `yaml:"tcpKeepAlive" json:"tcpKeepAlive"`
+	TCPKeepAlive *CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive,omitempty"`
 	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
-	NoHappyEyeballs *bool `yaml:"noHappyEyeballs" json:"noHappyEyeballs"`
+	NoHappyEyeballs *bool `yaml:"noHappyEyeballs" json:"noHappyEyeballs,omitempty"`
 	// HTTP proxy maximum keepalive connection pool size
-	KeepAliveConnections *int `yaml:"keepAliveConnections" json:"keepAliveConnections"`
+	KeepAliveConnections *int `yaml:"keepAliveConnections" json:"keepAliveConnections,omitempty"`
 	// HTTP proxy timeout for closing an idle connection
-	KeepAliveTimeout *time.Duration `yaml:"keepAliveTimeout" json:"keepAliveTimeout"`
+	KeepAliveTimeout *CustomDuration `yaml:"keepAliveTimeout" json:"keepAliveTimeout,omitempty"`
 	// Sets the HTTP Host header for the local webserver.
-	HTTPHostHeader *string `yaml:"httpHostHeader" json:"httpHostHeader"`
+	HTTPHostHeader *string `yaml:"httpHostHeader" json:"httpHostHeader,omitempty"`
 	// Hostname on the origin server certificate.
-	OriginServerName *string `yaml:"originServerName" json:"originServerName"`
+	OriginServerName *string `yaml:"originServerName" json:"originServerName,omitempty"`
 	// Path to the CA for the certificate of your origin.
 	// This option should be used only if your certificate is not signed by Cloudflare.
-	CAPool *string `yaml:"caPool" json:"caPool"`
+	CAPool *string `yaml:"caPool" json:"caPool,omitempty"`
 	// Disables TLS verification of the certificate presented by your origin.
 	// Will allow any certificate from the origin to be accepted.
 	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
-	NoTLSVerify *bool `yaml:"noTLSVerify" json:"noTLSVerify"`
+	NoTLSVerify *bool `yaml:"noTLSVerify" json:"noTLSVerify,omitempty"`
 	// Disables chunked transfer encoding.
 	// Useful if you are running a WSGI server.
-	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding" json:"disableChunkedEncoding"`
+	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding" json:"disableChunkedEncoding,omitempty"`
 	// Runs as jump host
-	BastionMode *bool `yaml:"bastionMode" json:"bastionMode"`
+	BastionMode *bool `yaml:"bastionMode" json:"bastionMode,omitempty"`
 	// Listen address for the proxy.
-	ProxyAddress *string `yaml:"proxyAddress" json:"proxyAddress"`
+	ProxyAddress *string `yaml:"proxyAddress" json:"proxyAddress,omitempty"`
 	// Listen port for the proxy.
-	ProxyPort *uint `yaml:"proxyPort" json:"proxyPort"`
+	ProxyPort *uint `yaml:"proxyPort" json:"proxyPort,omitempty"`
 	// Valid options are 'socks' or empty.
-	ProxyType *string `yaml:"proxyType" json:"proxyType"`
+	ProxyType *string `yaml:"proxyType" json:"proxyType,omitempty"`
 	// IP rules for the proxy service
-	IPRules []IngressIPRule `yaml:"ipRules" json:"ipRules"`
+	IPRules []IngressIPRule `yaml:"ipRules" json:"ipRules,omitempty"`
 }
 
 type IngressIPRule struct {
@@ -390,7 +392,7 @@ func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (settings *configFileSe
 	// Parse it again, with strict mode, to find warnings.
 	if file, err := os.Open(configFile); err == nil {
 		decoder := yaml.NewDecoder(file)
-		decoder.SetStrict(true)
+		decoder.KnownFields(true)
 		var unusedConfig configFileSettings
 		if err := decoder.Decode(&unusedConfig); err != nil {
 			warnings = err.Error()
@@ -399,3 +401,34 @@ func ReadConfigFile(c *cli.Context, log *zerolog.Logger) (settings *configFileSe
 
 	return &configuration, warnings, nil
 }
+
+// A CustomDuration is a Duration that has custom serialization for JSON.
+// JSON in Javascript assumes that int fields are 32 bits and Duration fields are deserialized assuming that numbers
+// are in nanoseconds, which in 32bit integers limits to just 2 seconds.
+// This type assumes that when serializing/deserializing from JSON, that the number is in seconds, while it maintains
+// the YAML serde assumptions.
+type CustomDuration struct {
+	time.Duration
+}
+
+func (s CustomDuration) MarshalJSON() ([]byte, error) {
+	return json.Marshal(s.Duration.Seconds())
+}
+
+func (s *CustomDuration) UnmarshalJSON(data []byte) error {
+	seconds, err := strconv.ParseInt(string(data), 10, 64)
+	if err != nil {
+		return err
+	}
+
+	s.Duration = time.Duration(seconds * int64(time.Second))
+	return nil
+}
+
+func (s *CustomDuration) MarshalYAML() (interface{}, error) {
+	return s.Duration.String(), nil
+}
+
+func (s *CustomDuration) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	return unmarshal(&s.Duration)
+}

config/configuration_test.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -110,14 +111,13 @@ counters:
 
 }
 
-func TestUnmarshalOriginRequestConfig(t *testing.T) {
-	raw := []byte(`
+var rawConfig = []byte(`
 {
-	"connectTimeout": 10000000000,
-	"tlsTimeout": 30000000000,
-	"tcpKeepAlive": 30000000000,
+	"connectTimeout": 10,
+	"tlsTimeout": 30,
+	"tcpKeepAlive": 30,
 	"noHappyEyeballs": true,
-	"keepAliveTimeout": 60000000000,
+	"keepAliveTimeout": 60,
 	"keepAliveConnections": 10,
 	"httpHostHeader": "app.tunnel.com",
 	"originServerName": "app.tunnel.com",
@@ -142,13 +142,41 @@ func TestUnmarshalOriginRequestConfig(t *testing.T) {
 	]
 }
 `)
+
+func TestMarshalUnmarshalOriginRequest(t *testing.T) {
+	testCases := []struct {
+		name          string
+		marshalFunc   func(in interface{}) (out []byte, err error)
+		unMarshalFunc func(in []byte, out interface{}) (err error)
+		baseUnit      time.Duration
+	}{
+		{"json", json.Marshal, json.Unmarshal, time.Second},
+		{"yaml", yaml.Marshal, yaml.Unmarshal, time.Nanosecond},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assertConfig(t, tc.marshalFunc, tc.unMarshalFunc, tc.baseUnit)
+		})
+	}
+}
+
+func assertConfig(
+	t *testing.T,
+	marshalFunc func(in interface{}) (out []byte, err error),
+	unMarshalFunc func(in []byte, out interface{}) (err error),
+	baseUnit time.Duration,
+) {
 	var config OriginRequestConfig
-	assert.NoError(t, json.Unmarshal(raw, &config))
-	assert.Equal(t, time.Second*10, *config.ConnectTimeout)
-	assert.Equal(t, time.Second*30, *config.TLSTimeout)
-	assert.Equal(t, time.Second*30, *config.TCPKeepAlive)
+	var config2 OriginRequestConfig
+
+	assert.NoError(t, unMarshalFunc(rawConfig, &config))
+
+	assert.Equal(t, baseUnit*10, config.ConnectTimeout.Duration)
+	assert.Equal(t, baseUnit*30, config.TLSTimeout.Duration)
+	assert.Equal(t, baseUnit*30, config.TCPKeepAlive.Duration)
 	assert.Equal(t, true, *config.NoHappyEyeballs)
-	assert.Equal(t, time.Second*60, *config.KeepAliveTimeout)
+	assert.Equal(t, baseUnit*60, config.KeepAliveTimeout.Duration)
 	assert.Equal(t, 10, *config.KeepAliveConnections)
 	assert.Equal(t, "app.tunnel.com", *config.HTTPHostHeader)
 	assert.Equal(t, "app.tunnel.com", *config.OriginServerName)
@@ -176,4 +204,12 @@ func TestUnmarshalOriginRequestConfig(t *testing.T) {
 		},
 	}
 	assert.Equal(t, ipRules, config.IPRules)
+
+	// validate that serializing and deserializing again matches the deserialization from raw string
+	result, err := marshalFunc(config)
+	require.NoError(t, err)
+	err = unMarshalFunc(result, &config2)
+	require.NoError(t, err)
+
+	require.Equal(t, config2, config)
 }

connection/connection.go

@@ -2,6 +2,7 @@ package connection
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"io"
 	"math"
@@ -11,7 +12,9 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/pkg/errors"
 
+	"github.com/cloudflare/cloudflared/tracing"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/websocket"
 )
@@ -25,13 +28,13 @@ const (
 
 var switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols, http.StatusText(http.StatusSwitchingProtocols))
 
-type Config struct {
-	OriginProxy     OriginProxy
-	GracePeriod     time.Duration
-	ReplaceExisting bool
+type Orchestrator interface {
+	UpdateConfig(version int32, config []byte) *pogs.UpdateConfigurationResponse
+	GetConfigJSON() ([]byte, error)
+	GetOriginProxy() (OriginProxy, error)
 }
 
-type NamedTunnelConfig struct {
+type NamedTunnelProperties struct {
 	Credentials    Credentials
 	Client         pogs.ClientInfo
 	QuickTunnelUrl string
@@ -42,7 +45,6 @@ type Credentials struct {
 	AccountTag   string
 	TunnelSecret []byte
 	TunnelID     uuid.UUID
-	TunnelName   string
 }
 
 func (c *Credentials) Auth() pogs.TunnelAuth {
@@ -52,7 +54,31 @@ func (c *Credentials) Auth() pogs.TunnelAuth {
 	}
 }
 
-type ClassicTunnelConfig struct {
+// TunnelToken are Credentials but encoded with custom fields namings.
+type TunnelToken struct {
+	AccountTag   string    `json:"a"`
+	TunnelSecret []byte    `json:"s"`
+	TunnelID     uuid.UUID `json:"t"`
+}
+
+func (t TunnelToken) Credentials() Credentials {
+	return Credentials{
+		AccountTag:   t.AccountTag,
+		TunnelSecret: t.TunnelSecret,
+		TunnelID:     t.TunnelID,
+	}
+}
+
+func (t TunnelToken) Encode() (string, error) {
+	val, err := json.Marshal(t)
+	if err != nil {
+		return "", errors.Wrap(err, "could not JSON encode token")
+	}
+
+	return base64.StdEncoding.EncodeToString(val), nil
+}
+
+type ClassicTunnelProperties struct {
 	Hostname   string
 	OriginCert []byte
 	// feature-flag to use new edge reconnect tokens
@@ -67,6 +93,7 @@ const (
 	TypeTCP
 	TypeControlStream
 	TypeHTTP
+	TypeConfiguration
 )
 
 // ShouldFlush returns whether this kind of connection should actively flush data
@@ -96,7 +123,7 @@ func (t Type) String() string {
 
 // OriginProxy is how data flows from cloudflared to the origin services running behind it.
 type OriginProxy interface {
-	ProxyHTTP(w ResponseWriter, req *http.Request, isWebsocket bool) error
+	ProxyHTTP(w ResponseWriter, tr *tracing.TracedRequest, isWebsocket bool) error
 	ProxyTCP(ctx context.Context, rwa ReadWriteAcker, req *TCPRequest) error
 }
 

connection/connection_test.go

@@ -4,33 +4,29 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"math/rand"
 	"net/http"
-	"net/url"
 	"testing"
 	"time"
 
-	"github.com/gobwas/ws/wsutil"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/tracing"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
 )
 
 const (
-	largeFileSize = 2 * 1024 * 1024
+	largeFileSize   = 2 * 1024 * 1024
+	testGracePeriod = time.Millisecond * 100
 )
 
 var (
-	unusedWarpRoutingService = (*ingress.WarpRoutingService)(nil)
-	testConfig               = &Config{
-		OriginProxy: &mockOriginProxy{},
-		GracePeriod: time.Millisecond * 100,
+	testOrchestrator = &mockOrchestrator{
+		originProxy: &mockOriginProxy{},
 	}
 	log           = zerolog.Nop()
-	testOriginURL = &url.URL{
-		Scheme: "https",
-		Host:   "connectiontest.argotunnel.com",
-	}
 	testLargeResp = make([]byte, largeFileSize)
 )
 
@@ -42,15 +38,42 @@ type testRequest struct {
 	isProxyError   bool
 }
 
+type mockOrchestrator struct {
+	originProxy OriginProxy
+}
+
+func (mcr *mockOrchestrator) GetConfigJSON() ([]byte, error) {
+	return nil, fmt.Errorf("not implemented")
+}
+
+func (*mockOrchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
+	return &tunnelpogs.UpdateConfigurationResponse{
+		LastAppliedVersion: version,
+	}
+}
+
+func (mcr *mockOrchestrator) GetOriginProxy() (OriginProxy, error) {
+	return mcr.originProxy, nil
+}
+
 type mockOriginProxy struct{}
 
 func (moc *mockOriginProxy) ProxyHTTP(
 	w ResponseWriter,
-	req *http.Request,
+	tr *tracing.TracedRequest,
 	isWebsocket bool,
 ) error {
+	req := tr.Request
 	if isWebsocket {
-		return wsEndpoint(w, req)
+		switch req.URL.Path {
+		case "/ws/echo":
+			return wsEchoEndpoint(w, req)
+		case "/ws/flaky":
+			return wsFlakyEndpoint(w, req)
+		default:
+			originRespEndpoint(w, http.StatusNotFound, []byte("ws endpoint not found"))
+			return fmt.Errorf("Unknwon websocket endpoint %s", req.URL.Path)
+		}
 	}
 	switch req.URL.Path {
 	case "/ok":
@@ -78,32 +101,82 @@ func (moc *mockOriginProxy) ProxyTCP(
 	return nil
 }
 
-type nowriter struct {
-	io.Reader
+type echoPipe struct {
+	reader *io.PipeReader
+	writer *io.PipeWriter
 }
 
-func (nowriter) Write(p []byte) (int, error) {
-	return 0, fmt.Errorf("Writer not implemented")
+func (ep *echoPipe) Read(p []byte) (int, error) {
+	return ep.reader.Read(p)
 }
 
-func wsEndpoint(w ResponseWriter, r *http.Request) error {
+func (ep *echoPipe) Write(p []byte) (int, error) {
+	return ep.writer.Write(p)
+}
+
+// A mock origin that echos data by streaming like a tcpOverWSConnection
+// https://github.com/cloudflare/cloudflared/blob/master/ingress/origin_connection.go
+func wsEchoEndpoint(w ResponseWriter, r *http.Request) error {
 	resp := &http.Response{
 		StatusCode: http.StatusSwitchingProtocols,
 	}
-	_ = w.WriteRespHeaders(resp.StatusCode, resp.Header)
-	clientReader := nowriter{r.Body}
+	if err := w.WriteRespHeaders(resp.StatusCode, resp.Header); err != nil {
+		return err
+	}
+	wsCtx, cancel := context.WithCancel(r.Context())
+	readPipe, writePipe := io.Pipe()
+	wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, r), &log)
 	go func() {
-		for {
-			data, err := wsutil.ReadClientText(clientReader)
-			if err != nil {
-				return
-			}
-			if err := wsutil.WriteServerText(w, data); err != nil {
-				return
-			}
+		select {
+		case <-wsCtx.Done():
+		case <-r.Context().Done():
 		}
+		readPipe.Close()
+		writePipe.Close()
 	}()
-	<-r.Context().Done()
+
+	originConn := &echoPipe{reader: readPipe, writer: writePipe}
+	websocket.Stream(wsConn, originConn, &log)
+	cancel()
+	wsConn.Close()
+	return nil
+}
+
+type flakyConn struct {
+	closeAt time.Time
+}
+
+func (fc *flakyConn) Read(p []byte) (int, error) {
+	if time.Now().After(fc.closeAt) {
+		return 0, io.EOF
+	}
+	n := copy(p, "Read from flaky connection")
+	return n, nil
+}
+
+func (fc *flakyConn) Write(p []byte) (int, error) {
+	if time.Now().After(fc.closeAt) {
+		return 0, fmt.Errorf("flaky connection closed")
+	}
+	return len(p), nil
+}
+
+func wsFlakyEndpoint(w ResponseWriter, r *http.Request) error {
+	resp := &http.Response{
+		StatusCode: http.StatusSwitchingProtocols,
+	}
+	if err := w.WriteRespHeaders(resp.StatusCode, resp.Header); err != nil {
+		return err
+	}
+	wsCtx, cancel := context.WithCancel(r.Context())
+
+	wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, r), &log)
+
+	closedAfter := time.Millisecond * time.Duration(rand.Intn(50))
+	originConn := &flakyConn{closeAt: time.Now().Add(closedAfter)}
+	websocket.Stream(wsConn, originConn, &log)
+	cancel()
+	wsConn.Close()
 	return nil
 }
 

connection/control.go

@@ -16,9 +16,9 @@ type RPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) Na
 type controlStream struct {
 	observer *Observer
 
-	connectedFuse     ConnectedFuse
-	namedTunnelConfig *NamedTunnelConfig
-	connIndex         uint8
+	connectedFuse         ConnectedFuse
+	namedTunnelProperties *NamedTunnelProperties
+	connIndex             uint8
 
 	newRPCClientFunc RPCClientFunc
 
@@ -30,16 +30,20 @@ type controlStream struct {
 // ControlStreamHandler registers connections with origintunneld and initiates graceful shutdown.
 type ControlStreamHandler interface {
 	// ServeControlStream handles the control plane of the transport in the current goroutine calling this
-	ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions) error
+	ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions, tunnelConfigGetter TunnelConfigJSONGetter) error
 	// IsStopped tells whether the method above has finished
 	IsStopped() bool
 }
 
+type TunnelConfigJSONGetter interface {
+	GetConfigJSON() ([]byte, error)
+}
+
 // NewControlStream returns a new instance of ControlStreamHandler
 func NewControlStream(
 	observer *Observer,
 	connectedFuse ConnectedFuse,
-	namedTunnelConfig *NamedTunnelConfig,
+	namedTunnelConfig *NamedTunnelProperties,
 	connIndex uint8,
 	newRPCClientFunc RPCClientFunc,
 	gracefulShutdownC <-chan struct{},
@@ -49,13 +53,13 @@ func NewControlStream(
 		newRPCClientFunc = newRegistrationRPCClient
 	}
 	return &controlStream{
-		observer:          observer,
-		connectedFuse:     connectedFuse,
-		namedTunnelConfig: namedTunnelConfig,
-		newRPCClientFunc:  newRPCClientFunc,
-		connIndex:         connIndex,
-		gracefulShutdownC: gracefulShutdownC,
-		gracePeriod:       gracePeriod,
+		observer:              observer,
+		connectedFuse:         connectedFuse,
+		namedTunnelProperties: namedTunnelConfig,
+		newRPCClientFunc:      newRPCClientFunc,
+		connIndex:             connIndex,
+		gracefulShutdownC:     gracefulShutdownC,
+		gracePeriod:           gracePeriod,
 	}
 }
 
@@ -63,15 +67,28 @@ func (c *controlStream) ServeControlStream(
 	ctx context.Context,
 	rw io.ReadWriteCloser,
 	connOptions *tunnelpogs.ConnectionOptions,
+	tunnelConfigGetter TunnelConfigJSONGetter,
 ) error {
 	rpcClient := c.newRPCClientFunc(ctx, rw, c.observer.log)
 
-	if err := rpcClient.RegisterConnection(ctx, c.namedTunnelConfig, connOptions, c.connIndex, c.observer); err != nil {
+	registrationDetails, err := rpcClient.RegisterConnection(ctx, c.namedTunnelProperties, connOptions, c.connIndex, c.observer)
+	if err != nil {
 		rpcClient.Close()
 		return err
 	}
 	c.connectedFuse.Connected()
 
+	// if conn index is 0 and tunnel is not remotely managed, then send local ingress rules configuration
+	if c.connIndex == 0 && !registrationDetails.TunnelIsRemotelyManaged {
+		if tunnelConfig, err := tunnelConfigGetter.GetConfigJSON(); err == nil {
+			if err := rpcClient.SendLocalConfiguration(ctx, tunnelConfig, c.observer); err != nil {
+				c.observer.log.Err(err).Msg("unable to send local configuration")
+			}
+		} else {
+			c.observer.log.Err(err).Msg("failed to obtain current configuration")
+		}
+	}
+
 	c.waitForUnregister(ctx, rpcClient)
 	return nil
 }

connection/h2mux.go

@@ -12,6 +12,7 @@ import (
 	"golang.org/x/sync/errgroup"
 
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/websocket"
 )
@@ -22,9 +23,10 @@ const (
 )
 
 type h2muxConnection struct {
-	config      *Config
-	muxerConfig *MuxerConfig
-	muxer       *h2mux.Muxer
+	orchestrator Orchestrator
+	gracePeriod  time.Duration
+	muxerConfig  *MuxerConfig
+	muxer        *h2mux.Muxer
 	// connectionID is only used by metrics, and prometheus requires labels to be string
 	connIndexStr string
 	connIndex    uint8
@@ -60,7 +62,8 @@ func (mc *MuxerConfig) H2MuxerConfig(h h2mux.MuxedStreamHandler, log *zerolog.Lo
 
 // NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
 func NewH2muxConnection(
-	config *Config,
+	orchestrator Orchestrator,
+	gracePeriod time.Duration,
 	muxerConfig *MuxerConfig,
 	edgeConn net.Conn,
 	connIndex uint8,
@@ -68,7 +71,8 @@ func NewH2muxConnection(
 	gracefulShutdownC <-chan struct{},
 ) (*h2muxConnection, error, bool) {
 	h := &h2muxConnection{
-		config:            config,
+		orchestrator:      orchestrator,
+		gracePeriod:       gracePeriod,
 		muxerConfig:       muxerConfig,
 		connIndexStr:      uint8ToString(connIndex),
 		connIndex:         connIndex,
@@ -88,7 +92,7 @@ func NewH2muxConnection(
 	return h, nil, false
 }
 
-func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelConfig, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
+func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelProperties, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {
 		return h.serveMuxer(serveCtx)
@@ -117,7 +121,7 @@ func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *Nam
 	return err
 }
 
-func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel *ClassicTunnelConfig, credentialManager CredentialManager, registrationOptions *tunnelpogs.RegistrationOptions, connectedFuse ConnectedFuse) error {
+func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel *ClassicTunnelProperties, credentialManager CredentialManager, registrationOptions *tunnelpogs.RegistrationOptions, connectedFuse ConnectedFuse) error {
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {
 		return h.serveMuxer(serveCtx)
@@ -224,7 +228,13 @@ func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
 		sourceConnectionType = TypeWebsocket
 	}
 
-	err := h.config.OriginProxy.ProxyHTTP(respWriter, req, sourceConnectionType == TypeWebsocket)
+	originProxy, err := h.orchestrator.GetOriginProxy()
+	if err != nil {
+		respWriter.WriteErrorResponse()
+		return err
+	}
+
+	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), sourceConnectionType == TypeWebsocket)
 	if err != nil {
 		respWriter.WriteErrorResponse()
 	}

connection/h2mux_test.go

@@ -48,7 +48,7 @@ func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
 	}()
 	var connIndex = uint8(0)
 	testObserver := NewObserver(&log, &log, false)
-	h2muxConn, err, _ := NewH2muxConnection(testConfig, testMuxerConfig, originConn, connIndex, testObserver, nil)
+	h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil)
 	require.NoError(t, err)
 	return h2muxConn, <-edgeMuxChan
 }
@@ -147,7 +147,7 @@ func TestServeStreamWS(t *testing.T) {
 	headers := []h2mux.Header{
 		{
 			Name:  ":path",
-			Value: "/ws",
+			Value: "/ws/echo",
 		},
 		{
 			Name:  "connection",
@@ -167,10 +167,10 @@ func TestServeStreamWS(t *testing.T) {
 	assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
 
 	data := []byte("test websocket")
-	err = wsutil.WriteClientText(writePipe, data)
+	err = wsutil.WriteClientBinary(writePipe, data)
 	require.NoError(t, err)
 
-	respBody, err := wsutil.ReadServerText(stream)
+	respBody, err := wsutil.ReadServerBinary(stream)
 	require.NoError(t, err)
 	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
 

connection/http2.go

@@ -2,6 +2,7 @@ package connection
 
 import (
 	"context"
+	gojson "encoding/json"
 	"fmt"
 	"io"
 	"net"
@@ -14,6 +15,7 @@ import (
 	"github.com/rs/zerolog"
 	"golang.org/x/net/http2"
 
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
@@ -23,6 +25,7 @@ const (
 	InternalTCPProxySrcHeader = "Cf-Cloudflared-Proxy-Src"
 	WebsocketUpgrade          = "websocket"
 	ControlStreamUpgrade      = "control-stream"
+	ConfigurationUpdate       = "update-configuration"
 )
 
 var errEdgeConnectionClosed = fmt.Errorf("connection with edge closed")
@@ -30,12 +33,12 @@ var errEdgeConnectionClosed = fmt.Errorf("connection with edge closed")
 // HTTP2Connection represents a net.Conn that uses HTTP2 frames to proxy traffic from the edge to cloudflared on the
 // origin.
 type HTTP2Connection struct {
-	conn        net.Conn
-	server      *http2.Server
-	config      *Config
-	connOptions *tunnelpogs.ConnectionOptions
-	observer    *Observer
-	connIndex   uint8
+	conn         net.Conn
+	server       *http2.Server
+	orchestrator Orchestrator
+	connOptions  *tunnelpogs.ConnectionOptions
+	observer     *Observer
+	connIndex    uint8
 	// newRPCClientFunc allows us to mock RPCs during testing
 	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
 
@@ -49,7 +52,7 @@ type HTTP2Connection struct {
 // NewHTTP2Connection returns a new instance of HTTP2Connection.
 func NewHTTP2Connection(
 	conn net.Conn,
-	config *Config,
+	orchestrator Orchestrator,
 	connOptions *tunnelpogs.ConnectionOptions,
 	observer *Observer,
 	connIndex uint8,
@@ -61,7 +64,7 @@ func NewHTTP2Connection(
 		server: &http2.Server{
 			MaxConcurrentStreams: MaxConcurrentStreams,
 		},
-		config:               config,
+		orchestrator:         orchestrator,
 		connOptions:          connOptions,
 		observer:             observer,
 		connIndex:            connIndex,
@@ -100,7 +103,13 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	connType := determineHTTP2Type(r)
 	handleMissingRequestParts(connType, r)
 
-	respWriter, err := NewHTTP2RespWriter(r, w, connType)
+	respWriter, err := NewHTTP2RespWriter(r, w, connType, c.log)
+	if err != nil {
+		c.observer.log.Error().Msg(err.Error())
+		return
+	}
+
+	originProxy, err := c.orchestrator.GetOriginProxy()
 	if err != nil {
 		c.observer.log.Error().Msg(err.Error())
 		return
@@ -108,15 +117,23 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	switch connType {
 	case TypeControlStream:
-		if err := c.controlStreamHandler.ServeControlStream(r.Context(), respWriter, c.connOptions); err != nil {
+		if err := c.controlStreamHandler.ServeControlStream(r.Context(), respWriter, c.connOptions, c.orchestrator); err != nil {
 			c.controlStreamErr = err
 			c.log.Error().Err(err)
 			respWriter.WriteErrorResponse()
 		}
 
+	case TypeConfiguration:
+		if err := c.handleConfigurationUpdate(respWriter, r); err != nil {
+			c.log.Error().Err(err)
+			respWriter.WriteErrorResponse()
+		}
+
 	case TypeWebsocket, TypeHTTP:
 		stripWebsocketUpgradeHeader(r)
-		if err := c.config.OriginProxy.ProxyHTTP(respWriter, r, connType == TypeWebsocket); err != nil {
+		// Check for tracing on request
+		tr := tracing.NewTracedRequest(r)
+		if err := originProxy.ProxyHTTP(respWriter, tr, connType == TypeWebsocket); err != nil {
 			err := fmt.Errorf("Failed to proxy HTTP: %w", err)
 			c.log.Error().Err(err)
 			respWriter.WriteErrorResponse()
@@ -131,7 +148,7 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 
 		rws := NewHTTPResponseReadWriterAcker(respWriter, r)
-		if err := c.config.OriginProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
+		if err := originProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
 			Dest:    host,
 			CFRay:   FindCfRayHeader(r),
 			LBProbe: IsLBProbeRequest(r),
@@ -146,6 +163,26 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// ConfigurationUpdateBody is the representation followed by the edge to send updates to cloudflared.
+type ConfigurationUpdateBody struct {
+	Version int32             `json:"version"`
+	Config  gojson.RawMessage `json:"config"`
+}
+
+func (c *HTTP2Connection) handleConfigurationUpdate(respWriter *http2RespWriter, r *http.Request) error {
+	var configBody ConfigurationUpdateBody
+	if err := json.NewDecoder(r.Body).Decode(&configBody); err != nil {
+		return err
+	}
+	resp := c.orchestrator.UpdateConfig(configBody.Version, configBody.Config)
+	bdy, err := json.Marshal(resp)
+	if err != nil {
+		return err
+	}
+	_, err = respWriter.Write(bdy)
+	return err
+}
+
 func (c *HTTP2Connection) close() {
 	// Wait for all serve HTTP handlers to return
 	c.activeRequestsWG.Wait()
@@ -157,14 +194,16 @@ type http2RespWriter struct {
 	w           http.ResponseWriter
 	flusher     http.Flusher
 	shouldFlush bool
+	log         *zerolog.Logger
 }
 
-func NewHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type) (*http2RespWriter, error) {
+func NewHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type, log *zerolog.Logger) (*http2RespWriter, error) {
 	flusher, isFlusher := w.(http.Flusher)
 	if !isFlusher {
 		respWriter := &http2RespWriter{
-			r: r.Body,
-			w: w,
+			r:   r.Body,
+			w:   w,
+			log: log,
 		}
 		respWriter.WriteErrorResponse()
 		return nil, fmt.Errorf("%T doesn't implement http.Flusher", w)
@@ -175,6 +214,7 @@ func NewHTTP2RespWriter(r *http.Request, w http.ResponseWriter, connType Type) (
 		w:           w,
 		flusher:     flusher,
 		shouldFlush: connType.shouldFlush(),
+		log:         log,
 	}, nil
 }
 
@@ -191,6 +231,12 @@ func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) erro
 			dest[name] = values
 		}
 
+		if h2name == tracing.IntCloudflaredTracingHeader {
+			// Add cf-int-cloudflared-tracing header outside of serialized userHeaders
+			rp.w.Header()[tracing.CanonicalCloudflaredTracingHeader] = values
+			continue
+		}
+
 		if !IsControlResponseHeader(h2name) || IsWebsocketClientHeader(h2name) {
 			// User headers, on the other hand, must all be serialized so that
 			// HTTP/2 header validation won't be applied to HTTP/1 header values
@@ -233,7 +279,7 @@ func (rp *http2RespWriter) Write(p []byte) (n int, err error) {
 		// Implementer of OriginClient should make sure it doesn't write to the connection after Proxy returns
 		// Register a recover routine just in case.
 		if r := recover(); r != nil {
-			println(fmt.Sprintf("Recover from http2 response writer panic, error %s", debug.Stack()))
+			rp.log.Debug().Msgf("Recover from http2 response writer panic, error %s", debug.Stack())
 		}
 	}()
 	n, err = rp.w.Write(p)
@@ -249,6 +295,8 @@ func (rp *http2RespWriter) Close() error {
 
 func determineHTTP2Type(r *http.Request) Type {
 	switch {
+	case isConfigurationUpdate(r):
+		return TypeConfiguration
 	case isWebsocketUpgrade(r):
 		return TypeWebsocket
 	case IsTCPStream(r):
@@ -282,6 +330,10 @@ func isWebsocketUpgrade(r *http.Request) bool {
 	return r.Header.Get(InternalUpgradeHeader) == WebsocketUpgrade
 }
 
+func isConfigurationUpdate(r *http.Request) bool {
+	return r.Header.Get(InternalUpgradeHeader) == ConfigurationUpdate
+}
+
 // IsTCPStream discerns if the connection request needs a tcp stream proxy.
 func IsTCPStream(r *http.Request) bool {
 	return r.Header.Get(InternalTCPProxySrcHeader) != ""

connection/http2_test.go

@@ -1,7 +1,9 @@
 package connection
 
 import (
+	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -13,6 +15,7 @@ import (
 	"time"
 
 	"github.com/gobwas/ws/wsutil"
+	"github.com/google/uuid"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -27,22 +30,23 @@ var (
 )
 
 func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
-	edgeConn, originConn := net.Pipe()
+	edgeConn, cfdConn := net.Pipe()
 	var connIndex = uint8(0)
 	log := zerolog.Nop()
 	obs := NewObserver(&log, &log, false)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelConfig{},
+		&NamedTunnelProperties{},
 		connIndex,
 		nil,
 		nil,
 		1*time.Second,
 	)
 	return NewHTTP2Connection(
-		originConn,
-		testConfig,
+		cfdConn,
+		// OriginProxy is set in testConfigManager
+		testOrchestrator,
 		&pogs.ConnectionOptions{},
 		obs,
 		connIndex,
@@ -51,6 +55,41 @@ func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
 	), edgeConn
 }
 
+func TestHTTP2ConfigurationSet(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	endpoint := fmt.Sprintf("http://localhost:8080/ok")
+	reqBody := []byte(`{
+"version": 2, 
+"config": {"warp-routing": {"enabled": true},  "originRequest" : {"connectTimeout": 10}, "ingress" : [ {"hostname": "test", "service": "https://localhost:8000" } , {"service": "http_status:404"} ]}}
+`)
+	reader := bytes.NewReader(reqBody)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, endpoint, reader)
+	require.NoError(t, err)
+	req.Header.Set(InternalUpgradeHeader, ConfigurationUpdate)
+
+	resp, err := edgeHTTP2Conn.RoundTrip(req)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	bdy, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	assert.Equal(t, `{"lastAppliedVersion":2,"err":null}`, string(bdy))
+	cancel()
+	wg.Wait()
+
+}
+
 func TestServeHTTP(t *testing.T) {
 	tests := []testRequest{
 		{
@@ -128,18 +167,26 @@ type mockNamedTunnelRPCClient struct {
 	unregistered chan struct{}
 }
 
+func (mc mockNamedTunnelRPCClient) SendLocalConfiguration(c context.Context, config []byte, observer *Observer) error {
+	return nil
+}
+
 func (mc mockNamedTunnelRPCClient) RegisterConnection(
 	c context.Context,
-	config *NamedTunnelConfig,
+	properties *NamedTunnelProperties,
 	options *tunnelpogs.ConnectionOptions,
 	connIndex uint8,
 	observer *Observer,
-) error {
+) (*tunnelpogs.ConnectionDetails, error) {
 	if mc.shouldFail != nil {
-		return mc.shouldFail
+		return nil, mc.shouldFail
 	}
 	close(mc.registered)
-	return nil
+	return &tunnelpogs.ConnectionDetails{
+		Location:                "LIS",
+		UUID:                    uuid.New(),
+		TunnelIsRemotelyManaged: false,
+	}, nil
 }
 
 func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
@@ -166,6 +213,8 @@ type wsRespWriter struct {
 	*httptest.ResponseRecorder
 	readPipe  *io.PipeReader
 	writePipe *io.PipeWriter
+	closed    bool
+	panicked  bool
 }
 
 func newWSRespWriter() *wsRespWriter {
@@ -174,46 +223,59 @@ func newWSRespWriter() *wsRespWriter {
 		httptest.NewRecorder(),
 		readPipe,
 		writePipe,
+		false,
+		false,
 	}
 }
 
+type nowriter struct {
+	io.Reader
+}
+
+func (nowriter) Write(_ []byte) (int, error) {
+	return 0, fmt.Errorf("writer not implemented")
+}
+
 func (w *wsRespWriter) RespBody() io.ReadWriter {
 	return nowriter{w.readPipe}
 }
 
 func (w *wsRespWriter) Write(data []byte) (n int, err error) {
+	if w.closed {
+		w.panicked = true
+		return 0, errors.New("wsRespWriter panicked")
+	}
 	return w.writePipe.Write(data)
 }
 
+func (w *wsRespWriter) close() {
+	w.closed = true
+}
+
 func TestServeWS(t *testing.T) {
 	http2Conn, _ := newTestHTTP2Connection()
 
 	ctx, cancel := context.WithCancel(context.Background())
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		http2Conn.Serve(ctx)
-	}()
 
 	respWriter := newWSRespWriter()
 	readPipe, writePipe := io.Pipe()
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/ws", readPipe)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/ws/echo", readPipe)
 	require.NoError(t, err)
 	req.Header.Set(InternalUpgradeHeader, WebsocketUpgrade)
 
-	wg.Add(1)
+	serveDone := make(chan struct{})
 	go func() {
-		defer wg.Done()
+		defer close(serveDone)
 		http2Conn.ServeHTTP(respWriter, req)
+		respWriter.close()
 	}()
 
 	data := []byte("test websocket")
-	err = wsutil.WriteClientText(writePipe, data)
+	err = wsutil.WriteClientBinary(writePipe, data)
 	require.NoError(t, err)
 
-	respBody, err := wsutil.ReadServerText(respWriter.RespBody())
+	respBody, err := wsutil.ReadServerBinary(respWriter.RespBody())
 	require.NoError(t, err)
 	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
 
@@ -223,7 +285,65 @@ func TestServeWS(t *testing.T) {
 	require.Equal(t, http.StatusOK, resp.StatusCode)
 	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeader))
 
+	<-serveDone
+	require.False(t, respWriter.panicked)
+}
+
+// TestNoWriteAfterServeHTTPReturns is a regression test of https://jira.cfops.it/browse/TUN-5184
+// to make sure we don't write to the ResponseWriter after the ServeHTTP method returns
+func TestNoWriteAfterServeHTTPReturns(t *testing.T) {
+	cfdHTTP2Conn, edgeTCPConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+
+	serverDone := make(chan struct{})
+	go func() {
+		defer close(serverDone)
+		cfdHTTP2Conn.Serve(ctx)
+	}()
+
+	edgeTransport := http2.Transport{}
+	edgeHTTP2Conn, err := edgeTransport.NewClientConn(edgeTCPConn)
+	require.NoError(t, err)
+	message := []byte(t.Name())
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			readPipe, writePipe := io.Pipe()
+			reqCtx, reqCancel := context.WithCancel(ctx)
+			req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, "http://localhost:8080/ws/flaky", readPipe)
+			require.NoError(t, err)
+			req.Header.Set(InternalUpgradeHeader, WebsocketUpgrade)
+
+			resp, err := edgeHTTP2Conn.RoundTrip(req)
+			require.NoError(t, err)
+			// http2RespWriter should rewrite status 101 to 200
+			require.Equal(t, http.StatusOK, resp.StatusCode)
+
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for {
+					select {
+					case <-reqCtx.Done():
+						return
+					default:
+					}
+					_ = wsutil.WriteClientBinary(writePipe, message)
+				}
+			}()
+
+			time.Sleep(time.Millisecond * 100)
+			reqCancel()
+		}()
+	}
+
 	wg.Wait()
+	cancel()
+	<-serverDone
 }
 
 func TestServeControlStream(t *testing.T) {
@@ -238,7 +358,7 @@ func TestServeControlStream(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelConfig{},
+		&NamedTunnelProperties{},
 		1,
 		rpcClientFactory.newMockRPCClient,
 		nil,
@@ -288,7 +408,7 @@ func TestFailRegistration(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelConfig{},
+		&NamedTunnelProperties{},
 		http2Conn.connIndex,
 		rpcClientFactory.newMockRPCClient,
 		nil,
@@ -334,7 +454,7 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelConfig{},
+		&NamedTunnelProperties{},
 		http2Conn.connIndex,
 		rpcClientFactory.newMockRPCClient,
 		shutdownC,
@@ -366,7 +486,7 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 
 	select {
 	case <-rpcClientFactory.registered:
-		break //ok
+		break // ok
 	case <-time.Tick(time.Second):
 		t.Fatal("timeout out waiting for registration")
 	}
@@ -376,7 +496,7 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 
 	select {
 	case <-rpcClientFactory.unregistered:
-		break //ok
+		break // ok
 	case <-time.Tick(time.Second):
 		t.Fatal("timeout out waiting for unregistered signal")
 	}

connection/metrics.go

@@ -13,6 +13,7 @@ const (
 	MetricsNamespace = "cloudflared"
 	TunnelSubsystem  = "tunnel"
 	muxerSubsystem   = "muxer"
+	configSubsystem  = "config"
 )
 
 type muxerMetrics struct {
@@ -36,6 +37,11 @@ type muxerMetrics struct {
 	compRateAve      *prometheus.GaugeVec
 }
 
+type localConfigMetrics struct {
+	pushes       prometheus.Counter
+	pushesErrors prometheus.Counter
+}
+
 type tunnelMetrics struct {
 	timerRetries    prometheus.Gauge
 	serverLocations *prometheus.GaugeVec
@@ -51,6 +57,39 @@ type tunnelMetrics struct {
 	muxerMetrics        *muxerMetrics
 	tunnelsHA           tunnelsForHA
 	userHostnamesCounts *prometheus.CounterVec
+
+	localConfigMetrics *localConfigMetrics
+}
+
+func newLocalConfigMetrics() *localConfigMetrics {
+
+	pushesMetric := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: configSubsystem,
+			Name:      "local_config_pushes",
+			Help:      "Number of local configuration pushes to the edge",
+		},
+	)
+
+	pushesErrorsMetric := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: configSubsystem,
+			Name:      "local_config_pushes_errors",
+			Help:      "Number of errors occurred during local configuration pushes",
+		},
+	)
+
+	prometheus.MustRegister(
+		pushesMetric,
+		pushesErrorsMetric,
+	)
+
+	return &localConfigMetrics{
+		pushes:       pushesMetric,
+		pushesErrors: pushesErrorsMetric,
+	}
 }
 
 func newMuxerMetrics() *muxerMetrics {
@@ -386,6 +425,7 @@ func initTunnelMetrics() *tunnelMetrics {
 		regFail:             registerFail,
 		rpcFail:             rpcFail,
 		userHostnamesCounts: userHostnamesCounts,
+		localConfigMetrics:  newLocalConfigMetrics(),
 	}
 }
 

connection/protocol.go

@@ -19,7 +19,7 @@ const (
 	edgeH2TLSServerName = "h2.cftunnel.com"
 	// edgeQUICServerName is the server name to establish quic connection with edge.
 	edgeQUICServerName = "quic.cftunnel.com"
-	autoSelectFlag     = "auto"
+	AutoSelectFlag     = "auto"
 )
 
 var (
@@ -195,7 +195,7 @@ type PercentageFetcher func() (edgediscovery.ProtocolPercents, error)
 func NewProtocolSelector(
 	protocolFlag string,
 	warpRoutingEnabled bool,
-	namedTunnel *NamedTunnelConfig,
+	namedTunnel *NamedTunnelProperties,
 	fetchFunc PercentageFetcher,
 	ttl time.Duration,
 	log *zerolog.Logger,
@@ -247,7 +247,7 @@ func selectNamedTunnelProtocols(
 
 	// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
 	// fallback on failures.
-	if protocolFlag == autoSelectFlag {
+	if protocolFlag == AutoSelectFlag {
 		return newAutoProtocolSelector(protocol, []Protocol{QUIC, HTTP2, H2mux}, threshold, fetchFunc, ttl, log), nil
 	}
 
@@ -272,7 +272,7 @@ func selectWarpRoutingProtocols(
 
 	// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
 	// fallback on failures.
-	if protocolFlag == autoSelectFlag {
+	if protocolFlag == AutoSelectFlag {
 		return newAutoProtocolSelector(protocol, []Protocol{QUICWarp, HTTP2Warp}, threshold, fetchFunc, ttl, log), nil
 	}
 

connection/protocol_test.go

@@ -16,7 +16,7 @@ const (
 )
 
 var (
-	testNamedTunnelConfig = &NamedTunnelConfig{
+	testNamedTunnelProperties = &NamedTunnelProperties{
 		Credentials: Credentials{
 			AccountTag: "testAccountTag",
 		},
@@ -51,7 +51,7 @@ func TestNewProtocolSelector(t *testing.T) {
 		hasFallback        bool
 		expectedFallback   Protocol
 		warpRoutingEnabled bool
-		namedTunnelConfig  *NamedTunnelConfig
+		namedTunnelConfig  *NamedTunnelProperties
 		fetchFunc          PercentageFetcher
 		wantErr            bool
 	}{
@@ -66,77 +66,77 @@ func TestNewProtocolSelector(t *testing.T) {
 			protocol:          "h2mux",
 			expectedProtocol:  H2mux,
 			fetchFunc:         func() (edgediscovery.ProtocolPercents, error) { return nil, nil },
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel over http2",
 			protocol:          "http2",
 			expectedProtocol:  HTTP2,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel http2 disabled still gets http2 because it is manually picked",
 			protocol:          "http2",
 			expectedProtocol:  HTTP2,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel quic disabled still gets quic because it is manually picked",
 			protocol:          "quic",
 			expectedProtocol:  QUIC,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel quic and http2 disabled",
-			protocol:          "auto",
+			protocol:          AutoSelectFlag,
 			expectedProtocol:  H2mux,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:             "named tunnel quic disabled",
-			protocol:         "auto",
+			protocol:         AutoSelectFlag,
 			expectedProtocol: HTTP2,
 			// Hasfallback true is because if http2 fails, then we further fallback to h2mux.
 			hasFallback:       true,
 			expectedFallback:  H2mux,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel auto all http2 disabled",
-			protocol:          "auto",
+			protocol:          AutoSelectFlag,
 			expectedProtocol:  H2mux,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel auto to h2mux",
-			protocol:          "auto",
+			protocol:          AutoSelectFlag,
 			expectedProtocol:  H2mux,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel auto to http2",
-			protocol:          "auto",
+			protocol:          AutoSelectFlag,
 			expectedProtocol:  HTTP2,
 			hasFallback:       true,
 			expectedFallback:  H2mux,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel auto to quic",
-			protocol:          "auto",
+			protocol:          AutoSelectFlag,
 			expectedProtocol:  QUIC,
 			hasFallback:       true,
 			expectedFallback:  HTTP2,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:               "warp routing requesting h2mux",
@@ -145,7 +145,7 @@ func TestNewProtocolSelector(t *testing.T) {
 			hasFallback:        false,
 			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
 			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelConfig,
+			namedTunnelConfig:  testNamedTunnelProperties,
 		},
 		{
 			name:               "warp routing requesting h2mux picks HTTP2 even if http2 percent is -1",
@@ -154,7 +154,7 @@ func TestNewProtocolSelector(t *testing.T) {
 			hasFallback:        false,
 			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
 			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelConfig,
+			namedTunnelConfig:  testNamedTunnelProperties,
 		},
 		{
 			name:               "warp routing http2",
@@ -163,36 +163,36 @@ func TestNewProtocolSelector(t *testing.T) {
 			hasFallback:        false,
 			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
 			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelConfig,
+			namedTunnelConfig:  testNamedTunnelProperties,
 		},
 		{
 			name:               "warp routing quic",
-			protocol:           "auto",
+			protocol:           AutoSelectFlag,
 			expectedProtocol:   QUICWarp,
 			hasFallback:        true,
 			expectedFallback:   HTTP2Warp,
 			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
 			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelConfig,
+			namedTunnelConfig:  testNamedTunnelProperties,
 		},
 		{
 			name:               "warp routing auto",
-			protocol:           "auto",
+			protocol:           AutoSelectFlag,
 			expectedProtocol:   HTTP2Warp,
 			hasFallback:        false,
 			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
 			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelConfig,
+			namedTunnelConfig:  testNamedTunnelProperties,
 		},
 		{
 			name:               "warp routing auto- quic",
-			protocol:           "auto",
+			protocol:           AutoSelectFlag,
 			expectedProtocol:   QUICWarp,
 			hasFallback:        true,
 			expectedFallback:   HTTP2Warp,
 			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
 			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelConfig,
+			namedTunnelConfig:  testNamedTunnelProperties,
 		},
 		{
 			// None named tunnel can only use h2mux, so specifying an unknown protocol is not an error
@@ -204,14 +204,14 @@ func TestNewProtocolSelector(t *testing.T) {
 			name:              "named tunnel unknown protocol",
 			protocol:          "unknown",
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 			wantErr:           true,
 		},
 		{
 			name:              "named tunnel fetch error",
-			protocol:          "auto",
+			protocol:          AutoSelectFlag,
 			fetchFunc:         mockFetcher(true),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 			expectedProtocol:  HTTP2,
 			wantErr:           false,
 		},
@@ -237,7 +237,7 @@ func TestNewProtocolSelector(t *testing.T) {
 
 func TestAutoProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
-	selector, err := NewProtocolSelector("auto", noWarpRoutingEnabled, testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
+	selector, err := NewProtocolSelector(AutoSelectFlag, noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, H2mux, selector.Current())
 
@@ -267,7 +267,7 @@ func TestAutoProtocolSelectorRefresh(t *testing.T) {
 func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
 	// Since the user chooses http2 on purpose, we always stick to it.
-	selector, err := NewProtocolSelector("http2", noWarpRoutingEnabled, testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
+	selector, err := NewProtocolSelector("http2", noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, HTTP2, selector.Current())
 
@@ -297,7 +297,7 @@ func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 func TestProtocolSelectorRefreshTTL(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}}
-	selector, err := NewProtocolSelector("auto", noWarpRoutingEnabled, testNamedTunnelConfig, fetcher.fetch(), time.Hour, &log)
+	selector, err := NewProtocolSelector(AutoSelectFlag, noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), time.Hour, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, QUIC, selector.Current())
 

connection/quic.go

@@ -20,6 +20,7 @@ import (
 	"github.com/cloudflare/cloudflared/datagramsession"
 	"github.com/cloudflare/cloudflared/ingress"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
@@ -36,7 +37,7 @@ const (
 type QUICConnection struct {
 	session              quic.Session
 	logger               *zerolog.Logger
-	httpProxy            OriginProxy
+	orchestrator         Orchestrator
 	sessionManager       datagramsession.Manager
 	controlStreamHandler ControlStreamHandler
 	connOptions          *tunnelpogs.ConnectionOptions
@@ -47,7 +48,7 @@ func NewQUICConnection(
 	quicConfig *quic.Config,
 	edgeAddr net.Addr,
 	tlsConfig *tls.Config,
-	httpProxy OriginProxy,
+	orchestrator Orchestrator,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler ControlStreamHandler,
 	logger *zerolog.Logger,
@@ -66,7 +67,7 @@ func NewQUICConnection(
 
 	return &QUICConnection{
 		session:              session,
-		httpProxy:            httpProxy,
+		orchestrator:         orchestrator,
 		logger:               logger,
 		sessionManager:       sessionManager,
 		controlStreamHandler: controlStreamHandler,
@@ -110,7 +111,7 @@ func (q *QUICConnection) Serve(ctx context.Context) error {
 
 func (q *QUICConnection) serveControlStream(ctx context.Context, controlStream quic.Stream) error {
 	// This blocks until the control plane is done.
-	err := q.controlStreamHandler.ServeControlStream(ctx, controlStream, q.connOptions)
+	err := q.controlStreamHandler.ServeControlStream(ctx, controlStream, q.connOptions, q.orchestrator)
 	if err != nil {
 		// Not wrapping error here to be consistent with the http2 message.
 		return err
@@ -175,18 +176,22 @@ func (q *QUICConnection) handleDataStream(stream *quicpogs.RequestServerStream)
 		return err
 	}
 
+	originProxy, err := q.orchestrator.GetOriginProxy()
+	if err != nil {
+		return err
+	}
 	switch connectRequest.Type {
 	case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
-		req, err := buildHTTPRequest(connectRequest, stream)
+		tracedReq, err := buildHTTPRequest(connectRequest, stream)
 		if err != nil {
 			return err
 		}
 
 		w := newHTTPResponseAdapter(stream)
-		return q.httpProxy.ProxyHTTP(w, req, connectRequest.Type == quicpogs.ConnectionTypeWebsocket)
+		return originProxy.ProxyHTTP(w, tracedReq, connectRequest.Type == quicpogs.ConnectionTypeWebsocket)
 	case quicpogs.ConnectionTypeTCP:
 		rwa := &streamReadWriteAcker{stream}
-		return q.httpProxy.ProxyTCP(context.Background(), rwa, &TCPRequest{Dest: connectRequest.Dest})
+		return originProxy.ProxyTCP(context.Background(), rwa, &TCPRequest{Dest: connectRequest.Dest})
 	}
 	return nil
 }
@@ -261,8 +266,8 @@ func (q *QUICConnection) UnregisterUdpSession(ctx context.Context, sessionID uui
 }
 
 // UpdateConfiguration is the RPC method invoked by edge when there is a new configuration
-func (q *QUICConnection) UpdateConfiguration(ctx context.Context, version int32, config []byte) (*tunnelpogs.UpdateConfigurationResponse, error) {
-	return nil, fmt.Errorf("TODO: TUN-5698")
+func (q *QUICConnection) UpdateConfiguration(ctx context.Context, version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
+	return q.orchestrator.UpdateConfig(version, config)
 }
 
 // streamReadWriteAcker is a light wrapper over QUIC streams with a callback to send response back to
@@ -301,7 +306,7 @@ func (hrw httpResponseAdapter) WriteErrorResponse(err error) {
 	hrw.WriteConnectResponseData(err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
 }
 
-func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadCloser) (*http.Request, error) {
+func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadCloser) (*tracing.TracedRequest, error) {
 	metadata := connectRequest.MetadataMap()
 	dest := connectRequest.Dest
 	method := metadata[HTTPMethodKey]
@@ -338,10 +343,13 @@ func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadClose
 	//   * there is no transfer-encoding=chunked already set.
 	// So, if transfer cannot be chunked and content length is 0, we dont set a request body.
 	if !isWebsocket && !isTransferEncodingChunked(req) && req.ContentLength == 0 {
-		req.Body = nil
+		req.Body = http.NoBody
 	}
 	stripWebsocketUpgradeHeader(req)
-	return req, err
+
+	// Check for tracing on request
+	tracedReq := tracing.NewTracedRequest(req)
+	return tracedReq, err
 }
 
 func setContentLength(req *http.Request) error {

connection/quic_test.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/datagramsession"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
@@ -47,7 +48,7 @@ func TestQUICServer(t *testing.T) {
 
 	// This is simply a sample websocket frame message.
 	wsBuf := &bytes.Buffer{}
-	wsutil.WriteClientText(wsBuf, []byte("Hello"))
+	wsutil.WriteClientBinary(wsBuf, []byte("Hello"))
 
 	var tests = []struct {
 		desc             string
@@ -104,7 +105,7 @@ func TestQUICServer(t *testing.T) {
 		},
 		{
 			desc:           "test ws proxy",
-			dest:           "/ok",
+			dest:           "/ws/echo",
 			connectionType: quicpogs.ConnectionTypeWebsocket,
 			metadata: []quicpogs.Metadata{
 				{
@@ -125,7 +126,7 @@ func TestQUICServer(t *testing.T) {
 				},
 			},
 			message:          wsBuf.Bytes(),
-			expectedResponse: []byte{0x81, 0x5, 0x48, 0x65, 0x6c, 0x6c, 0x6f},
+			expectedResponse: []byte{0x82, 0x5, 0x48, 0x65, 0x6c, 0x6c, 0x6f},
 		},
 		{
 			desc:             "test tcp proxy",
@@ -162,7 +163,7 @@ type fakeControlStream struct {
 	ControlStreamHandler
 }
 
-func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions) error {
+func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions, tunnelConfigGetter TunnelConfigJSONGetter) error {
 	<-ctx.Done()
 	return nil
 }
@@ -219,9 +220,10 @@ func quicServer(
 
 type mockOriginProxyWithRequest struct{}
 
-func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, r *http.Request, isWebsocket bool) error {
+func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, tr *tracing.TracedRequest, isWebsocket bool) error {
 	// These are a series of crude tests to ensure the headers and http related data is transferred from
 	// metadata.
+	r := tr.Request
 	if r.Method == "" {
 		return errors.New("method not sent")
 	}
@@ -233,7 +235,7 @@ func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, r *http.Reque
 	}
 
 	if isWebsocket {
-		return wsEndpoint(w, r)
+		return wsEchoEndpoint(w, r)
 	}
 	switch r.URL.Path {
 	case "/ok":
@@ -345,7 +347,7 @@ func TestBuildHTTPRequest(t *testing.T) {
 				},
 				ContentLength: 0,
 				Host:          "cf.host",
-				Body:          nil,
+				Body:          http.NoBody,
 			},
 			body: io.NopCloser(&bytes.Buffer{}),
 		},
@@ -478,7 +480,7 @@ func TestBuildHTTPRequest(t *testing.T) {
 			req, err := buildHTTPRequest(test.connectRequest, test.body)
 			assert.NoError(t, err)
 			test.req = test.req.WithContext(req.Context())
-			assert.Equal(t, test.req, req)
+			assert.Equal(t, test.req, req.Request)
 		})
 	}
 }
@@ -627,13 +629,12 @@ func testQUICConnection(udpListenerAddr net.Addr, t *testing.T) *QUICConnection
 		NextProtos:         []string{"argotunnel"},
 	}
 	// Start a mock httpProxy
-	originProxy := &mockOriginProxyWithRequest{}
 	log := zerolog.New(os.Stdout)
 	qc, err := NewQUICConnection(
 		testQUICConfig,
 		udpListenerAddr,
 		tlsClientConfig,
-		originProxy,
+		&mockOrchestrator{originProxy: &mockOriginProxyWithRequest{}},
 		&tunnelpogs.ConnectionOptions{},
 		fakeControlStream{},
 		&log,

connection/rpc.go

@@ -37,7 +37,7 @@ func NewTunnelServerClient(
 	}
 }
 
-func (tsc *tunnelServerClient) Authenticate(ctx context.Context, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) (tunnelpogs.AuthOutcome, error) {
+func (tsc *tunnelServerClient) Authenticate(ctx context.Context, classicTunnel *ClassicTunnelProperties, registrationOptions *tunnelpogs.RegistrationOptions) (tunnelpogs.AuthOutcome, error) {
 	authResp, err := tsc.client.Authenticate(ctx, classicTunnel.OriginCert, classicTunnel.Hostname, registrationOptions)
 	if err != nil {
 		return nil, err
@@ -54,10 +54,15 @@ func (tsc *tunnelServerClient) Close() {
 type NamedTunnelRPCClient interface {
 	RegisterConnection(
 		c context.Context,
-		config *NamedTunnelConfig,
+		config *NamedTunnelProperties,
 		options *tunnelpogs.ConnectionOptions,
 		connIndex uint8,
 		observer *Observer,
+	) (*tunnelpogs.ConnectionDetails, error)
+	SendLocalConfiguration(
+		c context.Context,
+		config []byte,
+		observer *Observer,
 	) error
 	GracefulShutdown(ctx context.Context, gracePeriod time.Duration)
 	Close()
@@ -86,25 +91,25 @@ func newRegistrationRPCClient(
 
 func (rsc *registrationServerClient) RegisterConnection(
 	ctx context.Context,
-	config *NamedTunnelConfig,
+	properties *NamedTunnelProperties,
 	options *tunnelpogs.ConnectionOptions,
 	connIndex uint8,
 	observer *Observer,
-) error {
+) (*tunnelpogs.ConnectionDetails, error) {
 	conn, err := rsc.client.RegisterConnection(
 		ctx,
-		config.Credentials.Auth(),
-		config.Credentials.TunnelID,
+		properties.Credentials.Auth(),
+		properties.Credentials.TunnelID,
 		connIndex,
 		options,
 	)
 	if err != nil {
 		if err.Error() == DuplicateConnectionError {
 			observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
-			return errDuplicationConnection
+			return nil, errDuplicationConnection
 		}
 		observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
-		return serverRegistrationErrorFromRPC(err)
+		return nil, serverRegistrationErrorFromRPC(err)
 	}
 
 	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
@@ -112,7 +117,18 @@ func (rsc *registrationServerClient) RegisterConnection(
 	observer.logServerInfo(connIndex, conn.Location, fmt.Sprintf("Connection %s registered", conn.UUID))
 	observer.sendConnectedEvent(connIndex, conn.Location)
 
-	return nil
+	return conn, nil
+}
+
+func (rsc *registrationServerClient) SendLocalConfiguration(ctx context.Context, config []byte, observer *Observer) (err error) {
+	observer.metrics.localConfigMetrics.pushes.Inc()
+	defer func() {
+		if err != nil {
+			observer.metrics.localConfigMetrics.pushesErrors.Inc()
+		}
+	}()
+
+	return rsc.client.SendLocalConfiguration(ctx, config)
 }
 
 func (rsc *registrationServerClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
@@ -137,7 +153,7 @@ const (
 	authenticate rpcName = " authenticate"
 )
 
-func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter CredentialManager, classicTunnel *ClassicTunnelProperties, registrationOptions *tunnelpogs.RegistrationOptions) error {
 	h.observer.sendRegisteringEvent(registrationOptions.ConnectionID)
 
 	stream, err := h.newRPCStream(ctx, register)
@@ -174,7 +190,7 @@ type CredentialManager interface {
 func (h *h2muxConnection) processRegistrationSuccess(
 	registration *tunnelpogs.TunnelRegistration,
 	name rpcName,
-	credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig,
+	credentialManager CredentialManager, classicTunnel *ClassicTunnelProperties,
 ) error {
 	for _, logLine := range registration.LogLines {
 		h.observer.log.Info().Msg(logLine)
@@ -205,7 +221,7 @@ func (h *h2muxConnection) processRegisterTunnelError(err tunnelpogs.TunnelRegist
 	}
 }
 
-func (h *h2muxConnection) reconnectTunnel(ctx context.Context, credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+func (h *h2muxConnection) reconnectTunnel(ctx context.Context, credentialManager CredentialManager, classicTunnel *ClassicTunnelProperties, registrationOptions *tunnelpogs.RegistrationOptions) error {
 	token, err := credentialManager.ReconnectToken()
 	if err != nil {
 		return err
@@ -264,7 +280,7 @@ func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelSe
 
 func (h *h2muxConnection) registerNamedTunnel(
 	ctx context.Context,
-	namedTunnel *NamedTunnelConfig,
+	namedTunnel *NamedTunnelProperties,
 	connOptions *tunnelpogs.ConnectionOptions,
 ) error {
 	stream, err := h.newRPCStream(ctx, register)
@@ -274,7 +290,7 @@ func (h *h2muxConnection) registerNamedTunnel(
 	rpcClient := h.newRPCClientFunc(ctx, stream, h.observer.log)
 	defer rpcClient.Close()
 
-	if err = rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+	if _, err = rpcClient.RegisterConnection(ctx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
 		return err
 	}
 	return nil
@@ -283,7 +299,7 @@ func (h *h2muxConnection) registerNamedTunnel(
 func (h *h2muxConnection) unregister(isNamedTunnel bool) {
 	h.observer.sendUnregisteringEvent(h.connIndex)
 
-	unregisterCtx, cancel := context.WithTimeout(context.Background(), h.config.GracePeriod)
+	unregisterCtx, cancel := context.WithTimeout(context.Background(), h.gracePeriod)
 	defer cancel()
 
 	stream, err := h.newRPCStream(unregisterCtx, unregister)
@@ -296,13 +312,13 @@ func (h *h2muxConnection) unregister(isNamedTunnel bool) {
 		rpcClient := h.newRPCClientFunc(unregisterCtx, stream, h.observer.log)
 		defer rpcClient.Close()
 
-		rpcClient.GracefulShutdown(unregisterCtx, h.config.GracePeriod)
+		rpcClient.GracefulShutdown(unregisterCtx, h.gracePeriod)
 	} else {
 		rpcClient := NewTunnelServerClient(unregisterCtx, stream, h.observer.log)
 		defer rpcClient.Close()
 
 		// gracePeriod is encoded in int64 using capnproto
-		_ = rpcClient.client.UnregisterTunnel(unregisterCtx, h.config.GracePeriod.Nanoseconds())
+		_ = rpcClient.client.UnregisterTunnel(unregisterCtx, h.gracePeriod.Nanoseconds())
 	}
 
 	h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unregistered tunnel connection")

datagramsession/manager.go

@@ -2,7 +2,9 @@ package datagramsession
 
 import (
 	"context"
+	"fmt"
 	"io"
+	"time"
 
 	"github.com/google/uuid"
 	"github.com/lucas-clemente/quic-go"
@@ -12,6 +14,11 @@ import (
 
 const (
 	requestChanCapacity = 16
+	defaultReqTimeout   = time.Second * 5
+)
+
+var (
+	errSessionManagerClosed = fmt.Errorf("session manager closed")
 )
 
 // Manager defines the APIs to manage sessions from the same transport.
@@ -28,20 +35,25 @@ type manager struct {
 	registrationChan   chan *registerSessionEvent
 	unregistrationChan chan *unregisterSessionEvent
 	datagramChan       chan *newDatagram
+	closedChan         chan struct{}
 	transport          transport
 	sessions           map[uuid.UUID]*Session
 	log                *zerolog.Logger
+	// timeout waiting for an API to finish. This can be overriden in test
+	timeout time.Duration
 }
 
-func NewManager(transport transport, log *zerolog.Logger) Manager {
+func NewManager(transport transport, log *zerolog.Logger) *manager {
 	return &manager{
 		registrationChan:   make(chan *registerSessionEvent),
 		unregistrationChan: make(chan *unregisterSessionEvent),
 		// datagramChan is buffered, so it can read more datagrams from transport while the event loop is processing other events
 		datagramChan: make(chan *newDatagram, requestChanCapacity),
+		closedChan:   make(chan struct{}),
 		transport:    transport,
 		sessions:     make(map[uuid.UUID]*Session),
 		log:          log,
+		timeout:      defaultReqTimeout,
 	}
 }
 
@@ -85,27 +97,67 @@ func (m *manager) Serve(ctx context.Context) error {
 			}
 		}
 	})
-	return errGroup.Wait()
+	err := errGroup.Wait()
+	close(m.closedChan)
+	m.shutdownSessions(err)
+	return err
+}
+
+func (m *manager) shutdownSessions(err error) {
+	if err == nil {
+		err = errSessionManagerClosed
+	}
+	closeSessionErr := &errClosedSession{
+		message: err.Error(),
+		// Usually connection with remote has been closed, so set this to true to skip unregistering from remote
+		byRemote: true,
+	}
+	for _, s := range m.sessions {
+		s.close(closeSessionErr)
+	}
 }
 
 func (m *manager) RegisterSession(ctx context.Context, sessionID uuid.UUID, originProxy io.ReadWriteCloser) (*Session, error) {
+	ctx, cancel := context.WithTimeout(ctx, m.timeout)
+	defer cancel()
 	event := newRegisterSessionEvent(sessionID, originProxy)
 	select {
 	case <-ctx.Done():
+		m.log.Error().Msg("Datagram session registration timeout")
 		return nil, ctx.Err()
 	case m.registrationChan <- event:
 		session := <-event.resultChan
 		return session, nil
+	// Once closedChan is closed, manager won't accept more registration because nothing is
+	// reading from registrationChan and it's an unbuffered channel
+	case <-m.closedChan:
+		return nil, errSessionManagerClosed
 	}
 }
 
 func (m *manager) registerSession(ctx context.Context, registration *registerSessionEvent) {
-	session := newSession(registration.sessionID, m.transport, registration.originProxy, m.log)
+	session := m.newSession(registration.sessionID, registration.originProxy)
 	m.sessions[registration.sessionID] = session
 	registration.resultChan <- session
 }
 
+func (m *manager) newSession(id uuid.UUID, dstConn io.ReadWriteCloser) *Session {
+	return &Session{
+		ID:        id,
+		transport: m.transport,
+		dstConn:   dstConn,
+		// activeAtChan has low capacity. It can be full when there are many concurrent read/write. markActive() will
+		// drop instead of blocking because last active time only needs to be an approximation
+		activeAtChan: make(chan time.Time, 2),
+		// capacity is 2 because close() and dstToTransport routine in Serve() can write to this channel
+		closeChan: make(chan error, 2),
+		log:       m.log,
+	}
+}
+
 func (m *manager) UnregisterSession(ctx context.Context, sessionID uuid.UUID, message string, byRemote bool) error {
+	ctx, cancel := context.WithTimeout(ctx, m.timeout)
+	defer cancel()
 	event := &unregisterSessionEvent{
 		sessionID: sessionID,
 		err: &errClosedSession{
@@ -115,9 +167,12 @@ func (m *manager) UnregisterSession(ctx context.Context, sessionID uuid.UUID, me
 	}
 	select {
 	case <-ctx.Done():
+		m.log.Error().Msg("Datagram session unregistration timeout")
 		return ctx.Err()
 	case m.unregistrationChan <- event:
 		return nil
+	case <-m.closedChan:
+		return errSessionManagerClosed
 	}
 }
 

datagramsession/manager_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"sync"
 	"testing"
 	"time"
 
@@ -21,12 +22,8 @@ func TestManagerServe(t *testing.T) {
 		msgs                = 50
 		remoteUnregisterMsg = "eyeball closed connection"
 	)
-	log := zerolog.Nop()
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(1),
-		respChan: newDatagramChannel(1),
-	}
-	mg := NewManager(transport, &log)
+
+	mg, transport := newTestManager(1)
 
 	eyeballTracker := make(map[uuid.UUID]*datagramChannel)
 	for i := 0; i < sessions; i++ {
@@ -120,6 +117,65 @@ func TestManagerServe(t *testing.T) {
 	<-serveDone
 }
 
+func TestTimeout(t *testing.T) {
+	const (
+		testTimeout = time.Millisecond * 50
+	)
+
+	mg, _ := newTestManager(1)
+	mg.timeout = testTimeout
+	ctx := context.Background()
+	sessionID := uuid.New()
+	// session manager is not running, so event loop is not running and therefore calling the APIs should timeout
+	session, err := mg.RegisterSession(ctx, sessionID, nil)
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+	require.Nil(t, session)
+
+	err = mg.UnregisterSession(ctx, sessionID, "session gone", true)
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+}
+
+func TestCloseTransportCloseSessions(t *testing.T) {
+	mg, transport := newTestManager(1)
+	ctx := context.Background()
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		err := mg.Serve(ctx)
+		require.Error(t, err)
+	}()
+
+	cfdConn, eyeballConn := net.Pipe()
+	session, err := mg.RegisterSession(ctx, uuid.New(), cfdConn)
+	require.NoError(t, err)
+	require.NotNil(t, session)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		_, err := eyeballConn.Write([]byte(t.Name()))
+		require.NoError(t, err)
+		transport.close()
+	}()
+
+	closedByRemote, err := session.Serve(ctx, time.Minute)
+	require.True(t, closedByRemote)
+	require.Error(t, err)
+
+	wg.Wait()
+}
+
+func newTestManager(capacity uint) (*manager, *mockQUICTransport) {
+	log := zerolog.Nop()
+	transport := &mockQUICTransport{
+		reqChan:  newDatagramChannel(capacity),
+		respChan: newDatagramChannel(capacity),
+	}
+	return NewManager(transport, &log), transport
+}
+
 type mockOrigin struct {
 	expectMsgCount int
 	expectedMsg    []byte
@@ -199,7 +255,10 @@ func (rc *datagramChannel) Send(ctx context.Context, sessionID uuid.UUID, payloa
 	case <-ctx.Done():
 		return ctx.Err()
 	case <-rc.closedChan:
-		return fmt.Errorf("datagram channel closed")
+		return &errClosedSession{
+			message:  fmt.Errorf("datagram channel closed").Error(),
+			byRemote: true,
+		}
 	case rc.datagramChan <- &newDatagram{sessionID: sessionID, payload: payload}:
 		return nil
 	}
@@ -210,7 +269,11 @@ func (rc *datagramChannel) Receive(ctx context.Context) (uuid.UUID, []byte, erro
 	case <-ctx.Done():
 		return uuid.Nil, nil, ctx.Err()
 	case <-rc.closedChan:
-		return uuid.Nil, nil, fmt.Errorf("datagram channel closed")
+		err := &errClosedSession{
+			message:  fmt.Errorf("datagram channel closed").Error(),
+			byRemote: true,
+		}
+		return uuid.Nil, nil, err
 	case msg := <-rc.datagramChan:
 		return msg.sessionID, msg.payload, nil
 	}

datagramsession/session.go

@@ -39,20 +39,6 @@ type Session struct {
 	log          *zerolog.Logger
 }
 
-func newSession(id uuid.UUID, transport transport, dstConn io.ReadWriteCloser, log *zerolog.Logger) *Session {
-	return &Session{
-		ID:        id,
-		transport: transport,
-		dstConn:   dstConn,
-		// activeAtChan has low capacity. It can be full when there are many concurrent read/write. markActive() will
-		// drop instead of blocking because last active time only needs to be an approximation
-		activeAtChan: make(chan time.Time, 2),
-		// capacity is 2 because close() and dstToTransport routine in Serve() can write to this channel
-		closeChan: make(chan error, 2),
-		log:       log,
-	}
-}
-
 func (s *Session) Serve(ctx context.Context, closeAfterIdle time.Duration) (closedByRemote bool, err error) {
 	go func() {
 		// QUIC implementation copies data to another buffer before returning https://github.com/lucas-clemente/quic-go/blob/v0.24.0/session.go#L1967-L1975

datagramsession/session_test.go

@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/errgroup"
 )
@@ -41,12 +40,9 @@ func testSessionReturns(t *testing.T, closeBy closeMethod, closeAfterIdle time.D
 	sessionID := uuid.New()
 	cfdConn, originConn := net.Pipe()
 	payload := testPayload(sessionID)
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(1),
-		respChan: newDatagramChannel(1),
-	}
-	log := zerolog.Nop()
-	session := newSession(sessionID, transport, cfdConn, &log)
+
+	mg, _ := newTestManager(1)
+	session := mg.newSession(sessionID, cfdConn)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	sessionDone := make(chan struct{})
@@ -117,12 +113,9 @@ func testActiveSessionNotClosed(t *testing.T, readFromDst bool, writeToDst bool)
 	sessionID := uuid.New()
 	cfdConn, originConn := net.Pipe()
 	payload := testPayload(sessionID)
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(100),
-		respChan: newDatagramChannel(100),
-	}
-	log := zerolog.Nop()
-	session := newSession(sessionID, transport, cfdConn, &log)
+
+	mg, _ := newTestManager(100)
+	session := mg.newSession(sessionID, cfdConn)
 
 	startTime := time.Now()
 	activeUntil := startTime.Add(activeTime)
@@ -184,7 +177,8 @@ func testActiveSessionNotClosed(t *testing.T, readFromDst bool, writeToDst bool)
 
 func TestMarkActiveNotBlocking(t *testing.T) {
 	const concurrentCalls = 50
-	session := newSession(uuid.New(), nil, nil, nil)
+	mg, _ := newTestManager(1)
+	session := mg.newSession(uuid.New(), nil)
 	var wg sync.WaitGroup
 	wg.Add(concurrentCalls)
 	for i := 0; i < concurrentCalls; i++ {
@@ -199,12 +193,9 @@ func TestMarkActiveNotBlocking(t *testing.T) {
 func TestZeroBytePayload(t *testing.T) {
 	sessionID := uuid.New()
 	cfdConn, originConn := net.Pipe()
-	transport := &mockQUICTransport{
-		reqChan:  newDatagramChannel(1),
-		respChan: newDatagramChannel(1),
-	}
-	log := zerolog.Nop()
-	session := newSession(sessionID, transport, cfdConn, &log)
+
+	mg, transport := newTestManager(1)
+	session := mg.newSession(sessionID, cfdConn)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	errGroup, ctx := errgroup.WithContext(ctx)

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17.5 as builder
+FROM golang:1.17.10 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
@@ -6,4 +6,4 @@ RUN apt-get update
 COPY . .
 # compile cloudflared
 RUN make cloudflared
-RUN cp /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+RUN cp /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/

edgediscovery/edgediscovery.go

@@ -114,7 +114,8 @@ func (ed *Edge) GetDifferentAddr(connIndex int) (*allregions.EdgeAddr, error) {
 		// note: if oldAddr were not nil, it will become available on the next iteration
 		return nil, errNoAddressesLeft
 	}
-	log.Debug().Msg("edgediscovery - GetDifferentAddr: Giving connection its new address")
+	log.Debug().Msgf("edgediscovery - GetDifferentAddr: Giving connection its new address: %v from the address list: %v",
+		addr, ed.regions.AvailableAddrs())
 	return addr, nil
 }
 

go.mod

@@ -3,59 +3,49 @@ module github.com/cloudflare/cloudflared
 go 1.17
 
 require (
-	github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
 	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
 	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
 	github.com/coredns/coredns v1.8.7
-	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
-	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
-	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
-	github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
-	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
-	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/gdamore/tcell v1.3.0
 	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
-	github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 // indirect
-	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/json-iterator/go v1.1.12
-	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lucas-clemente/quic-go v0.24.0
 	github.com/mattn/go-colorable v0.1.8
 	github.com/miekg/dns v1.1.45
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pkg/errors v0.9.1
-	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
-	github.com/prometheus/client_golang v1.11.0
+	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/client_model v0.2.0
-	github.com/prometheus/common v0.32.1 // indirect
 	github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92
 	github.com/rs/zerolog v1.20.0
-	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/stretchr/testify v1.7.0
+	github.com/stretchr/testify v1.7.1
 	github.com/urfave/cli/v2 v2.3.0
+	go.opentelemetry.io/contrib/propagators v0.22.0
+	go.opentelemetry.io/otel v1.6.3
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3
+	go.opentelemetry.io/otel/sdk v1.6.3
+	go.opentelemetry.io/otel/trace v1.6.3
+	go.opentelemetry.io/proto/otlp v0.15.0
 	go.uber.org/automaxprocs v1.4.0
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
+	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f
 	golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
+	golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
 	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b
-	google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb // indirect
-	google.golang.org/grpc v1.43.0 // indirect
-	gopkg.in/coreos/go-oidc.v2 v2.1.0
+	google.golang.org/protobuf v1.28.0
+	gopkg.in/coreos/go-oidc.v2 v2.2.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
-	gopkg.in/square/go-jose.v2 v2.4.0 // indirect
+	gopkg.in/square/go-jose.v2 v2.6.0
 	gopkg.in/yaml.v2 v2.4.0
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	zombiezen.com/go/capnproto2 v2.18.0+incompatible
 )
 
@@ -63,16 +53,28 @@ require (
 	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/apparentlymart/go-cidr v1.1.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cheekybits/genny v1.0.0 // indirect
 	github.com/coredns/caddy v1.1.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
+	github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
+	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
+	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
 	github.com/francoispqt/gojay v1.2.13 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 // indirect
+	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
 	github.com/marten-seemann/qtls-go1-16 v0.1.4 // indirect
 	github.com/marten-seemann/qtls-go1-17 v0.1.0 // indirect
@@ -84,18 +86,27 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
+	github.com/prometheus/procfs v0.7.3 // indirect
 	github.com/rivo/uniseg v0.1.0 // indirect
+	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
+	google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb // indirect
+	google.golang.org/grpc v1.45.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 )
 
 replace github.com/urfave/cli/v2 => github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d
 
 replace github.com/lucas-clemente/quic-go => github.com/chungthuang/quic-go v0.24.1-0.20220110095058-981dc498cb62
+
+// Avoid 'CVE-2022-21698'
+replace github.com/prometheus/golang_client => github.com/prometheus/golang_client v1.12.1

go.sum

@@ -100,13 +100,15 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
+github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA=
 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
 github.com/chungthuang/quic-go v0.24.1-0.20220110095058-981dc498cb62 h1:PLTB4iA6sOgAItzQY642tYdcGKfG/7i2gu93JQGgUcM=
@@ -131,8 +133,6 @@ github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0=
 github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
 github.com/coredns/coredns v1.8.7 h1:wVMjAnyFnY7Mc18AFO+9qbGD6ODPtdVUIlzoWrHr3hk=
 github.com/coredns/coredns v1.8.7/go.mod h1:bFmbgEfeRz5aizL2VsQ5LRlsvJuXWkgG/MWG9zxqjVM=
-github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73 h1:7CNPV0LWRCa1FNmqg700pbXhzvmoaXKyfxWRkjRym7Q=
-github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -212,6 +212,11 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
 github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
+github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8=
@@ -234,6 +239,8 @@ github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
 github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -284,8 +291,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
@@ -336,7 +344,10 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
 github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
@@ -480,8 +491,9 @@ github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
-github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.12.1 h1:ZiaPsmm9uiBeaSMRznKsCDNtPCS0T3JVDGF+06gjBzk=
+github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
@@ -497,8 +509,9 @@ github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
-github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
+github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU=
+github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/rabbitmq/amqp091-go v1.1.0/go.mod h1:ogQDLSOACsLPsIq0NpbtiifNZi2YOz0VTJ0kHRghqbM=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
@@ -560,8 +573,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
 github.com/tinylib/msgp v1.1.2 h1:gWmO7n0Ys2RBEb7GPYB9Ujq8Mk5p2U08lRnmMcGy6BQ=
 github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
@@ -586,7 +600,22 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.opentelemetry.io/contrib/propagators v0.22.0 h1:KGdv58M2//veiYLIhb31mofaI2LgkIPXXAZVeYVyfd8=
+go.opentelemetry.io/contrib/propagators v0.22.0/go.mod h1:xGOuXr6lLIF9BXipA4pm6UuOSI0M98U6tsI3khbOiwU=
+go.opentelemetry.io/otel v1.0.0-RC2/go.mod h1:w1thVQ7qbAy8MHb0IFj8a5Q2QU0l2ksf8u/CN8m3NOM=
+go.opentelemetry.io/otel v1.6.3 h1:FLOfo8f9JzFVFVyU+MSRJc2HdEAXQgm7pIv2uFKRSZE=
+go.opentelemetry.io/otel v1.6.3/go.mod h1:7BgNga5fNlF/iZjG06hM3yofffp0ofKCDwSXx1GC4dI=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.6.3/go.mod h1:NEu79Xo32iVb+0gVNV8PMd7GoWqnyDXRlj04yFjqz40=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3 h1:4/UjHWMVVc5VwX/KAtqJOHErKigMCH8NexChMuanb/o=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3/go.mod h1:UJmXdiVVBaZ63umRUTwJuCMAV//GCMvDiQwn703/GoY=
+go.opentelemetry.io/otel/sdk v1.6.3 h1:prSHYdwCQOX5DrsEzxowH3nLhoAzEBdZhvrR79scfLs=
+go.opentelemetry.io/otel/sdk v1.6.3/go.mod h1:A4iWF7HTXa+GWL/AaqESz28VuSBIcZ+0CV+IzJ5NMiQ=
+go.opentelemetry.io/otel/trace v1.0.0-RC2/go.mod h1:JPQ+z6nNw9mqEGT8o3eoPTdnNI+Aj5JcxEsVGREIAy4=
+go.opentelemetry.io/otel/trace v1.6.3 h1:IqN4L+5b0mPNjdXIiZ90Ni4Bl5BRkDQywePLWemd9bc=
+go.opentelemetry.io/otel/trace v1.6.3/go.mod h1:GNJQusJlUgZl9/TQBPKU/Y/ty+0iVB5fjhKeJGZPGFs=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
+go.opentelemetry.io/proto/otlp v0.15.0 h1:h0bKrvdrT/9sBwEJ6iWUqT/N/xPcS66bL4u3isneJ6w=
+go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/automaxprocs v1.4.0 h1:CpDZl6aOlLhReez+8S3eEotD7Jx0Os++lemPlMULQP0=
 go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q=
@@ -607,8 +636,9 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210920023735-84f357641f63/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc=
+golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -694,6 +724,7 @@ golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20210917221730-978cfadd31cf/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d h1:1n1fc535VhN8SYtD4cDUyNlfpAF2ROMM9+11equK3hs=
@@ -788,6 +819,7 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -800,8 +832,9 @@ golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM=
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
+golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b h1:9zKuko04nR4gjZ4+DNjHqRlAJqbJETHwiNKDqTfOjfE=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1027,8 +1060,10 @@ google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD
 google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k=
-google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM=
+google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
+google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M=
+google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -1042,8 +1077,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/DataDog/dd-trace-go.v1 v1.34.0/go.mod h1:HtrC65fyJ6lWazShCC9rlOeiTSZJ0XtZhkwjZM2WpC4=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1052,15 +1088,15 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/coreos/go-oidc.v2 v2.1.0 h1:E8PjVFdj/SLDKB0hvb70KTbMbYVHjqztiQdSkIg8E+I=
-gopkg.in/coreos/go-oidc.v2 v2.1.0/go.mod h1:fYaTe2FS96wZZwR17YTDHwG+Mw6fmyqJNxN2eNCGPCI=
+gopkg.in/coreos/go-oidc.v2 v2.2.1 h1:MY5SZClJ7vhjKfr64a4nHAOV/c3WH2gB9BMrR64J1Mc=
+gopkg.in/coreos/go-oidc.v2 v2.2.1/go.mod h1:fYaTe2FS96wZZwR17YTDHwG+Mw6fmyqJNxN2eNCGPCI=
 gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
-gopkg.in/square/go-jose.v2 v2.4.0 h1:0kXPskUMGAXXWJlP05ktEMOV0vmzFQUWw6d+aZJQU8A=
-gopkg.in/square/go-jose.v2 v2.4.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=
+gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

ingress/config.go

@@ -11,14 +11,16 @@ import (
 	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
-const (
-	defaultConnectTimeout       = 30 * time.Second
-	defaultTLSTimeout           = 10 * time.Second
-	defaultTCPKeepAlive         = 30 * time.Second
-	defaultKeepAliveConnections = 100
-	defaultKeepAliveTimeout     = 90 * time.Second
-	defaultProxyAddress         = "127.0.0.1"
+var (
+	defaultConnectTimeout   = config.CustomDuration{Duration: 30 * time.Second}
+	defaultTLSTimeout       = config.CustomDuration{Duration: 10 * time.Second}
+	defaultTCPKeepAlive     = config.CustomDuration{Duration: 30 * time.Second}
+	defaultKeepAliveTimeout = config.CustomDuration{Duration: 90 * time.Second}
+)
 
+const (
+	defaultProxyAddress           = "127.0.0.1"
+	defaultKeepAliveConnections   = 100
 	SSHServerFlag                 = "ssh-server"
 	Socks5Flag                    = "socks5"
 	ProxyConnectTimeoutFlag       = "proxy-connect-timeout"
@@ -45,18 +47,26 @@ type RemoteConfig struct {
 	WarpRouting config.WarpRoutingConfig
 }
 
-type remoteConfigJSON struct {
-	GlobalOriginRequest config.OriginRequestConfig      `json:"originRequest"`
+type RemoteConfigJSON struct {
+	GlobalOriginRequest *config.OriginRequestConfig     `json:"originRequest,omitempty"`
 	IngressRules        []config.UnvalidatedIngressRule `json:"ingress"`
 	WarpRouting         config.WarpRoutingConfig        `json:"warp-routing"`
 }
 
 func (rc *RemoteConfig) UnmarshalJSON(b []byte) error {
-	var rawConfig remoteConfigJSON
+	var rawConfig RemoteConfigJSON
+
 	if err := json.Unmarshal(b, &rawConfig); err != nil {
 		return err
 	}
-	ingress, err := validateIngress(rawConfig.IngressRules, originRequestFromConfig(rawConfig.GlobalOriginRequest))
+
+	// if nil, just assume the default values.
+	globalOriginRequestConfig := rawConfig.GlobalOriginRequest
+	if globalOriginRequestConfig == nil {
+		globalOriginRequestConfig = &config.OriginRequestConfig{}
+	}
+
+	ingress, err := validateIngress(rawConfig.IngressRules, originRequestFromConfig(*globalOriginRequestConfig))
 	if err != nil {
 		return err
 	}
@@ -68,12 +78,12 @@ func (rc *RemoteConfig) UnmarshalJSON(b []byte) error {
 }
 
 func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
-	var connectTimeout time.Duration = defaultConnectTimeout
-	var tlsTimeout time.Duration = defaultTLSTimeout
-	var tcpKeepAlive time.Duration = defaultTCPKeepAlive
+	var connectTimeout config.CustomDuration = defaultConnectTimeout
+	var tlsTimeout config.CustomDuration = defaultTLSTimeout
+	var tcpKeepAlive config.CustomDuration = defaultTCPKeepAlive
 	var noHappyEyeballs bool
 	var keepAliveConnections int = defaultKeepAliveConnections
-	var keepAliveTimeout time.Duration = defaultKeepAliveTimeout
+	var keepAliveTimeout config.CustomDuration = defaultKeepAliveTimeout
 	var httpHostHeader string
 	var originServerName string
 	var caPool string
@@ -84,13 +94,13 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 	var proxyPort uint
 	var proxyType string
 	if flag := ProxyConnectTimeoutFlag; c.IsSet(flag) {
-		connectTimeout = c.Duration(flag)
+		connectTimeout = config.CustomDuration{Duration: c.Duration(flag)}
 	}
 	if flag := ProxyTLSTimeoutFlag; c.IsSet(flag) {
-		tlsTimeout = c.Duration(flag)
+		tlsTimeout = config.CustomDuration{Duration: c.Duration(flag)}
 	}
 	if flag := ProxyTCPKeepAliveFlag; c.IsSet(flag) {
-		tcpKeepAlive = c.Duration(flag)
+		tcpKeepAlive = config.CustomDuration{Duration: c.Duration(flag)}
 	}
 	if flag := ProxyNoHappyEyeballsFlag; c.IsSet(flag) {
 		noHappyEyeballs = c.Bool(flag)
@@ -99,7 +109,7 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 		keepAliveConnections = c.Int(flag)
 	}
 	if flag := ProxyKeepAliveTimeoutFlag; c.IsSet(flag) {
-		keepAliveTimeout = c.Duration(flag)
+		keepAliveTimeout = config.CustomDuration{Duration: c.Duration(flag)}
 	}
 	if flag := HTTPHostHeaderFlag; c.IsSet(flag) {
 		httpHostHeader = c.String(flag)
@@ -218,41 +228,41 @@ func originRequestFromConfig(c config.OriginRequestConfig) OriginRequestConfig {
 // Note: To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
 type OriginRequestConfig struct {
 	// HTTP proxy timeout for establishing a new connection
-	ConnectTimeout time.Duration `yaml:"connectTimeout"`
+	ConnectTimeout config.CustomDuration `yaml:"connectTimeout" json:"connectTimeout"`
 	// HTTP proxy timeout for completing a TLS handshake
-	TLSTimeout time.Duration `yaml:"tlsTimeout"`
+	TLSTimeout config.CustomDuration `yaml:"tlsTimeout" json:"tlsTimeout"`
 	// HTTP proxy TCP keepalive duration
-	TCPKeepAlive time.Duration `yaml:"tcpKeepAlive"`
+	TCPKeepAlive config.CustomDuration `yaml:"tcpKeepAlive" json:"tcpKeepAlive"`
 	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
-	NoHappyEyeballs bool `yaml:"noHappyEyeballs"`
+	NoHappyEyeballs bool `yaml:"noHappyEyeballs" json:"noHappyEyeballs"`
 	// HTTP proxy timeout for closing an idle connection
-	KeepAliveTimeout time.Duration `yaml:"keepAliveTimeout"`
+	KeepAliveTimeout config.CustomDuration `yaml:"keepAliveTimeout" json:"keepAliveTimeout"`
 	// HTTP proxy maximum keepalive connection pool size
-	KeepAliveConnections int `yaml:"keepAliveConnections"`
+	KeepAliveConnections int `yaml:"keepAliveConnections" json:"keepAliveConnections"`
 	// Sets the HTTP Host header for the local webserver.
-	HTTPHostHeader string `yaml:"httpHostHeader"`
+	HTTPHostHeader string `yaml:"httpHostHeader" json:"httpHostHeader"`
 	// Hostname on the origin server certificate.
-	OriginServerName string `yaml:"originServerName"`
+	OriginServerName string `yaml:"originServerName" json:"originServerName"`
 	// Path to the CA for the certificate of your origin.
 	// This option should be used only if your certificate is not signed by Cloudflare.
-	CAPool string `yaml:"caPool"`
+	CAPool string `yaml:"caPool" json:"caPool"`
 	// Disables TLS verification of the certificate presented by your origin.
 	// Will allow any certificate from the origin to be accepted.
 	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
-	NoTLSVerify bool `yaml:"noTLSVerify"`
+	NoTLSVerify bool `yaml:"noTLSVerify" json:"noTLSVerify"`
 	// Disables chunked transfer encoding.
 	// Useful if you are running a WSGI server.
-	DisableChunkedEncoding bool `yaml:"disableChunkedEncoding"`
+	DisableChunkedEncoding bool `yaml:"disableChunkedEncoding" json:"disableChunkedEncoding"`
 	// Runs as jump host
-	BastionMode bool `yaml:"bastionMode"`
+	BastionMode bool `yaml:"bastionMode" json:"bastionMode"`
 	// Listen address for the proxy.
-	ProxyAddress string `yaml:"proxyAddress"`
+	ProxyAddress string `yaml:"proxyAddress" json:"proxyAddress"`
 	// Listen port for the proxy.
-	ProxyPort uint `yaml:"proxyPort"`
+	ProxyPort uint `yaml:"proxyPort" json:"proxyPort"`
 	// What sort of proxy should be started
-	ProxyType string `yaml:"proxyType"`
+	ProxyType string `yaml:"proxyType" json:"proxyType"`
 	// IP rules for the proxy service
-	IPRules []ipaccess.Rule `yaml:"ipRules"`
+	IPRules []ipaccess.Rule `yaml:"ipRules" json:"ipRules"`
 }
 
 func (defaults *OriginRequestConfig) setConnectTimeout(overrides config.OriginRequestConfig) {
@@ -385,3 +395,91 @@ func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfi
 	cfg.setIPRules(overrides)
 	return cfg
 }
+
+func ConvertToRawOriginConfig(c OriginRequestConfig) config.OriginRequestConfig {
+	var connectTimeout *config.CustomDuration
+	var tlsTimeout *config.CustomDuration
+	var tcpKeepAlive *config.CustomDuration
+	var keepAliveConnections *int
+	var keepAliveTimeout *config.CustomDuration
+	var proxyAddress *string
+
+	if c.ConnectTimeout != defaultConnectTimeout {
+		connectTimeout = &c.ConnectTimeout
+	}
+	if c.TLSTimeout != defaultTLSTimeout {
+		tlsTimeout = &c.TLSTimeout
+	}
+	if c.TCPKeepAlive != defaultTCPKeepAlive {
+		tcpKeepAlive = &c.TCPKeepAlive
+	}
+	if c.KeepAliveConnections != defaultKeepAliveConnections {
+		keepAliveConnections = &c.KeepAliveConnections
+	}
+	if c.KeepAliveTimeout != defaultKeepAliveTimeout {
+		keepAliveTimeout = &c.KeepAliveTimeout
+	}
+	if c.ProxyAddress != defaultProxyAddress {
+		proxyAddress = &c.ProxyAddress
+	}
+
+	return config.OriginRequestConfig{
+		ConnectTimeout:         connectTimeout,
+		TLSTimeout:             tlsTimeout,
+		TCPKeepAlive:           tcpKeepAlive,
+		NoHappyEyeballs:        defaultBoolToNil(c.NoHappyEyeballs),
+		KeepAliveConnections:   keepAliveConnections,
+		KeepAliveTimeout:       keepAliveTimeout,
+		HTTPHostHeader:         emptyStringToNil(c.HTTPHostHeader),
+		OriginServerName:       emptyStringToNil(c.OriginServerName),
+		CAPool:                 emptyStringToNil(c.CAPool),
+		NoTLSVerify:            defaultBoolToNil(c.NoTLSVerify),
+		DisableChunkedEncoding: defaultBoolToNil(c.DisableChunkedEncoding),
+		BastionMode:            defaultBoolToNil(c.BastionMode),
+		ProxyAddress:           proxyAddress,
+		ProxyPort:              zeroUIntToNil(c.ProxyPort),
+		ProxyType:              emptyStringToNil(c.ProxyType),
+		IPRules:                convertToRawIPRules(c.IPRules),
+	}
+}
+
+func convertToRawIPRules(ipRules []ipaccess.Rule) []config.IngressIPRule {
+	result := make([]config.IngressIPRule, 0)
+	for _, r := range ipRules {
+		cidr := r.StringCIDR()
+
+		newRule := config.IngressIPRule{
+			Prefix: &cidr,
+			Ports:  r.Ports(),
+			Allow:  r.RulePolicy(),
+		}
+
+		result = append(result, newRule)
+	}
+
+	return result
+}
+
+func defaultBoolToNil(b bool) *bool {
+	if b == false {
+		return nil
+	}
+
+	return &b
+}
+
+func emptyStringToNil(s string) *string {
+	if s == "" {
+		return nil
+	}
+
+	return &s
+}
+
+func zeroUIntToNil(v uint) *uint {
+	if v == 0 {
+		return nil
+	}
+
+	return &v
+}

ingress/config_test.go

@@ -65,7 +65,7 @@ func TestUnmarshalRemoteConfigOverridesGlobal(t *testing.T) {
 	err := json.Unmarshal(rawConfig, &remoteConfig)
 	require.NoError(t, err)
 	require.True(t, remoteConfig.Ingress.Rules[0].Config.NoTLSVerify)
-	require.True(t, remoteConfig.Ingress.defaults.NoHappyEyeballs)
+	require.True(t, remoteConfig.Ingress.Defaults.NoHappyEyeballs)
 }
 
 func TestOriginRequestConfigOverrides(t *testing.T) {
@@ -74,11 +74,11 @@ func TestOriginRequestConfigOverrides(t *testing.T) {
 		// root-level configuration.
 		actual0 := ing.Rules[0].Config
 		expected0 := OriginRequestConfig{
-			ConnectTimeout:         1 * time.Minute,
-			TLSTimeout:             1 * time.Second,
-			TCPKeepAlive:           1 * time.Second,
+			ConnectTimeout:         config.CustomDuration{Duration: 1 * time.Minute},
+			TLSTimeout:             config.CustomDuration{Duration: 1 * time.Second},
+			TCPKeepAlive:           config.CustomDuration{Duration: 1 * time.Second},
 			NoHappyEyeballs:        true,
-			KeepAliveTimeout:       1 * time.Second,
+			KeepAliveTimeout:       config.CustomDuration{Duration: 1 * time.Second},
 			KeepAliveConnections:   1,
 			HTTPHostHeader:         "abc",
 			OriginServerName:       "a1",
@@ -99,11 +99,11 @@ func TestOriginRequestConfigOverrides(t *testing.T) {
 		// Rule 1 overrode all the root-level config.
 		actual1 := ing.Rules[1].Config
 		expected1 := OriginRequestConfig{
-			ConnectTimeout:         2 * time.Minute,
-			TLSTimeout:             2 * time.Second,
-			TCPKeepAlive:           2 * time.Second,
+			ConnectTimeout:         config.CustomDuration{Duration: 2 * time.Minute},
+			TLSTimeout:             config.CustomDuration{Duration: 2 * time.Second},
+			TCPKeepAlive:           config.CustomDuration{Duration: 2 * time.Second},
 			NoHappyEyeballs:        false,
-			KeepAliveTimeout:       2 * time.Second,
+			KeepAliveTimeout:       config.CustomDuration{Duration: 2 * time.Second},
 			KeepAliveConnections:   2,
 			HTTPHostHeader:         "def",
 			OriginServerName:       "b2",
@@ -191,12 +191,12 @@ ingress:
 	rawConfig := []byte(`
 {
     "originRequest": {
-        "connectTimeout": 60000000000,
-		"tlsTimeout": 1000000000,
+        "connectTimeout": 60,
+		"tlsTimeout": 1,
 		"noHappyEyeballs": true,
-		"tcpKeepAlive": 1000000000,
+		"tcpKeepAlive": 1,
 		"keepAliveConnections": 1,
-		"keepAliveTimeout": 1000000000,
+		"keepAliveTimeout": 1,
 		"httpHostHeader": "abc",
 		"originServerName": "a1",
 		"caPool": "/tmp/path0",
@@ -228,12 +228,12 @@ ingress:
 			"hostname": "*",
             "service": "https://localhost:8001",
 			"originRequest": {
-				"connectTimeout": 120000000000,
-				"tlsTimeout": 2000000000,
+				"connectTimeout": 120,
+				"tlsTimeout": 2,
 				"noHappyEyeballs": false,
-				"tcpKeepAlive": 2000000000,
+				"tcpKeepAlive": 2,
 				"keepAliveConnections": 2,
-				"keepAliveTimeout": 2000000000,
+				"keepAliveTimeout": 2,
 				"httpHostHeader": "def",
 				"originServerName": "b2",
 				"caPool": "/tmp/path1",
@@ -286,11 +286,11 @@ func TestOriginRequestConfigDefaults(t *testing.T) {
 		// Rule 1 overrode all defaults.
 		actual1 := ing.Rules[1].Config
 		expected1 := OriginRequestConfig{
-			ConnectTimeout:         2 * time.Minute,
-			TLSTimeout:             2 * time.Second,
-			TCPKeepAlive:           2 * time.Second,
+			ConnectTimeout:         config.CustomDuration{Duration: 2 * time.Minute},
+			TLSTimeout:             config.CustomDuration{Duration: 2 * time.Second},
+			TCPKeepAlive:           config.CustomDuration{Duration: 2 * time.Second},
 			NoHappyEyeballs:        false,
-			KeepAliveTimeout:       2 * time.Second,
+			KeepAliveTimeout:       config.CustomDuration{Duration: 2 * time.Second},
 			KeepAliveConnections:   2,
 			HTTPHostHeader:         "def",
 			OriginServerName:       "b2",
@@ -360,12 +360,12 @@ ingress:
 			"hostname": "*",
             "service": "https://localhost:8001",
 			"originRequest": {
-				"connectTimeout": 120000000000,
-				"tlsTimeout": 2000000000,
+				"connectTimeout": 120,
+				"tlsTimeout": 2,
 				"noHappyEyeballs": false,
-				"tcpKeepAlive": 2000000000,
+				"tcpKeepAlive": 2,
 				"keepAliveConnections": 2,
-				"keepAliveTimeout": 2000000000,
+				"keepAliveTimeout": 2,
 				"httpHostHeader": "def",
 				"originServerName": "b2",
 				"caPool": "/tmp/path1",

ingress/ingress.go

@@ -7,7 +7,6 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"sync"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -65,8 +64,8 @@ func matchHost(ruleHost, reqHost string) bool {
 
 // Ingress maps eyeball requests to origins.
 type Ingress struct {
-	Rules    []Rule
-	defaults OriginRequestConfig
+	Rules    []Rule              `json:"ingress"`
+	Defaults OriginRequestConfig `json:"originRequest"`
 }
 
 // NewSingleOrigin constructs an Ingress set with only one rule, constructed from
@@ -87,7 +86,7 @@ func NewSingleOrigin(c *cli.Context, allowURLFromArgs bool) (Ingress, error) {
 				Config:  setConfig(defaults, config.OriginRequestConfig{}),
 			},
 		},
-		defaults: defaults,
+		Defaults: defaults,
 	}
 	return ing, err
 }
@@ -127,7 +126,7 @@ func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (OriginServ
 		if err != nil {
 			return nil, errors.Wrap(err, "Error validating --unix-socket")
 		}
-		return &unixSocketPath{path: path}, nil
+		return &unixSocketPath{path: path, scheme: "http"}, nil
 	}
 	u, err := url.Parse("http://localhost:8080")
 	return &httpService{url: u}, err
@@ -145,13 +144,11 @@ func (ing Ingress) IsSingleRule() bool {
 
 // StartOrigins will start any origin services managed by cloudflared, e.g. proxy servers or Hello World.
 func (ing Ingress) StartOrigins(
-	wg *sync.WaitGroup,
 	log *zerolog.Logger,
 	shutdownC <-chan struct{},
-	errC chan error,
 ) error {
 	for _, rule := range ing.Rules {
-		if err := rule.Service.start(wg, log, shutdownC, errC, rule.Config); err != nil {
+		if err := rule.Service.start(log, shutdownC, rule.Config); err != nil {
 			return errors.Wrapf(err, "Error starting local service %s", rule.Service)
 		}
 	}
@@ -172,7 +169,10 @@ func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginReq
 		if prefix := "unix:"; strings.HasPrefix(r.Service, prefix) {
 			// No validation necessary for unix socket filepath services
 			path := strings.TrimPrefix(r.Service, prefix)
-			service = &unixSocketPath{path: path}
+			service = &unixSocketPath{path: path, scheme: "http"}
+		} else if prefix := "unix+tls:"; strings.HasPrefix(r.Service, prefix) {
+			path := strings.TrimPrefix(r.Service, prefix)
+			service = &unixSocketPath{path: path, scheme: "https"}
 		} else if prefix := "http_status:"; strings.HasPrefix(r.Service, prefix) {
 			status, err := strconv.Atoi(strings.TrimPrefix(r.Service, prefix))
 			if err != nil {
@@ -180,7 +180,7 @@ func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginReq
 			}
 			srv := newStatusCode(status)
 			service = &srv
-		} else if r.Service == "hello_world" || r.Service == "hello-world" || r.Service == "helloworld" {
+		} else if r.Service == HelloWorldService || r.Service == "hello-world" || r.Service == "helloworld" {
 			service = new(helloWorld)
 		} else if r.Service == ServiceSocksProxy {
 			rules := make([]ipaccess.Rule, len(r.OriginRequest.IPRules))
@@ -230,23 +230,24 @@ func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginReq
 			return Ingress{}, err
 		}
 
-		var pathRegex *regexp.Regexp
+		var pathRegexp *Regexp
 		if r.Path != "" {
 			var err error
-			pathRegex, err = regexp.Compile(r.Path)
+			regex, err := regexp.Compile(r.Path)
 			if err != nil {
 				return Ingress{}, errors.Wrapf(err, "Rule #%d has an invalid regex", i+1)
 			}
+			pathRegexp = &Regexp{Regexp: regex}
 		}
 
 		rules[i] = Rule{
 			Hostname: r.Hostname,
 			Service:  service,
-			Path:     pathRegex,
+			Path:     pathRegexp,
 			Config:   cfg,
 		}
 	}
-	return Ingress{Rules: rules, defaults: defaults}, nil
+	return Ingress{Rules: rules, Defaults: defaults}, nil
 }
 
 func validateHostname(r config.UnvalidatedIngressRule, ruleIndex, totalRules int) error {

ingress/ingress_test.go

@@ -26,8 +26,21 @@ ingress:
 `
 	ing, err := ParseIngress(MustReadIngress(rawYAML))
 	require.NoError(t, err)
-	_, ok := ing.Rules[0].Service.(*unixSocketPath)
+	s, ok := ing.Rules[0].Service.(*unixSocketPath)
 	require.True(t, ok)
+	require.Equal(t, "http", s.scheme)
+}
+
+func TestParseUnixSocketTLS(t *testing.T) {
+	rawYAML := `
+ingress:
+- service: unix+tls:/tmp/echo.sock
+`
+	ing, err := ParseIngress(MustReadIngress(rawYAML))
+	require.NoError(t, err)
+	s, ok := ing.Rules[0].Service.(*unixSocketPath)
+	require.True(t, ok)
+	require.Equal(t, "https", s.scheme)
 }
 
 func Test_parseIngress(t *testing.T) {
@@ -469,12 +482,12 @@ func TestSingleOriginSetsConfig(t *testing.T) {
 	ingress, err := NewSingleOrigin(cliCtx, allowURLFromArgs)
 	require.NoError(t, err)
 
-	assert.Equal(t, time.Second, ingress.Rules[0].Config.ConnectTimeout)
-	assert.Equal(t, time.Second, ingress.Rules[0].Config.TLSTimeout)
-	assert.Equal(t, time.Second, ingress.Rules[0].Config.TCPKeepAlive)
+	assert.Equal(t, config.CustomDuration{Duration: time.Second}, ingress.Rules[0].Config.ConnectTimeout)
+	assert.Equal(t, config.CustomDuration{Duration: time.Second}, ingress.Rules[0].Config.TLSTimeout)
+	assert.Equal(t, config.CustomDuration{Duration: time.Second}, ingress.Rules[0].Config.TCPKeepAlive)
 	assert.True(t, ingress.Rules[0].Config.NoHappyEyeballs)
 	assert.Equal(t, 10, ingress.Rules[0].Config.KeepAliveConnections)
-	assert.Equal(t, time.Second, ingress.Rules[0].Config.KeepAliveTimeout)
+	assert.Equal(t, config.CustomDuration{Duration: time.Second}, ingress.Rules[0].Config.KeepAliveTimeout)
 	assert.Equal(t, "example.com:8080", ingress.Rules[0].Config.HTTPHostHeader)
 	assert.Equal(t, "example.com", ingress.Rules[0].Config.OriginServerName)
 	assert.Equal(t, "/etc/certs/ca.pem", ingress.Rules[0].Config.CAPool)
@@ -495,7 +508,7 @@ func TestFindMatchingRule(t *testing.T) {
 			},
 			{
 				Hostname: "tunnel-b.example.com",
-				Path:     mustParsePath(t, "/health"),
+				Path:     MustParsePath(t, "/health"),
 			},
 			{
 				Hostname: "*",
@@ -578,10 +591,10 @@ func TestIsHTTPService(t *testing.T) {
 	}
 }
 
-func mustParsePath(t *testing.T, path string) *regexp.Regexp {
+func MustParsePath(t *testing.T, path string) *Regexp {
 	regexp, err := regexp.Compile(path)
 	assert.NoError(t, err)
-	return regexp
+	return &Regexp{Regexp: regexp}
 }
 
 func MustParseURL(t *testing.T, rawURL string) *url.URL {

ingress/origin_connection.go

@@ -53,7 +53,7 @@ func (wc *tcpOverWSConnection) Stream(ctx context.Context, tunnelConn io.ReadWri
 	wc.streamHandler(wsConn, wc.conn, log)
 	cancel()
 	// Makes sure wsConn stops sending ping before terminating the stream
-	wsConn.WaitForShutdown()
+	wsConn.Close()
 }
 
 func (wc *tcpOverWSConnection) Close() {
@@ -73,21 +73,8 @@ func (sp *socksProxyOverWSConnection) Stream(ctx context.Context, tunnelConn io.
 	socks.StreamNetHandler(wsConn, sp.accessPolicy, log)
 	cancel()
 	// Makes sure wsConn stops sending ping before terminating the stream
-	wsConn.WaitForShutdown()
+	wsConn.Close()
 }
 
 func (sp *socksProxyOverWSConnection) Close() {
 }
-
-// wsProxyConnection represents a bidirectional stream for a websocket connection to the origin
-type wsProxyConnection struct {
-	rwc io.ReadWriteCloser
-}
-
-func (conn *wsProxyConnection) Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger) {
-	websocket.Stream(tunnelConn, conn.rwc, log)
-}
-
-func (conn *wsProxyConnection) Close() {
-	conn.rwc.Close()
-}

ingress/origin_proxy.go

@@ -1,6 +1,7 @@
 package ingress
 
 import (
+	"fmt"
 	"net"
 	"net/http"
 
@@ -23,7 +24,7 @@ type StreamBasedOriginProxy interface {
 }
 
 func (o *unixSocketPath) RoundTrip(req *http.Request) (*http.Response, error) {
-	req.URL.Scheme = "http"
+	req.URL.Scheme = o.scheme
 	return o.transport.RoundTrip(req)
 }
 
@@ -49,7 +50,13 @@ func (o *httpService) RoundTrip(req *http.Request) (*http.Response, error) {
 }
 
 func (o *statusCode) RoundTrip(_ *http.Request) (*http.Response, error) {
-	return o.resp, nil
+	resp := &http.Response{
+		StatusCode: o.code,
+		Status:     fmt.Sprintf("%d %s", o.code, http.StatusText(o.code)),
+		Body:       new(NopReadCloser),
+	}
+
+	return resp, nil
 }
 
 func (o *rawTCPService) EstablishConnection(dest string) (OriginConnection, error) {

ingress/origin_proxy_test.go

@@ -8,7 +8,6 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
-	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -132,10 +131,8 @@ func TestHTTPServiceHostHeaderOverride(t *testing.T) {
 	httpService := &httpService{
 		url: originURL,
 	}
-	var wg sync.WaitGroup
 	shutdownC := make(chan struct{})
-	errC := make(chan error)
-	require.NoError(t, httpService.start(&wg, testLogger, shutdownC, errC, cfg))
+	require.NoError(t, httpService.start(testLogger, shutdownC, cfg))
 
 	req, err := http.NewRequest(http.MethodGet, originURL.String(), nil)
 	require.NoError(t, err)
@@ -147,7 +144,46 @@ func TestHTTPServiceHostHeaderOverride(t *testing.T) {
 	respBody, err := ioutil.ReadAll(resp.Body)
 	require.NoError(t, err)
 	require.Equal(t, respBody, []byte(originURL.Host))
+}
 
+// TestHTTPServiceUsesIngressRuleScheme makes sure httpService uses scheme defined in ingress rule and not by eyeball request
+func TestHTTPServiceUsesIngressRuleScheme(t *testing.T) {
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		require.NotNil(t, r.TLS)
+		// Echo the X-Forwarded-Proto header for assertions
+		w.Write([]byte(r.Header.Get("X-Forwarded-Proto")))
+	}
+	origin := httptest.NewTLSServer(http.HandlerFunc(handler))
+	defer origin.Close()
+
+	originURL, err := url.Parse(origin.URL)
+	require.NoError(t, err)
+	require.Equal(t, "https", originURL.Scheme)
+
+	cfg := OriginRequestConfig{
+		NoTLSVerify: true,
+	}
+	httpService := &httpService{
+		url: originURL,
+	}
+	shutdownC := make(chan struct{})
+	require.NoError(t, httpService.start(testLogger, shutdownC, cfg))
+
+	// Tunnel uses scheme defined in the service field of the ingress rule, independent of the X-Forwarded-Proto header
+	protos := []string{"https", "http", "dne"}
+	for _, p := range protos {
+		req, err := http.NewRequest(http.MethodGet, originURL.String(), nil)
+		require.NoError(t, err)
+		req.Header.Add("X-Forwarded-Proto", p)
+
+		resp, err := httpService.RoundTrip(req)
+		require.NoError(t, err)
+		require.Equal(t, http.StatusOK, resp.StatusCode)
+
+		respBody, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+		require.Equal(t, respBody, []byte(p))
+	}
 }
 
 func tcpListenRoutine(listener net.Listener, closeChan chan struct{}) {

ingress/origin_service.go

@@ -3,12 +3,12 @@ package ingress
 import (
 	"context"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"net/url"
-	"sync"
 	"time"
 
 	"github.com/pkg/errors"
@@ -20,26 +20,38 @@ import (
 	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
+const (
+	HelloWorldService = "hello_world"
+	HttpStatusService = "http_status"
+)
+
 // OriginService is something a tunnel can proxy traffic to.
 type OriginService interface {
 	String() string
 	// Start the origin service if it's managed by cloudflared, e.g. proxy servers or Hello World.
 	// If it's not managed by cloudflared, this is a no-op because the user is responsible for
 	// starting the origin service.
-	start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error
+	// Implementor of services managed by cloudflared should terminate the service if shutdownC is closed
+	start(log *zerolog.Logger, shutdownC <-chan struct{}, cfg OriginRequestConfig) error
+	MarshalJSON() ([]byte, error)
 }
 
-// unixSocketPath is an OriginService representing a unix socket (which accepts HTTP)
+// unixSocketPath is an OriginService representing a unix socket (which accepts HTTP or HTTPS)
 type unixSocketPath struct {
 	path      string
+	scheme    string
 	transport *http.Transport
 }
 
 func (o *unixSocketPath) String() string {
-	return "unix socket: " + o.path
+	scheme := ""
+	if o.scheme == "https" {
+		scheme = "+tls"
+	}
+	return fmt.Sprintf("unix%s:%s", scheme, o.path)
 }
 
-func (o *unixSocketPath) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (o *unixSocketPath) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
 	transport, err := newHTTPTransport(o, cfg, log)
 	if err != nil {
 		return err
@@ -48,13 +60,17 @@ func (o *unixSocketPath) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdown
 	return nil
 }
 
+func (o unixSocketPath) MarshalJSON() ([]byte, error) {
+	return json.Marshal(o.String())
+}
+
 type httpService struct {
 	url        *url.URL
 	hostHeader string
 	transport  *http.Transport
 }
 
-func (o *httpService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (o *httpService) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
 	transport, err := newHTTPTransport(o, cfg, log)
 	if err != nil {
 		return err
@@ -68,6 +84,10 @@ func (o *httpService) String() string {
 	return o.url.String()
 }
 
+func (o httpService) MarshalJSON() ([]byte, error) {
+	return json.Marshal(o.String())
+}
+
 // rawTCPService dials TCP to the destination specified by the client
 // It's used by warp routing
 type rawTCPService struct {
@@ -78,13 +98,18 @@ func (o *rawTCPService) String() string {
 	return o.name
 }
 
-func (o *rawTCPService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (o *rawTCPService) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
 	return nil
 }
 
+func (o rawTCPService) MarshalJSON() ([]byte, error) {
+	return json.Marshal(o.String())
+}
+
 // tcpOverWSService models TCP origins serving eyeballs connecting over websocket, such as
 // cloudflared access commands.
 type tcpOverWSService struct {
+	scheme        string
 	dest          string
 	isBastion     bool
 	streamHandler streamHandlerFunc
@@ -106,7 +131,8 @@ func newTCPOverWSService(url *url.URL) *tcpOverWSService {
 		addPortIfMissing(url, 7864) // just a random port since there isn't a default in this case
 	}
 	return &tcpOverWSService{
-		dest: url.Host,
+		scheme: url.Scheme,
+		dest:   url.Host,
 	}
 }
 
@@ -136,10 +162,15 @@ func (o *tcpOverWSService) String() string {
 	if o.isBastion {
 		return ServiceBastion
 	}
-	return o.dest
+
+	if o.scheme != "" {
+		return fmt.Sprintf("%s://%s", o.scheme, o.dest)
+	} else {
+		return o.dest
+	}
 }
 
-func (o *tcpOverWSService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (o *tcpOverWSService) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
 	if cfg.ProxyType == socksProxy {
 		o.streamHandler = socks.StreamHandler
 	} else {
@@ -148,7 +179,11 @@ func (o *tcpOverWSService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdo
 	return nil
 }
 
-func (o *socksProxyOverWSService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (o tcpOverWSService) MarshalJSON() ([]byte, error) {
+	return json.Marshal(o.String())
+}
+
+func (o *socksProxyOverWSService) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
 	return nil
 }
 
@@ -156,6 +191,10 @@ func (o *socksProxyOverWSService) String() string {
 	return ServiceSocksProxy
 }
 
+func (o socksProxyOverWSService) MarshalJSON() ([]byte, error) {
+	return json.Marshal(o.String())
+}
+
 // HelloWorld is an OriginService for the built-in Hello World server.
 // Users only use this for testing and experimenting with cloudflared.
 type helloWorld struct {
@@ -164,18 +203,16 @@ type helloWorld struct {
 }
 
 func (o *helloWorld) String() string {
-	return "Hello World test origin"
+	return HelloWorldService
 }
 
 // Start starts a HelloWorld server and stores its address in the Service receiver.
 func (o *helloWorld) start(
-	wg *sync.WaitGroup,
 	log *zerolog.Logger,
 	shutdownC <-chan struct{},
-	errC chan error,
 	cfg OriginRequestConfig,
 ) error {
-	if err := o.httpService.start(wg, log, shutdownC, errC, cfg); err != nil {
+	if err := o.httpService.start(log, shutdownC, cfg); err != nil {
 		return err
 	}
 
@@ -183,11 +220,7 @@ func (o *helloWorld) start(
 	if err != nil {
 		return errors.Wrap(err, "Cannot start Hello World Server")
 	}
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		_ = hello.StartHelloWorldServer(log, helloListener, shutdownC)
-	}()
+	go hello.StartHelloWorldServer(log, helloListener, shutdownC)
 	o.server = helloListener
 
 	o.httpService.url = &url.URL{
@@ -198,35 +231,36 @@ func (o *helloWorld) start(
 	return nil
 }
 
+func (o helloWorld) MarshalJSON() ([]byte, error) {
+	return json.Marshal(o.String())
+}
+
 // statusCode is an OriginService that just responds with a given HTTP status.
 // Typical use-case is "user wants the catch-all rule to just respond 404".
 type statusCode struct {
-	resp *http.Response
+	code int
 }
 
 func newStatusCode(status int) statusCode {
-	resp := &http.Response{
-		StatusCode: status,
-		Status:     fmt.Sprintf("%d %s", status, http.StatusText(status)),
-		Body:       new(NopReadCloser),
-	}
-	return statusCode{resp: resp}
+	return statusCode{code: status}
 }
 
 func (o *statusCode) String() string {
-	return fmt.Sprintf("HTTP %d", o.resp.StatusCode)
+	return fmt.Sprintf("http_status:%d", o.code)
 }
 
 func (o *statusCode) start(
-	wg *sync.WaitGroup,
 	log *zerolog.Logger,
-	shutdownC <-chan struct{},
-	errC chan error,
+	_ <-chan struct{},
 	cfg OriginRequestConfig,
 ) error {
 	return nil
 }
 
+func (o statusCode) MarshalJSON() ([]byte, error) {
+	return json.Marshal(o.String())
+}
+
 type NopReadCloser struct{}
 
 // Read always returns EOF to signal end of input
@@ -248,8 +282,8 @@ func newHTTPTransport(service OriginService, cfg OriginRequestConfig, log *zerol
 		Proxy:                 http.ProxyFromEnvironment,
 		MaxIdleConns:          cfg.KeepAliveConnections,
 		MaxIdleConnsPerHost:   cfg.KeepAliveConnections,
-		IdleConnTimeout:       cfg.KeepAliveTimeout,
-		TLSHandshakeTimeout:   cfg.TLSTimeout,
+		IdleConnTimeout:       cfg.KeepAliveTimeout.Duration,
+		TLSHandshakeTimeout:   cfg.TLSTimeout.Duration,
 		ExpectContinueTimeout: 1 * time.Second,
 		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: cfg.NoTLSVerify},
 	}
@@ -258,8 +292,8 @@ func newHTTPTransport(service OriginService, cfg OriginRequestConfig, log *zerol
 	}
 
 	dialer := &net.Dialer{
-		Timeout:   cfg.ConnectTimeout,
-		KeepAlive: cfg.TCPKeepAlive,
+		Timeout:   cfg.ConnectTimeout.Duration,
+		KeepAlive: cfg.TCPKeepAlive.Duration,
 	}
 	if cfg.NoHappyEyeballs {
 		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
@@ -296,6 +330,10 @@ func (mos MockOriginHTTPService) String() string {
 	return "MockOriginService"
 }
 
-func (mos MockOriginHTTPService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (mos MockOriginHTTPService) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
 	return nil
 }
+
+func (mos MockOriginHTTPService) MarshalJSON() ([]byte, error) {
+	return json.Marshal(mos.String())
+}

ingress/rule.go

@@ -1,6 +1,7 @@
 package ingress
 
 import (
+	"encoding/json"
 	"regexp"
 	"strings"
 )
@@ -9,18 +10,18 @@ import (
 // service running on the given URL.
 type Rule struct {
 	// Requests for this hostname will be proxied to this rule's service.
-	Hostname string
+	Hostname string `json:"hostname"`
 
 	// Path is an optional regex that can specify path-driven ingress rules.
-	Path *regexp.Regexp
+	Path *Regexp `json:"path"`
 
 	// A (probably local) address. Requests for a hostname which matches this
 	// rule's hostname pattern will be proxied to the service running on this
 	// address.
-	Service OriginService
+	Service OriginService `json:"service"`
 
 	// Configure the request cloudflared sends to this specific origin.
-	Config OriginRequestConfig
+	Config OriginRequestConfig `json:"originRequest"`
 }
 
 // MultiLineString is for outputting rules in a human-friendly way when Cloudflared
@@ -32,9 +33,9 @@ func (r Rule) MultiLineString() string {
 		out.WriteString(r.Hostname)
 		out.WriteRune('\n')
 	}
-	if r.Path != nil {
+	if r.Path != nil && r.Path.Regexp != nil {
 		out.WriteString("\tpath: ")
-		out.WriteString(r.Path.String())
+		out.WriteString(r.Path.Regexp.String())
 		out.WriteRune('\n')
 	}
 	out.WriteString("\tservice: ")
@@ -45,6 +46,18 @@ func (r Rule) MultiLineString() string {
 // Matches checks if the rule matches a given hostname/path combination.
 func (r *Rule) Matches(hostname, path string) bool {
 	hostMatch := r.Hostname == "" || r.Hostname == "*" || matchHost(r.Hostname, hostname)
-	pathMatch := r.Path == nil || r.Path.MatchString(path)
+	pathMatch := r.Path == nil || r.Path.Regexp == nil || r.Path.Regexp.MatchString(path)
 	return hostMatch && pathMatch
 }
+
+// Regexp adds unmarshalling from json for regexp.Regexp
+type Regexp struct {
+	*regexp.Regexp
+}
+
+func (r *Regexp) MarshalJSON() ([]byte, error) {
+	if r.Regexp == nil {
+		return json.Marshal(nil)
+	}
+	return json.Marshal(r.Regexp.String())
+}

ingress/rule_test.go

@@ -1,6 +1,7 @@
 package ingress
 
 import (
+	"encoding/json"
 	"io"
 	"net/http/httptest"
 	"net/url"
@@ -8,12 +9,14 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/config"
 )
 
 func Test_rule_matches(t *testing.T) {
 	type fields struct {
 		Hostname string
-		Path     *regexp.Regexp
+		Path     *Regexp
 		Service  OriginService
 	}
 	type args struct {
@@ -99,13 +102,35 @@ func Test_rule_matches(t *testing.T) {
 			name: "Hostname and path",
 			fields: fields{
 				Hostname: "*.example.com",
-				Path:     regexp.MustCompile("/static/.*\\.html"),
+				Path:     &Regexp{Regexp: regexp.MustCompile("/static/.*\\.html")},
 			},
 			args: args{
 				requestURL: MustParseURL(t, "https://www.example.com/static/index.html"),
 			},
 			want: true,
 		},
+		{
+			name: "Hostname and empty Regex",
+			fields: fields{
+				Hostname: "example.com",
+				Path:     &Regexp{},
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://example.com/"),
+			},
+			want: true,
+		},
+		{
+			name: "Hostname and nil path",
+			fields: fields{
+				Hostname: "example.com",
+				Path:     nil,
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://example.com/"),
+			},
+			want: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -144,3 +169,52 @@ func TestStaticHTTPStatus(t *testing.T) {
 	sendReq()
 	sendReq()
 }
+
+func TestMarshalJSON(t *testing.T) {
+	localhost8000 := MustParseURL(t, "https://localhost:8000")
+	defaultConfig := setConfig(originRequestFromConfig(config.OriginRequestConfig{}), config.OriginRequestConfig{})
+	tests := []struct {
+		name     string
+		path     *Regexp
+		expected string
+		want     bool
+	}{
+		{
+			name:     "Nil",
+			path:     nil,
+			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
+			want:     true,
+		},
+		{
+			name:     "Nil regex",
+			path:     &Regexp{Regexp: nil},
+			expected: `{"hostname":"example.com","path":null,"service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
+			want:     true,
+		},
+		{
+			name:     "Empty",
+			path:     &Regexp{Regexp: regexp.MustCompile("")},
+			expected: `{"hostname":"example.com","path":"","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
+			want:     true,
+		},
+		{
+			name:     "Basic",
+			path:     &Regexp{Regexp: regexp.MustCompile("/echo")},
+			expected: `{"hostname":"example.com","path":"/echo","service":"https://localhost:8000","originRequest":{"connectTimeout":30,"tlsTimeout":10,"tcpKeepAlive":30,"noHappyEyeballs":false,"keepAliveTimeout":90,"keepAliveConnections":100,"httpHostHeader":"","originServerName":"","caPool":"","noTLSVerify":false,"disableChunkedEncoding":false,"bastionMode":false,"proxyAddress":"127.0.0.1","proxyPort":0,"proxyType":"","ipRules":null}}`,
+			want:     true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Rule{
+				Hostname: "example.com",
+				Service:  &httpService{url: localhost8000},
+				Path:     tt.path,
+				Config:   defaultConfig,
+			}
+			bytes, err := json.Marshal(r)
+			require.NoError(t, err)
+			require.Equal(t, tt.expected, string(bytes))
+		})
+	}
+}

ipaccess/access.go

@@ -99,3 +99,15 @@ func (ipr *Rule) PortsString() string {
 	}
 	return "all"
 }
+
+func (ipr *Rule) Ports() []int {
+	return ipr.ports
+}
+
+func (ipr *Rule) RulePolicy() bool {
+	return ipr.allow
+}
+
+func (ipr *Rule) StringCIDR() string {
+	return ipr.ipNet.String()
+}

metrics/metrics.go

@@ -22,7 +22,16 @@ const (
 	startupTime     = time.Millisecond * 500
 )
 
-func newMetricsHandler(readyServer *ReadyServer, quickTunnelHostname string) *mux.Router {
+type orchestrator interface {
+	GetVersionedConfigJSON() ([]byte, error)
+}
+
+func newMetricsHandler(
+	readyServer *ReadyServer,
+	quickTunnelHostname string,
+	orchestrator orchestrator,
+	log *zerolog.Logger,
+) *mux.Router {
 	router := mux.NewRouter()
 	router.PathPrefix("/debug/").Handler(http.DefaultServeMux)
 
@@ -36,6 +45,18 @@ func newMetricsHandler(readyServer *ReadyServer, quickTunnelHostname string) *mu
 	router.HandleFunc("/quicktunnel", func(w http.ResponseWriter, r *http.Request) {
 		_, _ = fmt.Fprintf(w, `{"hostname":"%s"}`, quickTunnelHostname)
 	})
+	if orchestrator != nil {
+		router.HandleFunc("/config", func(w http.ResponseWriter, r *http.Request) {
+			json, err := orchestrator.GetVersionedConfigJSON()
+			if err != nil {
+				w.WriteHeader(500)
+				_, _ = fmt.Fprintf(w, "ERR: %v", err)
+				log.Err(err).Msg("Failed to serve config")
+				return
+			}
+			_, _ = w.Write(json)
+		})
+	}
 
 	return router
 }
@@ -45,6 +66,7 @@ func ServeMetrics(
 	shutdownC <-chan struct{},
 	readyServer *ReadyServer,
 	quickTunnelHostname string,
+	orchestrator orchestrator,
 	log *zerolog.Logger,
 ) (err error) {
 	var wg sync.WaitGroup
@@ -52,7 +74,7 @@ func ServeMetrics(
 	trace.AuthRequest = func(*http.Request) (bool, bool) { return true, true }
 	// TODO: parameterize ReadTimeout and WriteTimeout. The maximum time we can
 	// profile CPU usage depends on WriteTimeout
-	h := newMetricsHandler(readyServer, quickTunnelHostname)
+	h := newMetricsHandler(readyServer, quickTunnelHostname, orchestrator, log)
 	server := &http.Server{
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 10 * time.Second,

orchestration/config.go

@@ -0,0 +1,66 @@
+package orchestration
+
+import (
+	"encoding/json"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ingress"
+)
+
+type newRemoteConfig struct {
+	ingress.RemoteConfig
+	// Add more fields when we support other settings in tunnel orchestration
+}
+
+type newLocalConfig struct {
+	RemoteConfig       ingress.RemoteConfig
+	ConfigurationFlags map[string]string `json:"__configuration_flags,omitempty"`
+}
+
+// Config is the original config as read and parsed by cloudflared.
+type Config struct {
+	Ingress            *ingress.Ingress
+	WarpRoutingEnabled bool
+
+	// Extra settings used to configure this instance but that are not eligible for remotely management
+	// ie. (--protocol, --loglevel, ...)
+	ConfigurationFlags map[string]string
+}
+
+func (rc *newLocalConfig) MarshalJSON() ([]byte, error) {
+	var r = struct {
+		ConfigurationFlags map[string]string `json:"__configuration_flags,omitempty"`
+		ingress.RemoteConfigJSON
+	}{
+		ConfigurationFlags: rc.ConfigurationFlags,
+		RemoteConfigJSON: ingress.RemoteConfigJSON{
+			// UI doesn't support top level configs, so we reconcile to individual ingress configs.
+			GlobalOriginRequest: nil,
+			IngressRules:        convertToUnvalidatedIngressRules(rc.RemoteConfig.Ingress),
+			WarpRouting:         rc.RemoteConfig.WarpRouting,
+		},
+	}
+
+	return json.Marshal(r)
+}
+
+func convertToUnvalidatedIngressRules(i ingress.Ingress) []config.UnvalidatedIngressRule {
+	result := make([]config.UnvalidatedIngressRule, 0)
+	for _, rule := range i.Rules {
+		var path string
+		if rule.Path != nil {
+			path = rule.Path.String()
+		}
+
+		newRule := config.UnvalidatedIngressRule{
+			Hostname:      rule.Hostname,
+			Path:          path,
+			Service:       rule.Service.String(),
+			OriginRequest: ingress.ConvertToRawOriginConfig(rule.Config),
+		}
+
+		result = append(result, newRule)
+	}
+
+	return result
+}

orchestration/config_test.go

@@ -0,0 +1,82 @@
+package orchestration
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/ingress"
+)
+
+// TestNewLocalConfig_MarshalJSON tests that we are able to converte a compiled and validated config back
+// into an "unvalidated" format which is compatible with Remote Managed configurations.
+func TestNewLocalConfig_MarshalJSON(t *testing.T) {
+
+	rawConfig := []byte(`
+	{
+		"originRequest": {
+					"connectTimeout": 160,
+					"httpHostHeader": "default"
+		},
+		"ingress": [
+			{
+				"hostname": "tun.example.com",
+				"service": "https://localhost:8000"
+			},
+			{
+				"hostname": "*",
+				"service": "https://localhost:8001",
+				"originRequest": {
+					"connectTimeout": 121,
+					"tlsTimeout": 2,
+					"noHappyEyeballs": false,
+					"tcpKeepAlive": 2,
+					"keepAliveConnections": 2,
+					"keepAliveTimeout": 2,
+					"httpHostHeader": "def",
+					"originServerName": "b2",
+					"caPool": "/tmp/path1",
+					"noTLSVerify": false,
+					"disableChunkedEncoding": false,
+					"bastionMode": false,
+					"proxyAddress": "interface",
+					"proxyPort": 200,
+					"proxyType": "",
+					"ipRules": [
+						{
+							"prefix": "10.0.0.0/16",
+							"ports": [3000, 3030],
+							"allow": false
+						},
+						{
+							"prefix": "192.16.0.0/24",
+							"ports": [5000, 5050],
+							"allow": true
+						}
+					]
+				}
+			}
+		]
+	}
+	`)
+
+	var expectedConfig ingress.RemoteConfig
+	err := json.Unmarshal(rawConfig, &expectedConfig)
+	require.NoError(t, err)
+
+	c := &newLocalConfig{
+		RemoteConfig:       expectedConfig,
+		ConfigurationFlags: nil,
+	}
+
+	jsonSerde, err := json.Marshal(c)
+	require.NoError(t, err)
+
+	var config ingress.RemoteConfig
+	err = json.Unmarshal(jsonSerde, &config)
+	require.NoError(t, err)
+
+	require.Equal(t, config.WarpRouting.Enabled, false)
+	require.Equal(t, config.Ingress.Rules, expectedConfig.Ingress.Rules)
+}

orchestration/metrics.go

@@ -0,0 +1,25 @@
+package orchestration
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	MetricsNamespace = "cloudflared"
+	MetricsSubsystem = "orchestration"
+)
+
+var (
+	configVersion = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: MetricsSubsystem,
+			Name:      "config_version",
+			Help:      "Configuration Version",
+		},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(configVersion)
+}

orchestration/orchestrator.go

@@ -0,0 +1,202 @@
+package orchestration
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"sync/atomic"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/proxy"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// Orchestrator manages configurations so they can be updatable during runtime
+// properties are static, so it can be read without lock
+// currentVersion and config are read/write infrequently, so their access are synchronized with RWMutex
+// access to proxy is synchronized with atmoic.Value, because it uses copy-on-write to provide scalable frequently
+// read when update is infrequent
+type Orchestrator struct {
+	currentVersion int32
+	// Used by UpdateConfig to make sure one update at a time
+	lock sync.RWMutex
+	// Underlying value is proxy.Proxy, can be read without the lock, but still needs the lock to update
+	proxy  atomic.Value
+	config *Config
+	tags   []tunnelpogs.Tag
+	log    *zerolog.Logger
+
+	// orchestrator must not handle any more updates after shutdownC is closed
+	shutdownC <-chan struct{}
+	// Closing proxyShutdownC will close the previous proxy
+	proxyShutdownC chan<- struct{}
+}
+
+func NewOrchestrator(ctx context.Context, config *Config, tags []tunnelpogs.Tag, log *zerolog.Logger) (*Orchestrator, error) {
+	o := &Orchestrator{
+		// Lowest possible version, any remote configuration will have version higher than this
+		currentVersion: 0,
+		config:         config,
+		tags:           tags,
+		log:            log,
+		shutdownC:      ctx.Done(),
+	}
+	if err := o.updateIngress(*config.Ingress, config.WarpRoutingEnabled); err != nil {
+		return nil, err
+	}
+	go o.waitToCloseLastProxy()
+	return o, nil
+}
+
+// UpdateConfig creates a new proxy with the new ingress rules
+func (o *Orchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
+	o.lock.Lock()
+	defer o.lock.Unlock()
+
+	if o.currentVersion >= version {
+		o.log.Debug().
+			Int32("current_version", o.currentVersion).
+			Int32("received_version", version).
+			Msg("Current version is equal or newer than received version")
+		return &tunnelpogs.UpdateConfigurationResponse{
+			LastAppliedVersion: o.currentVersion,
+		}
+	}
+	var newConf newRemoteConfig
+	if err := json.Unmarshal(config, &newConf); err != nil {
+		o.log.Err(err).
+			Int32("version", version).
+			Str("config", string(config)).
+			Msgf("Failed to deserialize new configuration")
+		return &tunnelpogs.UpdateConfigurationResponse{
+			LastAppliedVersion: o.currentVersion,
+			Err:                err,
+		}
+	}
+
+	if err := o.updateIngress(newConf.Ingress, newConf.WarpRouting.Enabled); err != nil {
+		o.log.Err(err).
+			Int32("version", version).
+			Str("config", string(config)).
+			Msgf("Failed to update ingress")
+		return &tunnelpogs.UpdateConfigurationResponse{
+			LastAppliedVersion: o.currentVersion,
+			Err:                err,
+		}
+	}
+	o.currentVersion = version
+
+	o.log.Info().
+		Int32("version", version).
+		Str("config", string(config)).
+		Msg("Updated to new configuration")
+	configVersion.Set(float64(version))
+	return &tunnelpogs.UpdateConfigurationResponse{
+		LastAppliedVersion: o.currentVersion,
+	}
+}
+
+// The caller is responsible to make sure there is no concurrent access
+func (o *Orchestrator) updateIngress(ingressRules ingress.Ingress, warpRoutingEnabled bool) error {
+	select {
+	case <-o.shutdownC:
+		return fmt.Errorf("cloudflared already shutdown")
+	default:
+	}
+
+	// Start new proxy before closing the ones from last version.
+	// The upside is we don't need to restart proxy from last version, which can fail
+	// The downside is new version might have ingress rule that require previous version to be shutdown first
+	// The downside is minimized because none of the ingress.OriginService implementation have that requirement
+	proxyShutdownC := make(chan struct{})
+	if err := ingressRules.StartOrigins(o.log, proxyShutdownC); err != nil {
+		return errors.Wrap(err, "failed to start origin")
+	}
+	newProxy := proxy.NewOriginProxy(ingressRules, warpRoutingEnabled, o.tags, o.log)
+	o.proxy.Store(newProxy)
+	o.config.Ingress = &ingressRules
+	o.config.WarpRoutingEnabled = warpRoutingEnabled
+
+	// If proxyShutdownC is nil, there is no previous running proxy
+	if o.proxyShutdownC != nil {
+		close(o.proxyShutdownC)
+	}
+	o.proxyShutdownC = proxyShutdownC
+	return nil
+}
+
+// GetConfigJSON returns the current json serialization of the config as the edge understands it
+func (o *Orchestrator) GetConfigJSON() ([]byte, error) {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+
+	c := &newLocalConfig{
+		RemoteConfig: ingress.RemoteConfig{
+			Ingress:     *o.config.Ingress,
+			WarpRouting: config.WarpRoutingConfig{Enabled: o.config.WarpRoutingEnabled},
+		},
+		ConfigurationFlags: o.config.ConfigurationFlags,
+	}
+
+	return json.Marshal(c)
+}
+
+// GetVersionedConfigJSON returns the current version and configuration as JSON
+func (o *Orchestrator) GetVersionedConfigJSON() ([]byte, error) {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	var currentConfiguration = struct {
+		Version int32 `json:"version"`
+		Config  struct {
+			Ingress       []ingress.Rule              `json:"ingress"`
+			WarpRouting   config.WarpRoutingConfig    `json:"warp-routing"`
+			OriginRequest ingress.OriginRequestConfig `json:"originRequest"`
+		} `json:"config"`
+	}{
+		Version: o.currentVersion,
+		Config: struct {
+			Ingress       []ingress.Rule              `json:"ingress"`
+			WarpRouting   config.WarpRoutingConfig    `json:"warp-routing"`
+			OriginRequest ingress.OriginRequestConfig `json:"originRequest"`
+		}{
+			Ingress:       o.config.Ingress.Rules,
+			WarpRouting:   config.WarpRoutingConfig{Enabled: o.config.WarpRoutingEnabled},
+			OriginRequest: o.config.Ingress.Defaults,
+		},
+	}
+	return json.Marshal(currentConfiguration)
+}
+
+// GetOriginProxy returns an interface to proxy to origin. It satisfies connection.ConfigManager interface
+func (o *Orchestrator) GetOriginProxy() (connection.OriginProxy, error) {
+	val := o.proxy.Load()
+	if val == nil {
+		err := fmt.Errorf("origin proxy not configured")
+		o.log.Error().Msg(err.Error())
+		return nil, err
+	}
+	proxy, ok := val.(*proxy.Proxy)
+	if !ok {
+		err := fmt.Errorf("origin proxy has unexpected value %+v", val)
+		o.log.Error().Msg(err.Error())
+		return nil, err
+	}
+	return proxy, nil
+}
+
+func (o *Orchestrator) waitToCloseLastProxy() {
+	<-o.shutdownC
+	o.lock.Lock()
+	defer o.lock.Unlock()
+
+	if o.proxyShutdownC != nil {
+		close(o.proxyShutdownC)
+		o.proxyShutdownC = nil
+	}
+}

orchestration/orchestrator_test.go

@@ -0,0 +1,724 @@
+package orchestration
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/gobwas/ws/wsutil"
+	gows "github.com/gorilla/websocket"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/proxy"
+	"github.com/cloudflare/cloudflared/tracing"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+var (
+	testLogger = zerolog.Logger{}
+	testTags   = []tunnelpogs.Tag{
+		{
+			Name:  "package",
+			Value: "orchestration",
+		},
+		{
+			Name:  "purpose",
+			Value: "test",
+		},
+	}
+)
+
+// TestUpdateConfiguration tests that
+// - configurations can be deserialized
+// - proxy can be updated
+// - last applied version and error are returned
+// - configurations can be deserialized
+// - receiving an old version is noop
+func TestUpdateConfiguration(t *testing.T) {
+	initConfig := &Config{
+		Ingress:            &ingress.Ingress{},
+		WarpRoutingEnabled: false,
+	}
+	orchestrator, err := NewOrchestrator(context.Background(), initConfig, testTags, &testLogger)
+	require.NoError(t, err)
+	initOriginProxy, err := orchestrator.GetOriginProxy()
+	require.NoError(t, err)
+	require.IsType(t, &proxy.Proxy{}, initOriginProxy)
+
+	configJSONV2 := []byte(`
+{
+	"unknown_field": "not_deserialized",
+    "originRequest": {
+        "connectTimeout": 90,
+		"noHappyEyeballs": true
+    },
+    "ingress": [
+        {
+            "hostname": "jira.tunnel.org",
+			"path": "^\/login",
+            "service": "http://192.16.19.1:443",
+            "originRequest": {
+                "noTLSVerify": true,
+                "connectTimeout": 10
+            }
+        },
+		{
+            "hostname": "jira.tunnel.org",
+            "service": "http://172.32.20.6:80",
+            "originRequest": {
+                "noTLSVerify": true,
+                "connectTimeout": 30
+            }
+        },
+        {
+            "service": "http_status:404"
+        }
+    ],
+    "warp-routing": {
+        "enabled": true
+    }
+}	
+`)
+
+	updateWithValidation(t, orchestrator, 2, configJSONV2)
+	configV2 := orchestrator.config
+	// Validate ingress rule 0
+	require.Equal(t, "jira.tunnel.org", configV2.Ingress.Rules[0].Hostname)
+	require.True(t, configV2.Ingress.Rules[0].Matches("jira.tunnel.org", "/login"))
+	require.True(t, configV2.Ingress.Rules[0].Matches("jira.tunnel.org", "/login/2fa"))
+	require.False(t, configV2.Ingress.Rules[0].Matches("jira.tunnel.org", "/users"))
+	require.Equal(t, "http://192.16.19.1:443", configV2.Ingress.Rules[0].Service.String())
+	require.Len(t, configV2.Ingress.Rules, 3)
+	// originRequest of this ingress rule overrides global default
+	require.Equal(t, config.CustomDuration{Duration: time.Second * 10}, configV2.Ingress.Rules[0].Config.ConnectTimeout)
+	require.Equal(t, true, configV2.Ingress.Rules[0].Config.NoTLSVerify)
+	// Inherited from global default
+	require.Equal(t, true, configV2.Ingress.Rules[0].Config.NoHappyEyeballs)
+	// Validate ingress rule 1
+	require.Equal(t, "jira.tunnel.org", configV2.Ingress.Rules[1].Hostname)
+	require.True(t, configV2.Ingress.Rules[1].Matches("jira.tunnel.org", "/users"))
+	require.Equal(t, "http://172.32.20.6:80", configV2.Ingress.Rules[1].Service.String())
+	// originRequest of this ingress rule overrides global default
+	require.Equal(t, config.CustomDuration{Duration: time.Second * 30}, configV2.Ingress.Rules[1].Config.ConnectTimeout)
+	require.Equal(t, true, configV2.Ingress.Rules[1].Config.NoTLSVerify)
+	// Inherited from global default
+	require.Equal(t, true, configV2.Ingress.Rules[1].Config.NoHappyEyeballs)
+	// Validate ingress rule 2, it's the catch-all rule
+	require.True(t, configV2.Ingress.Rules[2].Matches("blogs.tunnel.io", "/2022/02/10"))
+	// Inherited from global default
+	require.Equal(t, config.CustomDuration{Duration: time.Second * 90}, configV2.Ingress.Rules[2].Config.ConnectTimeout)
+	require.Equal(t, false, configV2.Ingress.Rules[2].Config.NoTLSVerify)
+	require.Equal(t, true, configV2.Ingress.Rules[2].Config.NoHappyEyeballs)
+	require.True(t, configV2.WarpRoutingEnabled)
+
+	originProxyV2, err := orchestrator.GetOriginProxy()
+	require.NoError(t, err)
+	require.IsType(t, &proxy.Proxy{}, originProxyV2)
+	require.NotEqual(t, originProxyV2, initOriginProxy)
+
+	// Should not downgrade to an older version
+	resp := orchestrator.UpdateConfig(1, nil)
+	require.NoError(t, resp.Err)
+	require.Equal(t, int32(2), resp.LastAppliedVersion)
+
+	invalidJSON := []byte(`
+{
+	"originRequest":
+}
+	
+`)
+
+	resp = orchestrator.UpdateConfig(3, invalidJSON)
+	require.Error(t, resp.Err)
+	require.Equal(t, int32(2), resp.LastAppliedVersion)
+	originProxyV3, err := orchestrator.GetOriginProxy()
+	require.NoError(t, err)
+	require.Equal(t, originProxyV2, originProxyV3)
+
+	configJSONV10 := []byte(`
+{
+    "ingress": [
+        {
+            "service": "hello-world"
+        }
+    ],
+    "warp-routing": {
+        "enabled": false
+    }
+}	
+`)
+	updateWithValidation(t, orchestrator, 10, configJSONV10)
+	configV10 := orchestrator.config
+	require.Len(t, configV10.Ingress.Rules, 1)
+	require.True(t, configV10.Ingress.Rules[0].Matches("blogs.tunnel.io", "/2022/02/10"))
+	require.Equal(t, ingress.HelloWorldService, configV10.Ingress.Rules[0].Service.String())
+	require.False(t, configV10.WarpRoutingEnabled)
+
+	originProxyV10, err := orchestrator.GetOriginProxy()
+	require.NoError(t, err)
+	require.IsType(t, &proxy.Proxy{}, originProxyV10)
+	require.NotEqual(t, originProxyV10, originProxyV2)
+}
+
+// TestConcurrentUpdateAndRead makes sure orchestrator can receive updates and return origin proxy concurrently
+func TestConcurrentUpdateAndRead(t *testing.T) {
+	const (
+		concurrentRequests = 200
+		hostname           = "public.tunnels.org"
+		expectedHost       = "internal.tunnels.svc.cluster.local"
+		tcpBody            = "testProxyTCP"
+	)
+
+	httpOrigin := httptest.NewServer(&validateHostHandler{
+		expectedHost: expectedHost,
+		body:         t.Name(),
+	})
+	defer httpOrigin.Close()
+
+	tcpOrigin, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	defer tcpOrigin.Close()
+
+	var (
+		configJSONV1 = []byte(fmt.Sprintf(`
+{
+    "originRequest": {
+        "connectTimeout": 90,
+		"noHappyEyeballs": true
+    },
+    "ingress": [
+        {
+            "hostname": "%s",
+            "service": "%s",
+            "originRequest": {
+				"httpHostHeader": "%s",
+                "connectTimeout": 10
+            }
+        },
+        {
+            "service": "http_status:404"
+        }
+    ],
+    "warp-routing": {
+        "enabled": true
+    }
+}
+`, hostname, httpOrigin.URL, expectedHost))
+		configJSONV2 = []byte(`
+{
+    "ingress": [
+        {
+            "service": "http_status:204"
+        }
+    ],
+    "warp-routing": {
+        "enabled": false
+    }
+}
+`)
+
+		configJSONV3 = []byte(`
+{
+    "ingress": [
+        {
+            "service": "http_status:418"
+        }
+    ],
+    "warp-routing": {
+        "enabled": true
+    }
+}
+`)
+
+		// appliedV2 makes sure v3 is applied after v2
+		appliedV2 = make(chan struct{})
+
+		initConfig = &Config{
+			Ingress:            &ingress.Ingress{},
+			WarpRoutingEnabled: false,
+		}
+	)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	orchestrator, err := NewOrchestrator(ctx, initConfig, testTags, &testLogger)
+	require.NoError(t, err)
+
+	updateWithValidation(t, orchestrator, 1, configJSONV1)
+
+	var wg sync.WaitGroup
+	// tcpOrigin will be closed when the test exits. Only the handler routines are included in the wait group
+	go func() {
+		serveTCPOrigin(t, tcpOrigin, &wg)
+	}()
+	for i := 0; i < concurrentRequests; i++ {
+		originProxy, err := orchestrator.GetOriginProxy()
+		require.NoError(t, err)
+		wg.Add(1)
+		go func(i int, originProxy connection.OriginProxy) {
+			defer wg.Done()
+			resp, err := proxyHTTP(originProxy, hostname)
+			require.NoError(t, err, "proxyHTTP %d failed %v", i, err)
+			defer resp.Body.Close()
+
+			var warpRoutingDisabled bool
+			// The response can be from initOrigin, http_status:204 or http_status:418
+			switch resp.StatusCode {
+			// v1 proxy, warp enabled
+			case 200:
+				body, err := ioutil.ReadAll(resp.Body)
+				require.NoError(t, err)
+				require.Equal(t, t.Name(), string(body))
+				warpRoutingDisabled = false
+			// v2 proxy, warp disabled
+			case 204:
+				require.Greater(t, i, concurrentRequests/4)
+				warpRoutingDisabled = true
+			// v3 proxy, warp enabled
+			case 418:
+				require.Greater(t, i, concurrentRequests/2)
+				warpRoutingDisabled = false
+			}
+
+			// Once we have originProxy, it won't be changed by configuration updates.
+			// We can infer the version by the ProxyHTTP response code
+			pr, pw := io.Pipe()
+
+			w := newRespReadWriteFlusher()
+
+			// Write TCP message and make sure it's echo back. This has to be done in a go routune since ProxyTCP doesn't
+			// return until the stream is closed.
+			if !warpRoutingDisabled {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					defer pw.Close()
+					tcpEyeball(t, pw, tcpBody, w)
+				}()
+			}
+
+			err = proxyTCP(ctx, originProxy, tcpOrigin.Addr().String(), w, pr)
+			if warpRoutingDisabled {
+				require.Error(t, err, "expect proxyTCP %d to return error", i)
+			} else {
+				require.NoError(t, err, "proxyTCP %d failed %v", i, err)
+			}
+
+		}(i, originProxy)
+
+		if i == concurrentRequests/4 {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				updateWithValidation(t, orchestrator, 2, configJSONV2)
+				close(appliedV2)
+			}()
+		}
+
+		if i == concurrentRequests/2 {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				// Makes sure v2 is applied before v3
+				<-appliedV2
+				updateWithValidation(t, orchestrator, 3, configJSONV3)
+			}()
+		}
+	}
+
+	wg.Wait()
+}
+
+func proxyHTTP(originProxy connection.OriginProxy, hostname string) (*http.Response, error) {
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s", hostname), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	w := httptest.NewRecorder()
+	log := zerolog.Nop()
+	respWriter, err := connection.NewHTTP2RespWriter(req, w, connection.TypeHTTP, &log)
+	if err != nil {
+		return nil, err
+	}
+
+	err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), false)
+	if err != nil {
+		return nil, err
+	}
+
+	return w.Result(), nil
+}
+
+func tcpEyeball(t *testing.T, reqWriter io.WriteCloser, body string, respReadWriter *respReadWriteFlusher) {
+	writeN, err := reqWriter.Write([]byte(body))
+	require.NoError(t, err)
+
+	readBuffer := make([]byte, writeN)
+	n, err := respReadWriter.Read(readBuffer)
+	require.NoError(t, err)
+	require.Equal(t, body, string(readBuffer[:n]))
+	require.Equal(t, writeN, n)
+}
+
+func proxyTCP(ctx context.Context, originProxy connection.OriginProxy, originAddr string, w http.ResponseWriter, reqBody io.ReadCloser) error {
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s", originAddr), reqBody)
+	if err != nil {
+		return err
+	}
+
+	log := zerolog.Nop()
+	respWriter, err := connection.NewHTTP2RespWriter(req, w, connection.TypeTCP, &log)
+	if err != nil {
+		return err
+	}
+
+	tcpReq := &connection.TCPRequest{
+		Dest:    originAddr,
+		CFRay:   "123",
+		LBProbe: false,
+	}
+	rws := connection.NewHTTPResponseReadWriterAcker(respWriter, req)
+
+	return originProxy.ProxyTCP(ctx, rws, tcpReq)
+}
+
+func serveTCPOrigin(t *testing.T, tcpOrigin net.Listener, wg *sync.WaitGroup) {
+	for {
+		conn, err := tcpOrigin.Accept()
+		if err != nil {
+			return
+		}
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			defer conn.Close()
+
+			echoTCP(t, conn)
+		}()
+	}
+}
+
+func echoTCP(t *testing.T, conn net.Conn) {
+	readBuf := make([]byte, 1000)
+	readN, err := conn.Read(readBuf)
+	require.NoError(t, err)
+
+	writeN, err := conn.Write(readBuf[:readN])
+	require.NoError(t, err)
+	require.Equal(t, readN, writeN)
+}
+
+type validateHostHandler struct {
+	expectedHost string
+	body         string
+}
+
+func (vhh *validateHostHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Host != vhh.expectedHost {
+		w.WriteHeader(http.StatusBadRequest)
+		return
+	}
+	w.WriteHeader(http.StatusOK)
+	w.Write([]byte(vhh.body))
+}
+
+func updateWithValidation(t *testing.T, orchestrator *Orchestrator, version int32, config []byte) {
+	resp := orchestrator.UpdateConfig(version, config)
+	require.NoError(t, resp.Err)
+	require.Equal(t, version, resp.LastAppliedVersion)
+}
+
+// TestClosePreviousProxies makes sure proxies started in the pervious configuration version are shutdown
+func TestClosePreviousProxies(t *testing.T) {
+	var (
+		hostname             = "hello.tunnel1.org"
+		configWithHelloWorld = []byte(fmt.Sprintf(`
+{
+    "ingress": [
+        {
+			"hostname": "%s",
+            "service": "hello-world"
+        },
+		{
+			"service": "http_status:404"
+		}
+    ],
+    "warp-routing": {
+        "enabled": true
+    }
+}
+`, hostname))
+
+		configTeapot = []byte(`
+{
+    "ingress": [
+		{
+			"service": "http_status:418"
+		}
+    ],
+    "warp-routing": {
+        "enabled": true
+    }
+}
+`)
+		initConfig = &Config{
+			Ingress:            &ingress.Ingress{},
+			WarpRoutingEnabled: false,
+		}
+	)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	orchestrator, err := NewOrchestrator(ctx, initConfig, testTags, &testLogger)
+	require.NoError(t, err)
+
+	updateWithValidation(t, orchestrator, 1, configWithHelloWorld)
+
+	originProxyV1, err := orchestrator.GetOriginProxy()
+	require.NoError(t, err)
+	resp, err := proxyHTTP(originProxyV1, hostname)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+
+	updateWithValidation(t, orchestrator, 2, configTeapot)
+
+	originProxyV2, err := orchestrator.GetOriginProxy()
+	require.NoError(t, err)
+	resp, err = proxyHTTP(originProxyV2, hostname)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusTeapot, resp.StatusCode)
+
+	// The hello-world server in config v1 should have been stopped
+	resp, err = proxyHTTP(originProxyV1, hostname)
+	require.Error(t, err)
+	require.Nil(t, resp)
+
+	// Apply the config with hello world server again, orchestrator should spin up another hello world server
+	updateWithValidation(t, orchestrator, 3, configWithHelloWorld)
+
+	originProxyV3, err := orchestrator.GetOriginProxy()
+	require.NoError(t, err)
+	require.NotEqual(t, originProxyV1, originProxyV3)
+
+	resp, err = proxyHTTP(originProxyV3, hostname)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+
+	// cancel the context should terminate the last proxy
+	cancel()
+	// Wait for proxies to shutdown
+	time.Sleep(time.Millisecond * 10)
+
+	resp, err = proxyHTTP(originProxyV3, hostname)
+	require.Error(t, err)
+	require.Nil(t, resp)
+}
+
+// TestPersistentConnection makes sure updating the ingress doesn't intefere with existing connections
+func TestPersistentConnection(t *testing.T) {
+	const (
+		hostname = "http://ws.tunnel.org"
+	)
+	msg := t.Name()
+	initConfig := &Config{
+		Ingress:            &ingress.Ingress{},
+		WarpRoutingEnabled: false,
+	}
+	orchestrator, err := NewOrchestrator(context.Background(), initConfig, testTags, &testLogger)
+	require.NoError(t, err)
+
+	wsOrigin := httptest.NewServer(http.HandlerFunc(wsEcho))
+	defer wsOrigin.Close()
+
+	tcpOrigin, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	defer tcpOrigin.Close()
+
+	configWithWSAndWarp := []byte(fmt.Sprintf(`
+{
+    "ingress": [
+        {
+            "service": "%s"
+        }
+    ],
+    "warp-routing": {
+        "enabled": true
+    }
+}
+`, wsOrigin.URL))
+
+	updateWithValidation(t, orchestrator, 1, configWithWSAndWarp)
+
+	originProxy, err := orchestrator.GetOriginProxy()
+	require.NoError(t, err)
+
+	wsReqReader, wsReqWriter := io.Pipe()
+	wsRespReadWriter := newRespReadWriteFlusher()
+
+	tcpReqReader, tcpReqWriter := io.Pipe()
+	tcpRespReadWriter := newRespReadWriteFlusher()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	var wg sync.WaitGroup
+	wg.Add(3)
+	// Start TCP origin
+	go func() {
+		defer wg.Done()
+		conn, err := tcpOrigin.Accept()
+		require.NoError(t, err)
+		defer conn.Close()
+
+		// Expect 3 TCP messages
+		for i := 0; i < 3; i++ {
+			echoTCP(t, conn)
+		}
+	}()
+	// Simulate cloudflared recieving a TCP connection
+	go func() {
+		defer wg.Done()
+		require.NoError(t, proxyTCP(ctx, originProxy, tcpOrigin.Addr().String(), tcpRespReadWriter, tcpReqReader))
+	}()
+	// Simulate cloudflared recieving a WS connection
+	go func() {
+		defer wg.Done()
+
+		req, err := http.NewRequest(http.MethodGet, hostname, wsReqReader)
+		require.NoError(t, err)
+		// ProxyHTTP will add Connection, Upgrade and Sec-Websocket-Version headers
+		req.Header.Add("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ==")
+
+		log := zerolog.Nop()
+		respWriter, err := connection.NewHTTP2RespWriter(req, wsRespReadWriter, connection.TypeWebsocket, &log)
+		require.NoError(t, err)
+
+		err = originProxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), true)
+		require.NoError(t, err)
+	}()
+
+	// Simulate eyeball WS and TCP connections
+	validateWsEcho(t, msg, wsReqWriter, wsRespReadWriter)
+	tcpEyeball(t, tcpReqWriter, msg, tcpRespReadWriter)
+
+	configNoWSAndWarp := []byte(`
+{
+    "ingress": [
+        {
+            "service": "http_status:404"
+        }
+    ],
+    "warp-routing": {
+        "enabled": false
+    }
+}
+`)
+
+	updateWithValidation(t, orchestrator, 2, configNoWSAndWarp)
+	// Make sure connection is still up
+	validateWsEcho(t, msg, wsReqWriter, wsRespReadWriter)
+	tcpEyeball(t, tcpReqWriter, msg, tcpRespReadWriter)
+
+	updateWithValidation(t, orchestrator, 3, configWithWSAndWarp)
+	// Make sure connection is still up
+	validateWsEcho(t, msg, wsReqWriter, wsRespReadWriter)
+	tcpEyeball(t, tcpReqWriter, msg, tcpRespReadWriter)
+
+	wsReqWriter.Close()
+	tcpReqWriter.Close()
+	wg.Wait()
+}
+
+func TestSerializeLocalConfig(t *testing.T) {
+	c := &newLocalConfig{
+		RemoteConfig: ingress.RemoteConfig{
+			Ingress:     ingress.Ingress{},
+			WarpRouting: config.WarpRoutingConfig{},
+		},
+		ConfigurationFlags: map[string]string{"a": "b"},
+	}
+
+	result, _ := json.Marshal(c)
+	fmt.Println(string(result))
+}
+
+func wsEcho(w http.ResponseWriter, r *http.Request) {
+	upgrader := gows.Upgrader{}
+
+	conn, err := upgrader.Upgrade(w, r, nil)
+	if err != nil {
+		return
+	}
+	defer conn.Close()
+	for {
+		mt, message, err := conn.ReadMessage()
+		if err != nil {
+			fmt.Println("read message err", err)
+			break
+		}
+		err = conn.WriteMessage(mt, message)
+		if err != nil {
+			fmt.Println("write message err", err)
+			break
+		}
+	}
+}
+
+func validateWsEcho(t *testing.T, msg string, reqWriter io.Writer, respReadWriter io.ReadWriter) {
+	err := wsutil.WriteClientText(reqWriter, []byte(msg))
+	require.NoError(t, err)
+
+	receivedMsg, err := wsutil.ReadServerText(respReadWriter)
+	require.NoError(t, err)
+	require.Equal(t, msg, string(receivedMsg))
+}
+
+type respReadWriteFlusher struct {
+	io.Reader
+	w             io.Writer
+	headers       http.Header
+	statusCode    int
+	setStatusOnce sync.Once
+	hasStatus     chan struct{}
+}
+
+func newRespReadWriteFlusher() *respReadWriteFlusher {
+	pr, pw := io.Pipe()
+	return &respReadWriteFlusher{
+		Reader:    pr,
+		w:         pw,
+		headers:   make(http.Header),
+		hasStatus: make(chan struct{}),
+	}
+}
+
+func (rrw *respReadWriteFlusher) Write(buf []byte) (int, error) {
+	rrw.WriteHeader(http.StatusOK)
+	return rrw.w.Write(buf)
+}
+
+func (rrw *respReadWriteFlusher) Flush() {}
+
+func (rrw *respReadWriteFlusher) Header() http.Header {
+	return rrw.headers
+}
+
+func (rrw *respReadWriteFlusher) WriteHeader(statusCode int) {
+	rrw.setStatusOnce.Do(func() {
+		rrw.statusCode = statusCode
+		close(rrw.hasStatus)
+	})
+}

origin/pool.go

@@ -1,29 +0,0 @@
-package origin
-
-import (
-	"sync"
-)
-
-type bufferPool struct {
-	// A bufferPool must not be copied after first use.
-	// https://golang.org/pkg/sync/#Pool
-	buffers sync.Pool
-}
-
-func newBufferPool(bufferSize int) *bufferPool {
-	return &bufferPool{
-		buffers: sync.Pool{
-			New: func() interface{} {
-				return make([]byte, bufferSize)
-			},
-		},
-	}
-}
-
-func (p *bufferPool) Get() []byte {
-	return p.buffers.Get().([]byte)
-}
-
-func (p *bufferPool) Put(buf []byte) {
-	p.buffers.Put(buf)
-}

proxy/metrics.go

@@ -1,4 +1,4 @@
-package origin
+package proxy
 
 import (
 	"github.com/prometheus/client_golang/prometheus"
@@ -43,14 +43,6 @@ var (
 			Help:      "Count of error proxying to origin",
 		},
 	)
-	haConnections = prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: connection.MetricsNamespace,
-			Subsystem: connection.TunnelSubsystem,
-			Name:      "ha_connections",
-			Help:      "Number of active ha connections",
-		},
-	)
 )
 
 func init() {
@@ -59,7 +51,6 @@ func init() {
 		concurrentRequests,
 		responseByCode,
 		requestErrors,
-		haConnections,
 	)
 }
 

proxy/proxy.go

@@ -1,4 +1,4 @@
-package origin
+package proxy
 
 import (
 	"bufio"
@@ -10,10 +10,14 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/cloudflare/cloudflared/carrier"
+	"github.com/cloudflare/cloudflared/cfio"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/websocket"
 )
@@ -32,39 +36,45 @@ type Proxy struct {
 	warpRouting  *ingress.WarpRoutingService
 	tags         []tunnelpogs.Tag
 	log          *zerolog.Logger
-	bufferPool   *bufferPool
 }
 
 // NewOriginProxy returns a new instance of the Proxy struct.
 func NewOriginProxy(
 	ingressRules ingress.Ingress,
-	warpRouting *ingress.WarpRoutingService,
+	warpRoutingEnabled bool,
 	tags []tunnelpogs.Tag,
 	log *zerolog.Logger,
 ) *Proxy {
-	return &Proxy{
+	proxy := &Proxy{
 		ingressRules: ingressRules,
-		warpRouting:  warpRouting,
 		tags:         tags,
 		log:          log,
-		bufferPool:   newBufferPool(512 * 1024),
 	}
+	if warpRoutingEnabled {
+		proxy.warpRouting = ingress.NewWarpRoutingService()
+		log.Info().Msgf("Warp-routing is enabled")
+	}
+
+	return proxy
 }
 
 // ProxyHTTP further depends on ingress rules to establish a connection with the origin service. This may be
 // a simple roundtrip or a tcp/websocket dial depending on ingres rule setup.
 func (p *Proxy) ProxyHTTP(
 	w connection.ResponseWriter,
-	req *http.Request,
+	tr *tracing.TracedRequest,
 	isWebsocket bool,
 ) error {
 	incrementRequests()
 	defer decrementConcurrentRequests()
 
+	req := tr.Request
 	cfRay := connection.FindCfRayHeader(req)
 	lbProbe := connection.IsLBProbeRequest(req)
 	p.appendTagHeaders(req)
 
+	_, ruleSpan := tr.Tracer().Start(req.Context(), "ingress_match",
+		trace.WithAttributes(attribute.String("req-host", req.Host)))
 	rule, ruleNum := p.ingressRules.FindMatchingRule(req.Host, req.URL.Path)
 	logFields := logFields{
 		cfRay:   cfRay,
@@ -72,12 +82,14 @@ func (p *Proxy) ProxyHTTP(
 		rule:    ruleNum,
 	}
 	p.logRequest(req, logFields)
+	ruleSpan.SetAttributes(attribute.Int("rule-num", ruleNum))
+	ruleSpan.End()
 
 	switch originProxy := rule.Service.(type) {
 	case ingress.HTTPOriginProxy:
 		if err := p.proxyHTTPRequest(
 			w,
-			req,
+			tr,
 			originProxy,
 			isWebsocket,
 			rule.Config.DisableChunkedEncoding,
@@ -88,7 +100,6 @@ func (p *Proxy) ProxyHTTP(
 			return err
 		}
 		return nil
-
 	case ingress.StreamBasedOriginProxy:
 		dest, err := getDestFromRule(rule, req)
 		if err != nil {
@@ -150,15 +161,15 @@ func ruleField(ing ingress.Ingress, ruleNum int) (ruleID string, srv string) {
 // ProxyHTTPRequest proxies requests of underlying type http and websocket to the origin service.
 func (p *Proxy) proxyHTTPRequest(
 	w connection.ResponseWriter,
-	req *http.Request,
+	tr *tracing.TracedRequest,
 	httpService ingress.HTTPOriginProxy,
 	isWebsocket bool,
 	disableChunkedEncoding bool,
 	fields logFields,
 ) error {
-	roundTripReq := req
+	roundTripReq := tr.Request
 	if isWebsocket {
-		roundTripReq = req.Clone(req.Context())
+		roundTripReq = tr.Clone(tr.Request.Context())
 		roundTripReq.Header.Set("Connection", "Upgrade")
 		roundTripReq.Header.Set("Upgrade", "websocket")
 		roundTripReq.Header.Set("Sec-Websocket-Version", "13")
@@ -168,7 +179,7 @@ func (p *Proxy) proxyHTTPRequest(
 		// Support for WSGI Servers by switching transfer encoding from chunked to gzip/deflate
 		if disableChunkedEncoding {
 			roundTripReq.TransferEncoding = []string{"gzip", "deflate"}
-			cLength, err := strconv.Atoi(req.Header.Get("Content-Length"))
+			cLength, err := strconv.Atoi(tr.Request.Header.Get("Content-Length"))
 			if err == nil {
 				roundTripReq.ContentLength = int64(cLength)
 			}
@@ -182,12 +193,24 @@ func (p *Proxy) proxyHTTPRequest(
 		roundTripReq.Header.Set("User-Agent", "")
 	}
 
+	_, ttfbSpan := tr.Tracer().Start(tr.Context(), "ttfb_origin")
 	resp, err := httpService.RoundTrip(roundTripReq)
 	if err != nil {
+		tracing.EndWithErrorStatus(ttfbSpan, err)
 		return errors.Wrap(err, "Unable to reach the origin service. The service may be down or it may not be responding to traffic from cloudflared")
 	}
+
+	tracing.EndWithStatusCode(ttfbSpan, resp.StatusCode)
 	defer resp.Body.Close()
 
+	// resp headers can be nil
+	if resp.Header == nil {
+		resp.Header = make(http.Header)
+	}
+
+	// Add spans to response header (if available)
+	tr.AddSpans(resp.Header, p.log)
+
 	err = w.WriteRespHeaders(resp.StatusCode, resp.Header)
 	if err != nil {
 		return errors.Wrap(err, "Error writing response header")
@@ -202,7 +225,7 @@ func (p *Proxy) proxyHTTPRequest(
 
 		eyeballStream := &bidirectionalStream{
 			writer: w,
-			reader: req.Body,
+			reader: tr.Request.Body,
 		}
 
 		websocket.Stream(eyeballStream, rwc, p.log)
@@ -213,11 +236,7 @@ func (p *Proxy) proxyHTTPRequest(
 		p.log.Debug().Msg("Detected Server-Side Events from Origin")
 		p.writeEventStream(w, resp.Body)
 	} else {
-		// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
-		// compression generates dictionary on first write
-		buf := p.bufferPool.Get()
-		defer p.bufferPool.Put(buf)
-		_, _ = io.CopyBuffer(w, resp.Body, buf)
+		_, _ = cfio.Copy(w, resp.Body)
 	}
 
 	p.logOriginResponse(resp, fields)

proxy/proxy_posix_test.go

@@ -1,7 +1,7 @@
 //go:build !windows
 // +build !windows
 
-package origin
+package proxy
 
 import (
 	"io/ioutil"

proxy/proxy_test.go

@@ -1,4 +1,4 @@
-package origin
+package proxy
 
 import (
 	"bytes"
@@ -27,12 +27,12 @@ import (
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tracing"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
-	testTags                 = []tunnelpogs.Tag{tunnelpogs.Tag{Name: "Name", Value: "value"}}
-	unusedWarpRoutingService = (*ingress.WarpRoutingService)(nil)
+	testTags = []tunnelpogs.Tag{tunnelpogs.Tag{Name: "Name", Value: "value"}}
 )
 
 type mockHTTPRespWriter struct {
@@ -61,6 +61,11 @@ func (w *mockHTTPRespWriter) Read(data []byte) (int, error) {
 	return 0, fmt.Errorf("mockHTTPRespWriter doesn't implement io.Reader")
 }
 
+// respHeaders is a test function to read respHeaders
+func (w *mockHTTPRespWriter) headers() http.Header {
+	return w.Header()
+}
+
 type mockWSRespWriter struct {
 	*mockHTTPRespWriter
 	writeNotification chan []byte
@@ -131,17 +136,14 @@ func TestProxySingleOrigin(t *testing.T) {
 	ingressRule, err := ingress.NewSingleOrigin(cliCtx, allowURLFromArgs)
 	require.NoError(t, err)
 
-	var wg sync.WaitGroup
-	errC := make(chan error)
-	require.NoError(t, ingressRule.StartOrigins(&wg, &log, ctx.Done(), errC))
+	require.NoError(t, ingressRule.StartOrigins(&log, ctx.Done()))
 
-	proxy := NewOriginProxy(ingressRule, unusedWarpRoutingService, testTags, &log)
+	proxy := NewOriginProxy(ingressRule, false, testTags, &log)
 	t.Run("testProxyHTTP", testProxyHTTP(proxy))
 	t.Run("testProxyWebsocket", testProxyWebsocket(proxy))
 	t.Run("testProxySSE", testProxySSE(proxy))
 	t.Run("testProxySSEAllData", testProxySSEAllData(proxy))
 	cancel()
-	wg.Wait()
 }
 
 func testProxyHTTP(proxy connection.OriginProxy) func(t *testing.T) {
@@ -150,7 +152,7 @@ func testProxyHTTP(proxy connection.OriginProxy) func(t *testing.T) {
 		req, err := http.NewRequest(http.MethodGet, "http://localhost:8080", nil)
 		require.NoError(t, err)
 
-		err = proxy.ProxyHTTP(responseWriter, req, false)
+		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false)
 		require.NoError(t, err)
 		for _, tag := range testTags {
 			assert.Equal(t, tag.Value, req.Header.Get(TagHeaderNamePrefix+tag.Name))
@@ -177,7 +179,7 @@ func testProxyWebsocket(proxy connection.OriginProxy) func(t *testing.T) {
 
 		errGroup, ctx := errgroup.WithContext(ctx)
 		errGroup.Go(func() error {
-			err = proxy.ProxyHTTP(responseWriter, req, true)
+			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), true)
 			require.NoError(t, err)
 
 			require.Equal(t, http.StatusSwitchingProtocols, responseWriter.Code)
@@ -238,7 +240,7 @@ func testProxySSE(proxy connection.OriginProxy) func(t *testing.T) {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			err = proxy.ProxyHTTP(responseWriter, req, false)
+			err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false)
 			require.NoError(t, err)
 
 			require.Equal(t, http.StatusOK, responseWriter.Code)
@@ -341,18 +343,16 @@ func runIngressTestScenarios(t *testing.T, unvalidatedIngress []config.Unvalidat
 	log := zerolog.Nop()
 
 	ctx, cancel := context.WithCancel(context.Background())
-	errC := make(chan error)
-	var wg sync.WaitGroup
-	require.NoError(t, ingress.StartOrigins(&wg, &log, ctx.Done(), errC))
+	require.NoError(t, ingress.StartOrigins(&log, ctx.Done()))
 
-	proxy := NewOriginProxy(ingress, unusedWarpRoutingService, testTags, &log)
+	proxy := NewOriginProxy(ingress, false, testTags, &log)
 
 	for _, test := range tests {
 		responseWriter := newMockHTTPRespWriter()
 		req, err := http.NewRequest(http.MethodGet, test.url, nil)
 		require.NoError(t, err)
 
-		err = proxy.ProxyHTTP(responseWriter, req, false)
+		err = proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false)
 		require.NoError(t, err)
 
 		assert.Equal(t, test.expectedStatus, responseWriter.Code)
@@ -363,7 +363,6 @@ func runIngressTestScenarios(t *testing.T, unvalidatedIngress []config.Unvalidat
 		}
 	}
 	cancel()
-	wg.Wait()
 }
 
 type mockAPI struct{}
@@ -394,13 +393,13 @@ func TestProxyError(t *testing.T) {
 
 	log := zerolog.Nop()
 
-	proxy := NewOriginProxy(ing, unusedWarpRoutingService, testTags, &log)
+	proxy := NewOriginProxy(ing, false, testTags, &log)
 
 	responseWriter := newMockHTTPRespWriter()
 	req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
 	assert.NoError(t, err)
 
-	assert.Error(t, proxy.ProxyHTTP(responseWriter, req, false))
+	assert.Error(t, proxy.ProxyHTTP(responseWriter, tracing.NewTracedRequest(req), false))
 }
 
 type replayer struct {
@@ -561,6 +560,24 @@ func TestConnections(t *testing.T) {
 				},
 			},
 		},
+		{
+			// Send (unexpected) HTTP when origin expects WS (to unwrap for raw TCP)
+			name: "http-(ws)tcp proxy",
+			args: args{
+				ingressServiceScheme:  "tcp://",
+				originService:         runEchoTCPService,
+				eyeballResponseWriter: newMockHTTPRespWriter(),
+				eyeballRequestBody:    http.NoBody,
+				connectionType:        connection.TypeHTTP,
+				requestHeaders: map[string][]string{
+					"Cf-Cloudflared-Proxy-Src": {"non-blank-value"},
+				},
+			},
+			want: want{
+				message: []byte{},
+				headers: map[string][]string{},
+			},
+		},
 		{
 			name: "tcp-tcp proxy without warpRoutingService enabled",
 			args: args{
@@ -634,10 +651,9 @@ func TestConnections(t *testing.T) {
 			test.args.originService(t, ln)
 
 			ingressRule := createSingleIngressConfig(t, test.args.ingressServiceScheme+ln.Addr().String())
-			var wg sync.WaitGroup
-			errC := make(chan error)
-			ingressRule.StartOrigins(&wg, logger, ctx.Done(), errC)
-			proxy := NewOriginProxy(ingressRule, test.args.warpRoutingService, testTags, logger)
+			ingressRule.StartOrigins(logger, ctx.Done())
+			proxy := NewOriginProxy(ingressRule, true, testTags, logger)
+			proxy.warpRouting = test.args.warpRoutingService
 
 			dest := ln.Addr().String()
 			req, err := http.NewRequest(
@@ -658,10 +674,10 @@ func TestConnections(t *testing.T) {
 				}()
 			}
 			if test.args.connectionType == connection.TypeTCP {
-				rws := connection.NewHTTPResponseReadWriterAcker(respWriter, req)
-				err = proxy.ProxyTCP(ctx, rws, &connection.TCPRequest{Dest: dest})
+				rwa := connection.NewHTTPResponseReadWriterAcker(respWriter, req)
+				err = proxy.ProxyTCP(ctx, rwa, &connection.TCPRequest{Dest: dest})
 			} else {
-				err = proxy.ProxyHTTP(respWriter, req, test.args.connectionType == connection.TypeWebsocket)
+				err = proxy.ProxyHTTP(respWriter, tracing.NewTracedRequest(req), test.args.connectionType == connection.TypeWebsocket)
 			}
 
 			cancel()

quic/quic_protocol_test.go

@@ -231,14 +231,18 @@ type mockConfigRPCServer struct {
 	config  []byte
 }
 
-func (s mockConfigRPCServer) UpdateConfiguration(_ context.Context, version int32, config []byte) (*tunnelpogs.UpdateConfigurationResponse, error) {
+func (s mockConfigRPCServer) UpdateConfiguration(_ context.Context, version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
 	if s.version != version {
-		return nil, fmt.Errorf("expect version %d, got %d", s.version, version)
+		return &tunnelpogs.UpdateConfigurationResponse{
+			Err: fmt.Errorf("expect version %d, got %d", s.version, version),
+		}
 	}
 	if !bytes.Equal(s.config, config) {
-		return nil, fmt.Errorf("expect config %v, got %v", s.config, config)
+		return &tunnelpogs.UpdateConfigurationResponse{
+			Err: fmt.Errorf("expect config %v, got %v", s.config, config),
+		}
 	}
-	return &tunnelpogs.UpdateConfigurationResponse{LastAppliedVersion: version}, nil
+	return &tunnelpogs.UpdateConfigurationResponse{LastAppliedVersion: version}
 }
 
 type mockRPCStream struct {

quic/tracing.go

@@ -3,9 +3,9 @@ package quic
 import (
 	"context"
 	"net"
+	"time"
 
 	"github.com/lucas-clemente/quic-go/logging"
-	"github.com/lucas-clemente/quic-go/qlog"
 	"github.com/rs/zerolog"
 )
 
@@ -40,75 +40,116 @@ func NewServerTracer(logger *zerolog.Logger) logging.Tracer {
 	}
 }
 
-func (t *tracer) TracerForConnection(_ context.Context, p logging.Perspective, odcid logging.ConnectionID) logging.ConnectionTracer {
-	connID := logging.ConnectionID(odcid).String()
-	ql := &quicLogger{
-		logger:       t.logger,
-		connectionID: connID,
-	}
+func (t *tracer) TracerForConnection(_ctx context.Context, _p logging.Perspective, _odcid logging.ConnectionID) logging.ConnectionTracer {
 	if t.config.isClient {
-		return newConnTracer(ql, p, odcid, newClientCollector(t.config.index))
+		return newConnTracer(newClientCollector(t.config.index))
 	}
-	return newConnTracer(ql, p, odcid, newServiceCollector())
+	return newConnTracer(newServiceCollector())
 }
 
 func (*tracer) SentPacket(net.Addr, *logging.Header, logging.ByteCount, []logging.Frame) {}
 func (*tracer) DroppedPacket(net.Addr, logging.PacketType, logging.ByteCount, logging.PacketDropReason) {
 }
 
-// connTracer is a wrapper around https://pkg.go.dev/github.com/lucas-clemente/quic-go@v0.23.0/qlog#NewConnectionTracer to collect metrics
+var _ logging.Tracer = (*tracer)(nil)
+
+// connTracer collects connection level metrics
 type connTracer struct {
-	logging.ConnectionTracer
 	metricsCollector MetricsCollector
-	connectionID     string
 }
 
-func newConnTracer(ql *quicLogger, p logging.Perspective, odcid logging.ConnectionID, metricsCollector MetricsCollector) logging.ConnectionTracer {
+var _ logging.ConnectionTracer = (*connTracer)(nil)
+
+func newConnTracer(metricsCollector MetricsCollector) logging.ConnectionTracer {
 	return &connTracer{
-		qlog.NewConnectionTracer(ql, p, odcid),
-		metricsCollector,
-		logging.ConnectionID(odcid).String(),
+		metricsCollector: metricsCollector,
 	}
 }
 
 func (ct *connTracer) StartedConnection(local, remote net.Addr, srcConnID, destConnID logging.ConnectionID) {
 	ct.metricsCollector.startedConnection()
-	ct.ConnectionTracer.StartedConnection(local, remote, srcConnID, destConnID)
 }
 
 func (ct *connTracer) ClosedConnection(err error) {
 	ct.metricsCollector.closedConnection(err)
-	ct.ConnectionTracer.ClosedConnection(err)
 }
 
 func (ct *connTracer) SentPacket(hdr *logging.ExtendedHeader, packetSize logging.ByteCount, ack *logging.AckFrame, frames []logging.Frame) {
 	ct.metricsCollector.sentPackets(packetSize)
-	ct.ConnectionTracer.SentPacket(hdr, packetSize, ack, frames)
 }
 
 func (ct *connTracer) ReceivedPacket(hdr *logging.ExtendedHeader, size logging.ByteCount, frames []logging.Frame) {
 	ct.metricsCollector.receivedPackets(size)
-	ct.ConnectionTracer.ReceivedPacket(hdr, size, frames)
 }
 
 func (ct *connTracer) BufferedPacket(pt logging.PacketType) {
 	ct.metricsCollector.bufferedPackets(pt)
-	ct.ConnectionTracer.BufferedPacket(pt)
 }
 
 func (ct *connTracer) DroppedPacket(pt logging.PacketType, size logging.ByteCount, reason logging.PacketDropReason) {
 	ct.metricsCollector.droppedPackets(pt, size, reason)
-	ct.ConnectionTracer.DroppedPacket(pt, size, reason)
 }
 
 func (ct *connTracer) LostPacket(level logging.EncryptionLevel, number logging.PacketNumber, reason logging.PacketLossReason) {
 	ct.metricsCollector.lostPackets(reason)
-	ct.ConnectionTracer.LostPacket(level, number, reason)
 }
 
 func (ct *connTracer) UpdatedMetrics(rttStats *logging.RTTStats, cwnd, bytesInFlight logging.ByteCount, packetsInFlight int) {
 	ct.metricsCollector.updatedRTT(rttStats)
-	ct.ConnectionTracer.UpdatedMetrics(rttStats, cwnd, bytesInFlight, packetsInFlight)
+}
+
+func (ct *connTracer) NegotiatedVersion(chosen logging.VersionNumber, clientVersions, serverVersions []logging.VersionNumber) {
+}
+
+func (ct *connTracer) SentTransportParameters(parameters *logging.TransportParameters) {
+}
+
+func (ct *connTracer) ReceivedTransportParameters(parameters *logging.TransportParameters) {
+}
+
+func (ct *connTracer) RestoredTransportParameters(parameters *logging.TransportParameters) {
+}
+
+func (ct *connTracer) ReceivedVersionNegotiationPacket(header *logging.Header, numbers []logging.VersionNumber) {
+}
+
+func (ct *connTracer) ReceivedRetry(header *logging.Header) {
+}
+
+func (ct *connTracer) AcknowledgedPacket(level logging.EncryptionLevel, number logging.PacketNumber) {
+}
+
+func (ct *connTracer) UpdatedCongestionState(state logging.CongestionState) {
+}
+
+func (ct *connTracer) UpdatedPTOCount(value uint32) {
+}
+
+func (ct *connTracer) UpdatedKeyFromTLS(level logging.EncryptionLevel, perspective logging.Perspective) {
+}
+
+func (ct *connTracer) UpdatedKey(generation logging.KeyPhase, remote bool) {
+}
+
+func (ct *connTracer) DroppedEncryptionLevel(level logging.EncryptionLevel) {
+}
+
+func (ct *connTracer) DroppedKey(generation logging.KeyPhase) {
+}
+
+func (ct *connTracer) SetLossTimer(timerType logging.TimerType, level logging.EncryptionLevel, time time.Time) {
+}
+
+func (ct *connTracer) LossTimerExpired(timerType logging.TimerType, level logging.EncryptionLevel) {
+}
+
+func (ct *connTracer) LossTimerCanceled() {
+}
+
+func (ct *connTracer) Close() {
+}
+
+func (ct *connTracer) Debug(name, msg string) {
 }
 
 type quicLogger struct {

release_pkgs.py

@@ -0,0 +1,327 @@
+"""
+    This is a utility for creating deb and rpm packages, signing them 
+    and uploading them to a storage and adding metadata to workers KV.
+
+    It has two over-arching responsiblities:
+    1. Create deb and yum repositories from .deb and .rpm files. 
+       This is also responsible for signing the packages and generally preparing 
+       them to be in an uploadable state.
+    2. Upload these packages to a storage in a format that apt and yum expect.
+"""
+from subprocess import Popen, PIPE
+import os
+import argparse
+import base64
+from pathlib import Path
+import logging
+import shutil
+from hashlib import sha256
+
+import gnupg
+import boto3
+from botocore.client import Config
+from botocore.exceptions import ClientError
+
+# The front facing R2 URL to access assets from.
+R2_ASSET_URL = 'https://demo-r2-worker.cloudflare-tunnel.workers.dev/'
+
+class PkgUploader:
+    def __init__(self, account_id, bucket_name, client_id, client_secret):
+        self.account_id = account_id
+        self.bucket_name = bucket_name
+        self.client_id = client_id
+        self.client_secret = client_secret
+
+    def upload_pkg_to_r2(self, filename, upload_file_path):
+        endpoint_url = f"https://{self.account_id}.r2.cloudflarestorage.com"
+        token_secret_hash = sha256(self.client_secret.encode()).hexdigest()
+         
+        config = Config(
+            region_name = 'auto',
+            s3={
+                "addressing_style": "path",
+            }
+        )
+         
+        r2 = boto3.client(
+            "s3",
+            endpoint_url=endpoint_url,
+            aws_access_key_id=self.client_id,
+            aws_secret_access_key=token_secret_hash,
+            config=config,
+        )
+
+        print(f"uploading asset: {filename} to {upload_file_path} in bucket{self.bucket_name}...")
+        try:
+            r2.upload_file(filename, self.bucket_name, upload_file_path)
+        except ClientError as e:
+            raise e
+
+                  
+class PkgCreator:
+    """
+        The distribution conf is what dictates to reprepro, the debian packaging building
+        and signing tool we use, what distros to support, what GPG key to use for signing
+        and what to call the debian binary etc. This function creates it "./conf/distributions".
+
+        origin - name of your package (String)
+        label - label of your package (could be same as the name) (String)
+        release - release you want this to be distributed for (List of Strings)
+        components - could be a channel like main/stable/beta
+        archs - Architecture (List of Strings)
+        description - (String)
+        gpg_key_id - gpg key id of what you want to use to sign the packages.(String) 
+    """
+    def create_distribution_conf(self, 
+            file_path,
+            origin,
+            label,
+            releases,
+            archs,
+            components, 
+            description,
+            gpg_key_id ):
+        with open(file_path, "w+") as distributions_file:
+            for release in releases:
+                distributions_file.write(f"Origin: {origin}\n")
+                distributions_file.write(f"Label: {label}\n")
+                distributions_file.write(f"Codename: {release}\n")
+                archs_list = " ".join(archs)
+                distributions_file.write(f"Architectures: {archs_list}\n")
+                distributions_file.write(f"Components: {components}\n")
+                distributions_file.write(f"Description: {description} - {release}\n")
+                distributions_file.write(f"SignWith: {gpg_key_id}\n")
+                distributions_file.write("\n")
+        return distributions_file
+
+    """
+        Uses the reprepro tool to generate packages, sign them and create the InRelease as specified
+        by the distribution_conf file. 
+
+        This function creates three folders db, pool and dist. 
+        db and pool contain information and metadata about builds. We can ignore these.
+        dist: contains all the pkgs and signed releases that are necessary for an apt download.
+    """
+    def create_deb_pkgs(self, release, deb_file):
+        print(f"creating deb pkgs: {release} : {deb_file}")
+        p = Popen(["reprepro", "includedeb", release, deb_file], stdout=PIPE, stderr=PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"create deb_pkgs result => {out}, {err}")
+            raise
+
+    def create_rpm_pkgs(self, artifacts_path, gpg_key_name):
+        self._setup_rpm_pkg_directories(artifacts_path, gpg_key_name)
+        p = Popen(["createrepo", "./rpm"], stdout=PIPE, stderr=PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"create rpm_pkgs result => {out}, {err}")
+            raise
+
+        self._sign_repomd()
+
+    def _sign_rpms(self, file_path):
+        p = Popen(["rpm" , "--define", f"_gpg_name {gpg_key_name}", "--addsign", file_path], stdout=PIPE, stderr=PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"rpm sign result result => {out}, {err}")
+            raise
+
+    def _sign_repomd(self):
+        p = Popen(["gpg", "--batch", "--detach-sign", "--armor", "./rpm/repodata/repomd.xml"], stdout=PIPE, stderr=PIPE) 
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"sign repomd result => {out}, {err}")
+            raise
+        
+    """
+        sets up and signs the RPM directories in the following format:
+        - rpm 
+           - aarch64
+           - x86_64
+           - 386
+
+        this assumes the assets are in the format <prefix>-<aarch64/x86_64/386>.rpm
+    """
+    def _setup_rpm_pkg_directories(self, artifacts_path, gpg_key_name, archs=["aarch64", "x86_64", "386"]):
+        for arch in archs:
+            for root, _ , files in os.walk(artifacts_path):
+                for file in files:
+                    if file.endswith(f"{arch}.rpm"):
+                        new_dir = f"./rpm/{arch}"
+                        os.makedirs(new_dir, exist_ok=True)
+                        old_path = os.path.join(root, file)
+                        new_path = os.path.join(new_dir, file)
+                        shutil.copyfile(old_path, new_path)
+                        self._sign_rpms(new_path)
+    
+    """
+        imports gpg keys into the system so reprepro and createrepo can use it to sign packages.
+        it returns the GPG ID after a successful import
+    """
+    def import_gpg_keys(self, private_key, public_key):
+        gpg = gnupg.GPG()
+        private_key = base64.b64decode(private_key)
+        gpg.import_keys(private_key)
+        public_key = base64.b64decode(public_key)
+        gpg.import_keys(public_key)
+        data = gpg.list_keys(secret=True)
+        return (data[0]["fingerprint"], data[0]["uids"][0])
+
+    """
+        basically rpm --import <key_file>
+        This enables us to sign rpms.
+    """
+    def import_rpm_key(self, public_key):
+        file_name = "pb.key"
+        with open(file_name, "wb") as f:
+            public_key = base64.b64decode(public_key)
+            f.write(public_key)
+        
+        p = Popen(["rpm", "--import", file_name], stdout=PIPE, stderr=PIPE)
+        out, err = p.communicate()
+        if p.returncode != 0:
+            print(f"create rpm import result => {out}, {err}")
+            raise
+
+
+"""
+    Walks through a directory and uploads it's assets to R2.
+    directory : root directory to walk through (String).
+    release: release string. If this value is none, a specific release path will not be created 
+              and the release will be uploaded to the default path. 
+    binary: name of the binary to upload
+"""
+def upload_from_directories(pkg_uploader, directory, release, binary):
+     for root, _ , files in os.walk(directory):
+        for file in files:
+            upload_file_name = os.path.join(binary, root, file)
+            if release:
+                upload_file_name = os.path.join(release, upload_file_name)
+            filename = os.path.join(root,file)
+            try: 
+                pkg_uploader.upload_pkg_to_r2(filename, upload_file_name)
+            except ClientError as e:
+                logging.error(e)
+                return 
+
+""" 
+    1. looks into a built_artifacts folder for cloudflared debs
+    2. creates Packages.gz, InRelease (signed) files
+    3. uploads them to Cloudflare R2 
+
+    pkg_creator, pkg_uploader: are instantiations of the two classes above.
+
+    gpg_key_id: is an id indicating the key the package should be signed with. The public key of this id will be 
+    uploaded to R2 so it can be presented to apt downloaders.
+
+    release_version: is the cloudflared release version. Only provide this if you want a permanent backup.
+"""
+def create_deb_packaging(pkg_creator, pkg_uploader, releases, gpg_key_id, binary_name, archs, package_component, release_version):
+    # set configuration for package creation.
+    print(f"initialising configuration for {binary_name} , {archs}")
+    Path("./conf").mkdir(parents=True, exist_ok=True)
+    pkg_creator.create_distribution_conf(
+    "./conf/distributions",
+    binary_name,
+    binary_name,
+    releases,
+    archs,
+    package_component,
+    f"apt repository for {binary_name}",
+    gpg_key_id)
+
+    # create deb pkgs
+    for release in releases:
+        for arch in archs:
+            print(f"creating deb pkgs for {release} and {arch}...")
+            pkg_creator.create_deb_pkgs(release, f"./built_artifacts/cloudflared-linux-{arch}.deb")
+
+    print("uploading latest to r2...")
+    upload_from_directories(pkg_uploader, "dists", None, binary_name)
+    upload_from_directories(pkg_uploader, "pool", None, binary_name)
+
+    if release_version:
+        print(f"uploading versioned release {release_version} to r2...")
+        upload_from_directories(pkg_uploader, "dists", release_version, binary_name)
+        upload_from_directories(pkg_uploader, "pool", release_version, binary_name)
+
+def create_rpm_packaging(pkg_creator, pkg_uploader, artifacts_path, release_version, binary_name, gpg_key_name):
+    print(f"creating rpm pkgs...")
+    pkg_creator.create_rpm_pkgs(artifacts_path, gpg_key_name)
+
+    print("uploading latest to r2...")
+    upload_from_directories(pkg_uploader, "rpm", None, binary_name)
+
+    if release_version:
+        print(f"uploading versioned release {release_version} to r2...")
+        upload_from_directories(pkg_uploader, "rpm", release_version, binary_name)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description="Creates linux releases and uploads them in a packaged format"
+    )
+
+    parser.add_argument(
+            "--bucket", default=os.environ.get("R2_BUCKET"), help="R2 Bucket name"
+    )
+    parser.add_argument(
+            "--id", default=os.environ.get("R2_CLIENT_ID"), help="R2 Client ID"
+    )
+    parser.add_argument(
+            "--secret", default=os.environ.get("R2_CLIENT_SECRET"), help="R2 Client Secret"
+    )
+    parser.add_argument(
+            "--account", default=os.environ.get("R2_ACCOUNT_ID"), help="R2 Account Tag"
+    )
+    parser.add_argument(
+            "--release-tag", default=os.environ.get("RELEASE_VERSION"), help="Release version you want your pkgs to be\
+            prefixed with. Leave empty if you don't want tagged release versions backed up to R2."
+    )
+
+    parser.add_argument(
+           "--binary", default=os.environ.get("BINARY_NAME"), help="The name of the binary the packages are for"
+    )
+
+    parser.add_argument(
+            "--gpg-private-key", default=os.environ.get("LINUX_SIGNING_PRIVATE_KEY"), help="GPG private key to sign the\
+            packages"
+    )
+
+    parser.add_argument(
+            "--gpg-public-key", default=os.environ.get("LINUX_SIGNING_PUBLIC_KEY"), help="GPG public key used for\
+            signing packages"
+    )
+
+    parser.add_argument(
+            "--deb-based-releases", default=["bookworm", "bullseye", "buster", "jammy", "impish", "focal", "bionic"],
+            help="list of debian based releases that need to be packaged for"
+    )
+
+    parser.add_argument(
+            "--archs", default=["amd64", "386", "arm64"], help="list of architectures we want to package for. Note that\
+            it is the caller's responsiblity to ensure that these debs are already present in a directory. This script\
+            will not build binaries or create their debs."
+            )
+    args = parser.parse_args()
+
+    return args
+
+if __name__ == "__main__":
+    try:
+        args = parse_args()
+    except Exception as e:
+        logging.exception(e)
+        exit(1)
+
+    pkg_creator = PkgCreator()
+    (gpg_key_id, gpg_key_name) = pkg_creator.import_gpg_keys(args.gpg_private_key, args.gpg_public_key)
+    pkg_creator.import_rpm_key(args.gpg_public_key)
+
+    pkg_uploader = PkgUploader(args.account, args.bucket, args.id, args.secret)
+    print(f"signing with gpg_key: {gpg_key_id}")
+    create_deb_packaging(pkg_creator, pkg_uploader, args.deb_based_releases, gpg_key_id, args.binary, args.archs,
+            "main", args.release_tag)
+    
+    create_rpm_packaging(pkg_creator, pkg_uploader, "./built_artifacts", args.release_tag, args.binary, gpg_key_name)

sshgen/sshgen.go

@@ -15,10 +15,10 @@ import (
 	"net/url"
 	"time"
 
-	"github.com/coreos/go-oidc/jose"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	gossh "golang.org/x/crypto/ssh"
+	"gopkg.in/square/go-jose.v2/jwt"
 
 	"github.com/cloudflare/cloudflared/config"
 	cfpath "github.com/cloudflare/cloudflared/token"
@@ -87,37 +87,33 @@ func SignCert(token, pubKey string) (string, error) {
 		return "", errors.New("invalid token")
 	}
 
-	jwt, err := jose.ParseJWT(token)
+	parsedToken, err := jwt.ParseSigned(token)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to parse JWT")
 	}
 
-	claims, err := jwt.Claims()
+	claims := jwt.Claims{}
+	err = parsedToken.UnsafeClaimsWithoutVerification(&claims)
 	if err != nil {
 		return "", errors.Wrap(err, "failed to retrieve JWT claims")
 	}
 
-	issuer, _, err := claims.StringClaim("iss")
-	if err != nil {
-		return "", errors.Wrap(err, "failed to retrieve JWT iss")
-	}
-
 	buf, err := json.Marshal(&signPayload{
 		PublicKey: pubKey,
 		JWT:       token,
-		Issuer:    issuer,
+		Issuer:    claims.Issuer,
 	})
 	if err != nil {
 		return "", errors.Wrap(err, "failed to marshal signPayload")
 	}
 	var res *http.Response
 	if mockRequest != nil {
-		res, err = mockRequest(issuer+signEndpoint, "application/json", bytes.NewBuffer(buf))
+		res, err = mockRequest(claims.Issuer+signEndpoint, "application/json", bytes.NewBuffer(buf))
 	} else {
 		client := http.Client{
 			Timeout: 10 * time.Second,
 		}
-		res, err = client.Post(issuer+signEndpoint, "application/json", bytes.NewBuffer(buf))
+		res, err = client.Post(claims.Issuer+signEndpoint, "application/json", bytes.NewBuffer(buf))
 	}
 
 	if err != nil {

sshgen/sshgen_test.go

@@ -4,8 +4,6 @@
 package sshgen
 
 import (
-	"crypto/rand"
-	"crypto/rsa"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -18,8 +16,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/coreos/go-oidc/jose"
 	"github.com/stretchr/testify/assert"
+	"gopkg.in/square/go-jose.v2"
+	"gopkg.in/square/go-jose.v2/jwt"
 
 	"github.com/cloudflare/cloudflared/config"
 	cfpath "github.com/cloudflare/cloudflared/token"
@@ -97,22 +96,25 @@ func TestCertGenSuccess(t *testing.T) {
 }
 
 func tokenGenerator() string {
-	iat := time.Now().Unix()
-	exp := time.Now().Add(time.Minute * 5).Unix()
-	claims := jose.Claims{}
-	claims.Add("aud", audTest)
-	claims.Add("iat", iat)
-	claims.Add("nonce", nonceTest)
-	claims.Add("exp", exp)
+	iat := time.Now()
+	exp := time.Now().Add(time.Minute * 5)
 
-	k, err := rsa.GenerateKey(rand.Reader, 512)
+	claims := jwt.Claims{
+		Audience: jwt.Audience{audTest},
+		IssuedAt: jwt.NewNumericDate(iat),
+		Expiry:   jwt.NewNumericDate(exp),
+	}
+
+	key := []byte("secret")
+	signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.HS256, Key: key}, (&jose.SignerOptions{}).WithType("JWT"))
 	if err != nil {
-		return ""
+		panic(err)
 	}
-	signer := jose.NewSignerRSA("asdf", *k)
-	token, terr := jose.NewSignedJWT(claims, signer)
-	if terr != nil {
-		return ""
+
+	signedToken, err := jwt.Signed(signer).Claims(claims).CompactSerialize()
+	if err != nil {
+		panic(err)
 	}
-	return token.Encode()
+
+	return signedToken
 }

supervisor/cloudflare_status_page.go

@@ -1,4 +1,4 @@
-package origin
+package supervisor
 
 import (
 	"encoding/json"

supervisor/cloudflare_status_page_test.go

@@ -1,4 +1,4 @@
-package origin
+package supervisor
 
 import (
 	"testing"

supervisor/conn_aware_logger.go

@@ -1,4 +1,4 @@
-package origin
+package supervisor
 
 import (
 	"github.com/rs/zerolog"

supervisor/external_control.go

@@ -1,4 +1,4 @@
-package origin
+package supervisor
 
 import (
 	"time"

supervisor/metrics.go

@@ -0,0 +1,27 @@
+package supervisor
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/cloudflare/cloudflared/connection"
+)
+
+// Metrics uses connection.MetricsNamespace(aka cloudflared) as namespace and connection.TunnelSubsystem
+// (tunnel) as subsystem to keep them consistent with the previous qualifier.
+
+var (
+	haConnections = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
+			Name:      "ha_connections",
+			Help:      "Number of active ha connections",
+		},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(
+		haConnections,
+	)
+}

supervisor/reconnect.go

@@ -1,4 +1,4 @@
-package origin
+package supervisor
 
 import (
 	"context"

supervisor/reconnect_test.go

@@ -1,4 +1,4 @@
-package origin
+package supervisor
 
 import (
 	"context"

supervisor/supervisor.go

@@ -1,4 +1,4 @@
-package origin
+package supervisor
 
 import (
 	"context"
@@ -7,12 +7,14 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/lucas-clemente/quic-go"
 	"github.com/rs/zerolog"
 
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/orchestration"
 	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -38,6 +40,7 @@ const (
 type Supervisor struct {
 	cloudflaredUUID   uuid.UUID
 	config            *TunnelConfig
+	orchestrator      *orchestration.Orchestrator
 	edgeIPs           *edgediscovery.Edge
 	tunnelErrors      chan tunnelError
 	tunnelsConnecting map[int]chan struct{}
@@ -64,7 +67,7 @@ type tunnelError struct {
 	err   error
 }
 
-func NewSupervisor(config *TunnelConfig, reconnectCh chan ReconnectSignal, gracefulShutdownC <-chan struct{}) (*Supervisor, error) {
+func NewSupervisor(config *TunnelConfig, orchestrator *orchestration.Orchestrator, reconnectCh chan ReconnectSignal, gracefulShutdownC <-chan struct{}) (*Supervisor, error) {
 	cloudflaredUUID, err := uuid.NewRandom()
 	if err != nil {
 		return nil, fmt.Errorf("failed to generate cloudflared instance ID: %w", err)
@@ -88,6 +91,7 @@ func NewSupervisor(config *TunnelConfig, reconnectCh chan ReconnectSignal, grace
 	return &Supervisor{
 		cloudflaredUUID:            cloudflaredUUID,
 		config:                     config,
+		orchestrator:               orchestrator,
 		edgeIPs:                    edgeIPs,
 		tunnelErrors:               make(chan tunnelError),
 		tunnelsConnecting:          map[int]chan struct{}{},
@@ -243,6 +247,7 @@ func (s *Supervisor) startFirstTunnel(
 		ctx,
 		s.reconnectCredentialManager,
 		s.config,
+		s.orchestrator,
 		addr,
 		s.log,
 		firstConnIndex,
@@ -260,9 +265,9 @@ func (s *Supervisor) startFirstTunnel(
 		switch err.(type) {
 		case nil:
 			return
-		// try the next address if it was a dialError(network problem) or
+		// try the next address if it was a quic.IdleTimeoutError, dialError(network problem) or
 		// dupConnRegisterTunnelError
-		case edgediscovery.DialError, connection.DupConnRegisterTunnelError:
+		case *quic.IdleTimeoutError, edgediscovery.DialError, connection.DupConnRegisterTunnelError:
 			edgeErrors++
 		default:
 			return
@@ -277,6 +282,7 @@ func (s *Supervisor) startFirstTunnel(
 			ctx,
 			s.reconnectCredentialManager,
 			s.config,
+			s.orchestrator,
 			addr,
 			s.log,
 			firstConnIndex,
@@ -311,6 +317,7 @@ func (s *Supervisor) startTunnel(
 		ctx,
 		s.reconnectCredentialManager,
 		s.config,
+		s.orchestrator,
 		addr,
 		s.log,
 		uint8(index),
@@ -380,7 +387,7 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 	defer rpcClient.Close()
 
 	const arbitraryConnectionID = uint8(0)
-	registrationOptions := s.config.RegistrationOptions(arbitraryConnectionID, edgeConn.LocalAddr().String(), s.cloudflaredUUID)
+	registrationOptions := s.config.registrationOptions(arbitraryConnectionID, edgeConn.LocalAddr().String(), s.cloudflaredUUID)
 	registrationOptions.NumPreviousAttempts = uint8(numPreviousAttempts)
 	return rpcClient.Authenticate(ctx, s.config.ClassicTunnel, registrationOptions)
 }

supervisor/tunnel.go

@@ -1,4 +1,4 @@
-package origin
+package supervisor
 
 import (
 	"context"
@@ -20,6 +20,7 @@ import (
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/orchestration"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	"github.com/cloudflare/cloudflared/retry"
 	"github.com/cloudflare/cloudflared/signal"
@@ -31,35 +32,37 @@ const (
 	dialTimeout              = 15 * time.Second
 	FeatureSerializedHeaders = "serialized_headers"
 	FeatureQuickReconnects   = "quick_reconnects"
+	FeatureAllowRemoteConfig = "allow_remote_config"
 )
 
 type TunnelConfig struct {
-	ConnectionConfig *connection.Config
-	OSArch           string
-	ClientID         string
-	CloseConnOnce    *sync.Once // Used to close connectedSignal no more than once
-	EdgeAddrs        []string
-	Region           string
-	HAConnections    int
-	IncidentLookup   IncidentLookup
-	IsAutoupdated    bool
-	LBPool           string
-	Tags             []tunnelpogs.Tag
-	Log              *zerolog.Logger
-	LogTransport     *zerolog.Logger
-	Observer         *connection.Observer
-	ReportedVersion  string
-	Retries          uint
-	RunFromTerminal  bool
+	GracePeriod     time.Duration
+	ReplaceExisting bool
+	OSArch          string
+	ClientID        string
+	CloseConnOnce   *sync.Once // Used to close connectedSignal no more than once
+	EdgeAddrs       []string
+	Region          string
+	HAConnections   int
+	IncidentLookup  IncidentLookup
+	IsAutoupdated   bool
+	LBPool          string
+	Tags            []tunnelpogs.Tag
+	Log             *zerolog.Logger
+	LogTransport    *zerolog.Logger
+	Observer        *connection.Observer
+	ReportedVersion string
+	Retries         uint
+	RunFromTerminal bool
 
-	NamedTunnel      *connection.NamedTunnelConfig
-	ClassicTunnel    *connection.ClassicTunnelConfig
+	NamedTunnel      *connection.NamedTunnelProperties
+	ClassicTunnel    *connection.ClassicTunnelProperties
 	MuxerConfig      *connection.MuxerConfig
 	ProtocolSelector connection.ProtocolSelector
 	EdgeTLSConfigs   map[connection.Protocol]*tls.Config
 }
 
-func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
+func (c *TunnelConfig) registrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
 	policy := tunnelrpc.ExistingTunnelPolicy_balance
 	if c.HAConnections <= 1 && c.LBPool == "" {
 		policy = tunnelrpc.ExistingTunnelPolicy_disconnect
@@ -81,7 +84,7 @@ func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP str
 	}
 }
 
-func (c *TunnelConfig) ConnectionOptions(originLocalAddr string, numPreviousAttempts uint8) *tunnelpogs.ConnectionOptions {
+func (c *TunnelConfig) connectionOptions(originLocalAddr string, numPreviousAttempts uint8) *tunnelpogs.ConnectionOptions {
 	// attempt to parse out origin IP, but don't fail since it's informational field
 	host, _, _ := net.SplitHostPort(originLocalAddr)
 	originIP := net.ParseIP(host)
@@ -89,7 +92,7 @@ func (c *TunnelConfig) ConnectionOptions(originLocalAddr string, numPreviousAtte
 	return &tunnelpogs.ConnectionOptions{
 		Client:              c.NamedTunnel.Client,
 		OriginLocalIP:       originIP,
-		ReplaceExisting:     c.ConnectionConfig.ReplaceExisting,
+		ReplaceExisting:     c.ReplaceExisting,
 		CompressionQuality:  uint8(c.MuxerConfig.CompressionSetting),
 		NumPreviousAttempts: numPreviousAttempts,
 	}
@@ -106,11 +109,12 @@ func (c *TunnelConfig) SupportedFeatures() []string {
 func StartTunnelDaemon(
 	ctx context.Context,
 	config *TunnelConfig,
+	orchestrator *orchestration.Orchestrator,
 	connectedSignal *signal.Signal,
 	reconnectCh chan ReconnectSignal,
 	graceShutdownC <-chan struct{},
 ) error {
-	s, err := NewSupervisor(config, reconnectCh, graceShutdownC)
+	s, err := NewSupervisor(config, orchestrator, reconnectCh, graceShutdownC)
 	if err != nil {
 		return err
 	}
@@ -121,6 +125,7 @@ func ServeTunnelLoop(
 	ctx context.Context,
 	credentialManager *reconnectCredentialManager,
 	config *TunnelConfig,
+	orchestrator *orchestration.Orchestrator,
 	addr *allregions.EdgeAddr,
 	connAwareLogger *ConnAwareLogger,
 	connIndex uint8,
@@ -156,6 +161,7 @@ func ServeTunnelLoop(
 			connLog,
 			credentialManager,
 			config,
+			orchestrator,
 			addr,
 			connIndex,
 			connectedFuse,
@@ -165,17 +171,15 @@ func ServeTunnelLoop(
 			protocolFallback.protocol,
 			gracefulShutdownC,
 		)
-		if !recoverable {
-			return err
-		}
 
-		config.Observer.SendReconnect(connIndex)
-
-		duration, ok := protocolFallback.GetMaxBackoffDuration(ctx)
-		if !ok {
-			return err
+		if recoverable {
+			duration, ok := protocolFallback.GetMaxBackoffDuration(ctx)
+			if !ok {
+				return err
+			}
+			config.Observer.SendReconnect(connIndex)
+			connLog.Logger().Info().Msgf("Retrying connection in up to %s seconds", duration)
 		}
-		connLog.Logger().Info().Msgf("Retrying connection in up to %s seconds", duration)
 
 		select {
 		case <-ctx.Done():
@@ -183,6 +187,10 @@ func ServeTunnelLoop(
 		case <-gracefulShutdownC:
 			return nil
 		case <-protocolFallback.BackoffTimer():
+			if !recoverable {
+				return err
+			}
+
 			if !selectNextProtocol(
 				connLog.Logger(),
 				protocolFallback,
@@ -227,6 +235,15 @@ func selectNextProtocol(
 	_, hasFallback := selector.Fallback()
 
 	if protocolBackoff.ReachedMaxRetries() || (hasFallback && isNetworkActivityTimeout) {
+		if isNetworkActivityTimeout {
+			connLog.Warn().Msg("If this log occurs persistently, and cloudflared is unable to connect to " +
+				"Cloudflare Network with `quic` protocol, then most likely your machine/network is getting its egress " +
+				"UDP to port 7844 (or others) blocked or dropped. Make sure to allow egress connectivity as per " +
+				"https://developers.cloudflare.com/cloudflare-one/connections/connect-apps/configuration/ports-and-ips/\n" +
+				"If you are using private routing to this Tunnel, then UDP (and Private DNS Resolution) will not work" +
+				"unless your cloudflared can connect with Cloudflare Network with `quic`.")
+		}
+
 		fallback, hasFallback := selector.Fallback()
 		if !hasFallback {
 			return false
@@ -254,6 +271,7 @@ func ServeTunnel(
 	connLog *ConnAwareLogger,
 	credentialManager *reconnectCredentialManager,
 	config *TunnelConfig,
+	orchestrator *orchestration.Orchestrator,
 	addr *allregions.EdgeAddr,
 	connIndex uint8,
 	fuse *h2mux.BooleanFuse,
@@ -282,6 +300,7 @@ func ServeTunnel(
 		connLog,
 		credentialManager,
 		config,
+		orchestrator,
 		addr,
 		connIndex,
 		fuse,
@@ -330,6 +349,7 @@ func serveTunnel(
 	connLog *ConnAwareLogger,
 	credentialManager *reconnectCredentialManager,
 	config *TunnelConfig,
+	orchestrator *orchestration.Orchestrator,
 	addr *allregions.EdgeAddr,
 	connIndex uint8,
 	fuse *h2mux.BooleanFuse,
@@ -339,7 +359,6 @@ func serveTunnel(
 	protocol connection.Protocol,
 	gracefulShutdownC <-chan struct{},
 ) (err error, recoverable bool) {
-
 	connectedFuse := &connectedFuse{
 		fuse:    fuse,
 		backoff: backoff,
@@ -351,15 +370,16 @@ func serveTunnel(
 		connIndex,
 		nil,
 		gracefulShutdownC,
-		config.ConnectionConfig.GracePeriod,
+		config.GracePeriod,
 	)
 
 	switch protocol {
 	case connection.QUIC, connection.QUICWarp:
-		connOptions := config.ConnectionOptions(addr.UDP.String(), uint8(backoff.Retries()))
+		connOptions := config.connectionOptions(addr.UDP.String(), uint8(backoff.Retries()))
 		return ServeQUIC(ctx,
 			addr.UDP,
 			config,
+			orchestrator,
 			connLog,
 			connOptions,
 			controlStream,
@@ -374,11 +394,12 @@ func serveTunnel(
 			return err, true
 		}
 
-		connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.Retries()))
+		connOptions := config.connectionOptions(edgeConn.LocalAddr().String(), uint8(backoff.Retries()))
 		if err := ServeHTTP2(
 			ctx,
 			connLog,
 			config,
+			orchestrator,
 			edgeConn,
 			connOptions,
 			controlStream,
@@ -401,6 +422,7 @@ func serveTunnel(
 			connLog,
 			credentialManager,
 			config,
+			orchestrator,
 			edgeConn,
 			connIndex,
 			connectedFuse,
@@ -427,6 +449,7 @@ func ServeH2mux(
 	connLog *ConnAwareLogger,
 	credentialManager *reconnectCredentialManager,
 	config *TunnelConfig,
+	orchestrator *orchestration.Orchestrator,
 	edgeConn net.Conn,
 	connIndex uint8,
 	connectedFuse *connectedFuse,
@@ -437,7 +460,8 @@ func ServeH2mux(
 	connLog.Logger().Debug().Msgf("Connecting via h2mux")
 	// Returns error from parsing the origin URL or handshake errors
 	handler, err, recoverable := connection.NewH2muxConnection(
-		config.ConnectionConfig,
+		orchestrator,
+		config.GracePeriod,
 		config.MuxerConfig,
 		edgeConn,
 		connIndex,
@@ -455,10 +479,10 @@ func ServeH2mux(
 
 	errGroup.Go(func() error {
 		if config.NamedTunnel != nil {
-			connOptions := config.ConnectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.Retries()))
+			connOptions := config.connectionOptions(edgeConn.LocalAddr().String(), uint8(connectedFuse.backoff.Retries()))
 			return handler.ServeNamedTunnel(serveCtx, config.NamedTunnel, connOptions, connectedFuse)
 		}
-		registrationOptions := config.RegistrationOptions(connIndex, edgeConn.LocalAddr().String(), cloudflaredUUID)
+		registrationOptions := config.registrationOptions(connIndex, edgeConn.LocalAddr().String(), cloudflaredUUID)
 		return handler.ServeClassicTunnel(serveCtx, config.ClassicTunnel, credentialManager, registrationOptions, connectedFuse)
 	})
 
@@ -473,6 +497,7 @@ func ServeHTTP2(
 	ctx context.Context,
 	connLog *ConnAwareLogger,
 	config *TunnelConfig,
+	orchestrator *orchestration.Orchestrator,
 	tlsServerConn net.Conn,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler connection.ControlStreamHandler,
@@ -483,7 +508,7 @@ func ServeHTTP2(
 	connLog.Logger().Debug().Msgf("Connecting via http2")
 	h2conn := connection.NewHTTP2Connection(
 		tlsServerConn,
-		config.ConnectionConfig,
+		orchestrator,
 		connOptions,
 		config.Observer,
 		connIndex,
@@ -512,6 +537,7 @@ func ServeQUIC(
 	ctx context.Context,
 	edgeAddr *net.UDPAddr,
 	config *TunnelConfig,
+	orchestrator *orchestration.Orchestrator,
 	connLogger *ConnAwareLogger,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler connection.ControlStreamHandler,
@@ -535,7 +561,7 @@ func ServeQUIC(
 		quicConfig,
 		edgeAddr,
 		tlsConfig,
-		config.ConnectionConfig.OriginProxy,
+		orchestrator,
 		connOptions,
 		controlStreamHandler,
 		connLogger.Logger())

supervisor/tunnel_test.go

@@ -1,4 +1,4 @@
-package origin
+package supervisor
 
 import (
 	"testing"
@@ -32,11 +32,7 @@ func TestWaitForBackoffFallback(t *testing.T) {
 	}
 	log := zerolog.Nop()
 	resolveTTL := time.Duration(0)
-	namedTunnel := &connection.NamedTunnelConfig{
-		Credentials: connection.Credentials{
-			AccountTag: "test-account",
-		},
-	}
+	namedTunnel := &connection.NamedTunnelProperties{}
 	mockFetcher := dynamicMockFetcher{
 		protocolPercents: edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}},
 	}

supervisor/tunnelsforha.go

@@ -1,4 +1,4 @@
-package origin
+package supervisor
 
 import (
 	"fmt"

token/token.go

@@ -13,9 +13,9 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/coreos/go-oidc/jose"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"gopkg.in/square/go-jose.v2"
 
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/retry"
@@ -342,7 +342,7 @@ func GetOrgTokenIfExists(authDomain string) (string, error) {
 		return "", err
 	}
 	var payload jwtPayload
-	err = json.Unmarshal(token.Payload, &payload)
+	err = json.Unmarshal(token.UnsafePayloadWithoutVerification(), &payload)
 	if err != nil {
 		return "", err
 	}
@@ -351,7 +351,7 @@ func GetOrgTokenIfExists(authDomain string) (string, error) {
 		err := os.Remove(path)
 		return "", err
 	}
-	return token.Encode(), nil
+	return token.CompactSerialize()
 }
 
 func GetAppTokenIfExists(appInfo *AppInfo) (string, error) {
@@ -364,7 +364,7 @@ func GetAppTokenIfExists(appInfo *AppInfo) (string, error) {
 		return "", err
 	}
 	var payload jwtPayload
-	err = json.Unmarshal(token.Payload, &payload)
+	err = json.Unmarshal(token.UnsafePayloadWithoutVerification(), &payload)
 	if err != nil {
 		return "", err
 	}
@@ -373,22 +373,21 @@ func GetAppTokenIfExists(appInfo *AppInfo) (string, error) {
 		err := os.Remove(path)
 		return "", err
 	}
-	return token.Encode(), nil
+	return token.CompactSerialize()
 
 }
 
 // GetTokenIfExists will return the token from local storage if it exists and not expired
-func getTokenIfExists(path string) (*jose.JWT, error) {
+func getTokenIfExists(path string) (*jose.JSONWebSignature, error) {
 	content, err := ioutil.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
-	token, err := jose.ParseJWT(string(content))
+	token, err := jose.ParseSigned(string(content))
 	if err != nil {
 		return nil, err
 	}
-
-	return &token, nil
+	return token, nil
 }
 
 // RemoveTokenIfExists removes the a token from local storage if it exists

tracing/client.go

@@ -0,0 +1,91 @@
+package tracing
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"sync"
+
+	coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
+	tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
+	"google.golang.org/protobuf/proto"
+)
+
+const (
+	MaxTraceAmount = 20
+)
+
+var (
+	errNoTraces   = errors.New("no traces recorded to be exported")
+	errNoopTracer = errors.New("noop tracer has no traces")
+)
+
+type InMemoryClient interface {
+	// Spans returns a copy of the list of in-memory stored spans as a base64
+	// encoded otlp protobuf string.
+	Spans() (string, error)
+}
+
+// InMemoryOtlpClient is a client implementation for otlptrace.Client
+type InMemoryOtlpClient struct {
+	mu    sync.Mutex
+	spans []*tracepb.ResourceSpans
+}
+
+func (mc *InMemoryOtlpClient) Start(_ context.Context) error {
+	return nil
+}
+
+func (mc *InMemoryOtlpClient) Stop(_ context.Context) error {
+	return nil
+}
+
+// UploadTraces adds the provided list of spans to the in-memory list.
+func (mc *InMemoryOtlpClient) UploadTraces(_ context.Context, protoSpans []*tracepb.ResourceSpans) error {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+	// Catch to make sure too many traces aren't being added to response header.
+	// Returning nil makes sure we don't fail to send the traces we already recorded.
+	if len(mc.spans)+len(protoSpans) > MaxTraceAmount {
+		return nil
+	}
+	mc.spans = append(mc.spans, protoSpans...)
+	return nil
+}
+
+// Spans returns the list of in-memory stored spans as a base64 encoded otlp protobuf string.
+func (mc *InMemoryOtlpClient) Spans() (string, error) {
+	mc.mu.Lock()
+	defer mc.mu.Unlock()
+	if len(mc.spans) <= 0 {
+		return "", errNoTraces
+	}
+	pbRequest := &coltracepb.ExportTraceServiceRequest{
+		ResourceSpans: mc.spans,
+	}
+	data, err := proto.Marshal(pbRequest)
+	if err != nil {
+		return "", err
+	}
+	return base64.StdEncoding.EncodeToString(data), nil
+}
+
+// NoopOtlpClient is a client implementation for otlptrace.Client that does nothing
+type NoopOtlpClient struct{}
+
+func (mc *NoopOtlpClient) Start(_ context.Context) error {
+	return nil
+}
+
+func (mc *NoopOtlpClient) Stop(_ context.Context) error {
+	return nil
+}
+
+func (mc *NoopOtlpClient) UploadTraces(_ context.Context, _ []*tracepb.ResourceSpans) error {
+	return nil
+}
+
+// Spans always returns no traces error
+func (mc *NoopOtlpClient) Spans() (string, error) {
+	return "", errNoopTracer
+}

tracing/client_test.go

@@ -0,0 +1,161 @@
+package tracing
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
+	"go.opentelemetry.io/otel/trace"
+	commonpb "go.opentelemetry.io/proto/otlp/common/v1"
+	resourcepb "go.opentelemetry.io/proto/otlp/resource/v1"
+	tracepb "go.opentelemetry.io/proto/otlp/trace/v1"
+)
+
+const (
+	resourceSchemaUrl   = "http://example.com/custom-resource-schema"
+	instrumentSchemaUrl = semconv.SchemaURL
+)
+
+var (
+	traceId      = []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F}
+	spanId       = []byte{0xFF, 0xFE, 0xFD, 0xFC, 0xFB, 0xFA, 0xF9, 0xF8}
+	parentSpanId = []byte{0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08}
+	startTime    = time.Date(2022, 4, 4, 0, 0, 0, 0, time.UTC)
+	endTime      = startTime.Add(5 * time.Second)
+
+	traceState, _ = trace.ParseTraceState("key1=val1,key2=val2")
+	instrScope    = &commonpb.InstrumentationScope{Name: "go.opentelemetry.io/test/otel", Version: "v1.6.0"}
+	otlpKeyValues = []*commonpb.KeyValue{
+		{
+			Key: "string_key",
+			Value: &commonpb.AnyValue{
+				Value: &commonpb.AnyValue_StringValue{
+					StringValue: "string value",
+				},
+			},
+		},
+		{
+			Key: "bool_key",
+			Value: &commonpb.AnyValue{
+				Value: &commonpb.AnyValue_BoolValue{
+					BoolValue: true,
+				},
+			},
+		},
+	}
+	otlpResource = &resourcepb.Resource{
+		Attributes: []*commonpb.KeyValue{
+			{
+				Key: "service.name",
+				Value: &commonpb.AnyValue{
+					Value: &commonpb.AnyValue_StringValue{
+						StringValue: "service-name",
+					},
+				},
+			},
+		},
+	}
+)
+
+var _ otlptrace.Client = (*InMemoryOtlpClient)(nil)
+var _ InMemoryClient = (*InMemoryOtlpClient)(nil)
+var _ otlptrace.Client = (*NoopOtlpClient)(nil)
+var _ InMemoryClient = (*NoopOtlpClient)(nil)
+
+func TestUploadTraces(t *testing.T) {
+	client := &InMemoryOtlpClient{}
+	spans := createResourceSpans([]*tracepb.Span{createOtlpSpan(traceId)})
+	spans2 := createResourceSpans([]*tracepb.Span{createOtlpSpan(traceId)})
+	err := client.UploadTraces(context.Background(), spans)
+	assert.NoError(t, err)
+	err = client.UploadTraces(context.Background(), spans2)
+	assert.NoError(t, err)
+	assert.Len(t, client.spans, 2)
+}
+
+func TestSpans(t *testing.T) {
+	client := &InMemoryOtlpClient{}
+	spans := createResourceSpans([]*tracepb.Span{createOtlpSpan(traceId)})
+	err := client.UploadTraces(context.Background(), spans)
+	assert.NoError(t, err)
+	assert.Len(t, client.spans, 1)
+	enc, err := client.Spans()
+	assert.NoError(t, err)
+	expected := "CsECCiAKHgoMc2VydmljZS5uYW1lEg4KDHNlcnZpY2UtbmFtZRLxAQonCh1nby5vcGVudGVsZW1ldHJ5LmlvL3Rlc3Qvb3RlbBIGdjEuNi4wEp0BChAAAQIDBAUGBwgJCgsMDQ4PEgj//v38+/r5+BoTa2V5MT12YWwxLGtleTI9dmFsMiIIDw4NDAsKCQgqCnRyYWNlX25hbWUwATkAANJvaYjiFkEA8teZaojiFkocCgpzdHJpbmdfa2V5Eg4KDHN0cmluZyB2YWx1ZUoOCghib29sX2tleRICEAF6EhIOc3RhdHVzIG1lc3NhZ2UYARomaHR0cHM6Ly9vcGVudGVsZW1ldHJ5LmlvL3NjaGVtYXMvMS43LjAaKWh0dHA6Ly9leGFtcGxlLmNvbS9jdXN0b20tcmVzb3VyY2Utc2NoZW1h"
+	assert.Equal(t, expected, enc)
+}
+
+func TestSpansEmpty(t *testing.T) {
+	client := &InMemoryOtlpClient{}
+	err := client.UploadTraces(context.Background(), []*tracepb.ResourceSpans{})
+	assert.NoError(t, err)
+	assert.Len(t, client.spans, 0)
+	_, err = client.Spans()
+	assert.ErrorIs(t, err, errNoTraces)
+}
+
+func TestSpansNil(t *testing.T) {
+	client := &InMemoryOtlpClient{}
+	err := client.UploadTraces(context.Background(), nil)
+	assert.NoError(t, err)
+	assert.Len(t, client.spans, 0)
+	_, err = client.Spans()
+	assert.ErrorIs(t, err, errNoTraces)
+}
+
+func TestSpansTooManySpans(t *testing.T) {
+	client := &InMemoryOtlpClient{}
+	for i := 0; i < MaxTraceAmount+1; i++ {
+		spans := createResourceSpans([]*tracepb.Span{createOtlpSpan(traceId)})
+		err := client.UploadTraces(context.Background(), spans)
+		assert.NoError(t, err)
+	}
+	assert.Len(t, client.spans, MaxTraceAmount)
+	_, err := client.Spans()
+	assert.NoError(t, err)
+}
+
+func createResourceSpans(spans []*tracepb.Span) []*tracepb.ResourceSpans {
+	return []*tracepb.ResourceSpans{createResourceSpan(spans)}
+}
+
+func createResourceSpan(spans []*tracepb.Span) *tracepb.ResourceSpans {
+	return &tracepb.ResourceSpans{
+		Resource: otlpResource,
+		ScopeSpans: []*tracepb.ScopeSpans{
+			{
+				Scope:     instrScope,
+				Spans:     spans,
+				SchemaUrl: instrumentSchemaUrl,
+			},
+		},
+		InstrumentationLibrarySpans: nil,
+		SchemaUrl:                   resourceSchemaUrl,
+	}
+}
+
+func createOtlpSpan(tid []byte) *tracepb.Span {
+	return &tracepb.Span{
+		TraceId:                tid,
+		SpanId:                 spanId,
+		TraceState:             traceState.String(),
+		ParentSpanId:           parentSpanId,
+		Name:                   "trace_name",
+		Kind:                   tracepb.Span_SPAN_KIND_INTERNAL,
+		StartTimeUnixNano:      uint64(startTime.UnixNano()),
+		EndTimeUnixNano:        uint64(endTime.UnixNano()),
+		Attributes:             otlpKeyValues,
+		DroppedAttributesCount: 0,
+		Events:                 nil,
+		DroppedEventsCount:     0,
+		Links:                  nil,
+		DroppedLinksCount:      0,
+		Status: &tracepb.Status{
+			Message: "status message",
+			Code:    tracepb.Status_STATUS_CODE_OK,
+		},
+	}
+}