Release 2022.2.2

Ingress validate currently validates config from a file. This PR adds a
new --json/-j flag to provide the ingress/config data as a plaintext
command line argument.

.github/ISSUE_TEMPLATE/bug-report---.md

@@ -0,0 +1,31 @@
+---
+name: "Bug report \U0001F41B"
+about: Create a report to help us improve cloudflared
+title: ''
+labels: awaiting reply, bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Configure '...'
+2. Run '....'
+3. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Environment and versions**
+ - OS: [e.g. MacOS]
+ - Architecture: [e.g. AMD, ARM]
+ - Version: [e.g. 2022.02.0]
+
+**Logs and errors**
+If applicable, add logs or errors to help explain your problem.
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature-request---.md

@@ -0,0 +1,17 @@
+---
+name: "Feature request \U0001F4A1"
+about: Suggest a feature or enhancement for cloudflared
+title: ''
+labels: awaiting reply, feature-request
+assignees: ''
+
+---
+
+**Describe the feature you'd like**
+A clear and concise description of the feature. What problem does it solve for you?
+
+**Describe alternatives you've considered**
+Are there any alternatives to solving this problem? If so, what was your experience with them?
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

CHANGES.md

@@ -1,5 +1,40 @@
 **Experimental**: This is a new format for release notes. The format and availability is subject to change.
 
+## 2021.12.2
+### Bug Fixes
+- Fix logging when `quic` transport is used and UDP traffic is proxied.
+- FIPS compliant cloudflared binaries will now be released as separate artifacts. Recall that these are only for linux
+and amd64.
+
+## 2021.12.1
+### Bug Fixes
+ - Fixes Github issue #530 where cloudflared 2021.12.0 could not reach origins that were HTTPS and using certain encryption
+methods forbidden by FIPS compliance (such as Let's Encrypt certificates). To address this fix we have temporarily reverted
+FIPS compliance from amd64 linux binaries that was recently introduced (or fixed actually as it was never working before).
+
+## 2021.12.0
+### New Features
+- Cloudflared binary released for amd64 linux is now FIPS compliant.
+
+### Improvements
+- Logging about connectivity to Cloudflare edge now only yields `ERR` level logging if there are no connections to
+Cloudflare edge that are active. Otherwise it logs `WARN` level.
+ 
+### Bug Fixes
+- Fixes Github issue #501.
+
+## 2021.11.0
+### Improvements
+- Fallback from `protocol:quic` to `protocol:http2` immediately if UDP connectivity isn't available. This could be because of a firewall or 
+egress rule.
+
+## 2021.10.4
+### Improvements
+- Collect quic transport metrics on RTT, packets and bytes transferred.
+
+### Bug Fixes
+- Fix race condition that was writing to the connection after the http2 handler returns.
+
 ## 2021.9.2
 
 ### New features

LICENSE

@@ -1,155 +1,211 @@
-SERVICES AGREEMENT
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
 
-Your installation of this software is symbol of your signature indicating that
-you accept the terms of this Services Agreement (this "Agreement").  This
-Agreement is a legal agreement between you (either an individual or a single
-entity) and CloudFlare, Inc. for the services being provided to you by
-CloudFlare or its authorized representative (the "Services"), including any
-computer software and any associated media, printed materials, and "online" or
-electronic documentation provided in connection with the Services (the
-"Software" and together with the Services are hereinafter collectively referred
-to as the "Solution").  If the user is not an individual, then "you" means your
-company, its officers, members, employees, agents, representatives, successors
-and assigns.  BY USING THE SOLUTION, YOU ARE INDICATING THAT YOU HAVE READ, AND
-AGREE TO BE BOUND BY, THE POLICIES, TERMS, AND CONDITIONS SET FORTH BELOW IN
-THEIR ENTIRETY WITHOUT LIMITATION OR QUALIFICATION, AS WELL AS BY ALL APPLICABLE
-LAWS AND REGULATIONS, AS IF YOU HAD HANDWRITTEN YOUR NAME ON A CONTRACT.  IF YOU
-DO NOT AGREE TO THESE TERMS AND CONDITIONS, YOU MAY NOT USE THE SOLUTION.
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-1.    GRANT OF RIGHTS
+1. Definitions.
 
-1.1	Grant of License.  The Solution is licensed by CloudFlare and its
-licensors, not sold.  Subject to the terms and conditions of this Agreement,
-CloudFlare hereby grants you a nonexclusive, nonsublicensable, nontransferable
-license to use the Solution.  You may examine source code, if provided to you,
-solely for the limited purpose of evaluating the Software for security flaws.
-You may also use the Service to create derivative works which are exclusively
-compatible with any CloudFlare product serviceand no other product or service.
-This license applies to the parts of the Solution developed by CloudFlare.  The
-Solution may also incorporate externally maintained libraries and other open software.
-These resources may be governed by other licenses.
+"License" shall mean the terms and conditions for use, reproduction,
+and distribution as defined by Sections 1 through 9 of this document.
 
-1.2	Restrictions.  The license granted herein is granted solely to you and
-not, by implication or otherwise, to any of your parents, subsidiaries or
-affiliates.  No right is granted hereunder to use the Solution to perform
-services for third parties. All rights not expressly granted hereunder are
-reserved to CloudFlare.  You may not use the Solution except as explicitly
-permitted under this Agreement.  You are expressly prohibited from modifying,
-adapting, translating, preparing derivative works from, decompiling, reverse
-engineering, disassembling or otherwise attempting to derive source code from
-the Software used to provide the Services or any internal data files generated
-by the Solution.  You are also prohibited from removing, obscuring or altering
-any copyright notice, trademarks, or other proprietary rights notices affixed to
-or associated with the Solution.
+"Licensor" shall mean the copyright owner or entity authorized by
+the copyright owner that is granting the License.
 
-1.3	Ownership.  As between the parties, CloudFlare and/or its licensors own
-and shall retain all right, title, and interest in and to the Solution,
-including any and all technology embodied therein, including all copyrights,
-patents, trade secrets, trade dress and other proprietary rights associated
-therewith, and any derivative works created there from.
+"Legal Entity" shall mean the union of the acting entity and all
+other entities that control, are controlled by, or are under common
+control with that entity. For the purposes of this definition,
+"control" means (i) the power, direct or indirect, to cause the
+direction or management of such entity, whether by contract or
+otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
 
-2.	LIMITATION OF LIABILITY
+"You" (or "Your") shall mean an individual or Legal Entity
+exercising permissions granted by this License.
 
-YOU EXPRESSLY ACKNOWLEDGE AND AGREE THAT DOWNLOADING THE SOFTWARE IS AT YOUR
-SOLE RISK.  THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTY OF ANY KIND
-AND CLOUDFLARE, ITS LICENSORS AND ITS AUTHORIZED REPRESENTATIVES (TOGETHER FOR
-PURPOSES HEREOF, "CLOUDFLARE") EXPRESSLY DISCLAIM ALL WARRANTIES, EXPRESS OR
-IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  CLOUDFLARE DOES NOT
-WARRANT THAT THE FUNCTIONS CONTAINED IN THE SOFTWARE WILL MEET YOUR
-REQUIREMENTS, OR THAT THE OPERATION OF THE SOFTWARE WILL BE UNINTERRUPTED OR
-ERROR-FREE, OR THAT DEFECTS IN THE SOFTWARE WILL BE CORRECTED.  FURTHERMORE,
-CLOUDFLARE DOES NOT WARRANT OR MAKE ANY REPRESENTATIONS REGARDING THE SOFTWARE
-OR RELATED DOCUMENTATION IN TERMS OF THEIR CORRECTNESS, ACCURACY, RELIABILITY,
-OR OTHERWISE. NO ORAL OR WRITTEN INFORMATION OR ADVICE GIVEN BY CLOUDFLARE SHALL
-CREATE A WARRANTY OR IN ANY WAY INCREASE THE SCOPE OF THIS WARRANTY.
+"Source" form shall mean the preferred form for making modifications,
+including but not limited to software source code, documentation
+source, and configuration files.
 
-3.	CONFIDENTIALITY
+"Object" form shall mean any form resulting from mechanical
+transformation or translation of a Source form, including but
+not limited to compiled object code, generated documentation,
+and conversions to other media types.
 
-It may be necessary during the set up and performance of the Solution for the
-parties to exchange Confidential Information. "Confidential Information" means
-any information whether oral, or written, of a private, secret, proprietary or
-confidential nature, concerning either party or its business operations,
-including without limitation: (a) your data and (b) CloudFlare's access control
-systems, specialized network equipment and techniques related to the Solution,
-use policies, which include trade secrets of CloudFlare and its licensors. Each
-party agrees to use the same degree of care to protect the confidentiality of
-the Confidential Information of the other party and to prevent its unauthorized
-use or dissemination as it uses to protect its own Confidential Information of a
-similar nature, but in no event shall exercise less than due diligence and
-reasonable care. Each party agrees to use the Confidential Information of the
-other party only for purposes related to the performance of this Agreement. All
-Confidential Information remains the property of the party disclosing the
-information and no license or other rights to Confidential Information is
-granted or implied hereby.
+"Work" shall mean the work of authorship, whether in Source or
+Object form, made available under the License, as indicated by a
+copyright notice that is included in or attached to the work
+(an example is provided in the Appendix below).
 
-4.	TERM AND TERMINATION
+"Derivative Works" shall mean any work, whether in Source or Object
+form, that is based on (or derived from) the Work and for which the
+editorial revisions, annotations, elaborations, or other modifications
+represent, as a whole, an original work of authorship. For the purposes
+of this License, Derivative Works shall not include works that remain
+separable from, or merely link (or bind by name) to the interfaces of,
+the Work and Derivative Works thereof.
 
-4.1	Term.  This Agreement shall be effective upon download or install of the
-Software.
+"Contribution" shall mean any work of authorship, including
+the original version of the Work and any modifications or additions
+to that Work or Derivative Works thereof, that is intentionally
+submitted to Licensor for inclusion in the Work by the copyright owner
+or by an individual or Legal Entity authorized to submit on behalf of
+the copyright owner. For the purposes of this definition, "submitted"
+means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems,
+and issue tracking systems that are managed by, or on behalf of, the
+Licensor for the purpose of discussing and improving the Work, but
+excluding communication that is conspicuously marked or otherwise
+designated in writing by the copyright owner as "Not a Contribution."
 
-4.2	Termination. This Agreement may be terminated by CloudFlare or its
-authorized representative by written notice to you if any of the following
-events occur:  (i) you fail to pay any amounts due for the Services and the
-Solution when due and after written notice of such nonpayment has been given to
-you; (ii) you are in material breach of any term, condition, or provision of
-this Agreement or any other agreement executed by you with CloudFlare or its
-authorized representative in connection with the provision of the Solution and
-Services (a "Related Agreement"); or (iii) you terminate or suspend your
-business, becomes subject to any bankruptcy or insolvency proceeding under
-federal or state statutes, or become insolvent or subject to direct control by a
-trustee, receiver or similar authority.
+"Contributor" shall mean Licensor and any individual or Legal Entity
+on behalf of whom a Contribution has been received by Licensor and
+subsequently incorporated within the Work.
 
-4.3	Effect of Termination.  Upon the termination of this Agreement for any
-reason: (1) all license rights granted hereunder shall terminate and (2) all
-Confidential Information shall be returned to the disclosing party or destroyed.
+2. Grant of Copyright License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the
+Work and such Derivative Works in Source or Object form.
 
-5.	MISCELLANEOUS
+3. Grant of Patent License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+(except as stated in this section) patent license to make, have made,
+use, offer to sell, sell, import, and otherwise transfer the Work,
+where such license applies only to those patent claims licensable
+by such Contributor that are necessarily infringed by their
+Contribution(s) alone or by combination of their Contribution(s)
+with the Work to which such Contribution(s) was submitted. If You
+institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work
+or a Contribution incorporated within the Work constitutes direct
+or contributory patent infringement, then any patent licenses
+granted to You under this License for that Work shall terminate
+as of the date such litigation is filed.
 
-5.1	Assignment.  You may not assign any of your rights or delegate any of
-your obligations under this Agreement, whether by operation of law or otherwise,
-without the prior express written consent of CloudFlare or its authorized
-representative.  Any such assignment without the prior express written consent
-of CloudFlare or its authorized representative shall be void.  Subject to the
-foregoing, this Agreement will bind and inure to the benefit of the parties,
-their respective successors and permitted assigns.
+4. Redistribution. You may reproduce and distribute copies of the
+Work or Derivative Works thereof in any medium, with or without
+modifications, and in Source or Object form, provided that You
+meet the following conditions:
 
-5.2	Waiver and Amendment.  No modification, amendment or waiver of any
-provision of this Agreement shall be effective unless in writing and signed by
-the party to be charged.  No failure or delay by either party in exercising any
-right, power, or remedy under this Agreement, except as specifically provided
-herein, shall operate as a waiver of any such right, power or remedy.  Without
-limiting the foregoing, terms and conditions on any purchase orders or similar
-materials submitted by you to CloudFlare or its authorized representative shall
-be of no force or effect.
+(a) You must give any other recipients of the Work or
+Derivative Works a copy of this License; and
 
-5.3	Governing Law.  This Agreement shall be governed by the laws of the State
-of California, USA, excluding conflict of laws and provisions, and excluding the
-United Nations Convention on Contracts for the International Sale of Goods.
+(b) You must cause any modified files to carry prominent notices
+stating that You changed the files; and
 
-5.4	Notices.  All notices, demands or consents required or permitted under
-this Agreement shall be in writing.  Notice shall be sent to you at the e-mail
-address provided by you to CloudFlare or its authorized representative in
-connection with the Solution.
+(c) You must retain, in the Source form of any Derivative Works
+that You distribute, all copyright, patent, trademark, and
+attribution notices from the Source form of the Work,
+excluding those notices that do not pertain to any part of
+the Derivative Works; and
 
-5.5	Independent Contractors. The parties are independent contractors.
-Neither party shall be deemed to be an employee, agent, partner or legal
-representative of the other for any purpose and neither shall have any right,
-power or authority to create any obligation or responsibility on behalf of the
-other.
+(d) If the Work includes a "NOTICE" text file as part of its
+distribution, then any Derivative Works that You distribute must
+include a readable copy of the attribution notices contained
+within such NOTICE file, excluding those notices that do not
+pertain to any part of the Derivative Works, in at least one
+of the following places: within a NOTICE text file distributed
+as part of the Derivative Works; within the Source form or
+documentation, if provided along with the Derivative Works; or,
+within a display generated by the Derivative Works, if and
+wherever such third-party notices normally appear. The contents
+of the NOTICE file are for informational purposes only and
+do not modify the License. You may add Your own attribution
+notices within Derivative Works that You distribute, alongside
+or as an addendum to the NOTICE text from the Work, provided
+that such additional attribution notices cannot be construed
+as modifying the License.
 
-5.6	Severability.  If any provision of this Agreement is held by a court of
-competent jurisdiction to be contrary to law, such provision shall be changed
-and interpreted so as to best accomplish the objectives of the original
-provision to the fullest extent allowed by law and the remaining provisions of
-this Agreement shall remain in full force and effect.
+You may add Your own copyright statement to Your modifications and
+may provide additional or different license terms and conditions
+for use, reproduction, or distribution of Your modifications, or
+for any such Derivative Works as a whole, provided Your use,
+reproduction, and distribution of the Work otherwise complies with
+the conditions stated in this License.
 
-5.7	Force Majeure.  CloudFlare shall not be liable to the other party for any
-failure or delay in performance caused by reasons beyond its reasonable control.
+5. Submission of Contributions. Unless You explicitly state otherwise,
+any Contribution intentionally submitted for inclusion in the Work
+by You to the Licensor shall be under the terms and conditions of
+this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify
+the terms of any separate license agreement you may have executed
+with Licensor regarding such Contributions.
 
-5.8	Complete Understanding.  This Agreement and the Related Agreement
-constitute the final, complete and exclusive agreement between the parties with
-respect to the subject matter hereof, and supersedes all previous written and
-oral agreements and communications related to the subject matter of this
-Agreement.  To the extent this Agreement and the Related Agreement conflict,
-this Agreement shall control.
+6. Trademarks. This License does not grant permission to use the trade
+names, trademarks, service marks, or product names of the Licensor,
+except as required for reasonable and customary use in describing the
+origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+agreed to in writing, Licensor provides the Work (and each
+Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied, including, without limitation, any warranties or conditions
+of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+PARTICULAR PURPOSE. You are solely responsible for determining the
+appropriateness of using or redistributing the Work and assume any
+risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+whether in tort (including negligence), contract, or otherwise,
+unless required by applicable law (such as deliberate and grossly
+negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special,
+incidental, or consequential damages of any character arising as a
+result of this License or out of the use or inability to use the
+Work (including but not limited to damages for loss of goodwill,
+work stoppage, computer failure or malfunction, or any and all
+other commercial damages or losses), even if such Contributor
+has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+the Work or Derivative Works thereof, You may choose to offer,
+and charge a fee for, acceptance of support, warranty, indemnity,
+or other liability obligations and/or rights consistent with this
+License. However, in accepting such obligations, You may act only
+on Your own behalf and on Your sole responsibility, not on behalf
+of any other Contributor, and only if You agree to indemnify,
+defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason
+of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+To apply the Apache License to your work, attach the following
+boilerplate notice, with the fields enclosed by brackets "[]"
+replaced with your own identifying information. (Don't include
+the brackets!)  The text should be enclosed in the appropriate
+comment syntax for the file format. We also recommend that a
+file or class name and description of purpose be included on the
+same "printed page" as the copyright notice for easier
+identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+
+## Runtime Library Exception to the Apache 2.0 License: ##
+
+
+As an exception, if you use this Software to compile your source code and
+portions of this Software are embedded into the binary product as a result,
+you may redistribute such product without providing attribution as would
+otherwise be required by Sections 4(a), 4(b) and 4(d) of the License.

Makefile

@@ -1,25 +1,41 @@
-VERSION       := $(shell git describe --tags --always --dirty="-dev" --match "[0-9][0-9][0-9][0-9].*.*")
+VERSION       := $(shell git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
 MSI_VERSION   := $(shell git tag -l --sort=v:refname | grep "w" | tail -1 | cut -c2-)
 #MSI_VERSION expects the format of the tag to be: (wX.X.X). Starts with the w character to not break cfsetup.
 #e.g. w3.0.1 or w4.2.10. It trims off the w character when creating the MSI.
 
-ifeq ($(FIPS), true)
-	GO_BUILD_TAGS := $(GO_BUILD_TAGS) fips
-endif
-
-ifneq ($(GO_BUILD_TAGS),)
-	GO_BUILD_TAGS := -tags $(GO_BUILD_TAGS)
+ifeq ($(ORIGINAL_NAME), true)
+	# Used for builds that want FIPS compilation but want the artifacts generated to still have the original name.
+	BINARY_NAME := cloudflared
+else ifeq ($(FIPS), true)
+	# Used for FIPS compliant builds that do not match the case above.
+	BINARY_NAME := cloudflared-fips
+else
+	# Used for all other (non-FIPS) builds.
+	BINARY_NAME := cloudflared
 endif
 
 ifeq ($(NIGHTLY), true)
-	DEB_PACKAGE_NAME := cloudflared-nightly
+	DEB_PACKAGE_NAME := $(BINARY_NAME)-nightly
 	NIGHTLY_FLAGS := --conflicts cloudflared --replaces cloudflared
 else
-	DEB_PACKAGE_NAME := cloudflared
+	DEB_PACKAGE_NAME := $(BINARY_NAME)
 endif
 
 DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
-VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
+VERSION_FLAGS := -X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"
+
+LINK_FLAGS :=
+ifeq ($(FIPS), true)
+	LINK_FLAGS := -linkmode=external -extldflags=-static $(LINK_FLAGS)
+	# Prevent linking with libc regardless of CGO enabled or not.
+	GO_BUILD_TAGS := $(GO_BUILD_TAGS) osusergo netgo fips
+	VERSION_FLAGS := $(VERSION_FLAGS) -X "main.BuildType=FIPS"
+endif
+
+LDFLAGS := -ldflags='$(VERSION_FLAGS) $(LINK_FLAGS)'
+ifneq ($(GO_BUILD_TAGS),)
+	GO_BUILD_TAGS := -tags "$(GO_BUILD_TAGS)"
+endif
 
 IMPORT_PATH   := github.com/cloudflare/cloudflared
 PACKAGE_DIR   := $(CURDIR)/packaging
@@ -61,9 +77,9 @@ else
 endif
 
 ifeq ($(TARGET_OS), windows)
-	EXECUTABLE_PATH=./cloudflared.exe
+	EXECUTABLE_PATH=./$(BINARY_NAME).exe
 else
-	EXECUTABLE_PATH=./cloudflared
+	EXECUTABLE_PATH=./$(BINARY_NAME)
 endif
 
 ifeq ($(FLAVOR), centos-7)
@@ -80,17 +96,15 @@ clean:
 	go clean
 
 .PHONY: cloudflared
-cloudflared: 
+cloudflared:
 ifeq ($(FIPS), true)
 	$(info Building cloudflared with go-fips)
-	-test -f fips/fips.go && mv fips/fips.go fips/fips.go.linux-amd64
-	mv fips/fips.go.linux-amd64 fips/fips.go
+	cp -f fips/fips.go.linux-amd64 cmd/cloudflared/fips.go
 endif
-
-	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
-
+	GOOS=$(TARGET_OS) GOARCH=$(TARGET_ARCH) go build -v -mod=vendor $(GO_BUILD_TAGS) $(LDFLAGS) $(IMPORT_PATH)/cmd/cloudflared
 ifeq ($(FIPS), true)
-	mv fips/fips.go fips/fips.go.linux-amd64
+	rm -f cmd/cloudflared/fips.go
+	./check-fips.sh cloudflared
 endif
 
 .PHONY: container
@@ -100,10 +114,10 @@ container:
 .PHONY: test
 test: vet
 ifndef CI
-	go test -v -mod=vendor -race $(VERSION_FLAGS) ./...
+	go test -v -mod=vendor -race $(LDFLAGS) ./...
 else
 	@mkdir -p .cover
-	go test -v -mod=vendor -race $(VERSION_FLAGS) -coverprofile=".cover/c.out" ./...
+	go test -v -mod=vendor -race $(LDFLAGS) -coverprofile=".cover/c.out" ./...
 	go tool cover -html ".cover/c.out" -o .cover/all.html
 endif
 
@@ -112,10 +126,10 @@ test-ssh-server:
 	docker-compose -f ssh_server_tests/docker-compose.yml up
 
 define publish_package
-	chmod 664 cloudflared*.$(1); \
+	chmod 664 $(BINARY_NAME)*.$(1); \
 	for HOST in $(CF_PKG_HOSTS); do \
-		ssh-keyscan -t rsa $$HOST >> ~/.ssh/known_hosts; \
-		scp -p -4 cloudflared*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/cloudflared/; \
+		ssh-keyscan -t ecdsa $$HOST >> ~/.ssh/known_hosts; \
+		scp -p -4 $(BINARY_NAME)*.$(1) cfsync@$$HOST:/state/cf-pkg/staging/$(2)/$(TARGET_PUBLIC_REPO)/$(BINARY_NAME)/; \
 	done
 endef
 
@@ -127,6 +141,8 @@ publish-deb: cloudflared-deb
 publish-rpm: cloudflared-rpm
 	$(call publish_package,rpm,yum)
 
+# When we build packages, the package name will be FIPS-aware.
+# But we keep the binary installed by it to be named "cloudflared" regardless.
 define build_package
 	mkdir -p $(PACKAGE_DIR)
 	cp cloudflared $(PACKAGE_DIR)/cloudflared
@@ -247,8 +263,8 @@ tunnelrpc-deps:
 	capnp compile -ogo tunnelrpc/tunnelrpc.capnp
 
 .PHONY: quic-deps
-quic-deps: 
-	which capnp 
+quic-deps:
+	which capnp
 	which capnpc-go
 	capnp compile -ogo quic/schema/quic_metadata_protocol.capnp
 
@@ -258,9 +274,9 @@ vet:
 	# go get github.com/sudarshan-reddy/go-sumtype (don't do this in build directory or this will cause vendor issues)
 	# Note: If you have github.com/BurntSushi/go-sumtype then you might have to use the repo above instead
 	# for now because it uses an older version of golang.org/x/tools.
-	which go-sumtype  
+	which go-sumtype
 	go-sumtype $$(go list -mod=vendor ./...)
 
 .PHONY: goimports
 goimports:
-	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done
+	for d in $$(go list -mod=readonly -f '{{.Dir}}' -a ./... | fgrep -v tunnelrpc) ; do goimports -format-only -local github.com/cloudflare/cloudflared -w $$d ; done

RELEASE_NOTES

@@ -1,3 +1,125 @@
+2022.2.2
+- 2022-02-22 TUN-5754: Allow ingress validate to take plaintext option
+- 2022-02-17 TUN-5678: Cloudflared uses typed tunnel API
+
+2022.2.1
+- 2022-02-10 TUN-5184: Handle errors in bidrectional streaming (websocket#Stream) gracefully when 1 side has ended
+- 2022-02-14 Update issue templates
+- 2022-02-14 Update issue templates
+- 2022-02-11 TUN-5768: Update cloudflared license file
+- 2022-02-11 TUN-5698: Make ingress rules and warp routing dynamically configurable
+- 2022-02-14 TUN-5678: Adapt cloudflared to use new typed APIs
+- 2022-02-17 Revert "TUN-5678: Adapt cloudflared to use new typed APIs"
+- 2022-02-11 TUN-5697: Listen for UpdateConfiguration RPC in quic transport
+- 2022-02-04 TUN-5744: Add a test to make sure cloudflared uses scheme defined in ingress rule, not X-Forwarded-Proto header
+- 2022-02-07 TUN-5749: Refactor cloudflared to pave way for reconfigurable ingress - Split origin into supervisor and proxy packages - Create configManager to handle dynamic config
+- 2021-10-19 TUN-5184: Make sure outstanding websocket write is finished, and no more writes after shutdown
+
+2022.2.0
+- 2022-02-02 TUN-4947: Use http when talking to Unix sockets origins
+- 2022-02-02 TUN-5695: Define RPC method to update configuration
+- 2022-01-27 TUN-5621: Correctly manage QUIC stream closing
+- 2022-01-28 TUN-5702: Allow to deserialize config from JSON
+
+2022.1.3
+- 2022-01-21 TUN-5477: Unhide vnet commands
+- 2022-01-24 TUN-5669: Change network command to vnet
+- 2022-01-25 TUN-5675: Remove github.com/dgrijalva/jwt-go dependency by upgrading coredns version
+- 2022-01-27 TUN-5719: Re-attempt connection to edge with QUIC despite network error when there is no fallback
+- 2022-01-28 TUN-5724: Fix SSE streaming by guaranteeing we write everything we read
+- 2022-01-17 TUN-5547: Bump golang x/net package to fix http2 transport bugs
+- 2022-01-19 TUN-5659: Proxy UDP with zero-byte payload
+- 2021-10-22 Add X-Forwarded-Host for http proxy
+
+2022.1.2
+- 2022-01-13 TUN-5650: Fix pynacl version to 1.4.0 and pygithub version to 1.55 so release doesn't break unexpectedly
+
+2022.1.1
+- 2022-01-10 TUN-5631: Build everything with go 1.17.5
+- 2022-01-06 TUN-5623: Configure quic max datagram frame size to 1350 bytes for none Windows platforms
+
+2022.1.0
+- 2022-01-03 TUN-5612: Add support for specifying TLS min/max version
+- 2022-01-03 TUN-5612: Make tls min/max version public visible
+- 2022-01-03 TUN-5551: Internally published debian artifacts are now named just cloudflared even though they are FIPS compliant
+- 2022-01-04 TUN-5600: Close QUIC transports as soon as possible while respecting graceful shutdown
+- 2022-01-05 TUN-5616: Never fallback transport if user chooses it on purpose
+- 2022-01-05 TUN-5204: Unregister QUIC transports on disconnect
+- 2022-01-04 TUN-5600: Add coverage to component tests for various transports
+
+2021.12.4
+- 2021-12-27 TUN-5482: Refactor tunnelstore client related packages for more coherent package
+- 2021-12-27 TUN-5551: Change internally published debian package to be FIPS compliant
+- 2021-12-27 TUN-5551: Show whether the binary was built for FIPS compliance
+
+2021.12.3
+- 2021-12-22 TUN-5584: Changes for release 2021.12.2
+- 2021-12-22 TUN-5590: QUIC datagram max user payload is 1217 bytes
+- 2021-12-22 TUN-5593: Read full packet from UDP connection, even if it exceeds MTU of the transport. When packet length is greater than the MTU of the transport, we will silently drop packets (for now).
+- 2021-12-23 TUN-5597: Log session ID when session is terminated by edge
+
+2021.12.2
+- 2021-12-20 TUN-5571: Remove redundant session manager log, it's already logged in origin/tunnel.ServeQUIC
+- 2021-12-20 TUN-5570: Only log RPC server events at error level to reduce noise
+- 2021-12-14 TUN-5494: Send a RPC with terminate reason to edge if the session is closed locally
+- 2021-11-09 TUN-5551: Reintroduce FIPS compliance for linux amd64 now as separate binaries
+
+2021.12.1
+- 2021-12-16 TUN-5549: Revert "TUN-5277: Ensure cloudflared binary is FIPS compliant on linux amd64"
+
+2021.12.0
+- 2021-12-13 TUN-5530: Get current time from ticker
+- 2021-12-15 TUN-5544: Update CHANGES.md for next release
+- 2021-12-07 TUN-5519: Adjust URL for virtual_networks endpoint to match what we will publish
+- 2021-12-02 TUN-5488: Close session after it's idle for a period defined by registerUdpSession RPC
+- 2021-12-09 TUN-5504: Fix upload of packages to public repo
+- 2021-11-30 TUN-5481: Create abstraction for Origin UDP Connection
+- 2021-11-30 TUN-5422: Define RPC to unregister session
+- 2021-11-26 TUN-5361: Commands for managing virtual networks
+- 2021-11-29 TUN-5362: Adjust route ip commands to be aware of virtual networks
+- 2021-11-23 TUN-5301: Separate datagram multiplex and session management logic from quic connection logic
+- 2021-11-10 TUN-5405: Update net package to v0.0.0-20211109214657-ef0fda0de508
+- 2021-11-10 TUN-5408: Update quic package to v0.24.0
+- 2021-11-12 Fix typos
+- 2021-11-13 Fix for Issue #501: Unexpected User-agent insertion when tunneling http request
+- 2021-11-16 TUN-5129: Remove `-dev` suffix when computing version and Git has uncommitted changes
+- 2021-11-18 TUN-5441: Fix message about available protocols
+- 2021-11-12 TUN-5300: Define RPC to register UDP sessions
+- 2021-11-14 TUN-5299: Send/receive QUIC datagram from edge and proxy to origin as UDP
+- 2021-11-04 TUN-5387: Updated CHANGES.md for 2021.11.0
+- 2021-11-08 TUN-5368: Log connection issues with LogLevel that depends on tunnel state
+- 2021-11-09 TUN-5397: Log cloudflared output when it fails to connect tunnel
+- 2021-11-09 TUN-5277: Ensure cloudflared binary is FIPS compliant on linux amd64
+- 2021-11-08 TUN-5393: Content-length is no longer a control header for non-h2mux transports
+
+2021.11.0
+- 2021-11-03 TUN-5285: Fallback to HTTP2 immediately if connection times out with no network activity
+- 2021-09-29 Add flag to 'tunnel create' subcommand to specify a base64-encoded secret
+
+2021.10.5
+- 2021-10-25 Update change log for release 2021.10.4
+- 2021-10-25 Revert "TUN-5184: Make sure outstanding websocket write is finished, and no more writes after shutdown"
+
+2021.10.4
+- 2021-10-21 TUN-5287: Fix misuse of wait group in TestQUICServer that caused the test to exit immediately
+- 2021-10-21 TUN-5286: Upgrade crypto/ssh package to fix CVE-2020-29652
+- 2021-10-18 TUN-5262: Allow to configure max fetch size for listing queries
+- 2021-10-19 TUN-5262: Improvements to `max-fetch-size` that allow to deal with large number of tunnels in account
+- 2021-10-15 TUN-5261: Collect QUIC metrics about RTT, packets and bytes transfered and log events at tracing level
+- 2021-10-19 TUN-5184: Make sure outstanding websocket write is finished, and no more writes after shutdown
+
+2021.10.3
+- 2021-10-14 TUN-5255: Fix potential panic if Cloudflare API fails to respond to GetTunnel(id) during delete command
+- 2021-10-14 TUN-5257: Fix more cfsetup targets that were broken by recent package changes
+
+2021.10.2
+- 2021-10-11 TUN-5138: Switch to QUIC on auto protocol based on threshold
+- 2021-10-14 TUN-5250: Add missing packages for cfsetup to succeed in github release pkgs target
+
+2021.10.1
+- 2021-10-12 TUN-5246: Use protocol: quic for Quick tunnels if one is not already set
+- 2021-10-13 TUN-5249: Revert "TUN-5138: Switch to QUIC on auto protocol based on threshold"
+
 2021.10.0
 - 2021-10-11 TUN-5138: Switch to QUIC on auto protocol based on threshold
 - 2021-10-07 TUN-5195: Do not set empty body if not applicable

build-packages-fips.sh

@@ -0,0 +1,25 @@
+VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
+echo $VERSION
+
+# This controls the directory the built artifacts go into
+export ARTIFACT_DIR=built_artifacts/
+mkdir -p $ARTIFACT_DIR
+
+arch=("amd64")
+export TARGET_ARCH=$arch
+export TARGET_OS=linux
+export FIPS=true
+# For BoringCrypto to link, we need CGO enabled. Otherwise compilation fails.
+export CGO_ENABLED=1
+
+make cloudflared-deb
+mv cloudflared-fips\_$VERSION\_$arch.deb $ARTIFACT_DIR/cloudflared-fips-linux-$arch.deb
+
+# rpm packages invert the - and _ and use x86_64 instead of amd64.
+RPMVERSION=$(echo $VERSION|sed -r 's/-/_/g')
+RPMARCH="x86_64"
+make cloudflared-rpm
+mv cloudflared-fips-$RPMVERSION-1.$RPMARCH.rpm $ARTIFACT_DIR/cloudflared-fips-linux-$RPMARCH.rpm
+
+# finally move the linux binary as well.
+mv ./cloudflared $ARTIFACT_DIR/cloudflared-fips-linux-$arch

build-packages.sh

@@ -1,12 +1,15 @@
-VERSION=$(git describe --tags --always --dirty="-dev" --match "[0-9][0-9][0-9][0-9].*.*")
+VERSION=$(git describe --tags --always --match "[0-9][0-9][0-9][0-9].*.*")
 echo $VERSION
+
+# Avoid depending on C code since we don't need it.
 export CGO_ENABLED=0
+
 # This controls the directory the built artifacts go into
 export ARTIFACT_DIR=built_artifacts/
 mkdir -p $ARTIFACT_DIR
 windowsArchs=("amd64" "386")
 export TARGET_OS=windows
-for arch in ${windowsArchs[@]}; do 
+for arch in ${windowsArchs[@]}; do
     export TARGET_ARCH=$arch
     make cloudflared-msi
     mv ./cloudflared.exe $ARTIFACT_DIR/cloudflared-windows-$arch.exe
@@ -14,15 +17,14 @@ for arch in ${windowsArchs[@]}; do
 done
 
 
-export FIPS=true
-linuxArchs=("amd64" "386" "arm" "arm64")
+linuxArchs=("386" "amd64" "arm" "arm64")
 export TARGET_OS=linux
-for arch in ${linuxArchs[@]}; do 
+for arch in ${linuxArchs[@]}; do
     export TARGET_ARCH=$arch
     make cloudflared-deb
     mv cloudflared\_$VERSION\_$arch.deb $ARTIFACT_DIR/cloudflared-linux-$arch.deb
 
-    # rpm packages invert the - and _ and use x86_64 instead of amd64. 
+    # rpm packages invert the - and _ and use x86_64 instead of amd64.
     RPMVERSION=$(echo $VERSION|sed -r 's/-/_/g')
     RPMARCH=$arch
     if [ $arch == "amd64" ];then
@@ -37,4 +39,3 @@ for arch in ${linuxArchs[@]}; do
     # finally move the linux binary as well.
     mv ./cloudflared $ARTIFACT_DIR/cloudflared-linux-$arch
 done
-

carrier/carrier.go

@@ -54,7 +54,7 @@ func (c *StdinoutStream) Write(p []byte) (int, error) {
 	return os.Stdout.Write(p)
 }
 
-// Helper to allow defering the response close with a check that the resp is not nil
+// Helper to allow deferring the response close with a check that the resp is not nil
 func closeRespBody(resp *http.Response) {
 	if resp != nil {
 		_ = resp.Body.Close()

certutil/certutil.go

@@ -66,7 +66,7 @@ func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
 				originCert.ServiceKey = ntt.ServiceKey
 				originCert.AccountID = ntt.AccountID
 			} else {
-				// Try the older format, where the zoneID and service key are seperated by
+				// Try the older format, where the zoneID and service key are separated by
 				// a new line character
 				token := string(block.Bytes)
 				s := strings.Split(token, "\n")

cfapi/base_client.go

@@ -0,0 +1,186 @@
+package cfapi
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog"
+	"golang.org/x/net/http2"
+)
+
+const (
+	defaultTimeout  = 15 * time.Second
+	jsonContentType = "application/json"
+)
+
+var (
+	ErrUnauthorized = errors.New("unauthorized")
+	ErrBadRequest   = errors.New("incorrect request parameters")
+	ErrNotFound     = errors.New("not found")
+	ErrAPINoSuccess = errors.New("API call failed")
+)
+
+type RESTClient struct {
+	baseEndpoints *baseEndpoints
+	authToken     string
+	userAgent     string
+	client        http.Client
+	log           *zerolog.Logger
+}
+
+type baseEndpoints struct {
+	accountLevel  url.URL
+	zoneLevel     url.URL
+	accountRoutes url.URL
+	accountVnets  url.URL
+}
+
+var _ Client = (*RESTClient)(nil)
+
+func NewRESTClient(baseURL, accountTag, zoneTag, authToken, userAgent string, log *zerolog.Logger) (*RESTClient, error) {
+	if strings.HasSuffix(baseURL, "/") {
+		baseURL = baseURL[:len(baseURL)-1]
+	}
+	accountLevelEndpoint, err := url.Parse(fmt.Sprintf("%s/accounts/%s/cfd_tunnel", baseURL, accountTag))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create account level endpoint")
+	}
+	accountRoutesEndpoint, err := url.Parse(fmt.Sprintf("%s/accounts/%s/teamnet/routes", baseURL, accountTag))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create route account-level endpoint")
+	}
+	accountVnetsEndpoint, err := url.Parse(fmt.Sprintf("%s/accounts/%s/teamnet/virtual_networks", baseURL, accountTag))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create virtual network account-level endpoint")
+	}
+	zoneLevelEndpoint, err := url.Parse(fmt.Sprintf("%s/zones/%s/tunnels", baseURL, zoneTag))
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create account level endpoint")
+	}
+	httpTransport := http.Transport{
+		TLSHandshakeTimeout:   defaultTimeout,
+		ResponseHeaderTimeout: defaultTimeout,
+	}
+	http2.ConfigureTransport(&httpTransport)
+	return &RESTClient{
+		baseEndpoints: &baseEndpoints{
+			accountLevel:  *accountLevelEndpoint,
+			zoneLevel:     *zoneLevelEndpoint,
+			accountRoutes: *accountRoutesEndpoint,
+			accountVnets:  *accountVnetsEndpoint,
+		},
+		authToken: authToken,
+		userAgent: userAgent,
+		client: http.Client{
+			Transport: &httpTransport,
+			Timeout:   defaultTimeout,
+		},
+		log: log,
+	}, nil
+}
+
+func (r *RESTClient) sendRequest(method string, url url.URL, body interface{}) (*http.Response, error) {
+	var bodyReader io.Reader
+	if body != nil {
+		if bodyBytes, err := json.Marshal(body); err != nil {
+			return nil, errors.Wrap(err, "failed to serialize json body")
+		} else {
+			bodyReader = bytes.NewBuffer(bodyBytes)
+		}
+	}
+
+	req, err := http.NewRequest(method, url.String(), bodyReader)
+	if err != nil {
+		return nil, errors.Wrapf(err, "can't create %s request", method)
+	}
+	req.Header.Set("User-Agent", r.userAgent)
+	if bodyReader != nil {
+		req.Header.Set("Content-Type", jsonContentType)
+	}
+	req.Header.Add("X-Auth-User-Service-Key", r.authToken)
+	req.Header.Add("Accept", "application/json;version=1")
+	return r.client.Do(req)
+}
+
+func parseResponse(reader io.Reader, data interface{}) error {
+	// Schema for Tunnelstore responses in the v1 API.
+	// Roughly, it's a wrapper around a particular result that adds failures/errors/etc
+	var result response
+	// First, parse the wrapper and check the API call succeeded
+	if err := json.NewDecoder(reader).Decode(&result); err != nil {
+		return errors.Wrap(err, "failed to decode response")
+	}
+	if err := result.checkErrors(); err != nil {
+		return err
+	}
+	if !result.Success {
+		return ErrAPINoSuccess
+	}
+	// At this point we know the API call succeeded, so, parse out the inner
+	// result into the datatype provided as a parameter.
+	if err := json.Unmarshal(result.Result, &data); err != nil {
+		return errors.Wrap(err, "the Cloudflare API response was an unexpected type")
+	}
+	return nil
+}
+
+type response struct {
+	Success  bool            `json:"success,omitempty"`
+	Errors   []apiErr        `json:"errors,omitempty"`
+	Messages []string        `json:"messages,omitempty"`
+	Result   json.RawMessage `json:"result,omitempty"`
+}
+
+func (r *response) checkErrors() error {
+	if len(r.Errors) == 0 {
+		return nil
+	}
+	if len(r.Errors) == 1 {
+		return r.Errors[0]
+	}
+	var messages string
+	for _, e := range r.Errors {
+		messages += fmt.Sprintf("%s; ", e)
+	}
+	return fmt.Errorf("API errors: %s", messages)
+}
+
+type apiErr struct {
+	Code    json.Number `json:"code,omitempty"`
+	Message string      `json:"message,omitempty"`
+}
+
+func (e apiErr) Error() string {
+	return fmt.Sprintf("code: %v, reason: %s", e.Code, e.Message)
+}
+
+func (r *RESTClient) statusCodeToError(op string, resp *http.Response) error {
+	if resp.Header.Get("Content-Type") == "application/json" {
+		var errorsResp response
+		if json.NewDecoder(resp.Body).Decode(&errorsResp) == nil {
+			if err := errorsResp.checkErrors(); err != nil {
+				return errors.Errorf("Failed to %s: %s", op, err)
+			}
+		}
+	}
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return nil
+	case http.StatusBadRequest:
+		return ErrBadRequest
+	case http.StatusUnauthorized, http.StatusForbidden:
+		return ErrUnauthorized
+	case http.StatusNotFound:
+		return ErrNotFound
+	}
+	return errors.Errorf("API call to %s failed with status %d: %s", op,
+		resp.StatusCode, http.StatusText(resp.StatusCode))
+}

cfapi/client.go

@@ -0,0 +1,39 @@
+package cfapi
+
+import (
+	"github.com/google/uuid"
+)
+
+type TunnelClient interface {
+	CreateTunnel(name string, tunnelSecret []byte) (*Tunnel, error)
+	GetTunnel(tunnelID uuid.UUID) (*Tunnel, error)
+	DeleteTunnel(tunnelID uuid.UUID) error
+	ListTunnels(filter *TunnelFilter) ([]*Tunnel, error)
+	ListActiveClients(tunnelID uuid.UUID) ([]*ActiveClient, error)
+	CleanupConnections(tunnelID uuid.UUID, params *CleanupParams) error
+}
+
+type HostnameClient interface {
+	RouteTunnel(tunnelID uuid.UUID, route HostnameRoute) (HostnameRouteResult, error)
+}
+
+type IPRouteClient interface {
+	ListRoutes(filter *IpRouteFilter) ([]*DetailedRoute, error)
+	AddRoute(newRoute NewRoute) (Route, error)
+	DeleteRoute(params DeleteRouteParams) error
+	GetByIP(params GetRouteByIpParams) (DetailedRoute, error)
+}
+
+type VnetClient interface {
+	CreateVirtualNetwork(newVnet NewVirtualNetwork) (VirtualNetwork, error)
+	ListVirtualNetworks(filter *VnetFilter) ([]*VirtualNetwork, error)
+	DeleteVirtualNetwork(id uuid.UUID) error
+	UpdateVirtualNetwork(id uuid.UUID, updates UpdateVirtualNetwork) error
+}
+
+type Client interface {
+	TunnelClient
+	HostnameClient
+	IPRouteClient
+	VnetClient
+}

cfapi/hostname.go

@@ -0,0 +1,192 @@
+package cfapi
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+)
+
+type Change = string
+
+const (
+	ChangeNew       = "new"
+	ChangeUpdated   = "updated"
+	ChangeUnchanged = "unchanged"
+)
+
+// HostnameRoute represents a record type that can route to a tunnel
+type HostnameRoute interface {
+	json.Marshaler
+	RecordType() string
+	UnmarshalResult(body io.Reader) (HostnameRouteResult, error)
+	String() string
+}
+
+type HostnameRouteResult interface {
+	// SuccessSummary explains what will route to this tunnel when it's provisioned successfully
+	SuccessSummary() string
+}
+
+type DNSRoute struct {
+	userHostname      string
+	overwriteExisting bool
+}
+
+type DNSRouteResult struct {
+	route *DNSRoute
+	CName Change `json:"cname"`
+	Name  string `json:"name"`
+}
+
+func NewDNSRoute(userHostname string, overwriteExisting bool) HostnameRoute {
+	return &DNSRoute{
+		userHostname:      userHostname,
+		overwriteExisting: overwriteExisting,
+	}
+}
+
+func (dr *DNSRoute) MarshalJSON() ([]byte, error) {
+	s := struct {
+		Type              string `json:"type"`
+		UserHostname      string `json:"user_hostname"`
+		OverwriteExisting bool   `json:"overwrite_existing"`
+	}{
+		Type:              dr.RecordType(),
+		UserHostname:      dr.userHostname,
+		OverwriteExisting: dr.overwriteExisting,
+	}
+	return json.Marshal(&s)
+}
+
+func (dr *DNSRoute) UnmarshalResult(body io.Reader) (HostnameRouteResult, error) {
+	var result DNSRouteResult
+	err := parseResponse(body, &result)
+	result.route = dr
+	return &result, err
+}
+
+func (dr *DNSRoute) RecordType() string {
+	return "dns"
+}
+
+func (dr *DNSRoute) String() string {
+	return fmt.Sprintf("%s %s", dr.RecordType(), dr.userHostname)
+}
+
+func (res *DNSRouteResult) SuccessSummary() string {
+	var msgFmt string
+	switch res.CName {
+	case ChangeNew:
+		msgFmt = "Added CNAME %s which will route to this tunnel"
+	case ChangeUpdated: // this is not currently returned by tunnelsore
+		msgFmt = "%s updated to route to your tunnel"
+	case ChangeUnchanged:
+		msgFmt = "%s is already configured to route to your tunnel"
+	}
+	return fmt.Sprintf(msgFmt, res.hostname())
+}
+
+// hostname yields the resulting name for the DNS route; if that is not available from Cloudflare API, then the
+// requested name is returned instead (should not be the common path, it is just a fall-back).
+func (res *DNSRouteResult) hostname() string {
+	if res.Name != "" {
+		return res.Name
+	}
+	return res.route.userHostname
+}
+
+type LBRoute struct {
+	lbName string
+	lbPool string
+}
+
+type LBRouteResult struct {
+	route        *LBRoute
+	LoadBalancer Change `json:"load_balancer"`
+	Pool         Change `json:"pool"`
+}
+
+func NewLBRoute(lbName, lbPool string) HostnameRoute {
+	return &LBRoute{
+		lbName: lbName,
+		lbPool: lbPool,
+	}
+}
+
+func (lr *LBRoute) MarshalJSON() ([]byte, error) {
+	s := struct {
+		Type   string `json:"type"`
+		LBName string `json:"lb_name"`
+		LBPool string `json:"lb_pool"`
+	}{
+		Type:   lr.RecordType(),
+		LBName: lr.lbName,
+		LBPool: lr.lbPool,
+	}
+	return json.Marshal(&s)
+}
+
+func (lr *LBRoute) RecordType() string {
+	return "lb"
+}
+
+func (lb *LBRoute) String() string {
+	return fmt.Sprintf("%s %s %s", lb.RecordType(), lb.lbName, lb.lbPool)
+}
+
+func (lr *LBRoute) UnmarshalResult(body io.Reader) (HostnameRouteResult, error) {
+	var result LBRouteResult
+	err := parseResponse(body, &result)
+	result.route = lr
+	return &result, err
+}
+
+func (res *LBRouteResult) SuccessSummary() string {
+	var msg string
+	switch res.LoadBalancer + "," + res.Pool {
+	case "new,new":
+		msg = "Created load balancer %s and added a new pool %s with this tunnel as an origin"
+	case "new,updated":
+		msg = "Created load balancer %s with an existing pool %s which was updated to use this tunnel as an origin"
+	case "new,unchanged":
+		msg = "Created load balancer %s with an existing pool %s which already has this tunnel as an origin"
+	case "updated,new":
+		msg = "Added new pool %[2]s with this tunnel as an origin to load balancer %[1]s"
+	case "updated,updated":
+		msg = "Updated pool %[2]s to use this tunnel as an origin and added it to load balancer %[1]s"
+	case "updated,unchanged":
+		msg = "Added pool %[2]s, which already has this tunnel as an origin, to load balancer %[1]s"
+	case "unchanged,updated":
+		msg = "Added this tunnel as an origin in pool %[2]s which is already used by load balancer %[1]s"
+	case "unchanged,unchanged":
+		msg = "Load balancer %s already uses pool %s which has this tunnel as an origin"
+	case "unchanged,new":
+		// this state is not possible
+		fallthrough
+	default:
+		msg = "Something went wrong: failed to modify load balancer %s with pool %s; please check traffic manager configuration in the dashboard"
+	}
+
+	return fmt.Sprintf(msg, res.route.lbName, res.route.lbPool)
+}
+
+func (r *RESTClient) RouteTunnel(tunnelID uuid.UUID, route HostnameRoute) (HostnameRouteResult, error) {
+	endpoint := r.baseEndpoints.zoneLevel
+	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/routes", tunnelID))
+	resp, err := r.sendRequest("PUT", endpoint, route)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return route.UnmarshalResult(resp.Body)
+	}
+
+	return nil, r.statusCodeToError("add route", resp)
+}

cfapi/hostname_test.go

@@ -1,17 +1,9 @@
-package tunnelstore
+package cfapi
 
 import (
-	"bytes"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net"
-	"reflect"
 	"strings"
 	"testing"
-	"time"
 
-	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -105,125 +97,3 @@ func TestLBRouteResultSuccessSummary(t *testing.T) {
 		assert.Equal(t, tt.expected, actual, "case %d", i+1)
 	}
 }
-
-func Test_parseListTunnels(t *testing.T) {
-	type args struct {
-		body string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    []*Tunnel
-		wantErr bool
-	}{
-		{
-			name: "empty list",
-			args: args{body: `{"success": true, "result": []}`},
-			want: []*Tunnel{},
-		},
-		{
-			name:    "success is false",
-			args:    args{body: `{"success": false, "result": []}`},
-			wantErr: true,
-		},
-		{
-			name:    "errors are present",
-			args:    args{body: `{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}], "result": []}`},
-			wantErr: true,
-		},
-		{
-			name:    "invalid response",
-			args:    args{body: `abc`},
-			wantErr: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			body := ioutil.NopCloser(bytes.NewReader([]byte(tt.args.body)))
-			got, err := parseListTunnels(body)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("parseListTunnels() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("parseListTunnels() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func Test_unmarshalTunnel(t *testing.T) {
-	type args struct {
-		reader io.Reader
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    *Tunnel
-		wantErr bool
-	}{
-		// TODO: Add test cases.
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := unmarshalTunnel(tt.args.reader)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("unmarshalTunnel() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("unmarshalTunnel() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
-func TestUnmarshalTunnelOk(t *testing.T) {
-
-	jsonBody := `{"success": true, "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}`
-	expected := Tunnel{
-		ID:          uuid.Nil,
-		Name:        "test",
-		CreatedAt:   time.Time{},
-		Connections: []Connection{},
-	}
-	actual, err := unmarshalTunnel(bytes.NewReader([]byte(jsonBody)))
-	assert.NoError(t, err)
-	assert.Equal(t, &expected, actual)
-}
-
-func TestUnmarshalTunnelErr(t *testing.T) {
-
-	tests := []string{
-		`abc`,
-		`{"success": true, "result": abc}`,
-		`{"success": false, "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}}`,
-		`{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}], "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}}`,
-	}
-
-	for i, test := range tests {
-		_, err := unmarshalTunnel(bytes.NewReader([]byte(test)))
-		assert.Error(t, err, fmt.Sprintf("Test #%v failed", i))
-	}
-}
-
-func TestUnmarshalConnections(t *testing.T) {
-	jsonBody := `{"success":true,"messages":[],"errors":[],"result":[{"id":"d4041254-91e3-4deb-bd94-b46e11680b1e","features":["ha-origin"],"version":"2021.2.5","arch":"darwin_amd64","conns":[{"colo_name":"LIS","id":"ac2286e5-c708-4588-a6a0-ba6b51940019","is_pending_reconnect":false,"origin_ip":"148.38.28.2","opened_at":"0001-01-01T00:00:00Z"}],"run_at":"0001-01-01T00:00:00Z"}]}`
-	expected := ActiveClient{
-		ID:       uuid.MustParse("d4041254-91e3-4deb-bd94-b46e11680b1e"),
-		Features: []string{"ha-origin"},
-		Version:  "2021.2.5",
-		Arch:     "darwin_amd64",
-		RunAt:    time.Time{},
-		Connections: []Connection{{
-			ID:                 uuid.MustParse("ac2286e5-c708-4588-a6a0-ba6b51940019"),
-			ColoName:           "LIS",
-			IsPendingReconnect: false,
-			OriginIP:           net.ParseIP("148.38.28.2"),
-			OpenedAt:           time.Time{},
-		}},
-	}
-	actual, err := parseConnectionsDetails(bytes.NewReader([]byte(jsonBody)))
-	assert.NoError(t, err)
-	assert.Equal(t, []*ActiveClient{&expected}, actual)
-}

cfapi/ip_route.go

@@ -0,0 +1,240 @@
+package cfapi
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"path"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+)
+
+// Route is a mapping from customer's IP space to a tunnel.
+// Each route allows the customer to route eyeballs in their corporate network
+// to certain private IP ranges. Each Route represents an IP range in their
+// network, and says that eyeballs can reach that route using the corresponding
+// tunnel.
+type Route struct {
+	Network  CIDR      `json:"network"`
+	TunnelID uuid.UUID `json:"tunnel_id"`
+	// Optional field. When unset, it means the Route belongs to the default virtual network.
+	VNetID    *uuid.UUID `json:"virtual_network_id,omitempty"`
+	Comment   string     `json:"comment"`
+	CreatedAt time.Time  `json:"created_at"`
+	DeletedAt time.Time  `json:"deleted_at"`
+}
+
+// CIDR is just a newtype wrapper around net.IPNet. It adds JSON unmarshalling.
+type CIDR net.IPNet
+
+func (c CIDR) String() string {
+	n := net.IPNet(c)
+	return n.String()
+}
+
+func (c CIDR) MarshalJSON() ([]byte, error) {
+	str := c.String()
+	json, err := json.Marshal(str)
+	if err != nil {
+		return nil, errors.Wrap(err, "error serializing CIDR into JSON")
+	}
+	return json, nil
+}
+
+// UnmarshalJSON parses a JSON string into net.IPNet
+func (c *CIDR) UnmarshalJSON(data []byte) error {
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return errors.Wrap(err, "error parsing cidr string")
+	}
+	_, network, err := net.ParseCIDR(s)
+	if err != nil {
+		return errors.Wrap(err, "error parsing invalid network from backend")
+	}
+	if network == nil {
+		return fmt.Errorf("backend returned invalid network %s", s)
+	}
+	*c = CIDR(*network)
+	return nil
+}
+
+// NewRoute has all the parameters necessary to add a new route to the table.
+type NewRoute struct {
+	Network  net.IPNet
+	TunnelID uuid.UUID
+	Comment  string
+	// Optional field. If unset, backend will assume the default vnet for the account.
+	VNetID *uuid.UUID
+}
+
+// MarshalJSON handles fields with non-JSON types (e.g. net.IPNet).
+func (r NewRoute) MarshalJSON() ([]byte, error) {
+	return json.Marshal(&struct {
+		TunnelID uuid.UUID  `json:"tunnel_id"`
+		Comment  string     `json:"comment"`
+		VNetID   *uuid.UUID `json:"virtual_network_id,omitempty"`
+	}{
+		TunnelID: r.TunnelID,
+		Comment:  r.Comment,
+		VNetID:   r.VNetID,
+	})
+}
+
+// DetailedRoute is just a Route with some extra fields, e.g. TunnelName.
+type DetailedRoute struct {
+	Network  CIDR      `json:"network"`
+	TunnelID uuid.UUID `json:"tunnel_id"`
+	// Optional field. When unset, it means the DetailedRoute belongs to the default virtual network.
+	VNetID     *uuid.UUID `json:"virtual_network_id,omitempty"`
+	Comment    string     `json:"comment"`
+	CreatedAt  time.Time  `json:"created_at"`
+	DeletedAt  time.Time  `json:"deleted_at"`
+	TunnelName string     `json:"tunnel_name"`
+}
+
+// IsZero checks if DetailedRoute is the zero value.
+func (r *DetailedRoute) IsZero() bool {
+	return r.TunnelID == uuid.Nil
+}
+
+// TableString outputs a table row summarizing the route, to be used
+// when showing the user their routing table.
+func (r DetailedRoute) TableString() string {
+	deletedColumn := "-"
+	if !r.DeletedAt.IsZero() {
+		deletedColumn = r.DeletedAt.Format(time.RFC3339)
+	}
+	vnetColumn := "default"
+	if r.VNetID != nil {
+		vnetColumn = r.VNetID.String()
+	}
+
+	return fmt.Sprintf(
+		"%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
+		r.Network.String(),
+		vnetColumn,
+		r.Comment,
+		r.TunnelID,
+		r.TunnelName,
+		r.CreatedAt.Format(time.RFC3339),
+		deletedColumn,
+	)
+}
+
+type DeleteRouteParams struct {
+	Network net.IPNet
+	// Optional field. If unset, backend will assume the default vnet for the account.
+	VNetID *uuid.UUID
+}
+
+type GetRouteByIpParams struct {
+	Ip net.IP
+	// Optional field. If unset, backend will assume the default vnet for the account.
+	VNetID *uuid.UUID
+}
+
+// ListRoutes calls the Tunnelstore GET endpoint for all routes under an account.
+func (r *RESTClient) ListRoutes(filter *IpRouteFilter) ([]*DetailedRoute, error) {
+	endpoint := r.baseEndpoints.accountRoutes
+	endpoint.RawQuery = filter.Encode()
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return parseListDetailedRoutes(resp.Body)
+	}
+
+	return nil, r.statusCodeToError("list routes", resp)
+}
+
+// AddRoute calls the Tunnelstore POST endpoint for a given route.
+func (r *RESTClient) AddRoute(newRoute NewRoute) (Route, error) {
+	endpoint := r.baseEndpoints.accountRoutes
+	endpoint.Path = path.Join(endpoint.Path, "network", url.PathEscape(newRoute.Network.String()))
+	resp, err := r.sendRequest("POST", endpoint, newRoute)
+	if err != nil {
+		return Route{}, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return parseRoute(resp.Body)
+	}
+
+	return Route{}, r.statusCodeToError("add route", resp)
+}
+
+// DeleteRoute calls the Tunnelstore DELETE endpoint for a given route.
+func (r *RESTClient) DeleteRoute(params DeleteRouteParams) error {
+	endpoint := r.baseEndpoints.accountRoutes
+	endpoint.Path = path.Join(endpoint.Path, "network", url.PathEscape(params.Network.String()))
+	setVnetParam(&endpoint, params.VNetID)
+
+	resp, err := r.sendRequest("DELETE", endpoint, nil)
+	if err != nil {
+		return errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		_, err := parseRoute(resp.Body)
+		return err
+	}
+
+	return r.statusCodeToError("delete route", resp)
+}
+
+// GetByIP checks which route will proxy a given IP.
+func (r *RESTClient) GetByIP(params GetRouteByIpParams) (DetailedRoute, error) {
+	endpoint := r.baseEndpoints.accountRoutes
+	endpoint.Path = path.Join(endpoint.Path, "ip", url.PathEscape(params.Ip.String()))
+	setVnetParam(&endpoint, params.VNetID)
+
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return DetailedRoute{}, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return parseDetailedRoute(resp.Body)
+	}
+
+	return DetailedRoute{}, r.statusCodeToError("get route by IP", resp)
+}
+
+func parseListDetailedRoutes(body io.ReadCloser) ([]*DetailedRoute, error) {
+	var routes []*DetailedRoute
+	err := parseResponse(body, &routes)
+	return routes, err
+}
+
+func parseRoute(body io.ReadCloser) (Route, error) {
+	var route Route
+	err := parseResponse(body, &route)
+	return route, err
+}
+
+func parseDetailedRoute(body io.ReadCloser) (DetailedRoute, error) {
+	var route DetailedRoute
+	err := parseResponse(body, &route)
+	return route, err
+}
+
+// setVnetParam overwrites the URL's query parameters with a query param to scope the HostnameRoute action to a certain
+// virtual network (if one is provided).
+func setVnetParam(endpoint *url.URL, vnetID *uuid.UUID) {
+	queryParams := url.Values{}
+	if vnetID != nil {
+		queryParams.Set("virtual_network_id", vnetID.String())
+	}
+	endpoint.RawQuery = queryParams.Encode()
+}

cfapi/ip_route_filter.go

@@ -0,0 +1,165 @@
+package cfapi
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	filterIpRouteDeleted = cli.BoolFlag{
+		Name:  "filter-is-deleted",
+		Usage: "If false (default), only show non-deleted routes. If true, only show deleted routes.",
+	}
+	filterIpRouteTunnelID = cli.StringFlag{
+		Name:  "filter-tunnel-id",
+		Usage: "Show only routes with the given tunnel ID.",
+	}
+	filterSubsetIpRoute = cli.StringFlag{
+		Name:    "filter-network-is-subset-of",
+		Aliases: []string{"nsub"},
+		Usage:   "Show only routes whose network is a subset of the given network.",
+	}
+	filterSupersetIpRoute = cli.StringFlag{
+		Name:    "filter-network-is-superset-of",
+		Aliases: []string{"nsup"},
+		Usage:   "Show only routes whose network is a superset of the given network.",
+	}
+	filterIpRouteComment = cli.StringFlag{
+		Name:  "filter-comment-is",
+		Usage: "Show only routes with this comment.",
+	}
+	filterIpRouteByVnet = cli.StringFlag{
+		Name:  "filter-vnet-id",
+		Usage: "Show only routes that are attached to the given virtual network ID.",
+	}
+
+	// Flags contains all filter flags.
+	IpRouteFilterFlags = []cli.Flag{
+		&filterIpRouteDeleted,
+		&filterIpRouteTunnelID,
+		&filterSubsetIpRoute,
+		&filterSupersetIpRoute,
+		&filterIpRouteComment,
+		&filterIpRouteByVnet,
+	}
+)
+
+// IpRouteFilter which routes get queried.
+type IpRouteFilter struct {
+	queryParams url.Values
+}
+
+// NewIpRouteFilterFromCLI parses CLI flags to discover which filters should get applied.
+func NewIpRouteFilterFromCLI(c *cli.Context) (*IpRouteFilter, error) {
+	f := &IpRouteFilter{
+		queryParams: url.Values{},
+	}
+
+	// Set deletion filter
+	if flag := filterIpRouteDeleted.Name; c.IsSet(flag) && c.Bool(flag) {
+		f.deleted()
+	} else {
+		f.notDeleted()
+	}
+
+	if subset, err := cidrFromFlag(c, filterSubsetIpRoute); err != nil {
+		return nil, err
+	} else if subset != nil {
+		f.networkIsSupersetOf(*subset)
+	}
+
+	if superset, err := cidrFromFlag(c, filterSupersetIpRoute); err != nil {
+		return nil, err
+	} else if superset != nil {
+		f.networkIsSupersetOf(*superset)
+	}
+
+	if comment := c.String(filterIpRouteComment.Name); comment != "" {
+		f.commentIs(comment)
+	}
+
+	if tunnelID := c.String(filterIpRouteTunnelID.Name); tunnelID != "" {
+		u, err := uuid.Parse(tunnelID)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Couldn't parse UUID from %s", filterIpRouteTunnelID.Name)
+		}
+		f.tunnelID(u)
+	}
+
+	if vnetId := c.String(filterIpRouteByVnet.Name); vnetId != "" {
+		u, err := uuid.Parse(vnetId)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Couldn't parse UUID from %s", filterIpRouteByVnet.Name)
+		}
+		f.vnetID(u)
+	}
+
+	if maxFetch := c.Int("max-fetch-size"); maxFetch > 0 {
+		f.MaxFetchSize(uint(maxFetch))
+	}
+
+	return f, nil
+}
+
+// Parses a CIDR from the flag. If the flag was unset, returns (nil, nil).
+func cidrFromFlag(c *cli.Context, flag cli.StringFlag) (*net.IPNet, error) {
+	if !c.IsSet(flag.Name) {
+		return nil, nil
+	}
+
+	_, subset, err := net.ParseCIDR(c.String(flag.Name))
+	if err != nil {
+		return nil, err
+	} else if subset == nil {
+		return nil, fmt.Errorf("Invalid CIDR supplied for %s", flag.Name)
+	}
+
+	return subset, nil
+}
+
+func (f *IpRouteFilter) commentIs(comment string) {
+	f.queryParams.Set("comment", comment)
+}
+
+func (f *IpRouteFilter) notDeleted() {
+	f.queryParams.Set("is_deleted", "false")
+}
+
+func (f *IpRouteFilter) deleted() {
+	f.queryParams.Set("is_deleted", "true")
+}
+
+func (f *IpRouteFilter) networkIsSubsetOf(superset net.IPNet) {
+	f.queryParams.Set("network_subset", superset.String())
+}
+
+func (f *IpRouteFilter) networkIsSupersetOf(subset net.IPNet) {
+	f.queryParams.Set("network_superset", subset.String())
+}
+
+func (f *IpRouteFilter) existedAt(existedAt time.Time) {
+	f.queryParams.Set("existed_at", existedAt.Format(time.RFC3339))
+}
+
+func (f *IpRouteFilter) tunnelID(id uuid.UUID) {
+	f.queryParams.Set("tunnel_id", id.String())
+}
+
+func (f *IpRouteFilter) vnetID(id uuid.UUID) {
+	f.queryParams.Set("virtual_network_id", id.String())
+}
+
+func (f *IpRouteFilter) MaxFetchSize(max uint) {
+	f.queryParams.Set("per_page", strconv.Itoa(int(max)))
+}
+
+func (f IpRouteFilter) Encode() string {
+	return f.queryParams.Encode()
+}

cfapi/ip_route_test.go

@@ -0,0 +1,175 @@
+package cfapi
+
+import (
+	"encoding/json"
+	"fmt"
+	"net"
+	"strings"
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUnmarshalRoute(t *testing.T) {
+	testCases := []struct {
+		Json    string
+		HasVnet bool
+	}{
+		{
+			`{
+				"network":"10.1.2.40/29",
+				"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
+				"comment":"test",
+				"created_at":"2020-12-22T02:00:15.587008Z",
+				"deleted_at":null
+			}`,
+			false,
+		},
+		{
+			`{
+				"network":"10.1.2.40/29",
+				"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
+				"comment":"test",
+				"created_at":"2020-12-22T02:00:15.587008Z",
+				"deleted_at":null,
+				"virtual_network_id":"38c95083-8191-4110-8339-3f438d44fdb9"
+			}`,
+			true,
+		},
+	}
+
+	for _, testCase := range testCases {
+		data := testCase.Json
+
+		var r Route
+		err := json.Unmarshal([]byte(data), &r)
+
+		// Check everything worked
+		require.NoError(t, err)
+		require.Equal(t, uuid.MustParse("fba6ffea-807f-4e7a-a740-4184ee1b82c8"), r.TunnelID)
+		require.Equal(t, "test", r.Comment)
+		_, cidr, err := net.ParseCIDR("10.1.2.40/29")
+		require.NoError(t, err)
+		require.Equal(t, CIDR(*cidr), r.Network)
+		require.Equal(t, "test", r.Comment)
+
+		if testCase.HasVnet {
+			require.Equal(t, uuid.MustParse("38c95083-8191-4110-8339-3f438d44fdb9"), *r.VNetID)
+		} else {
+			require.Nil(t, r.VNetID)
+		}
+	}
+}
+
+func TestDetailedRouteJsonRoundtrip(t *testing.T) {
+	testCases := []struct {
+		Json    string
+		HasVnet bool
+	}{
+		{
+			`{
+				"network":"10.1.2.40/29",
+				"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
+				"comment":"test",
+				"created_at":"2020-12-22T02:00:15.587008Z",
+				"deleted_at":"2021-01-14T05:01:42.183002Z",
+				"tunnel_name":"Mr. Tun"
+			}`,
+			false,
+		},
+		{
+			`{
+				"network":"10.1.2.40/29",
+				"tunnel_id":"fba6ffea-807f-4e7a-a740-4184ee1b82c8",
+				"virtual_network_id":"38c95083-8191-4110-8339-3f438d44fdb9",
+				"comment":"test",
+				"created_at":"2020-12-22T02:00:15.587008Z",
+				"deleted_at":"2021-01-14T05:01:42.183002Z",
+				"tunnel_name":"Mr. Tun"
+			}`,
+			true,
+		},
+	}
+
+	for _, testCase := range testCases {
+		data := testCase.Json
+
+		var r DetailedRoute
+		err := json.Unmarshal([]byte(data), &r)
+
+		// Check everything worked
+		require.NoError(t, err)
+		require.Equal(t, uuid.MustParse("fba6ffea-807f-4e7a-a740-4184ee1b82c8"), r.TunnelID)
+		require.Equal(t, "test", r.Comment)
+		_, cidr, err := net.ParseCIDR("10.1.2.40/29")
+		require.NoError(t, err)
+		require.Equal(t, CIDR(*cidr), r.Network)
+		require.Equal(t, "test", r.Comment)
+		require.Equal(t, "Mr. Tun", r.TunnelName)
+
+		if testCase.HasVnet {
+			require.Equal(t, uuid.MustParse("38c95083-8191-4110-8339-3f438d44fdb9"), *r.VNetID)
+		} else {
+			require.Nil(t, r.VNetID)
+		}
+
+		bytes, err := json.Marshal(r)
+		require.NoError(t, err)
+		obtainedJson := string(bytes)
+		data = strings.Replace(data, "\t", "", -1)
+		data = strings.Replace(data, "\n", "", -1)
+		require.Equal(t, data, obtainedJson)
+	}
+}
+
+func TestMarshalNewRoute(t *testing.T) {
+	_, network, err := net.ParseCIDR("1.2.3.4/32")
+	require.NoError(t, err)
+	require.NotNil(t, network)
+	vnetId := uuid.New()
+
+	newRoutes := []NewRoute{
+		{
+			Network:  *network,
+			TunnelID: uuid.New(),
+			Comment:  "hi",
+		},
+		{
+			Network:  *network,
+			TunnelID: uuid.New(),
+			Comment:  "hi",
+			VNetID:   &vnetId,
+		},
+	}
+
+	for _, newRoute := range newRoutes {
+		// Test where receiver is struct
+		serialized, err := json.Marshal(newRoute)
+		require.NoError(t, err)
+		require.True(t, strings.Contains(string(serialized), "tunnel_id"))
+
+		// Test where receiver is pointer to struct
+		serialized, err = json.Marshal(&newRoute)
+		require.NoError(t, err)
+		require.True(t, strings.Contains(string(serialized), "tunnel_id"))
+
+		if newRoute.VNetID == nil {
+			require.False(t, strings.Contains(string(serialized), "virtual_network_id"))
+		} else {
+			require.True(t, strings.Contains(string(serialized), "virtual_network_id"))
+		}
+	}
+}
+
+func TestRouteTableString(t *testing.T) {
+	_, network, err := net.ParseCIDR("1.2.3.4/32")
+	require.NoError(t, err)
+	require.NotNil(t, network)
+	r := DetailedRoute{
+		Network: CIDR(*network),
+	}
+	row := r.TableString()
+	fmt.Println(row)
+	require.True(t, strings.HasPrefix(row, "1.2.3.4/32"))
+}

cfapi/tunnel.go

@@ -0,0 +1,183 @@
+package cfapi
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"path"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+)
+
+var ErrTunnelNameConflict = errors.New("tunnel with name already exists")
+
+type Tunnel struct {
+	ID          uuid.UUID    `json:"id"`
+	Name        string       `json:"name"`
+	CreatedAt   time.Time    `json:"created_at"`
+	DeletedAt   time.Time    `json:"deleted_at"`
+	Connections []Connection `json:"connections"`
+}
+
+type Connection struct {
+	ColoName           string    `json:"colo_name"`
+	ID                 uuid.UUID `json:"id"`
+	IsPendingReconnect bool      `json:"is_pending_reconnect"`
+	OriginIP           net.IP    `json:"origin_ip"`
+	OpenedAt           time.Time `json:"opened_at"`
+}
+
+type ActiveClient struct {
+	ID          uuid.UUID    `json:"id"`
+	Features    []string     `json:"features"`
+	Version     string       `json:"version"`
+	Arch        string       `json:"arch"`
+	RunAt       time.Time    `json:"run_at"`
+	Connections []Connection `json:"conns"`
+}
+
+type newTunnel struct {
+	Name         string `json:"name"`
+	TunnelSecret []byte `json:"tunnel_secret"`
+}
+
+type CleanupParams struct {
+	queryParams url.Values
+}
+
+func NewCleanupParams() *CleanupParams {
+	return &CleanupParams{
+		queryParams: url.Values{},
+	}
+}
+
+func (cp *CleanupParams) ForClient(clientID uuid.UUID) {
+	cp.queryParams.Set("client_id", clientID.String())
+}
+
+func (cp CleanupParams) encode() string {
+	return cp.queryParams.Encode()
+}
+
+func (r *RESTClient) CreateTunnel(name string, tunnelSecret []byte) (*Tunnel, error) {
+	if name == "" {
+		return nil, errors.New("tunnel name required")
+	}
+	if _, err := uuid.Parse(name); err == nil {
+		return nil, errors.New("you cannot use UUIDs as tunnel names")
+	}
+	body := &newTunnel{
+		Name:         name,
+		TunnelSecret: tunnelSecret,
+	}
+
+	resp, err := r.sendRequest("POST", r.baseEndpoints.accountLevel, body)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	switch resp.StatusCode {
+	case http.StatusOK:
+		return unmarshalTunnel(resp.Body)
+	case http.StatusConflict:
+		return nil, ErrTunnelNameConflict
+	}
+
+	return nil, r.statusCodeToError("create tunnel", resp)
+}
+
+func (r *RESTClient) GetTunnel(tunnelID uuid.UUID) (*Tunnel, error) {
+	endpoint := r.baseEndpoints.accountLevel
+	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v", tunnelID))
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return unmarshalTunnel(resp.Body)
+	}
+
+	return nil, r.statusCodeToError("get tunnel", resp)
+}
+
+func (r *RESTClient) DeleteTunnel(tunnelID uuid.UUID) error {
+	endpoint := r.baseEndpoints.accountLevel
+	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v", tunnelID))
+	resp, err := r.sendRequest("DELETE", endpoint, nil)
+	if err != nil {
+		return errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	return r.statusCodeToError("delete tunnel", resp)
+}
+
+func (r *RESTClient) ListTunnels(filter *TunnelFilter) ([]*Tunnel, error) {
+	endpoint := r.baseEndpoints.accountLevel
+	endpoint.RawQuery = filter.encode()
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return parseListTunnels(resp.Body)
+	}
+
+	return nil, r.statusCodeToError("list tunnels", resp)
+}
+
+func parseListTunnels(body io.ReadCloser) ([]*Tunnel, error) {
+	var tunnels []*Tunnel
+	err := parseResponse(body, &tunnels)
+	return tunnels, err
+}
+
+func (r *RESTClient) ListActiveClients(tunnelID uuid.UUID) ([]*ActiveClient, error) {
+	endpoint := r.baseEndpoints.accountLevel
+	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/connections", tunnelID))
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return parseConnectionsDetails(resp.Body)
+	}
+
+	return nil, r.statusCodeToError("list connection details", resp)
+}
+
+func parseConnectionsDetails(reader io.Reader) ([]*ActiveClient, error) {
+	var clients []*ActiveClient
+	err := parseResponse(reader, &clients)
+	return clients, err
+}
+
+func (r *RESTClient) CleanupConnections(tunnelID uuid.UUID, params *CleanupParams) error {
+	endpoint := r.baseEndpoints.accountLevel
+	endpoint.RawQuery = params.encode()
+	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/connections", tunnelID))
+	resp, err := r.sendRequest("DELETE", endpoint, nil)
+	if err != nil {
+		return errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	return r.statusCodeToError("cleanup connections", resp)
+}
+
+func unmarshalTunnel(reader io.Reader) (*Tunnel, error) {
+	var tunnel Tunnel
+	err := parseResponse(reader, &tunnel)
+	return &tunnel, err
+}

cfapi/tunnel_filter.go

@@ -0,0 +1,55 @@
+package cfapi
+
+import (
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+const (
+	TimeLayout = time.RFC3339
+)
+
+type TunnelFilter struct {
+	queryParams url.Values
+}
+
+func NewTunnelFilter() *TunnelFilter {
+	return &TunnelFilter{
+		queryParams: url.Values{},
+	}
+}
+
+func (f *TunnelFilter) ByName(name string) {
+	f.queryParams.Set("name", name)
+}
+
+func (f *TunnelFilter) ByNamePrefix(namePrefix string) {
+	f.queryParams.Set("name_prefix", namePrefix)
+}
+
+func (f *TunnelFilter) ExcludeNameWithPrefix(excludePrefix string) {
+	f.queryParams.Set("exclude_prefix", excludePrefix)
+}
+
+func (f *TunnelFilter) NoDeleted() {
+	f.queryParams.Set("is_deleted", "false")
+}
+
+func (f *TunnelFilter) ByExistedAt(existedAt time.Time) {
+	f.queryParams.Set("existed_at", existedAt.Format(TimeLayout))
+}
+
+func (f *TunnelFilter) ByTunnelID(tunnelID uuid.UUID) {
+	f.queryParams.Set("uuid", tunnelID.String())
+}
+
+func (f *TunnelFilter) MaxFetchSize(max uint) {
+	f.queryParams.Set("per_page", strconv.Itoa(int(max)))
+}
+
+func (f TunnelFilter) encode() string {
+	return f.queryParams.Encode()
+}

cfapi/tunnel_test.go

@@ -0,0 +1,149 @@
+package cfapi
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+)
+
+var loc, _ = time.LoadLocation("UTC")
+
+func Test_parseListTunnels(t *testing.T) {
+	type args struct {
+		body string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []*Tunnel
+		wantErr bool
+	}{
+		{
+			name: "empty list",
+			args: args{body: `{"success": true, "result": []}`},
+			want: []*Tunnel{},
+		},
+		{
+			name:    "success is false",
+			args:    args{body: `{"success": false, "result": []}`},
+			wantErr: true,
+		},
+		{
+			name:    "errors are present",
+			args:    args{body: `{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}], "result": []}`},
+			wantErr: true,
+		},
+		{
+			name:    "invalid response",
+			args:    args{body: `abc`},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			body := ioutil.NopCloser(bytes.NewReader([]byte(tt.args.body)))
+			got, err := parseListTunnels(body)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseListTunnels() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("parseListTunnels() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_unmarshalTunnel(t *testing.T) {
+	type args struct {
+		body string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *Tunnel
+		wantErr bool
+	}{
+		{
+			name: "empty list",
+			args: args{body: `{"success": true, "result": {"id":"b34cc7ce-925b-46ee-bc23-4cb5c18d8292","created_at":"2021-07-29T13:46:14.090955Z","deleted_at":"2021-07-29T14:07:27.559047Z","name":"qt-bIWWN7D662ogh61pCPfu5s2XgqFY1OyV","account_id":6946212,"account_tag":"5ab4e9dfbd435d24068829fda0077963","conns_active_at":null,"conns_inactive_at":"2021-07-29T13:47:22.548482Z","tun_type":"cfd_tunnel","metadata":{"qtid":"a6fJROgkXutNruBGaJjD"}}}`},
+			want: &Tunnel{
+				ID:          uuid.MustParse("b34cc7ce-925b-46ee-bc23-4cb5c18d8292"),
+				Name:        "qt-bIWWN7D662ogh61pCPfu5s2XgqFY1OyV",
+				CreatedAt:   time.Date(2021, 07, 29, 13, 46, 14, 90955000, loc),
+				DeletedAt:   time.Date(2021, 07, 29, 14, 7, 27, 559047000, loc),
+				Connections: nil,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := unmarshalTunnel(strings.NewReader(tt.args.body))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("unmarshalTunnel() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("unmarshalTunnel() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestUnmarshalTunnelOk(t *testing.T) {
+
+	jsonBody := `{"success": true, "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}`
+	expected := Tunnel{
+		ID:          uuid.Nil,
+		Name:        "test",
+		CreatedAt:   time.Time{},
+		Connections: []Connection{},
+	}
+	actual, err := unmarshalTunnel(bytes.NewReader([]byte(jsonBody)))
+	assert.NoError(t, err)
+	assert.Equal(t, &expected, actual)
+}
+
+func TestUnmarshalTunnelErr(t *testing.T) {
+
+	tests := []string{
+		`abc`,
+		`{"success": true, "result": abc}`,
+		`{"success": false, "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}}`,
+		`{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}], "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}}`,
+	}
+
+	for i, test := range tests {
+		_, err := unmarshalTunnel(bytes.NewReader([]byte(test)))
+		assert.Error(t, err, fmt.Sprintf("Test #%v failed", i))
+	}
+}
+
+func TestUnmarshalConnections(t *testing.T) {
+	jsonBody := `{"success":true,"messages":[],"errors":[],"result":[{"id":"d4041254-91e3-4deb-bd94-b46e11680b1e","features":["ha-origin"],"version":"2021.2.5","arch":"darwin_amd64","conns":[{"colo_name":"LIS","id":"ac2286e5-c708-4588-a6a0-ba6b51940019","is_pending_reconnect":false,"origin_ip":"148.38.28.2","opened_at":"0001-01-01T00:00:00Z"}],"run_at":"0001-01-01T00:00:00Z"}]}`
+	expected := ActiveClient{
+		ID:       uuid.MustParse("d4041254-91e3-4deb-bd94-b46e11680b1e"),
+		Features: []string{"ha-origin"},
+		Version:  "2021.2.5",
+		Arch:     "darwin_amd64",
+		RunAt:    time.Time{},
+		Connections: []Connection{{
+			ID:                 uuid.MustParse("ac2286e5-c708-4588-a6a0-ba6b51940019"),
+			ColoName:           "LIS",
+			IsPendingReconnect: false,
+			OriginIP:           net.ParseIP("148.38.28.2"),
+			OpenedAt:           time.Time{},
+		}},
+	}
+	actual, err := parseConnectionsDetails(bytes.NewReader([]byte(jsonBody)))
+	assert.NoError(t, err)
+	assert.Equal(t, []*ActiveClient{&expected}, actual)
+}

cfapi/virtual_network.go

@@ -0,0 +1,127 @@
+package cfapi
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"strconv"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+)
+
+type NewVirtualNetwork struct {
+	Name      string `json:"name"`
+	Comment   string `json:"comment"`
+	IsDefault bool   `json:"is_default"`
+}
+
+type VirtualNetwork struct {
+	ID        uuid.UUID `json:"id"`
+	Comment   string    `json:"comment"`
+	Name      string    `json:"name"`
+	IsDefault bool      `json:"is_default_network"`
+	CreatedAt time.Time `json:"created_at"`
+	DeletedAt time.Time `json:"deleted_at"`
+}
+
+type UpdateVirtualNetwork struct {
+	Name      *string `json:"name,omitempty"`
+	Comment   *string `json:"comment,omitempty"`
+	IsDefault *bool   `json:"is_default_network,omitempty"`
+}
+
+func (virtualNetwork VirtualNetwork) TableString() string {
+	deletedColumn := "-"
+	if !virtualNetwork.DeletedAt.IsZero() {
+		deletedColumn = virtualNetwork.DeletedAt.Format(time.RFC3339)
+	}
+	return fmt.Sprintf(
+		"%s\t%s\t%s\t%s\t%s\t%s\t",
+		virtualNetwork.ID,
+		virtualNetwork.Name,
+		strconv.FormatBool(virtualNetwork.IsDefault),
+		virtualNetwork.Comment,
+		virtualNetwork.CreatedAt.Format(time.RFC3339),
+		deletedColumn,
+	)
+}
+
+func (r *RESTClient) CreateVirtualNetwork(newVnet NewVirtualNetwork) (VirtualNetwork, error) {
+	resp, err := r.sendRequest("POST", r.baseEndpoints.accountVnets, newVnet)
+	if err != nil {
+		return VirtualNetwork{}, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return parseVnet(resp.Body)
+	}
+
+	return VirtualNetwork{}, r.statusCodeToError("add virtual network", resp)
+}
+
+func (r *RESTClient) ListVirtualNetworks(filter *VnetFilter) ([]*VirtualNetwork, error) {
+	endpoint := r.baseEndpoints.accountVnets
+	endpoint.RawQuery = filter.Encode()
+	resp, err := r.sendRequest("GET", endpoint, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		return parseListVnets(resp.Body)
+	}
+
+	return nil, r.statusCodeToError("list virtual networks", resp)
+}
+
+func (r *RESTClient) DeleteVirtualNetwork(id uuid.UUID) error {
+	endpoint := r.baseEndpoints.accountVnets
+	endpoint.Path = path.Join(endpoint.Path, url.PathEscape(id.String()))
+	resp, err := r.sendRequest("DELETE", endpoint, nil)
+	if err != nil {
+		return errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		_, err := parseVnet(resp.Body)
+		return err
+	}
+
+	return r.statusCodeToError("delete virtual network", resp)
+}
+
+func (r *RESTClient) UpdateVirtualNetwork(id uuid.UUID, updates UpdateVirtualNetwork) error {
+	endpoint := r.baseEndpoints.accountVnets
+	endpoint.Path = path.Join(endpoint.Path, url.PathEscape(id.String()))
+	resp, err := r.sendRequest("PATCH", endpoint, updates)
+	if err != nil {
+		return errors.Wrap(err, "REST request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == http.StatusOK {
+		_, err := parseVnet(resp.Body)
+		return err
+	}
+
+	return r.statusCodeToError("update virtual network", resp)
+}
+
+func parseListVnets(body io.ReadCloser) ([]*VirtualNetwork, error) {
+	var vnets []*VirtualNetwork
+	err := parseResponse(body, &vnets)
+	return vnets, err
+}
+
+func parseVnet(body io.ReadCloser) (VirtualNetwork, error) {
+	var vnet VirtualNetwork
+	err := parseResponse(body, &vnet)
+	return vnet, err
+}

cfapi/virtual_network_filter.go

@@ -0,0 +1,99 @@
+package cfapi
+
+import (
+	"net/url"
+	"strconv"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	filterVnetId = cli.StringFlag{
+		Name:  "id",
+		Usage: "List virtual networks with the given `ID`",
+	}
+	filterVnetByName = cli.StringFlag{
+		Name:  "name",
+		Usage: "List virtual networks with the given `NAME`",
+	}
+	filterDefaultVnet = cli.BoolFlag{
+		Name:  "is-default",
+		Usage: "If true, lists the virtual network that is the default one. If false, lists all non-default virtual networks for the account. If absent, all are included in the results regardless of their default status.",
+	}
+	filterDeletedVnet = cli.BoolFlag{
+		Name:  "show-deleted",
+		Usage: "If false (default), only show non-deleted virtual networks. If true, only show deleted virtual networks.",
+	}
+	VnetFilterFlags = []cli.Flag{
+		&filterVnetId,
+		&filterVnetByName,
+		&filterDefaultVnet,
+		&filterDeletedVnet,
+	}
+)
+
+// VnetFilter which virtual networks get queried.
+type VnetFilter struct {
+	queryParams url.Values
+}
+
+func NewVnetFilter() *VnetFilter {
+	return &VnetFilter{
+		queryParams: url.Values{},
+	}
+}
+
+func (f *VnetFilter) ById(vnetId uuid.UUID) {
+	f.queryParams.Set("id", vnetId.String())
+}
+
+func (f *VnetFilter) ByName(name string) {
+	f.queryParams.Set("name", name)
+}
+
+func (f *VnetFilter) ByDefaultStatus(isDefault bool) {
+	f.queryParams.Set("is_default", strconv.FormatBool(isDefault))
+}
+
+func (f *VnetFilter) WithDeleted(isDeleted bool) {
+	f.queryParams.Set("is_deleted", strconv.FormatBool(isDeleted))
+}
+
+func (f *VnetFilter) MaxFetchSize(max uint) {
+	f.queryParams.Set("per_page", strconv.Itoa(int(max)))
+}
+
+func (f VnetFilter) Encode() string {
+	return f.queryParams.Encode()
+}
+
+// NewFromCLI parses CLI flags to discover which filters should get applied to list virtual networks.
+func NewFromCLI(c *cli.Context) (*VnetFilter, error) {
+	f := NewVnetFilter()
+
+	if id := c.String("id"); id != "" {
+		vnetId, err := uuid.Parse(id)
+		if err != nil {
+			return nil, errors.Wrapf(err, "%s is not a valid virtual network ID", id)
+		}
+		f.ById(vnetId)
+	}
+
+	if name := c.String("name"); name != "" {
+		f.ByName(name)
+	}
+
+	if c.IsSet("is-default") {
+		f.ByDefaultStatus(c.Bool("is-default"))
+	}
+
+	f.WithDeleted(c.Bool("show-deleted"))
+
+	if maxFetch := c.Int("max-fetch-size"); maxFetch > 0 {
+		f.MaxFetchSize(uint(maxFetch))
+	}
+
+	return f, nil
+}

cfapi/virtual_network_test.go

@@ -0,0 +1,79 @@
+package cfapi
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/require"
+)
+
+func TestVirtualNetworkJsonRoundtrip(t *testing.T) {
+	data := `{
+		"id":"74fce949-351b-4752-b261-81a56cfd3130",
+		"comment":"New York DC1",
+		"name":"us-east-1",
+		"is_default_network":true,
+		"created_at":"2021-11-26T14:40:02.600673Z",
+		"deleted_at":"2021-12-01T10:23:13.102645Z"
+	}`
+	var v VirtualNetwork
+	err := json.Unmarshal([]byte(data), &v)
+
+	require.NoError(t, err)
+	require.Equal(t, uuid.MustParse("74fce949-351b-4752-b261-81a56cfd3130"), v.ID)
+	require.Equal(t, "us-east-1", v.Name)
+	require.Equal(t, "New York DC1", v.Comment)
+	require.Equal(t, true, v.IsDefault)
+
+	bytes, err := json.Marshal(v)
+	require.NoError(t, err)
+	obtainedJson := string(bytes)
+	data = strings.Replace(data, "\t", "", -1)
+	data = strings.Replace(data, "\n", "", -1)
+	require.Equal(t, data, obtainedJson)
+}
+
+func TestMarshalNewVnet(t *testing.T) {
+	newVnet := NewVirtualNetwork{
+		Name:      "eu-west-1",
+		Comment:   "London office",
+		IsDefault: true,
+	}
+
+	serialized, err := json.Marshal(newVnet)
+	require.NoError(t, err)
+	require.True(t, strings.Contains(string(serialized), newVnet.Name))
+}
+
+func TestMarshalUpdateVnet(t *testing.T) {
+	newName := "bulgaria-1"
+	updates := UpdateVirtualNetwork{
+		Name: &newName,
+	}
+
+	// Test where receiver is struct
+	serialized, err := json.Marshal(updates)
+	require.NoError(t, err)
+	require.True(t, strings.Contains(string(serialized), newName))
+}
+
+func TestVnetTableString(t *testing.T) {
+	virtualNet := VirtualNetwork{
+		ID:        uuid.New(),
+		Name:      "us-east-1",
+		Comment:   "New York DC1",
+		IsDefault: true,
+		CreatedAt: time.Now(),
+		DeletedAt: time.Time{},
+	}
+
+	row := virtualNet.TableString()
+	require.True(t, strings.HasPrefix(row, virtualNet.ID.String()))
+	require.True(t, strings.Contains(row, virtualNet.Name))
+	require.True(t, strings.Contains(row, virtualNet.Comment))
+	require.True(t, strings.Contains(row, "true"))
+	require.True(t, strings.HasSuffix(row, "-\t"))
+}

cfsetup.yaml

@@ -1,20 +1,10 @@
-pinned_go: &pinned_go go=1.17-1
-pinned_go_fips: &pinned_go_fips go-boring=1.16.6-6
+pinned_go: &pinned_go go=1.17.5-1
+pinned_go_fips: &pinned_go_fips go-boring=1.17.5-1
 
 build_dir: &build_dir /cfsetup_build
 default-flavor: buster
 stretch: &stretch
   build:
-    build_dir: *build_dir
-    builddeps:
-      - *pinned_go_fips
-      - build-essential
-    post-cache:
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - export FIPS=true
-      - make cloudflared
-  build-non-fips:  # helpful to catch problems with non-fips (only used for releasing non-linux artifacts) before releases
     build_dir: *build_dir
     builddeps:
       - *pinned_go
@@ -23,27 +13,45 @@ stretch: &stretch
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared
-  build-all-packages: #except osxpkg
+  build-fips:
     build_dir: *build_dir
     builddeps:
       - *pinned_go_fips
       - build-essential
-      - fakeroot
-      - rubygem-fpm
-      - rpm
-      - wget
-      # libmsi and libgcab are libraries the wixl binary depends on.
-      - libmsi-dev
-      - libgcab-dev
-    pre-cache:
-      # TODO: https://jira.cfops.it/browse/TUN-4792 Replace this wixl with the official one once msitools supports
-      # environment.
-      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
-      - chmod a+x /usr/local/bin/wixl
     post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
       - export FIPS=true
-      - ./build-packages.sh
+      - make cloudflared
+  # except FIPS (handled in github-fips-release-pkgs) and macos (handled in github-release-macos-amd64)
   github-release-pkgs:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+      - rpm
+      - wget
+      # libmsi and libgcab are libraries the wixl binary depends on.
+      - libmsi-dev
+      - libgcab-dev
+      - python3-dev
+      - libffi-dev
+      - python3-setuptools
+      - python3-pip
+    pre-cache: &github_release_pkgs_pre_cache
+      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
+      - chmod a+x /usr/local/bin/wixl
+      - pip3 install pynacl==1.4.0
+      - pip3 install pygithub==1.55
+    post-cache:
+      # build all packages (except macos and FIPS) and move them to /cfsetup/built_artifacts
+      - ./build-packages.sh
+      # release the packages built and moved to /cfsetup/built_artifacts
+      - make github-release-built-pkgs
+  # handle FIPS separately so that we built with gofips compiler
+  github-fips-release-pkgs:
     build_dir: *build_dir
     builddeps:
       - *pinned_go_fips
@@ -55,20 +63,29 @@ stretch: &stretch
       # libmsi and libgcab are libraries the wixl binary depends on.
       - libmsi-dev
       - libgcab-dev
+      - python3-dev
+      - libffi-dev
       - python3-setuptools
       - python3-pip
-    pre-cache:
-      - wget https://github.com/sudarshan-reddy/msitools/releases/download/v0.101b/wixl -P /usr/local/bin
-      - chmod a+x /usr/local/bin/wixl
-      - pip3 install pygithub
+    pre-cache: *github_release_pkgs_pre_cache
     post-cache:
-      # build all packages and move them to /cfsetup/built_artifacts
-      - ./build-packages.sh
-      # release the packages built and moved to /cfsetup/built_artifacts
+      # same logic as above, but for FIPS packages only
+      - ./build-packages-fips.sh
       - make github-release-built-pkgs
   build-deb:
     build_dir: *build_dir
     builddeps: &build_deb_deps
+      - *pinned_go
+      - build-essential
+      - fakeroot
+      - rubygem-fpm
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - make cloudflared-deb
+  build-fips-internal-deb:
+    build_dir: *build_dir
+    builddeps: &build_fips_deb_deps
       - *pinned_go_fips
       - build-essential
       - fakeroot
@@ -77,15 +94,17 @@ stretch: &stretch
       - export GOOS=linux
       - export GOARCH=amd64
       - export FIPS=true
+      - export ORIGINAL_NAME=true
       - make cloudflared-deb
-  build-deb-nightly:
+  build-fips-internal-deb-nightly:
     build_dir: *build_dir
-    builddeps: *build_deb_deps
+    builddeps: *build_fips_deb_deps
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
-      - export FIPS=true
       - export NIGHTLY=true
+      - export FIPS=true
+      - export ORIGINAL_NAME=true
       - make cloudflared-deb
   build-deb-arm64:
     build_dir: *build_dir
@@ -97,7 +116,7 @@ stretch: &stretch
   publish-deb:
     build_dir: *build_dir
     builddeps:
-      - *pinned_go_fips
+      - *pinned_go
       - build-essential
       - fakeroot
       - rubygem-fpm
@@ -105,27 +124,43 @@ stretch: &stretch
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
-      - export FIPS=true
       - make publish-deb
   github-release-macos-amd64:
     build_dir: *build_dir
     builddeps:
       - *pinned_go
+      - build-essential
+      - python3-dev
+      - libffi-dev
       - python3-setuptools
       - python3-pip
     pre-cache: &install_pygithub
-      - pip3 install pygithub
+      - pip3 install pynacl==1.4.0
+      - pip3 install pygithub==1.55
     post-cache:
       - make github-mac-upload
   test:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - gotest-to-teamcity
+    pre-cache: &test_pre_cache
+      - go get golang.org/x/tools/cmd/goimports
+      - go get github.com/sudarshan-reddy/go-sumtype@v0.0.0-20210827105221-82eca7e5abb1
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=amd64
+      - export PATH="$HOME/go/bin:$PATH"
+      - ./fmt-check.sh
+      - make test | gotest-to-teamcity
+  test-fips:
     build_dir: *build_dir
     builddeps:
       - *pinned_go_fips
       - build-essential
       - gotest-to-teamcity
-    pre-cache:
-      - go get golang.org/x/tools/cmd/goimports
-      - go get github.com/sudarshan-reddy/go-sumtype@v0.0.0-20210827105221-82eca7e5abb1
+    pre-cache: *test_pre_cache
     post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
@@ -150,7 +185,7 @@ stretch: &stretch
     post-cache:
       # Creates and routes a Named Tunnel for this build. Also constructs config file from env vars.
       - python3 component-tests/setup.py --type create
-      - pytest component-tests
+      - pytest component-tests -o log_cli=true --log-cli-level=INFO
       # The Named Tunnel is deleted and its route unprovisioned here.
       - python3 component-tests/setup.py --type cleanup
   update-homebrew:
@@ -163,6 +198,9 @@ stretch: &stretch
     build_dir: *build_dir
     builddeps:
       - *pinned_go
+      - build-essential
+      - python3-dev
+      - libffi-dev
       - python3-setuptools
       - python3-pip
     pre-cache: *install_pygithub
@@ -208,23 +246,10 @@ centos-7:
     pre-cache:
       - yum install -y fakeroot
       - yum upgrade -y binutils-2.27-44.base.el7.x86_64
-      - wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.16.3.linux-amd64.tar.gz
+      - wget https://go.dev/dl/go1.17.5.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.17.5.linux-amd64.tar.gz
     post-cache:
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
       - export GOARCH=amd64
-      - make publish-rpm
-  build-rpm:
-    build_dir: *build_dir
-    builddeps: *el7_builddeps
-    pre-cache:
-      - yum install -y fakeroot
-      - yum upgrade -y binutils-2.27-44.base.el7.x86_64
-      - wget https://golang.org/dl/go1.16.3.linux-amd64.tar.gz -P /tmp/
-      - tar -C /usr/local -xzf /tmp/go1.16.3.linux-amd64.tar.gz
-    post-cache:
-      - export PATH=$PATH:/usr/local/go/bin
-      - export GOOS=linux
-      - export GOARCH=amd64
-      - make cloudflared-rpm
+      - make publish-rpm

check-fips.sh

@@ -0,0 +1,15 @@
+# Pass the path to the executable to check for FIPS compliance
+exe=$1
+
+if [ "$(go tool nm "${exe}" | grep -c '_Cfunc__goboringcrypto_')" -eq 0 ]; then
+    # Asserts that executable is using FIPS-compliant boringcrypto
+    echo "${exe}: missing goboring symbols" >&2
+    exit 1
+fi
+if [ "$(go tool nm "${exe}" | grep -c 'crypto/internal/boring/sig.FIPSOnly')" -eq 0 ]; then
+    # Asserts that executable is using FIPS-only schemes
+    echo "${exe}: missing fipsonly symbols" >&2
+    exit 1
+fi
+
+echo "${exe} is FIPS-compliant"

cmd/cloudflared/access/cmd.go

@@ -240,7 +240,7 @@ func login(c *cli.Context) error {
 	return nil
 }
 
-// ensureURLScheme prepends a URL with https:// if it doesnt have a scheme. http:// URLs will not be converted.
+// ensureURLScheme prepends a URL with https:// if it doesn't have a scheme. http:// URLs will not be converted.
 func ensureURLScheme(url string) string {
 	url = strings.Replace(strings.ToLower(url), "http://", "https://", 1)
 	if !strings.HasPrefix(url, "https://") {

cmd/cloudflared/cliutil/build_info.go

@@ -1,4 +1,4 @@
-package buildinfo
+package cliutil
 
 import (
 	"fmt"
@@ -11,23 +11,39 @@ type BuildInfo struct {
 	GoOS               string `json:"go_os"`
 	GoVersion          string `json:"go_version"`
 	GoArch             string `json:"go_arch"`
+	BuildType          string `json:"build_type"`
 	CloudflaredVersion string `json:"cloudflared_version"`
 }
 
-func GetBuildInfo(cloudflaredVersion string) *BuildInfo {
+func GetBuildInfo(buildType, version string) *BuildInfo {
 	return &BuildInfo{
 		GoOS:               runtime.GOOS,
 		GoVersion:          runtime.Version(),
 		GoArch:             runtime.GOARCH,
-		CloudflaredVersion: cloudflaredVersion,
+		BuildType:          buildType,
+		CloudflaredVersion: version,
 	}
 }
 
 func (bi *BuildInfo) Log(log *zerolog.Logger) {
 	log.Info().Msgf("Version %s", bi.CloudflaredVersion)
+	if bi.BuildType != "" {
+		log.Info().Msgf("Built%s", bi.GetBuildTypeMsg())
+	}
 	log.Info().Msgf("GOOS: %s, GOVersion: %s, GoArch: %s", bi.GoOS, bi.GoVersion, bi.GoArch)
 }
 
 func (bi *BuildInfo) OSArch() string {
 	return fmt.Sprintf("%s_%s", bi.GoOS, bi.GoArch)
 }
+
+func (bi *BuildInfo) Version() string {
+	return bi.CloudflaredVersion
+}
+
+func (bi *BuildInfo) GetBuildTypeMsg() string {
+	if bi.BuildType == "" {
+		return ""
+	}
+	return fmt.Sprintf(" with %s", bi.BuildType)
+}

cmd/cloudflared/generic_service.go

@@ -1,3 +1,4 @@
+//go:build !windows && !darwin && !linux
 // +build !windows,!darwin,!linux
 
 package main

cmd/cloudflared/linux_service.go

@@ -1,3 +1,4 @@
+//go:build linux
 // +build linux
 
 package main

cmd/cloudflared/macos_service.go

@@ -1,3 +1,4 @@
+//go:build darwin
 // +build darwin
 
 package main

cmd/cloudflared/main.go

@@ -31,6 +31,7 @@ const (
 var (
 	Version   = "DEV"
 	BuildTime = "unknown"
+	BuildType = ""
 	// Mostly network errors that we don't want reported back to Sentry, this is done by substring match.
 	ignoredErrors = []string{
 		"connection reset by peer",
@@ -46,9 +47,10 @@ var (
 
 func main() {
 	rand.Seed(time.Now().UnixNano())
-	metrics.RegisterBuildInfo(BuildTime, Version)
+	metrics.RegisterBuildInfo(BuildType, BuildTime, Version)
 	raven.SetRelease(Version)
 	maxprocs.Set()
+	bInfo := cliutil.GetBuildInfo(BuildType, Version)
 
 	// Graceful shutdown channel used by the app. When closed, app must terminate gracefully.
 	// Windows service manager closes this channel when it receives stop command.
@@ -71,7 +73,7 @@ func main() {
    Terms (https://www.cloudflare.com/terms/) and Privacy Policy (https://www.cloudflare.com/privacypolicy/).`,
 		time.Now().Year(),
 	)
-	app.Version = fmt.Sprintf("%s (built %s)", Version, BuildTime)
+	app.Version = fmt.Sprintf("%s (built %s%s)", Version, BuildTime, bInfo.GetBuildTypeMsg())
 	app.Description = `cloudflared connects your machine or user identity to Cloudflare's global network.
 	You can use it to authenticate a session to reach an API behind Access, route web traffic to this machine,
 	and configure access control.
@@ -81,7 +83,7 @@ func main() {
 	app.Action = action(graceShutdownC)
 	app.Commands = commands(cli.ShowVersion)
 
-	tunnel.Init(Version, graceShutdownC) // we need this to support the tunnel sub command...
+	tunnel.Init(bInfo, graceShutdownC) // we need this to support the tunnel sub command...
 	access.Init(graceShutdownC)
 	updater.Init(Version)
 	runApp(app, graceShutdownC)

cmd/cloudflared/tunnel/cmd.go

@@ -21,7 +21,7 @@ import (
 	"github.com/urfave/cli/v2"
 	"github.com/urfave/cli/v2/altsrc"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/proxydns"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
@@ -31,11 +31,11 @@ import (
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
-	"github.com/cloudflare/cloudflared/origin"
+	"github.com/cloudflare/cloudflared/orchestration"
 	"github.com/cloudflare/cloudflared/signal"
+	"github.com/cloudflare/cloudflared/supervisor"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunneldns"
-	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 const (
@@ -86,7 +86,7 @@ const (
 
 var (
 	graceShutdownC chan struct{}
-	version        string
+	buildInfo      *cliutil.BuildInfo
 
 	routeFailMsg = fmt.Sprintf("failed to provision routing, please create it manually via Cloudflare dashboard or UI; "+
 		"most likely you already have a conflicting record there. You can also rerun this command with --%s to overwrite "+
@@ -102,6 +102,7 @@ func Commands() []*cli.Command {
 		buildLoginSubcommand(false),
 		buildCreateCommand(),
 		buildRouteCommand(),
+		buildVirtualNetworkSubcommand(false),
 		buildRunCommand(),
 		buildListCommand(),
 		buildInfoCommand(),
@@ -173,15 +174,16 @@ func TunnelCommand(c *cli.Context) error {
 	return runClassicTunnel(sc)
 }
 
-func Init(ver string, gracefulShutdown chan struct{}) {
-	version, graceShutdownC = ver, gracefulShutdown
+func Init(info *cliutil.BuildInfo, gracefulShutdown chan struct{}) {
+	buildInfo, graceShutdownC = info, gracefulShutdown
 }
 
 // runAdhocNamedTunnel create, route and run a named tunnel in one command
 func runAdhocNamedTunnel(sc *subcommandContext, name, credentialsOutputPath string) error {
 	tunnel, ok, err := sc.tunnelActive(name)
 	if err != nil || !ok {
-		tunnel, err = sc.create(name, credentialsOutputPath)
+		// pass empty string as secret to generate one
+		tunnel, err = sc.create(name, credentialsOutputPath, "")
 		if err != nil {
 			return errors.Wrap(err, "failed to create tunnel")
 		}
@@ -206,23 +208,23 @@ func runAdhocNamedTunnel(sc *subcommandContext, name, credentialsOutputPath stri
 
 // runClassicTunnel creates a "classic" non-named tunnel
 func runClassicTunnel(sc *subcommandContext) error {
-	return StartServer(sc.c, version, nil, sc.log, sc.isUIEnabled)
+	return StartServer(sc.c, buildInfo, nil, sc.log, sc.isUIEnabled)
 }
 
-func routeFromFlag(c *cli.Context) (route tunnelstore.Route, ok bool) {
+func routeFromFlag(c *cli.Context) (route cfapi.HostnameRoute, ok bool) {
 	if hostname := c.String("hostname"); hostname != "" {
 		if lbPool := c.String("lb-pool"); lbPool != "" {
-			return tunnelstore.NewLBRoute(hostname, lbPool), true
+			return cfapi.NewLBRoute(hostname, lbPool), true
 		}
-		return tunnelstore.NewDNSRoute(hostname, c.Bool(overwriteDNSFlagName)), true
+		return cfapi.NewDNSRoute(hostname, c.Bool(overwriteDNSFlagName)), true
 	}
 	return nil, false
 }
 
 func StartServer(
 	c *cli.Context,
-	version string,
-	namedTunnel *connection.NamedTunnelConfig,
+	info *cliutil.BuildInfo,
+	namedTunnel *connection.NamedTunnelProperties,
 	log *zerolog.Logger,
 	isUIEnabled bool,
 ) error {
@@ -268,8 +270,7 @@ func StartServer(
 		defer trace.Stop()
 	}
 
-	buildInfo := buildinfo.GetBuildInfo(version)
-	buildInfo.Log(log)
+	info.Log(log)
 	logClientOptions(c, log)
 
 	// this context drives the server, when it's cancelled tunnel and all other components (origins, dns, etc...) should stop
@@ -333,7 +334,7 @@ func StartServer(
 		observer.SendURL(quickTunnelURL)
 	}
 
-	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, log, logTransport, observer, namedTunnel)
+	tunnelConfig, dynamicConfig, err := prepareTunnelConfig(c, info, log, logTransport, observer, namedTunnel)
 	if err != nil {
 		log.Err(err).Msg("Couldn't start tunnel")
 		return err
@@ -353,11 +354,12 @@ func StartServer(
 		errC <- metrics.ServeMetrics(metricsListener, ctx.Done(), readinessServer, quickTunnelURL, log)
 	}()
 
-	if err := ingressRules.StartOrigins(&wg, log, ctx.Done(), errC); err != nil {
+	orchestrator, err := orchestration.NewOrchestrator(ctx, dynamicConfig, tunnelConfig.Tags, tunnelConfig.Log)
+	if err != nil {
 		return err
 	}
 
-	reconnectCh := make(chan origin.ReconnectSignal, 1)
+	reconnectCh := make(chan supervisor.ReconnectSignal, 1)
 	if c.IsSet("stdin-control") {
 		log.Info().Msg("Enabling control through stdin")
 		go stdinControl(reconnectCh, log)
@@ -369,15 +371,15 @@ func StartServer(
 			wg.Done()
 			log.Info().Msg("Tunnel server stopped")
 		}()
-		errC <- origin.StartTunnelDaemon(ctx, tunnelConfig, connectedSignal, reconnectCh, graceShutdownC)
+		errC <- supervisor.StartTunnelDaemon(ctx, tunnelConfig, orchestrator, connectedSignal, reconnectCh, graceShutdownC)
 	}()
 
 	if isUIEnabled {
 		tunnelUI := ui.NewUIModel(
-			version,
+			info.Version(),
 			hostname,
 			metricsListener.Addr().String(),
-			&ingressRules,
+			dynamicConfig.Ingress,
 			tunnelConfig.HAConnections,
 		)
 		app := tunnelUI.Launch(ctx, log, logTransport)
@@ -646,6 +648,12 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  "https://api.trycloudflare.com",
 			Hidden: true,
 		}),
+		altsrc.NewIntFlag(&cli.IntFlag{
+			Name:    "max-fetch-size",
+			Usage:   `The maximum number of results that cloudflared can fetch from Cloudflare API for any listing operations needed`,
+			EnvVars: []string{"TUNNEL_MAX_FETCH_SIZE"},
+			Hidden:  true,
+		}),
 		selectProtocolFlag,
 		overwriteDNSFlag,
 	}...)
@@ -992,7 +1000,7 @@ func configureProxyDNSFlags(shouldHide bool) []cli.Flag {
 	}
 }
 
-func stdinControl(reconnectCh chan origin.ReconnectSignal, log *zerolog.Logger) {
+func stdinControl(reconnectCh chan supervisor.ReconnectSignal, log *zerolog.Logger) {
 	for {
 		scanner := bufio.NewScanner(os.Stdin)
 		for scanner.Scan() {
@@ -1003,7 +1011,7 @@ func stdinControl(reconnectCh chan origin.ReconnectSignal, log *zerolog.Logger)
 			case "":
 				break
 			case "reconnect":
-				var reconnect origin.ReconnectSignal
+				var reconnect supervisor.ReconnectSignal
 				if len(parts) > 1 {
 					var err error
 					if reconnect.Delay, err = time.ParseDuration(parts[1]); err != nil {

cmd/cloudflared/tunnel/configuration.go

@@ -16,13 +16,15 @@ import (
 	"github.com/urfave/cli/v2"
 	"golang.org/x/crypto/ssh/terminal"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/ingress"
-	"github.com/cloudflare/cloudflared/origin"
+	"github.com/cloudflare/cloudflared/orchestration"
+	"github.com/cloudflare/cloudflared/supervisor"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/validation"
@@ -86,7 +88,7 @@ func logClientOptions(c *cli.Context, log *zerolog.Logger) {
 	}
 }
 
-func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelConfig) bool {
+func dnsProxyStandAlone(c *cli.Context, namedTunnel *connection.NamedTunnelProperties) bool {
 	return c.IsSet("proxy-dns") && (!c.IsSet("hostname") && !c.IsSet("tag") && !c.IsSet("hello-world") && namedTunnel == nil)
 }
 
@@ -148,60 +150,59 @@ func getOriginCert(originCertPath string, log *zerolog.Logger) ([]byte, error) {
 
 func prepareTunnelConfig(
 	c *cli.Context,
-	buildInfo *buildinfo.BuildInfo,
-	version string,
+	info *cliutil.BuildInfo,
 	log, logTransport *zerolog.Logger,
 	observer *connection.Observer,
-	namedTunnel *connection.NamedTunnelConfig,
-) (*origin.TunnelConfig, ingress.Ingress, error) {
+	namedTunnel *connection.NamedTunnelProperties,
+) (*supervisor.TunnelConfig, *orchestration.Config, error) {
 	isNamedTunnel := namedTunnel != nil
 
 	configHostname := c.String("hostname")
 	hostname, err := validation.ValidateHostname(configHostname)
 	if err != nil {
 		log.Err(err).Str(LogFieldHostname, configHostname).Msg("Invalid hostname")
-		return nil, ingress.Ingress{}, errors.Wrap(err, "Invalid hostname")
+		return nil, nil, errors.Wrap(err, "Invalid hostname")
 	}
 	clientID := c.String("id")
 	if !c.IsSet("id") {
 		clientID, err = generateRandomClientID(log)
 		if err != nil {
-			return nil, ingress.Ingress{}, err
+			return nil, nil, err
 		}
 	}
 
 	tags, err := NewTagSliceFromCLI(c.StringSlice("tag"))
 	if err != nil {
 		log.Err(err).Msg("Tag parse failure")
-		return nil, ingress.Ingress{}, errors.Wrap(err, "Tag parse failure")
+		return nil, nil, errors.Wrap(err, "Tag parse failure")
 	}
 
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
 
 	var (
 		ingressRules  ingress.Ingress
-		classicTunnel *connection.ClassicTunnelConfig
+		classicTunnel *connection.ClassicTunnelProperties
 	)
 	cfg := config.GetConfiguration()
 	if isNamedTunnel {
 		clientUUID, err := uuid.NewRandom()
 		if err != nil {
-			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate connector UUID")
+			return nil, nil, errors.Wrap(err, "can't generate connector UUID")
 		}
 		log.Info().Msgf("Generated Connector ID: %s", clientUUID)
-		features := append(c.StringSlice("features"), origin.FeatureSerializedHeaders)
+		features := append(c.StringSlice("features"), supervisor.FeatureSerializedHeaders)
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],
 			Features: dedup(features),
-			Version:  version,
-			Arch:     buildInfo.OSArch(),
+			Version:  info.Version(),
+			Arch:     info.OSArch(),
 		}
 		ingressRules, err = ingress.ParseIngress(cfg)
 		if err != nil && err != ingress.ErrNoIngressRules {
-			return nil, ingress.Ingress{}, err
+			return nil, nil, err
 		}
 		if !ingressRules.IsEmpty() && c.IsSet("url") {
-			return nil, ingress.Ingress{}, ingress.ErrURLIncompatibleWithIngress
+			return nil, nil, ingress.ErrURLIncompatibleWithIngress
 		}
 	} else {
 
@@ -212,10 +213,10 @@ func prepareTunnelConfig(
 
 		originCert, err := getOriginCert(originCertPath, &originCertLog)
 		if err != nil {
-			return nil, ingress.Ingress{}, errors.Wrap(err, "Error getting origin cert")
+			return nil, nil, errors.Wrap(err, "Error getting origin cert")
 		}
 
-		classicTunnel = &connection.ClassicTunnelConfig{
+		classicTunnel = &connection.ClassicTunnelProperties{
 			Hostname:   hostname,
 			OriginCert: originCert,
 			// turn off use of reconnect token and auth refresh when using named tunnels
@@ -227,20 +228,14 @@ func prepareTunnelConfig(
 	if ingressRules.IsEmpty() {
 		ingressRules, err = ingress.NewSingleOrigin(c, !isNamedTunnel)
 		if err != nil {
-			return nil, ingress.Ingress{}, err
+			return nil, nil, err
 		}
 	}
 
-	var warpRoutingService *ingress.WarpRoutingService
 	warpRoutingEnabled := isWarpRoutingEnabled(cfg.WarpRouting, isNamedTunnel)
-	if warpRoutingEnabled {
-		warpRoutingService = ingress.NewWarpRoutingService()
-		log.Info().Msgf("Warp-routing is enabled")
-	}
-
-	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), warpRoutingEnabled, namedTunnel, edgediscovery.ProtocolPercentage, origin.ResolveTTL, log)
+	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), warpRoutingEnabled, namedTunnel, edgediscovery.ProtocolPercentage, supervisor.ResolveTTL, log)
 	if err != nil {
-		return nil, ingress.Ingress{}, err
+		return nil, nil, err
 	}
 	log.Info().Msgf("Initial protocol %s", protocolSelector.Current())
 
@@ -248,11 +243,11 @@ func prepareTunnelConfig(
 	for _, p := range connection.ProtocolList {
 		tlsSettings := p.TLSSettings()
 		if tlsSettings == nil {
-			return nil, ingress.Ingress{}, fmt.Errorf("%s has unknown TLS settings", p)
+			return nil, nil, fmt.Errorf("%s has unknown TLS settings", p)
 		}
 		edgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c, tlsSettings.ServerName)
 		if err != nil {
-			return nil, ingress.Ingress{}, errors.Wrap(err, "unable to create TLS config to connect with edge")
+			return nil, nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
 		}
 		if len(tlsSettings.NextProtos) > 0 {
 			edgeTLSConfig.NextProtos = tlsSettings.NextProtos
@@ -260,15 +255,9 @@ func prepareTunnelConfig(
 		edgeTLSConfigs[p] = edgeTLSConfig
 	}
 
-	originProxy := origin.NewOriginProxy(ingressRules, warpRoutingService, tags, log)
 	gracePeriod, err := gracePeriod(c)
 	if err != nil {
-		return nil, ingress.Ingress{}, err
-	}
-	connectionConfig := &connection.Config{
-		OriginProxy:     originProxy,
-		GracePeriod:     gracePeriod,
-		ReplaceExisting: c.Bool("force"),
+		return nil, nil, err
 	}
 	muxerConfig := &connection.MuxerConfig{
 		HeartbeatInterval: c.Duration("heartbeat-interval"),
@@ -279,21 +268,22 @@ func prepareTunnelConfig(
 		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
 	}
 
-	return &origin.TunnelConfig{
-		ConnectionConfig: connectionConfig,
-		OSArch:           buildInfo.OSArch(),
-		ClientID:         clientID,
-		EdgeAddrs:        c.StringSlice("edge"),
-		Region:           c.String("region"),
-		HAConnections:    c.Int("ha-connections"),
-		IncidentLookup:   origin.NewIncidentLookup(),
-		IsAutoupdated:    c.Bool("is-autoupdated"),
-		LBPool:           c.String("lb-pool"),
-		Tags:             tags,
-		Log:              log,
-		LogTransport:     logTransport,
-		Observer:         observer,
-		ReportedVersion:  version,
+	tunnelConfig := &supervisor.TunnelConfig{
+		GracePeriod:     gracePeriod,
+		ReplaceExisting: c.Bool("force"),
+		OSArch:          info.OSArch(),
+		ClientID:        clientID,
+		EdgeAddrs:       c.StringSlice("edge"),
+		Region:          c.String("region"),
+		HAConnections:   c.Int("ha-connections"),
+		IncidentLookup:  supervisor.NewIncidentLookup(),
+		IsAutoupdated:   c.Bool("is-autoupdated"),
+		LBPool:          c.String("lb-pool"),
+		Tags:            tags,
+		Log:             log,
+		LogTransport:    logTransport,
+		Observer:        observer,
+		ReportedVersion: info.Version(),
 		// Note TUN-3758 , we use Int because UInt is not supported with altsrc
 		Retries:          uint(c.Int("retries")),
 		RunFromTerminal:  isRunningFromTerminal(),
@@ -302,7 +292,12 @@ func prepareTunnelConfig(
 		MuxerConfig:      muxerConfig,
 		ProtocolSelector: protocolSelector,
 		EdgeTLSConfigs:   edgeTLSConfigs,
-	}, ingressRules, nil
+	}
+	dynamicConfig := &orchestration.Config{
+		Ingress:            &ingressRules,
+		WarpRoutingEnabled: warpRoutingEnabled,
+	}
+	return tunnelConfig, dynamicConfig, nil
 }
 
 func gracePeriod(c *cli.Context) (time.Duration, error) {

cmd/cloudflared/tunnel/configuration_test.go

@@ -1,4 +1,6 @@
+//go:build ignore
 // +build ignore
+
 // TODO: Remove the above build tag and include this test when we start compiling with Golang 1.10.0+
 
 package tunnel

cmd/cloudflared/tunnel/info.go

@@ -5,12 +5,12 @@ import (
 
 	"github.com/google/uuid"
 
-	"github.com/cloudflare/cloudflared/tunnelstore"
+	"github.com/cloudflare/cloudflared/cfapi"
 )
 
 type Info struct {
-	ID         uuid.UUID                   `json:"id"`
-	Name       string                      `json:"name"`
-	CreatedAt  time.Time                   `json:"createdAt"`
-	Connectors []*tunnelstore.ActiveClient `json:"conns"`
+	ID         uuid.UUID             `json:"id"`
+	Name       string                `json:"name"`
+	CreatedAt  time.Time             `json:"createdAt"`
+	Connectors []*cfapi.ActiveClient `json:"conns"`
 }

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -1,6 +1,7 @@
 package tunnel
 
 import (
+	"encoding/json"
 	"fmt"
 	"net/url"
 
@@ -12,6 +13,15 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
+const ingressDataJSONFlagName = "json"
+
+var ingressDataJSON = &cli.StringFlag{
+	Name:    ingressDataJSONFlagName,
+	Aliases: []string{"j"},
+	Usage:   `Accepts data in the form of json as an input rather than read from a file`,
+	EnvVars: []string{"TUNNEL_INGRESS_VALIDATE_JSON"},
+}
+
 func buildIngressSubcommand() *cli.Command {
 	return &cli.Command{
 		Name:      "ingress",
@@ -49,6 +59,7 @@ func buildValidateIngressCommand() *cli.Command {
 		Usage:       "Validate the ingress configuration ",
 		UsageText:   "cloudflared tunnel [--config FILEPATH] ingress validate",
 		Description: "Validates the configuration file, ensuring your ingress rules are OK.",
+		Flags:       []cli.Flag{ingressDataJSON},
 	}
 }
 
@@ -69,12 +80,11 @@ func buildTestURLCommand() *cli.Command {
 
 // validateIngressCommand check the syntax of the ingress rules in the cloudflared config file
 func validateIngressCommand(c *cli.Context, warnings string) error {
-	conf := config.GetConfiguration()
-	if conf.Source() == "" {
-		fmt.Println("No configuration file was found. Please create one, or use the --config flag to specify its filepath. You can use the help command to learn more about configuration files")
-		return nil
+	conf, err := getConfiguration(c)
+	if err != nil {
+		return err
 	}
-	fmt.Println("Validating rules from", conf.Source())
+
 	if _, err := ingress.ParseIngress(conf); err != nil {
 		return errors.Wrap(err, "Validation failed")
 	}
@@ -90,6 +100,22 @@ func validateIngressCommand(c *cli.Context, warnings string) error {
 	return nil
 }
 
+func getConfiguration(c *cli.Context) (*config.Configuration, error) {
+	var conf *config.Configuration
+	if c.IsSet(ingressDataJSONFlagName) {
+		ingressJSON := c.String(ingressDataJSONFlagName)
+		fmt.Println("Validating rules from cmdline flag --json")
+		err := json.Unmarshal([]byte(ingressJSON), &conf)
+		return conf, err
+	}
+	conf = config.GetConfiguration()
+	if conf.Source() == "" {
+		return nil, errors.New("No configuration file was found. Please create one, or use the --config flag to specify its filepath. You can use the help command to learn more about configuration files")
+	}
+	fmt.Println("Validating rules from", conf.Source())
+	return conf, nil
+}
+
 // testURLCommand checks which ingress rule matches the given URL.
 func testURLCommand(c *cli.Context) error {
 	requestArg := c.Args().First()

cmd/cloudflared/tunnel/quick_tunnel.go

@@ -70,10 +70,14 @@ func RunQuickTunnel(sc *subcommandContext) error {
 		sc.log.Info().Msg(line)
 	}
 
+	if !sc.c.IsSet("protocol") {
+		sc.c.Set("protocol", "quic")
+	}
+
 	return StartServer(
 		sc.c,
-		version,
-		&connection.NamedTunnelConfig{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
+		buildInfo,
+		&connection.NamedTunnelProperties{Credentials: credentials, QuickTunnelUrl: data.Result.Hostname},
 		sc.log,
 		sc.isUIEnabled,
 	)

cmd/cloudflared/tunnel/signal_test.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package tunnel

cmd/cloudflared/tunnel/subcommand_context.go

@@ -1,6 +1,7 @@
 package tunnel
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"os"
@@ -13,9 +14,9 @@ import (
 	"github.com/urfave/cli/v2"
 
 	"github.com/cloudflare/cloudflared/certutil"
+	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 type errInvalidJSONCredential struct {
@@ -36,7 +37,7 @@ type subcommandContext struct {
 	fs          fileSystem
 
 	// These fields should be accessed using their respective Getter
-	tunnelstoreClient tunnelstore.Client
+	tunnelstoreClient cfapi.Client
 	userCredential    *userCredential
 }
 
@@ -67,7 +68,7 @@ type userCredential struct {
 	certPath string
 }
 
-func (sc *subcommandContext) client() (tunnelstore.Client, error) {
+func (sc *subcommandContext) client() (cfapi.Client, error) {
 	if sc.tunnelstoreClient != nil {
 		return sc.tunnelstoreClient, nil
 	}
@@ -75,8 +76,8 @@ func (sc *subcommandContext) client() (tunnelstore.Client, error) {
 	if err != nil {
 		return nil, err
 	}
-	userAgent := fmt.Sprintf("cloudflared/%s", version)
-	client, err := tunnelstore.NewRESTClient(
+	userAgent := fmt.Sprintf("cloudflared/%s", buildInfo.Version())
+	client, err := cfapi.NewRESTClient(
 		sc.c.String("api-url"),
 		credential.cert.AccountID,
 		credential.cert.ZoneID,
@@ -148,15 +149,27 @@ func (sc *subcommandContext) readTunnelCredentials(credFinder CredFinder) (conne
 	return credentials, nil
 }
 
-func (sc *subcommandContext) create(name string, credentialsFilePath string) (*tunnelstore.Tunnel, error) {
+func (sc *subcommandContext) create(name string, credentialsFilePath string, secret string) (*cfapi.Tunnel, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't create client to talk to Cloudflare Tunnel backend")
 	}
 
-	tunnelSecret, err := generateTunnelSecret()
-	if err != nil {
-		return nil, errors.Wrap(err, "couldn't generate the secret for your new tunnel")
+	var tunnelSecret []byte
+	if secret == "" {
+		tunnelSecret, err = generateTunnelSecret()
+		if err != nil {
+			return nil, errors.Wrap(err, "couldn't generate the secret for your new tunnel")
+		}
+	} else {
+		decodedSecret, err := base64.StdEncoding.DecodeString(secret)
+		if err != nil {
+			return nil, errors.Wrap(err, "Couldn't decode tunnel secret from base64")
+		}
+		tunnelSecret = []byte(decodedSecret)
+		if len(tunnelSecret) < 32 {
+			return nil, errors.New("Decoded tunnel secret must be at least 32 bytes long")
+		}
 	}
 
 	tunnel, err := client.CreateTunnel(name, tunnelSecret)
@@ -211,7 +224,7 @@ func (sc *subcommandContext) create(name string, credentialsFilePath string) (*t
 	return tunnel, nil
 }
 
-func (sc *subcommandContext) list(filter *tunnelstore.Filter) ([]*tunnelstore.Tunnel, error) {
+func (sc *subcommandContext) list(filter *cfapi.TunnelFilter) ([]*cfapi.Tunnel, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, err
@@ -230,7 +243,7 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 	for _, id := range tunnelIDs {
 		tunnel, err := client.GetTunnel(id)
 		if err != nil {
-			return errors.Wrapf(err, "Can't get tunnel information. Please check tunnel id: %s", tunnel.ID)
+			return errors.Wrapf(err, "Can't get tunnel information. Please check tunnel id: %s", id)
 		}
 
 		// Check if tunnel DeletedAt field has already been set
@@ -238,7 +251,7 @@ func (sc *subcommandContext) delete(tunnelIDs []uuid.UUID) error {
 			return fmt.Errorf("Tunnel %s has already been deleted", tunnel.ID)
 		}
 		if forceFlagSet {
-			if err := client.CleanupConnections(tunnel.ID, tunnelstore.NewCleanupParams()); err != nil {
+			if err := client.CleanupConnections(tunnel.ID, cfapi.NewCleanupParams()); err != nil {
 				return errors.Wrapf(err, "Error cleaning up connections for tunnel %s", tunnel.ID)
 			}
 		}
@@ -290,15 +303,15 @@ func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 
 	return StartServer(
 		sc.c,
-		version,
-		&connection.NamedTunnelConfig{Credentials: credentials},
+		buildInfo,
+		&connection.NamedTunnelProperties{Credentials: credentials},
 		sc.log,
 		sc.isUIEnabled,
 	)
 }
 
 func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
-	params := tunnelstore.NewCleanupParams()
+	params := cfapi.NewCleanupParams()
 	extraLog := ""
 	if connector := sc.c.String("connector-id"); connector != "" {
 		connectorID, err := uuid.Parse(connector)
@@ -322,7 +335,7 @@ func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
 	return nil
 }
 
-func (sc *subcommandContext) route(tunnelID uuid.UUID, r tunnelstore.Route) (tunnelstore.RouteResult, error) {
+func (sc *subcommandContext) route(tunnelID uuid.UUID, r cfapi.HostnameRoute) (cfapi.HostnameRouteResult, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, err
@@ -332,8 +345,8 @@ func (sc *subcommandContext) route(tunnelID uuid.UUID, r tunnelstore.Route) (tun
 }
 
 // Query Tunnelstore to find the active tunnel with the given name.
-func (sc *subcommandContext) tunnelActive(name string) (*tunnelstore.Tunnel, bool, error) {
-	filter := tunnelstore.NewFilter()
+func (sc *subcommandContext) tunnelActive(name string) (*cfapi.Tunnel, bool, error) {
+	filter := cfapi.NewTunnelFilter()
 	filter.NoDeleted()
 	filter.ByName(name)
 	tunnels, err := sc.list(filter)
@@ -373,52 +386,42 @@ func (sc *subcommandContext) findID(input string) (uuid.UUID, error) {
 }
 
 // findIDs is just like mapping `findID` over a slice, but it only uses
-// one Tunnelstore API call.
+// one Tunnelstore API call per non-UUID input provided.
 func (sc *subcommandContext) findIDs(inputs []string) ([]uuid.UUID, error) {
+	uuids, names := splitUuids(inputs)
 
-	// Shortcut without Tunnelstore call if we find that all inputs are already UUIDs.
-	uuids, err := convertNamesToUuids(inputs, make(map[string]uuid.UUID))
-	if err == nil {
-		return uuids, nil
+	for _, name := range names {
+		filter := cfapi.NewTunnelFilter()
+		filter.NoDeleted()
+		filter.ByName(name)
+
+		tunnels, err := sc.list(filter)
+		if err != nil {
+			return nil, err
+		}
+
+		if len(tunnels) != 1 {
+			return nil, fmt.Errorf("there should only be 1 non-deleted Tunnel named %s", name)
+		}
+
+		uuids = append(uuids, tunnels[0].ID)
 	}
 
-	// First, look up all tunnels the user has
-	filter := tunnelstore.NewFilter()
-	filter.NoDeleted()
-	tunnels, err := sc.list(filter)
-	if err != nil {
-		return nil, err
-	}
-	// Do the pure list-processing in its own function, so that it can be
-	// unit tested easily.
-	return findIDs(tunnels, inputs)
+	return uuids, nil
 }
 
-func findIDs(tunnels []*tunnelstore.Tunnel, inputs []string) ([]uuid.UUID, error) {
-	// Put them into a dictionary for faster lookups
-	nameToID := make(map[string]uuid.UUID, len(tunnels))
-	for _, tunnel := range tunnels {
-		nameToID[tunnel.Name] = tunnel.ID
-	}
+func splitUuids(inputs []string) ([]uuid.UUID, []string) {
+	uuids := make([]uuid.UUID, 0)
+	names := make([]string, 0)
 
-	return convertNamesToUuids(inputs, nameToID)
-}
-
-func convertNamesToUuids(inputs []string, nameToID map[string]uuid.UUID) ([]uuid.UUID, error) {
-	tunnelIDs := make([]uuid.UUID, len(inputs))
-	var badInputs []string
-	for i, input := range inputs {
-		if id, err := uuid.Parse(input); err == nil {
-			tunnelIDs[i] = id
-		} else if id, ok := nameToID[input]; ok {
-			tunnelIDs[i] = id
+	for _, input := range inputs {
+		id, err := uuid.Parse(input)
+		if err != nil {
+			names = append(names, input)
 		} else {
-			badInputs = append(badInputs, input)
+			uuids = append(uuids, id)
 		}
 	}
-	if len(badInputs) > 0 {
-		msg := "Please specify either the ID or name of a tunnel. The following inputs were neither: %s"
-		return nil, fmt.Errorf(msg, strings.Join(badInputs, ", "))
-	}
-	return tunnelIDs, nil
+
+	return uuids, names
 }

cmd/cloudflared/tunnel/subcommand_context_teamnet.go

@@ -1,16 +1,14 @@
 package tunnel
 
 import (
-	"net"
-
 	"github.com/pkg/errors"
 
-	"github.com/cloudflare/cloudflared/teamnet"
+	"github.com/cloudflare/cloudflared/cfapi"
 )
 
 const noClientMsg = "error while creating backend client"
 
-func (sc *subcommandContext) listRoutes(filter *teamnet.Filter) ([]*teamnet.DetailedRoute, error) {
+func (sc *subcommandContext) listRoutes(filter *cfapi.IpRouteFilter) ([]*cfapi.DetailedRoute, error) {
 	client, err := sc.client()
 	if err != nil {
 		return nil, errors.Wrap(err, noClientMsg)
@@ -18,26 +16,26 @@ func (sc *subcommandContext) listRoutes(filter *teamnet.Filter) ([]*teamnet.Deta
 	return client.ListRoutes(filter)
 }
 
-func (sc *subcommandContext) addRoute(newRoute teamnet.NewRoute) (teamnet.Route, error) {
+func (sc *subcommandContext) addRoute(newRoute cfapi.NewRoute) (cfapi.Route, error) {
 	client, err := sc.client()
 	if err != nil {
-		return teamnet.Route{}, errors.Wrap(err, noClientMsg)
+		return cfapi.Route{}, errors.Wrap(err, noClientMsg)
 	}
 	return client.AddRoute(newRoute)
 }
 
-func (sc *subcommandContext) deleteRoute(network net.IPNet) error {
+func (sc *subcommandContext) deleteRoute(params cfapi.DeleteRouteParams) error {
 	client, err := sc.client()
 	if err != nil {
 		return errors.Wrap(err, noClientMsg)
 	}
-	return client.DeleteRoute(network)
+	return client.DeleteRoute(params)
 }
 
-func (sc *subcommandContext) getRouteByIP(ip net.IP) (teamnet.DetailedRoute, error) {
+func (sc *subcommandContext) getRouteByIP(params cfapi.GetRouteByIpParams) (cfapi.DetailedRoute, error) {
 	client, err := sc.client()
 	if err != nil {
-		return teamnet.DetailedRoute{}, errors.Wrap(err, noClientMsg)
+		return cfapi.DetailedRoute{}, errors.Wrap(err, noClientMsg)
 	}
-	return client.GetByIP(ip)
+	return client.GetByIP(params)
 }

cmd/cloudflared/tunnel/subcommand_context_test.go

@@ -11,85 +11,13 @@ import (
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/assert"
 	"github.com/urfave/cli/v2"
 
+	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
-func Test_findIDs(t *testing.T) {
-	type args struct {
-		tunnels []*tunnelstore.Tunnel
-		inputs  []string
-	}
-	tests := []struct {
-		name    string
-		args    args
-		want    []uuid.UUID
-		wantErr bool
-	}{
-		{
-			name: "input not found",
-			args: args{
-				inputs: []string{"asdf"},
-			},
-			wantErr: true,
-		},
-		{
-			name: "only UUID",
-			args: args{
-				inputs: []string{"a8398a0b-876d-48ed-b609-3fcfd67a4950"},
-			},
-			want: []uuid.UUID{uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950")},
-		},
-		{
-			name: "only name",
-			args: args{
-				tunnels: []*tunnelstore.Tunnel{
-					{
-						ID:   uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950"),
-						Name: "tunnel1",
-					},
-				},
-				inputs: []string{"tunnel1"},
-			},
-			want: []uuid.UUID{uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950")},
-		},
-		{
-			name: "both UUID and name",
-			args: args{
-				tunnels: []*tunnelstore.Tunnel{
-					{
-						ID:   uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950"),
-						Name: "tunnel1",
-					},
-					{
-						ID:   uuid.MustParse("bf028b68-744f-466e-97f8-c46161d80aa5"),
-						Name: "tunnel2",
-					},
-				},
-				inputs: []string{"tunnel1", "bf028b68-744f-466e-97f8-c46161d80aa5"},
-			},
-			want: []uuid.UUID{
-				uuid.MustParse("a8398a0b-876d-48ed-b609-3fcfd67a4950"),
-				uuid.MustParse("bf028b68-744f-466e-97f8-c46161d80aa5"),
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := findIDs(tt.args.tunnels, tt.args.inputs)
-			if (err != nil) != tt.wantErr {
-				t.Errorf("findIDs() error = %v, wantErr %v", err, tt.wantErr)
-				return
-			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("findIDs() = %v, want %v", got, tt.want)
-			}
-		})
-	}
-}
-
 type mockFileSystem struct {
 	rf  func(string) ([]byte, error)
 	vfp func(string) bool
@@ -109,7 +37,7 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 		log               *zerolog.Logger
 		isUIEnabled       bool
 		fs                fileSystem
-		tunnelstoreClient tunnelstore.Client
+		tunnelstoreClient cfapi.Client
 		userCredential    *userCredential
 	}
 	type args struct {
@@ -260,13 +188,13 @@ func Test_subcommandContext_findCredentials(t *testing.T) {
 }
 
 type deleteMockTunnelStore struct {
-	tunnelstore.Client
+	cfapi.Client
 	mockTunnels      map[uuid.UUID]mockTunnelBehaviour
 	deletedTunnelIDs []uuid.UUID
 }
 
 type mockTunnelBehaviour struct {
-	tunnel     tunnelstore.Tunnel
+	tunnel     cfapi.Tunnel
 	deleteErr  error
 	cleanupErr error
 }
@@ -282,7 +210,7 @@ func newDeleteMockTunnelStore(tunnels ...mockTunnelBehaviour) *deleteMockTunnelS
 	}
 }
 
-func (d *deleteMockTunnelStore) GetTunnel(tunnelID uuid.UUID) (*tunnelstore.Tunnel, error) {
+func (d *deleteMockTunnelStore) GetTunnel(tunnelID uuid.UUID) (*cfapi.Tunnel, error) {
 	tunnel, ok := d.mockTunnels[tunnelID]
 	if !ok {
 		return nil, fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
@@ -306,7 +234,7 @@ func (d *deleteMockTunnelStore) DeleteTunnel(tunnelID uuid.UUID) error {
 	return nil
 }
 
-func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID, _ *tunnelstore.CleanupParams) error {
+func (d *deleteMockTunnelStore) CleanupConnections(tunnelID uuid.UUID, _ *cfapi.CleanupParams) error {
 	tunnel, ok := d.mockTunnels[tunnelID]
 	if !ok {
 		return fmt.Errorf("Couldn't find tunnel: %v", tunnelID)
@@ -357,10 +285,10 @@ func Test_subcommandContext_Delete(t *testing.T) {
 				}(),
 				tunnelstoreClient: newDeleteMockTunnelStore(
 					mockTunnelBehaviour{
-						tunnel: tunnelstore.Tunnel{ID: tunnelID1},
+						tunnel: cfapi.Tunnel{ID: tunnelID1},
 					},
 					mockTunnelBehaviour{
-						tunnel: tunnelstore.Tunnel{ID: tunnelID2},
+						tunnel: cfapi.Tunnel{ID: tunnelID2},
 					},
 				),
 			},
@@ -395,3 +323,48 @@ func Test_subcommandContext_Delete(t *testing.T) {
 		})
 	}
 }
+
+func Test_subcommandContext_ValidateIngressCommand(t *testing.T) {
+	var tests = []struct {
+		name        string
+		c           *cli.Context
+		wantErr     bool
+		expectedErr error
+	}{
+		{
+			name: "read a valid configuration from data",
+			c: func() *cli.Context {
+				data := `{ "warp-routing": {"enabled": true},  "originRequest" : {"connectTimeout": 10}, "ingress" : [ {"hostname": "test", "service": "https://localhost:8000" } , {"service": "http_status:404"} ]}`
+				flagSet := flag.NewFlagSet("json", flag.PanicOnError)
+				flagSet.String(ingressDataJSONFlagName, data, "")
+				c := cli.NewContext(cli.NewApp(), flagSet, nil)
+				_ = c.Set(ingressDataJSONFlagName, data)
+				return c
+			}(),
+		},
+		{
+			name: "read an invalid configuration with multiple mistakes",
+			c: func() *cli.Context {
+				data := `{ "ingress" : [ {"hostname": "test", "service": "localhost:8000" } , {"service": "http_status:invalid_status"} ]}`
+				flagSet := flag.NewFlagSet("json", flag.PanicOnError)
+				flagSet.String(ingressDataJSONFlagName, data, "")
+				c := cli.NewContext(cli.NewApp(), flagSet, nil)
+				_ = c.Set(ingressDataJSONFlagName, data)
+				return c
+			}(),
+			wantErr:     true,
+			expectedErr: errors.New("Validation failed: localhost:8000 is an invalid address, please make sure it has a scheme and a hostname"),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateIngressCommand(tt.c, "")
+			if tt.wantErr {
+				assert.Equal(t, tt.expectedErr.Error(), err.Error())
+			} else {
+				assert.Nil(t, err)
+			}
+		})
+	}
+}

cmd/cloudflared/tunnel/subcommand_context_vnets.go

@@ -0,0 +1,40 @@
+package tunnel
+
+import (
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/cfapi"
+)
+
+func (sc *subcommandContext) addVirtualNetwork(newVnet cfapi.NewVirtualNetwork) (cfapi.VirtualNetwork, error) {
+	client, err := sc.client()
+	if err != nil {
+		return cfapi.VirtualNetwork{}, errors.Wrap(err, noClientMsg)
+	}
+	return client.CreateVirtualNetwork(newVnet)
+}
+
+func (sc *subcommandContext) listVirtualNetworks(filter *cfapi.VnetFilter) ([]*cfapi.VirtualNetwork, error) {
+	client, err := sc.client()
+	if err != nil {
+		return nil, errors.Wrap(err, noClientMsg)
+	}
+	return client.ListVirtualNetworks(filter)
+}
+
+func (sc *subcommandContext) deleteVirtualNetwork(vnetId uuid.UUID) error {
+	client, err := sc.client()
+	if err != nil {
+		return errors.Wrap(err, noClientMsg)
+	}
+	return client.DeleteVirtualNetwork(vnetId)
+}
+
+func (sc *subcommandContext) updateVirtualNetwork(vnetId uuid.UUID, updates cfapi.UpdateVirtualNetwork) error {
+	client, err := sc.client()
+	if err != nil {
+		return errors.Wrap(err, noClientMsg)
+	}
+	return client.UpdateVirtualNetwork(vnetId, updates)
+}

cmd/cloudflared/tunnel/subcommands.go

@@ -21,11 +21,11 @@ import (
 	"golang.org/x/net/idna"
 	yaml "gopkg.in/yaml.v2"
 
+	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/config"
 	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/tunnelstore"
 )
 
 const (
@@ -64,8 +64,8 @@ var (
 		Name:        "when",
 		Aliases:     []string{"w"},
 		Usage:       "List tunnels that are active at the given `TIME` in RFC3339 format",
-		Layout:      tunnelstore.TimeLayout,
-		DefaultText: fmt.Sprintf("current time, %s", time.Now().Format(tunnelstore.TimeLayout)),
+		Layout:      cfapi.TimeLayout,
+		DefaultText: fmt.Sprintf("current time, %s", time.Now().Format(cfapi.TimeLayout)),
 	}
 	listIDFlag = &cli.StringFlag{
 		Name:    "id",
@@ -156,6 +156,12 @@ var (
 		Usage:   `Overwrites existing DNS records with this hostname`,
 		EnvVars: []string{"TUNNEL_FORCE_PROVISIONING_DNS"},
 	}
+	createSecretFlag = &cli.StringFlag{
+		Name:    "secret",
+		Aliases: []string{"s"},
+		Usage:   "Base64 encoded secret to set for the tunnel. The decoded secret must be at least 32 bytes long. If not specified, a random 32-byte secret will be generated.",
+		EnvVars: []string{"TUNNEL_CREATE_SECRET"},
+	}
 )
 
 func buildCreateCommand() *cli.Command {
@@ -170,7 +176,7 @@ func buildCreateCommand() *cli.Command {
   For example, to create a tunnel named 'my-tunnel' run:
 
   $ cloudflared tunnel create my-tunnel`,
-		Flags:              []cli.Flag{outputFormatFlag, credentialsFileFlagCLIOnly},
+		Flags:              []cli.Flag{outputFormatFlag, credentialsFileFlagCLIOnly, createSecretFlag},
 		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
@@ -196,7 +202,7 @@ func createCommand(c *cli.Context) error {
 	warningChecker := updater.StartWarningCheck(c)
 	defer warningChecker.LogWarningIfAny(sc.log)
 
-	_, err = sc.create(name, c.String(CredFileFlag))
+	_, err = sc.create(name, c.String(CredFileFlag), c.String(createSecretFlag.Name))
 	return errors.Wrap(err, "failed to create tunnel")
 }
 
@@ -254,7 +260,7 @@ func listCommand(c *cli.Context) error {
 	warningChecker := updater.StartWarningCheck(c)
 	defer warningChecker.LogWarningIfAny(sc.log)
 
-	filter := tunnelstore.NewFilter()
+	filter := cfapi.NewTunnelFilter()
 	if !c.Bool("show-deleted") {
 		filter.NoDeleted()
 	}
@@ -277,6 +283,9 @@ func listCommand(c *cli.Context) error {
 		}
 		filter.ByTunnelID(tunnelID)
 	}
+	if maxFetch := c.Int("max-fetch-size"); maxFetch > 0 {
+		filter.MaxFetchSize(uint(maxFetch))
+	}
 
 	tunnels, err := sc.list(filter)
 	if err != nil {
@@ -320,13 +329,13 @@ func listCommand(c *cli.Context) error {
 	if len(tunnels) > 0 {
 		formatAndPrintTunnelList(tunnels, c.Bool("show-recently-disconnected"))
 	} else {
-		fmt.Println("You have no tunnels, use 'cloudflared tunnel create' to define a new tunnel")
+		fmt.Println("No tunnels were found for the given filter flags. You can use 'cloudflared tunnel create' to create a tunnel.")
 	}
 
 	return nil
 }
 
-func formatAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconnected bool) {
+func formatAndPrintTunnelList(tunnels []*cfapi.Tunnel, showRecentlyDisconnected bool) {
 	writer := tabWriter()
 	defer writer.Flush()
 
@@ -348,7 +357,7 @@ func formatAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconn
 	}
 }
 
-func fmtConnections(connections []tunnelstore.Connection, showRecentlyDisconnected bool) string {
+func fmtConnections(connections []cfapi.Connection, showRecentlyDisconnected bool) string {
 
 	// Count connections per colo
 	numConnsPerColo := make(map[string]uint, len(connections))
@@ -468,8 +477,8 @@ func tunnelInfo(c *cli.Context) error {
 	return nil
 }
 
-func getTunnel(sc *subcommandContext, tunnelID uuid.UUID) (*tunnelstore.Tunnel, error) {
-	filter := tunnelstore.NewFilter()
+func getTunnel(sc *subcommandContext, tunnelID uuid.UUID) (*cfapi.Tunnel, error) {
+	filter := cfapi.NewTunnelFilter()
 	filter.ByTunnelID(tunnelID)
 	tunnels, err := sc.list(filter)
 	if err != nil {
@@ -702,7 +711,7 @@ Further information about managing Cloudflare WARP traffic to your tunnel is ava
 			{
 				Name:        "dns",
 				Action:      cliutil.ConfiguredAction(routeDnsCommand),
-				Usage:       "Route a hostname by creating a DNS CNAME record to a tunnel",
+				Usage:       "HostnameRoute a hostname by creating a DNS CNAME record to a tunnel",
 				UsageText:   "cloudflared tunnel route dns [TUNNEL] [HOSTNAME]",
 				Description: `Creates a DNS CNAME record hostname that points to the tunnel.`,
 				Flags:       []cli.Flag{overwriteDNSFlag},
@@ -719,7 +728,7 @@ Further information about managing Cloudflare WARP traffic to your tunnel is ava
 	}
 }
 
-func dnsRouteFromArg(c *cli.Context, overwriteExisting bool) (tunnelstore.Route, error) {
+func dnsRouteFromArg(c *cli.Context, overwriteExisting bool) (cfapi.HostnameRoute, error) {
 	const (
 		userHostnameIndex = 1
 		expectedNArgs     = 2
@@ -733,10 +742,10 @@ func dnsRouteFromArg(c *cli.Context, overwriteExisting bool) (tunnelstore.Route,
 	} else if !validateHostname(userHostname, true) {
 		return nil, errors.Errorf("%s is not a valid hostname", userHostname)
 	}
-	return tunnelstore.NewDNSRoute(userHostname, overwriteExisting), nil
+	return cfapi.NewDNSRoute(userHostname, overwriteExisting), nil
 }
 
-func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
+func lbRouteFromArg(c *cli.Context) (cfapi.HostnameRoute, error) {
 	const (
 		lbNameIndex   = 1
 		lbPoolIndex   = 2
@@ -759,7 +768,7 @@ func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 		return nil, errors.Errorf("%s is not a valid pool name", lbPool)
 	}
 
-	return tunnelstore.NewLBRoute(lbName, lbPool), nil
+	return cfapi.NewLBRoute(lbName, lbPool), nil
 }
 
 var nameRegex = regexp.MustCompile("^[_a-zA-Z0-9][-_.a-zA-Z0-9]*$")
@@ -806,7 +815,7 @@ func routeCommand(c *cli.Context, routeType string) error {
 	if err != nil {
 		return err
 	}
-	var route tunnelstore.Route
+	var route cfapi.HostnameRoute
 	switch routeType {
 	case "dns":
 		route, err = dnsRouteFromArg(c, c.Bool(overwriteDNSFlagName))

cmd/cloudflared/tunnel/subcommands_test.go

@@ -8,12 +8,12 @@ import (
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/cloudflare/cloudflared/tunnelstore"
+	"github.com/cloudflare/cloudflared/cfapi"
 )
 
 func Test_fmtConnections(t *testing.T) {
 	type args struct {
-		connections []tunnelstore.Connection
+		connections []cfapi.Connection
 	}
 	tests := []struct {
 		name string
@@ -23,14 +23,14 @@ func Test_fmtConnections(t *testing.T) {
 		{
 			name: "empty",
 			args: args{
-				connections: []tunnelstore.Connection{},
+				connections: []cfapi.Connection{},
 			},
 			want: "",
 		},
 		{
 			name: "trivial",
 			args: args{
-				connections: []tunnelstore.Connection{
+				connections: []cfapi.Connection{
 					{
 						ColoName: "DFW",
 						ID:       uuid.MustParse("ea550130-57fd-4463-aab1-752822231ddd"),
@@ -42,7 +42,7 @@ func Test_fmtConnections(t *testing.T) {
 		{
 			name: "with a pending reconnect",
 			args: args{
-				connections: []tunnelstore.Connection{
+				connections: []cfapi.Connection{
 					{
 						ColoName:           "DFW",
 						ID:                 uuid.MustParse("ea550130-57fd-4463-aab1-752822231ddd"),
@@ -55,7 +55,7 @@ func Test_fmtConnections(t *testing.T) {
 		{
 			name: "many colos",
 			args: args{
-				connections: []tunnelstore.Connection{
+				connections: []cfapi.Connection{
 					{
 						ColoName: "YRV",
 						ID:       uuid.MustParse("ea550130-57fd-4463-aab1-752822231ddd"),

cmd/cloudflared/tunnel/teamnet_subcommands.go

@@ -6,35 +6,54 @@ import (
 	"os"
 	"text/tabwriter"
 
+	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
 
+	"github.com/cloudflare/cloudflared/cfapi"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
-	"github.com/cloudflare/cloudflared/teamnet"
+)
 
-	"github.com/urfave/cli/v2"
+var (
+	vnetFlag = &cli.StringFlag{
+		Name:    "vnet",
+		Aliases: []string{"vn"},
+		Usage:   "The ID or name of the virtual network to which the route is associated to.",
+	}
 )
 
 func buildRouteIPSubcommand() *cli.Command {
 	return &cli.Command{
 		Name:      "ip",
-		Usage:     "Configure and query Cloudflare WARP routing to services or private networks available through this tunnel.",
+		Usage:     "Configure and query Cloudflare WARP routing to private IP networks made available through Cloudflare Tunnels.",
 		UsageText: "cloudflared tunnel [--config FILEPATH] route COMMAND [arguments...]",
-		Description: `cloudflared can provision private routes from any IP space to origins in your corporate network.
-Users enrolled in your Cloudflare for Teams organization can reach those routes through the
-Cloudflare WARP client. You can also build rules to determine who can reach certain routes.`,
+		Description: `cloudflared can provision routes for any IP space in your corporate network. Users enrolled in
+your Cloudflare for Teams organization can reach those IPs through the Cloudflare WARP
+client. You can then configure L7/L4 filtering on https://dash.teams.cloudflare.com to
+determine who can reach certain routes.
+By default IP routes all exist within a single virtual network. If you use the same IP
+space(s) in different physical private networks, all meant to be reachable via IP routes,
+then you have to manage the ambiguous IP routes by associating them to virtual networks.
+See "cloudflared tunnel vnet --help" for more information.`,
 		Subcommands: []*cli.Command{
 			{
 				Name:      "add",
 				Action:    cliutil.ConfiguredAction(addRouteCommand),
-				Usage:     "Add any new network to the routing table reachable via the tunnel",
-				UsageText: "cloudflared tunnel [--config FILEPATH] route ip add [CIDR] [TUNNEL] [COMMENT?]",
-				Description: `Adds any network route space (represented as a CIDR) to your routing table.
-That network space becomes reachable for requests egressing from a user's machine
+				Usage:     "Add a new network to the routing table reachable via a Tunnel",
+				UsageText: "cloudflared tunnel [--config FILEPATH] route ip add [flags] [CIDR] [TUNNEL] [COMMENT?]",
+				Description: `Adds a network IP route space (represented as a CIDR) to your routing table.
+That network IP space becomes reachable for requests egressing from a user's machine
 as long as it is using Cloudflare WARP client and is enrolled in the same account
-that is running the tunnel chosen here. Further, those requests will be proxied to
-the specified tunnel, and reach an IP in the given CIDR, as long as that IP is
-reachable from the tunnel.`,
+that is running the Tunnel chosen here. Further, those requests will be proxied to
+the specified Tunnel, and reach an IP in the given CIDR, as long as that IP is
+reachable from cloudflared.
+If the CIDR exists in more than one private network, to be connected with Cloudflare
+Tunnels, then you have to manage those IP routes with virtual networks (see
+"cloudflared tunnel vnet --help)". In those cases, you then have to tell
+which virtual network's routing table you want to add the route to with:
+"cloudflared tunnel route ip add --vnet [ID/name] [CIDR] [TUNNEL]".`,
+				Flags: []cli.Flag{vnetFlag},
 			},
 			{
 				Name:        "show",
@@ -49,17 +68,22 @@ reachable from the tunnel.`,
 				Name:      "delete",
 				Action:    cliutil.ConfiguredAction(deleteRouteCommand),
 				Usage:     "Delete a row from your organization's private routing table",
-				UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [CIDR]",
-				Description: `Deletes the row for a given CIDR from your routing table. That portion
-of your network will no longer be reachable by the WARP clients.`,
+				UsageText: "cloudflared tunnel [--config FILEPATH] route ip delete [flags] [CIDR]",
+				Description: `Deletes the row for a given CIDR from your routing table. That portion of your network
+will no longer be reachable by the WARP clients. Note that if you use virtual
+networks, then you have to tell which virtual network whose routing table you
+have a row deleted from.`,
+				Flags: []cli.Flag{vnetFlag},
 			},
 			{
 				Name:      "get",
 				Action:    cliutil.ConfiguredAction(getRouteByIPCommand),
 				Usage:     "Check which row of the routing table matches a given IP.",
-				UsageText: "cloudflared tunnel [--config FILEPATH] route ip get [IP]",
-				Description: `Checks which row of the routing table will be used to proxy a given IP.
-				This helps check and validate your config.`,
+				UsageText: "cloudflared tunnel [--config FILEPATH] route ip get [flags] [IP]",
+				Description: `Checks which row of the routing table will be used to proxy a given IP. This helps check
+and validate your config. Note that if you use virtual networks, then you have
+to tell which virtual network whose routing table you want to use.`,
+				Flags: []cli.Flag{vnetFlag},
 			},
 		},
 	}
@@ -67,7 +91,7 @@ of your network will no longer be reachable by the WARP clients.`,
 
 func showRoutesFlags() []cli.Flag {
 	flags := make([]cli.Flag, 0)
-	flags = append(flags, teamnet.FilterFlags...)
+	flags = append(flags, cfapi.IpRouteFilterFlags...)
 	flags = append(flags, outputFormatFlag)
 	return flags
 }
@@ -78,7 +102,7 @@ func showRoutesCommand(c *cli.Context) error {
 		return err
 	}
 
-	filter, err := teamnet.NewFromCLI(c)
+	filter, err := cfapi.NewIpRouteFilterFromCLI(c)
 	if err != nil {
 		return errors.Wrap(err, "invalid config for routing filters")
 	}
@@ -98,7 +122,7 @@ func showRoutesCommand(c *cli.Context) error {
 	if len(routes) > 0 {
 		formatAndPrintRouteList(routes)
 	} else {
-		fmt.Println("You have no routes, use 'cloudflared tunnel route ip add' to add a route")
+		fmt.Println("No routes were found for the given filter flags. You can use 'cloudflared tunnel route ip add' to add a route.")
 	}
 
 	return nil
@@ -112,7 +136,9 @@ func addRouteCommand(c *cli.Context) error {
 	if c.NArg() < 2 {
 		return errors.New("You must supply at least 2 arguments, first the network you wish to route (in CIDR form e.g. 1.2.3.4/32) and then the tunnel ID to proxy with")
 	}
+
 	args := c.Args()
+
 	_, network, err := net.ParseCIDR(args.Get(0))
 	if err != nil {
 		return errors.Wrap(err, "Invalid network CIDR")
@@ -120,19 +146,32 @@ func addRouteCommand(c *cli.Context) error {
 	if network == nil {
 		return errors.New("Invalid network CIDR")
 	}
+
 	tunnelRef := args.Get(1)
 	tunnelID, err := sc.findID(tunnelRef)
 	if err != nil {
 		return errors.Wrap(err, "Invalid tunnel")
 	}
+
 	comment := ""
 	if c.NArg() >= 3 {
 		comment = args.Get(2)
 	}
-	_, err = sc.addRoute(teamnet.NewRoute{
+
+	var vnetId *uuid.UUID
+	if c.IsSet(vnetFlag.Name) {
+		id, err := getVnetId(sc, c.String(vnetFlag.Name))
+		if err != nil {
+			return err
+		}
+		vnetId = &id
+	}
+
+	_, err = sc.addRoute(cfapi.NewRoute{
 		Comment:  comment,
 		Network:  *network,
 		TunnelID: tunnelID,
+		VNetID:   vnetId,
 	})
 	if err != nil {
 		return errors.Wrap(err, "API error")
@@ -146,9 +185,11 @@ func deleteRouteCommand(c *cli.Context) error {
 	if err != nil {
 		return err
 	}
+
 	if c.NArg() != 1 {
 		return errors.New("You must supply exactly one argument, the network whose route you want to delete (in CIDR form e.g. 1.2.3.4/32)")
 	}
+
 	_, network, err := net.ParseCIDR(c.Args().First())
 	if err != nil {
 		return errors.Wrap(err, "Invalid network CIDR")
@@ -156,7 +197,20 @@ func deleteRouteCommand(c *cli.Context) error {
 	if network == nil {
 		return errors.New("Invalid network CIDR")
 	}
-	if err := sc.deleteRoute(*network); err != nil {
+
+	params := cfapi.DeleteRouteParams{
+		Network: *network,
+	}
+
+	if c.IsSet(vnetFlag.Name) {
+		vnetId, err := getVnetId(sc, c.String(vnetFlag.Name))
+		if err != nil {
+			return err
+		}
+		params.VNetID = &vnetId
+	}
+
+	if err := sc.deleteRoute(params); err != nil {
 		return errors.Wrap(err, "API error")
 	}
 	fmt.Printf("Successfully deleted route for %s\n", network)
@@ -177,19 +231,32 @@ func getRouteByIPCommand(c *cli.Context) error {
 	if ip == nil {
 		return fmt.Errorf("Invalid IP %s", ipInput)
 	}
-	route, err := sc.getRouteByIP(ip)
+
+	params := cfapi.GetRouteByIpParams{
+		Ip: ip,
+	}
+
+	if c.IsSet(vnetFlag.Name) {
+		vnetId, err := getVnetId(sc, c.String(vnetFlag.Name))
+		if err != nil {
+			return err
+		}
+		params.VNetID = &vnetId
+	}
+
+	route, err := sc.getRouteByIP(params)
 	if err != nil {
 		return errors.Wrap(err, "API error")
 	}
 	if route.IsZero() {
 		fmt.Printf("No route matches the IP %s\n", ip)
 	} else {
-		formatAndPrintRouteList([]*teamnet.DetailedRoute{&route})
+		formatAndPrintRouteList([]*cfapi.DetailedRoute{&route})
 	}
 	return nil
 }
 
-func formatAndPrintRouteList(routes []*teamnet.DetailedRoute) {
+func formatAndPrintRouteList(routes []*cfapi.DetailedRoute) {
 	const (
 		minWidth = 0
 		tabWidth = 8
@@ -202,7 +269,7 @@ func formatAndPrintRouteList(routes []*teamnet.DetailedRoute) {
 	defer writer.Flush()
 
 	// Print column headers with tabbed columns
-	_, _ = fmt.Fprintln(writer, "NETWORK\tCOMMENT\tTUNNEL ID\tTUNNEL NAME\tCREATED\tDELETED\t")
+	_, _ = fmt.Fprintln(writer, "NETWORK\tVIRTUAL NET ID\tCOMMENT\tTUNNEL ID\tTUNNEL NAME\tCREATED\tDELETED\t")
 
 	// Loop through routes, create formatted string for each, and print using tabwriter
 	for _, route := range routes {

cmd/cloudflared/tunnel/vnets_subcommands.go

@@ -0,0 +1,285 @@
+package tunnel
+
+import (
+	"fmt"
+	"os"
+	"text/tabwriter"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cfapi"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+)
+
+var (
+	makeDefaultFlag = &cli.BoolFlag{
+		Name:    "default",
+		Aliases: []string{"d"},
+		Usage: "The virtual network becomes the default one for the account. This means that all operations that " +
+			"omit a virtual network will now implicitly be using this virtual network (i.e., the default one) such " +
+			"as new IP routes that are created. When this flag is not set, the virtual network will not become the " +
+			"default one in the account.",
+	}
+	newNameFlag = &cli.StringFlag{
+		Name:    "name",
+		Aliases: []string{"n"},
+		Usage:   "The new name for the virtual network.",
+	}
+	newCommentFlag = &cli.StringFlag{
+		Name:    "comment",
+		Aliases: []string{"c"},
+		Usage:   "A new comment describing the purpose of the virtual network.",
+	}
+)
+
+func buildVirtualNetworkSubcommand(hidden bool) *cli.Command {
+	return &cli.Command{
+		Name:      "vnet",
+		Usage:     "Configure and query virtual networks to manage private IP routes with overlapping IPs.",
+		UsageText: "cloudflared tunnel [--config FILEPATH] network COMMAND [arguments...]",
+		Description: `cloudflared allows to manage IP routes that expose origins in your private network space via their IP directly
+to clients outside (e.g. using WARP client) --- those are configurable via "cloudflared tunnel route ip" commands.
+By default, all those IP routes live in the same virtual network. Managing virtual networks (e.g. by creating a
+new one) becomes relevant when you have different private networks that have overlapping IPs. E.g.: if you have
+a private network A running Tunnel 1, and private network B running Tunnel 2, it is possible that both Tunnels
+expose the same IP space (say 10.0.0.0/8); to handle that, you have to add each IP Route (one that points to
+Tunnel 1 and another that points to Tunnel 2) in different Virtual Networks. That way, if your clients are on
+Virtual Network X, they will see Tunnel 1 (via Route A) and not see Tunnel 2 (since its Route B is associated
+to another Virtual Network Y).`,
+		Hidden: hidden,
+		Subcommands: []*cli.Command{
+			{
+				Name:      "add",
+				Action:    cliutil.ConfiguredAction(addVirtualNetworkCommand),
+				Usage:     "Add a new virtual network to which IP routes can be attached",
+				UsageText: "cloudflared tunnel [--config FILEPATH] network add [flags] NAME [\"comment\"]",
+				Description: `Adds a new virtual network. You can then attach IP routes to this virtual network with "cloudflared tunnel route ip"
+commands. By doing so, such route(s) become segregated from route(s) in another virtual networks. Note that all
+routes exist within some virtual network. If you do not specify any, then the system pre-creates a default virtual
+network to which all routes belong. That is fine if you do not have overlapping IPs within different physical
+private networks in your infrastructure exposed via Cloudflare Tunnel. Note: if a virtual network is added as
+the new default, then the previous existing default virtual network will be automatically modified to no longer
+be the current default.`,
+				Flags:  []cli.Flag{makeDefaultFlag},
+				Hidden: hidden,
+			},
+			{
+				Name:        "list",
+				Action:      cliutil.ConfiguredAction(listVirtualNetworksCommand),
+				Usage:       "Lists the virtual networks",
+				UsageText:   "cloudflared tunnel [--config FILEPATH] network list [flags]",
+				Description: "Lists the virtual networks based on the given filter flags.",
+				Flags:       listVirtualNetworksFlags(),
+				Hidden:      hidden,
+			},
+			{
+				Name:      "delete",
+				Action:    cliutil.ConfiguredAction(deleteVirtualNetworkCommand),
+				Usage:     "Delete a virtual network",
+				UsageText: "cloudflared tunnel [--config FILEPATH] network delete VIRTUAL_NETWORK",
+				Description: `Deletes the virtual network (given its ID or name). This is only possible if that virtual network is unused. 
+A virtual network may be used by IP routes or by WARP devices.`,
+				Hidden: hidden,
+			},
+			{
+				Name:      "update",
+				Action:    cliutil.ConfiguredAction(updateVirtualNetworkCommand),
+				Usage:     "Update a virtual network",
+				UsageText: "cloudflared tunnel [--config FILEPATH] network update [flags] VIRTUAL_NETWORK",
+				Description: `Updates the virtual network (given its ID or name). If this virtual network is updated to become the new
+default, then the previously existing default virtual network will also be modified to no longer be the default.
+You cannot update a virtual network to not be the default anymore directly. Instead, you should create a new
+default or update an existing one to become the default.`,
+				Flags:  []cli.Flag{newNameFlag, newCommentFlag, makeDefaultFlag},
+				Hidden: hidden,
+			},
+		},
+	}
+}
+
+func listVirtualNetworksFlags() []cli.Flag {
+	flags := make([]cli.Flag, 0)
+	flags = append(flags, cfapi.VnetFilterFlags...)
+	flags = append(flags, outputFormatFlag)
+	return flags
+}
+
+func addVirtualNetworkCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+	if c.NArg() < 1 {
+		return errors.New("You must supply at least 1 argument, the name of the virtual network you wish to add.")
+	}
+
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
+	args := c.Args()
+
+	name := args.Get(0)
+
+	comment := ""
+	if c.NArg() >= 2 {
+		comment = args.Get(1)
+	}
+
+	newVnet := cfapi.NewVirtualNetwork{
+		Name:      name,
+		Comment:   comment,
+		IsDefault: c.Bool(makeDefaultFlag.Name),
+	}
+	createdVnet, err := sc.addVirtualNetwork(newVnet)
+
+	if err != nil {
+		return errors.Wrap(err, "Could not add virtual network")
+	}
+
+	extraMsg := ""
+	if createdVnet.IsDefault {
+		extraMsg = " (as the new default for this account) "
+	}
+	fmt.Printf(
+		"Successfully added virtual 'network' %s with ID: %s%s\n"+
+			"You can now add IP routes attached to this virtual network. See `cloudflared tunnel route ip add -help`\n",
+		name, createdVnet.ID, extraMsg,
+	)
+	return nil
+}
+
+func listVirtualNetworksCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+
+	warningChecker := updater.StartWarningCheck(c)
+	defer warningChecker.LogWarningIfAny(sc.log)
+
+	filter, err := cfapi.NewFromCLI(c)
+	if err != nil {
+		return errors.Wrap(err, "invalid flags for filtering virtual networks")
+	}
+
+	vnets, err := sc.listVirtualNetworks(filter)
+	if err != nil {
+		return err
+	}
+
+	if outputFormat := c.String(outputFormatFlag.Name); outputFormat != "" {
+		return renderOutput(outputFormat, vnets)
+	}
+
+	if len(vnets) > 0 {
+		formatAndPrintVnetsList(vnets)
+	} else {
+		fmt.Println("No virtual networks were found for the given filter flags. You can use 'cloudflared tunnel vnet add' to add a virtual network.")
+	}
+
+	return nil
+}
+
+func deleteVirtualNetworkCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+	if c.NArg() != 1 {
+		return errors.New("You must supply exactly one argument, either the ID or name of the virtual network to delete")
+	}
+
+	input := c.Args().Get(0)
+	vnetId, err := getVnetId(sc, input)
+	if err != nil {
+		return err
+	}
+
+	if err := sc.deleteVirtualNetwork(vnetId); err != nil {
+		return errors.Wrap(err, "API error")
+	}
+	fmt.Printf("Successfully deleted virtual network '%s'\n", input)
+	return nil
+}
+
+func updateVirtualNetworkCommand(c *cli.Context) error {
+	sc, err := newSubcommandContext(c)
+	if err != nil {
+		return err
+	}
+	if c.NArg() != 1 {
+		return errors.New(" You must supply exactly one argument, either the ID or (current) name of the virtual network to update")
+	}
+
+	input := c.Args().Get(0)
+	vnetId, err := getVnetId(sc, input)
+	if err != nil {
+		return err
+	}
+
+	updates := cfapi.UpdateVirtualNetwork{}
+
+	if c.IsSet(newNameFlag.Name) {
+		newName := c.String(newNameFlag.Name)
+		updates.Name = &newName
+	}
+	if c.IsSet(newCommentFlag.Name) {
+		newComment := c.String(newCommentFlag.Name)
+		updates.Comment = &newComment
+	}
+	if c.IsSet(makeDefaultFlag.Name) {
+		isDefault := c.Bool(makeDefaultFlag.Name)
+		updates.IsDefault = &isDefault
+	}
+
+	if err := sc.updateVirtualNetwork(vnetId, updates); err != nil {
+		return errors.Wrap(err, "API error")
+	}
+	fmt.Printf("Successfully updated virtual network '%s'\n", input)
+	return nil
+}
+
+func getVnetId(sc *subcommandContext, input string) (uuid.UUID, error) {
+	val, err := uuid.Parse(input)
+	if err == nil {
+		return val, nil
+	}
+
+	filter := cfapi.NewVnetFilter()
+	filter.WithDeleted(false)
+	filter.ByName(input)
+
+	vnets, err := sc.listVirtualNetworks(filter)
+	if err != nil {
+		return uuid.Nil, err
+	}
+
+	if len(vnets) != 1 {
+		return uuid.Nil, fmt.Errorf("there should only be 1 non-deleted virtual network named %s", input)
+	}
+
+	return vnets[0].ID, nil
+}
+
+func formatAndPrintVnetsList(vnets []*cfapi.VirtualNetwork) {
+	const (
+		minWidth = 0
+		tabWidth = 8
+		padding  = 1
+		padChar  = ' '
+		flags    = 0
+	)
+
+	writer := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
+	defer writer.Flush()
+
+	_, _ = fmt.Fprintln(writer, "ID\tNAME\tIS DEFAULT\tCOMMENT\tCREATED\tDELETED\t")
+
+	for _, virtualNetwork := range vnets {
+		formattedStr := virtualNetwork.TableString()
+		_, _ = fmt.Fprintln(writer, formattedStr)
+	}
+}

cmd/cloudflared/updater/workers_service_test.go

@@ -1,3 +1,4 @@
+//go:build !windows
 // +build !windows
 
 package updater

cmd/cloudflared/updater/workers_update.go

@@ -47,7 +47,7 @@ type batchData struct {
 }
 
 // WorkersVersion implements the Version interface.
-// It contains everything needed to preform a version upgrade
+// It contains everything needed to perform a version upgrade
 type WorkersVersion struct {
 	downloadURL  string
 	checksum     string

cmd/cloudflared/windows_service.go

@@ -1,3 +1,4 @@
+//go:build windows
 // +build windows
 
 package main

component-tests/constants.py

@@ -3,3 +3,7 @@ MAX_RETRIES = 5
 BACKOFF_SECS = 7
 
 PROXY_DNS_PORT = 9053
+
+
+def protocols():
+    return ["h2mux", "http2", "quic"]

component-tests/test_logging.py

@@ -75,7 +75,7 @@ class TestLogging:
         }
         config = component_tests_config(extra_config)
         with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
-            wait_tunnel_ready(tunnel_url=config.get_url())
+            wait_tunnel_ready(tunnel_url=config.get_url(), cfd_logs=str(log_file))
             assert_log_in_file(log_file)
             assert_json_log(log_file)
 
@@ -88,5 +88,5 @@ class TestLogging:
         }
         config = component_tests_config(extra_config)
         with start_cloudflared(tmp_path, config, new_process=True, capture_output=False):
-            wait_tunnel_ready(tunnel_url=config.get_url())
+            wait_tunnel_ready(tunnel_url=config.get_url(), cfd_logs=str(log_dir))
             assert_log_to_dir(config, log_dir)

component-tests/test_reconnect.py

@@ -7,6 +7,7 @@ import pytest
 from flaky import flaky
 
 from conftest import CfdModes
+from constants import protocols
 from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
 
 
@@ -18,9 +19,16 @@ class TestReconnect:
         "stdin-control": True,
     }
 
+    def _extra_config(self, protocol):
+        return {
+            "stdin-control": True,
+            "protocol": protocol,
+        }
+
     @pytest.mark.skipif(platform.system() == "Windows", reason=f"Currently buggy on Windows TUN-4584")
-    def test_named_reconnect(self, tmp_path, component_tests_config):
-        config = component_tests_config(self.extra_config)
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_named_reconnect(self, tmp_path, component_tests_config, protocol):
+        config = component_tests_config(self._extra_config(protocol))
         with start_cloudflared(tmp_path, config, new_process=True, allow_input=True, capture_output=False) as cloudflared:
             # Repeat the test multiple times because some issues only occur after multiple reconnects
             self.assert_reconnect(config, cloudflared, 5)

component-tests/test_termination.py

@@ -8,6 +8,7 @@ import time
 import pytest
 import requests
 
+from constants import protocols
 from util import start_cloudflared, wait_tunnel_ready, check_tunnel_not_connected
 
 
@@ -17,17 +18,21 @@ def supported_signals():
     return [signal.SIGTERM, signal.SIGINT]
 
 
-class TestTermination():
+class TestTermination:
     grace_period = 5
     timeout = 10
-    extra_config = {
-        "grace-period": f"{grace_period}s",
-    }
     sse_endpoint = "/sse?freq=1s"
 
+    def _extra_config(self, protocol):
+        return {
+            "grace-period": f"{self.grace_period}s",
+            "protocol": protocol,
+        }
+
     @pytest.mark.parametrize("signal", supported_signals())
-    def test_graceful_shutdown(self, tmp_path, component_tests_config, signal):
-        config = component_tests_config(self.extra_config)
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_graceful_shutdown(self, tmp_path, component_tests_config, signal, protocol):
+        config = component_tests_config(self._extra_config(protocol))
         with start_cloudflared(
                 tmp_path, config, new_process=True, capture_output=False) as cloudflared:
             wait_tunnel_ready(tunnel_url=config.get_url())
@@ -47,8 +52,9 @@ class TestTermination():
     # test cloudflared terminates before grace period expires when all eyeball
     # connections are drained
     @pytest.mark.parametrize("signal", supported_signals())
-    def test_shutdown_once_no_connection(self, tmp_path, component_tests_config, signal):
-        config = component_tests_config(self.extra_config)
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_shutdown_once_no_connection(self, tmp_path, component_tests_config, signal, protocol):
+        config = component_tests_config(self._extra_config(protocol))
         with start_cloudflared(
                 tmp_path, config, new_process=True, capture_output=False) as cloudflared:
             wait_tunnel_ready(tunnel_url=config.get_url())
@@ -66,8 +72,9 @@ class TestTermination():
                 self.wait_eyeball_thread(in_flight_req, self.grace_period)
 
     @pytest.mark.parametrize("signal", supported_signals())
-    def test_no_connection_shutdown(self, tmp_path, component_tests_config, signal):
-        config = component_tests_config(self.extra_config)
+    @pytest.mark.parametrize("protocol", protocols())
+    def test_no_connection_shutdown(self, tmp_path, component_tests_config, signal, protocol):
+        config = component_tests_config(self._extra_config(protocol))
         with start_cloudflared(
                 tmp_path, config, new_process=True, capture_output=False) as cloudflared:
             wait_tunnel_ready(tunnel_url=config.get_url())

component-tests/util.py

@@ -1,12 +1,13 @@
-from contextlib import contextmanager
 import logging
-import requests
-from retrying import retry
+import os
 import subprocess
-import yaml
-
+from contextlib import contextmanager
 from time import sleep
 
+import requests
+import yaml
+from retrying import retry
+
 from constants import METRICS_PORT, MAX_RETRIES, BACKOFF_SECS
 
 LOGGER = logging.getLogger(__name__)
@@ -19,7 +20,8 @@ def write_config(directory, config):
     return config_path
 
 
-def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False, allow_input=False, capture_output=True, root=False):
+def start_cloudflared(directory, config, cfd_args=["run"], cfd_pre_args=["tunnel"], new_process=False,
+                      allow_input=False, capture_output=True, root=False):
     config_path = write_config(directory, config.full_config)
     cmd = cloudflared_cmd(config, config_path, cfd_args, cfd_pre_args, root)
     if new_process:
@@ -53,18 +55,42 @@ def run_cloudflared_background(cmd, allow_input, capture_output):
             LOGGER.info(f"cloudflared log: {cfd.stderr.read()}")
 
 
+def wait_tunnel_ready(tunnel_url=None, require_min_connections=1, cfd_logs=None):
+    try:
+        inner_wait_tunnel_ready(tunnel_url, require_min_connections)
+    except Exception as e:
+        if cfd_logs is not None:
+            _log_cloudflared_logs(cfd_logs)
+        raise e
+
+
 @retry(stop_max_attempt_number=MAX_RETRIES, wait_fixed=BACKOFF_SECS * 1000)
-def wait_tunnel_ready(tunnel_url=None, require_min_connections=1):
+def inner_wait_tunnel_ready(tunnel_url=None, require_min_connections=1):
     metrics_url = f'http://localhost:{METRICS_PORT}/ready'
 
     with requests.Session() as s:
         resp = send_request(s, metrics_url, True)
-        assert resp.json()[
-            "readyConnections"] >= require_min_connections, f"Ready endpoint returned {resp.json()} but we expect at least {require_min_connections} connections"
+
+        assert resp.json()["readyConnections"] >= require_min_connections, \
+            f"Ready endpoint returned {resp.json()} but we expect at least {require_min_connections} connections"
+
         if tunnel_url is not None:
             send_request(s, tunnel_url, True)
 
 
+def _log_cloudflared_logs(cfd_logs):
+    log_file = cfd_logs
+    if os.path.isdir(cfd_logs):
+        files = os.listdir(cfd_logs)
+        if len(files) == 0:
+            return
+        log_file = os.path.join(cfd_logs, files[0])
+    with open(log_file, "r") as f:
+        LOGGER.warning("Cloudflared Tunnel was not ready:")
+        for line in f.readlines():
+            LOGGER.warning(line)
+
+
 @retry(stop_max_attempt_number=MAX_RETRIES * BACKOFF_SECS, wait_fixed=1000)
 def check_tunnel_not_connected():
     url = f'http://localhost:{METRICS_PORT}/ready'

config/configuration.go

@@ -175,60 +175,62 @@ func ValidateUrl(c *cli.Context, allowURLFromArgs bool) (*url.URL, error) {
 }
 
 type UnvalidatedIngressRule struct {
-	Hostname      string
-	Path          string
-	Service       string
-	OriginRequest OriginRequestConfig `yaml:"originRequest"`
+	Hostname      string              `json:"hostname"`
+	Path          string              `json:"path"`
+	Service       string              `json:"service"`
+	OriginRequest OriginRequestConfig `yaml:"originRequest" json:"originRequest"`
 }
 
 // OriginRequestConfig is a set of optional fields that users may set to
 // customize how cloudflared sends requests to origin services. It is used to set
 // up general config that apply to all rules, and also, specific per-rule
 // config.
-// Note: To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
+// Note:
+// - To specify a time.Duration in go-yaml, use e.g. "3s" or "24h".
+// - To specify a time.Duration in json, use int64 of the nanoseconds
 type OriginRequestConfig struct {
 	// HTTP proxy timeout for establishing a new connection
-	ConnectTimeout *time.Duration `yaml:"connectTimeout"`
+	ConnectTimeout *time.Duration `yaml:"connectTimeout" json:"connectTimeout"`
 	// HTTP proxy timeout for completing a TLS handshake
-	TLSTimeout *time.Duration `yaml:"tlsTimeout"`
+	TLSTimeout *time.Duration `yaml:"tlsTimeout" json:"tlsTimeout"`
 	// HTTP proxy TCP keepalive duration
-	TCPKeepAlive *time.Duration `yaml:"tcpKeepAlive"`
+	TCPKeepAlive *time.Duration `yaml:"tcpKeepAlive" json:"tcpKeepAlive"`
 	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
-	NoHappyEyeballs *bool `yaml:"noHappyEyeballs"`
+	NoHappyEyeballs *bool `yaml:"noHappyEyeballs" json:"noHappyEyeballs"`
 	// HTTP proxy maximum keepalive connection pool size
-	KeepAliveConnections *int `yaml:"keepAliveConnections"`
+	KeepAliveConnections *int `yaml:"keepAliveConnections" json:"keepAliveConnections"`
 	// HTTP proxy timeout for closing an idle connection
-	KeepAliveTimeout *time.Duration `yaml:"keepAliveTimeout"`
+	KeepAliveTimeout *time.Duration `yaml:"keepAliveTimeout" json:"keepAliveTimeout"`
 	// Sets the HTTP Host header for the local webserver.
-	HTTPHostHeader *string `yaml:"httpHostHeader"`
+	HTTPHostHeader *string `yaml:"httpHostHeader" json:"httpHostHeader"`
 	// Hostname on the origin server certificate.
-	OriginServerName *string `yaml:"originServerName"`
+	OriginServerName *string `yaml:"originServerName" json:"originServerName"`
 	// Path to the CA for the certificate of your origin.
 	// This option should be used only if your certificate is not signed by Cloudflare.
-	CAPool *string `yaml:"caPool"`
+	CAPool *string `yaml:"caPool" json:"caPool"`
 	// Disables TLS verification of the certificate presented by your origin.
 	// Will allow any certificate from the origin to be accepted.
 	// Note: The connection from your machine to Cloudflare's Edge is still encrypted.
-	NoTLSVerify *bool `yaml:"noTLSVerify"`
+	NoTLSVerify *bool `yaml:"noTLSVerify" json:"noTLSVerify"`
 	// Disables chunked transfer encoding.
 	// Useful if you are running a WSGI server.
-	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding"`
+	DisableChunkedEncoding *bool `yaml:"disableChunkedEncoding" json:"disableChunkedEncoding"`
 	// Runs as jump host
-	BastionMode *bool `yaml:"bastionMode"`
+	BastionMode *bool `yaml:"bastionMode" json:"bastionMode"`
 	// Listen address for the proxy.
-	ProxyAddress *string `yaml:"proxyAddress"`
+	ProxyAddress *string `yaml:"proxyAddress" json:"proxyAddress"`
 	// Listen port for the proxy.
-	ProxyPort *uint `yaml:"proxyPort"`
+	ProxyPort *uint `yaml:"proxyPort" json:"proxyPort"`
 	// Valid options are 'socks' or empty.
-	ProxyType *string `yaml:"proxyType"`
+	ProxyType *string `yaml:"proxyType" json:"proxyType"`
 	// IP rules for the proxy service
-	IPRules []IngressIPRule `yaml:"ipRules"`
+	IPRules []IngressIPRule `yaml:"ipRules" json:"ipRules"`
 }
 
 type IngressIPRule struct {
-	Prefix *string `yaml:"prefix"`
-	Ports  []int   `yaml:"ports"`
-	Allow  bool    `yaml:"allow"`
+	Prefix *string `yaml:"prefix" json:"prefix"`
+	Ports  []int   `yaml:"ports" json:"ports"`
+	Allow  bool    `yaml:"allow" json:"allow"`
 }
 
 type Configuration struct {
@@ -240,7 +242,7 @@ type Configuration struct {
 }
 
 type WarpRoutingConfig struct {
-	Enabled bool `yaml:"enabled"`
+	Enabled bool `yaml:"enabled" json:"enabled"`
 }
 
 type configFileSettings struct {

config/configuration_test.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"encoding/json"
 	"testing"
 	"time"
 
@@ -26,6 +27,18 @@ func TestConfigFileSettings(t *testing.T) {
 	)
 	rawYAML := `
 tunnel: config-file-test
+originRequest:
+  ipRules:
+  - prefix: "10.0.0.0/8"
+    ports:
+    - 80
+    - 8080
+    allow: false
+  - prefix: "fc00::/7"
+    ports:
+    - 443
+    - 4443
+    allow: true
 ingress:
  - hostname: tunnel1.example.com
    path: /id
@@ -53,6 +66,21 @@ counters:
 	assert.Equal(t, firstIngress, config.Ingress[0])
 	assert.Equal(t, secondIngress, config.Ingress[1])
 	assert.Equal(t, warpRouting, config.WarpRouting)
+	privateV4 := "10.0.0.0/8"
+	privateV6 := "fc00::/7"
+	ipRules := []IngressIPRule{
+		{
+			Prefix: &privateV4,
+			Ports:  []int{80, 8080},
+			Allow:  false,
+		},
+		{
+			Prefix: &privateV6,
+			Ports:  []int{443, 4443},
+			Allow:  true,
+		},
+	}
+	assert.Equal(t, ipRules, config.OriginRequest.IPRules)
 
 	retries, err := config.Int("retries")
 	assert.NoError(t, err)
@@ -81,3 +109,71 @@ counters:
 	assert.Equal(t, 456, counters[1])
 
 }
+
+func TestUnmarshalOriginRequestConfig(t *testing.T) {
+	raw := []byte(`
+{
+	"connectTimeout": 10000000000,
+	"tlsTimeout": 30000000000,
+	"tcpKeepAlive": 30000000000,
+	"noHappyEyeballs": true,
+	"keepAliveTimeout": 60000000000,
+	"keepAliveConnections": 10,
+	"httpHostHeader": "app.tunnel.com",
+	"originServerName": "app.tunnel.com",
+	"caPool": "/etc/capool",
+	"noTLSVerify": true,
+	"disableChunkedEncoding": true,
+	"bastionMode": true,
+	"proxyAddress": "127.0.0.3",
+	"proxyPort": 9000,
+	"proxyType": "socks",
+	"ipRules": [
+		{
+			"prefix": "10.0.0.0/8",
+			"ports": [80, 8080],
+			"allow": false
+		},
+		{
+			"prefix": "fc00::/7",
+			"ports": [443, 4443],
+			"allow": true
+		}
+	]
+}
+`)
+	var config OriginRequestConfig
+	assert.NoError(t, json.Unmarshal(raw, &config))
+	assert.Equal(t, time.Second*10, *config.ConnectTimeout)
+	assert.Equal(t, time.Second*30, *config.TLSTimeout)
+	assert.Equal(t, time.Second*30, *config.TCPKeepAlive)
+	assert.Equal(t, true, *config.NoHappyEyeballs)
+	assert.Equal(t, time.Second*60, *config.KeepAliveTimeout)
+	assert.Equal(t, 10, *config.KeepAliveConnections)
+	assert.Equal(t, "app.tunnel.com", *config.HTTPHostHeader)
+	assert.Equal(t, "app.tunnel.com", *config.OriginServerName)
+	assert.Equal(t, "/etc/capool", *config.CAPool)
+	assert.Equal(t, true, *config.NoTLSVerify)
+	assert.Equal(t, true, *config.DisableChunkedEncoding)
+	assert.Equal(t, true, *config.BastionMode)
+	assert.Equal(t, "127.0.0.3", *config.ProxyAddress)
+	assert.Equal(t, true, *config.NoTLSVerify)
+	assert.Equal(t, uint(9000), *config.ProxyPort)
+	assert.Equal(t, "socks", *config.ProxyType)
+
+	privateV4 := "10.0.0.0/8"
+	privateV6 := "fc00::/7"
+	ipRules := []IngressIPRule{
+		{
+			Prefix: &privateV4,
+			Ports:  []int{80, 8080},
+			Allow:  false,
+		},
+		{
+			Prefix: &privateV6,
+			Ports:  []int{443, 4443},
+			Allow:  true,
+		},
+	}
+	assert.Equal(t, ipRules, config.IPRules)
+}

connection/connection.go

@@ -25,13 +25,12 @@ const (
 
 var switchingProtocolText = fmt.Sprintf("%d %s", http.StatusSwitchingProtocols, http.StatusText(http.StatusSwitchingProtocols))
 
-type Config struct {
-	OriginProxy     OriginProxy
-	GracePeriod     time.Duration
-	ReplaceExisting bool
+type Orchestrator interface {
+	UpdateConfig(version int32, config []byte) *pogs.UpdateConfigurationResponse
+	GetOriginProxy() (OriginProxy, error)
 }
 
-type NamedTunnelConfig struct {
+type NamedTunnelProperties struct {
 	Credentials    Credentials
 	Client         pogs.ClientInfo
 	QuickTunnelUrl string
@@ -52,7 +51,7 @@ func (c *Credentials) Auth() pogs.TunnelAuth {
 	}
 }
 
-type ClassicTunnelConfig struct {
+type ClassicTunnelProperties struct {
 	Hostname   string
 	OriginCert []byte
 	// feature-flag to use new edge reconnect tokens

connection/connection_test.go

@@ -4,33 +4,28 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"math/rand"
 	"net/http"
-	"net/url"
 	"testing"
 	"time"
 
-	"github.com/gobwas/ws/wsutil"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/cloudflare/cloudflared/ingress"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
 )
 
 const (
-	largeFileSize = 2 * 1024 * 1024
+	largeFileSize   = 2 * 1024 * 1024
+	testGracePeriod = time.Millisecond * 100
 )
 
 var (
-	unusedWarpRoutingService = (*ingress.WarpRoutingService)(nil)
-	testConfig               = &Config{
-		OriginProxy: &mockOriginProxy{},
-		GracePeriod: time.Millisecond * 100,
+	testOrchestrator = &mockOrchestrator{
+		originProxy: &mockOriginProxy{},
 	}
 	log           = zerolog.Nop()
-	testOriginURL = &url.URL{
-		Scheme: "https",
-		Host:   "connectiontest.argotunnel.com",
-	}
 	testLargeResp = make([]byte, largeFileSize)
 )
 
@@ -42,6 +37,20 @@ type testRequest struct {
 	isProxyError   bool
 }
 
+type mockOrchestrator struct {
+	originProxy OriginProxy
+}
+
+func (*mockOrchestrator) UpdateConfig(version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
+	return &tunnelpogs.UpdateConfigurationResponse{
+		LastAppliedVersion: version,
+	}
+}
+
+func (mcr *mockOrchestrator) GetOriginProxy() (OriginProxy, error) {
+	return mcr.originProxy, nil
+}
+
 type mockOriginProxy struct{}
 
 func (moc *mockOriginProxy) ProxyHTTP(
@@ -50,7 +59,15 @@ func (moc *mockOriginProxy) ProxyHTTP(
 	isWebsocket bool,
 ) error {
 	if isWebsocket {
-		return wsEndpoint(w, req)
+		switch req.URL.Path {
+		case "/ws/echo":
+			return wsEchoEndpoint(w, req)
+		case "/ws/flaky":
+			return wsFlakyEndpoint(w, req)
+		default:
+			originRespEndpoint(w, http.StatusNotFound, []byte("ws endpoint not found"))
+			return fmt.Errorf("Unknwon websocket endpoint %s", req.URL.Path)
+		}
 	}
 	switch req.URL.Path {
 	case "/ok":
@@ -78,32 +95,82 @@ func (moc *mockOriginProxy) ProxyTCP(
 	return nil
 }
 
-type nowriter struct {
-	io.Reader
+type echoPipe struct {
+	reader *io.PipeReader
+	writer *io.PipeWriter
 }
 
-func (nowriter) Write(p []byte) (int, error) {
-	return 0, fmt.Errorf("Writer not implemented")
+func (ep *echoPipe) Read(p []byte) (int, error) {
+	return ep.reader.Read(p)
 }
 
-func wsEndpoint(w ResponseWriter, r *http.Request) error {
+func (ep *echoPipe) Write(p []byte) (int, error) {
+	return ep.writer.Write(p)
+}
+
+// A mock origin that echos data by streaming like a tcpOverWSConnection
+// https://github.com/cloudflare/cloudflared/blob/master/ingress/origin_connection.go
+func wsEchoEndpoint(w ResponseWriter, r *http.Request) error {
 	resp := &http.Response{
 		StatusCode: http.StatusSwitchingProtocols,
 	}
-	_ = w.WriteRespHeaders(resp.StatusCode, resp.Header)
-	clientReader := nowriter{r.Body}
+	if err := w.WriteRespHeaders(resp.StatusCode, resp.Header); err != nil {
+		return err
+	}
+	wsCtx, cancel := context.WithCancel(r.Context())
+	readPipe, writePipe := io.Pipe()
+	wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, r), &log)
 	go func() {
-		for {
-			data, err := wsutil.ReadClientText(clientReader)
-			if err != nil {
-				return
-			}
-			if err := wsutil.WriteServerText(w, data); err != nil {
-				return
-			}
+		select {
+		case <-wsCtx.Done():
+		case <-r.Context().Done():
 		}
+		readPipe.Close()
+		writePipe.Close()
 	}()
-	<-r.Context().Done()
+
+	originConn := &echoPipe{reader: readPipe, writer: writePipe}
+	websocket.Stream(wsConn, originConn, &log)
+	cancel()
+	wsConn.Close()
+	return nil
+}
+
+type flakyConn struct {
+	closeAt time.Time
+}
+
+func (fc *flakyConn) Read(p []byte) (int, error) {
+	if time.Now().After(fc.closeAt) {
+		return 0, io.EOF
+	}
+	n := copy(p, "Read from flaky connection")
+	return n, nil
+}
+
+func (fc *flakyConn) Write(p []byte) (int, error) {
+	if time.Now().After(fc.closeAt) {
+		return 0, fmt.Errorf("flaky connection closed")
+	}
+	return len(p), nil
+}
+
+func wsFlakyEndpoint(w ResponseWriter, r *http.Request) error {
+	resp := &http.Response{
+		StatusCode: http.StatusSwitchingProtocols,
+	}
+	if err := w.WriteRespHeaders(resp.StatusCode, resp.Header); err != nil {
+		return err
+	}
+	wsCtx, cancel := context.WithCancel(r.Context())
+
+	wsConn := websocket.NewConn(wsCtx, NewHTTPResponseReadWriterAcker(w, r), &log)
+
+	closedAfter := time.Millisecond * time.Duration(rand.Intn(50))
+	originConn := &flakyConn{closeAt: time.Now().Add(closedAfter)}
+	websocket.Stream(wsConn, originConn, &log)
+	cancel()
+	wsConn.Close()
 	return nil
 }
 

connection/control.go

@@ -16,9 +16,9 @@ type RPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) Na
 type controlStream struct {
 	observer *Observer
 
-	connectedFuse     ConnectedFuse
-	namedTunnelConfig *NamedTunnelConfig
-	connIndex         uint8
+	connectedFuse         ConnectedFuse
+	namedTunnelProperties *NamedTunnelProperties
+	connIndex             uint8
 
 	newRPCClientFunc RPCClientFunc
 
@@ -29,7 +29,9 @@ type controlStream struct {
 
 // ControlStreamHandler registers connections with origintunneld and initiates graceful shutdown.
 type ControlStreamHandler interface {
-	ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions, shouldWaitForUnregister bool) error
+	// ServeControlStream handles the control plane of the transport in the current goroutine calling this
+	ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions) error
+	// IsStopped tells whether the method above has finished
 	IsStopped() bool
 }
 
@@ -37,7 +39,7 @@ type ControlStreamHandler interface {
 func NewControlStream(
 	observer *Observer,
 	connectedFuse ConnectedFuse,
-	namedTunnelConfig *NamedTunnelConfig,
+	namedTunnelConfig *NamedTunnelProperties,
 	connIndex uint8,
 	newRPCClientFunc RPCClientFunc,
 	gracefulShutdownC <-chan struct{},
@@ -47,13 +49,13 @@ func NewControlStream(
 		newRPCClientFunc = newRegistrationRPCClient
 	}
 	return &controlStream{
-		observer:          observer,
-		connectedFuse:     connectedFuse,
-		namedTunnelConfig: namedTunnelConfig,
-		newRPCClientFunc:  newRPCClientFunc,
-		connIndex:         connIndex,
-		gracefulShutdownC: gracefulShutdownC,
-		gracePeriod:       gracePeriod,
+		observer:              observer,
+		connectedFuse:         connectedFuse,
+		namedTunnelProperties: namedTunnelConfig,
+		newRPCClientFunc:      newRPCClientFunc,
+		connIndex:             connIndex,
+		gracefulShutdownC:     gracefulShutdownC,
+		gracePeriod:           gracePeriod,
 	}
 }
 
@@ -61,22 +63,16 @@ func (c *controlStream) ServeControlStream(
 	ctx context.Context,
 	rw io.ReadWriteCloser,
 	connOptions *tunnelpogs.ConnectionOptions,
-	shouldWaitForUnregister bool,
 ) error {
 	rpcClient := c.newRPCClientFunc(ctx, rw, c.observer.log)
 
-	if err := rpcClient.RegisterConnection(ctx, c.namedTunnelConfig, connOptions, c.connIndex, c.observer); err != nil {
+	if err := rpcClient.RegisterConnection(ctx, c.namedTunnelProperties, connOptions, c.connIndex, c.observer); err != nil {
 		rpcClient.Close()
 		return err
 	}
 	c.connectedFuse.Connected()
 
-	if shouldWaitForUnregister {
-		c.waitForUnregister(ctx, rpcClient)
-	} else {
-		go c.waitForUnregister(ctx, rpcClient)
-	}
-
+	c.waitForUnregister(ctx, rpcClient)
 	return nil
 }
 

connection/h2mux.go

@@ -22,9 +22,10 @@ const (
 )
 
 type h2muxConnection struct {
-	config      *Config
-	muxerConfig *MuxerConfig
-	muxer       *h2mux.Muxer
+	orchestrator Orchestrator
+	gracePeriod  time.Duration
+	muxerConfig  *MuxerConfig
+	muxer        *h2mux.Muxer
 	// connectionID is only used by metrics, and prometheus requires labels to be string
 	connIndexStr string
 	connIndex    uint8
@@ -60,7 +61,8 @@ func (mc *MuxerConfig) H2MuxerConfig(h h2mux.MuxedStreamHandler, log *zerolog.Lo
 
 // NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
 func NewH2muxConnection(
-	config *Config,
+	orchestrator Orchestrator,
+	gracePeriod time.Duration,
 	muxerConfig *MuxerConfig,
 	edgeConn net.Conn,
 	connIndex uint8,
@@ -68,7 +70,8 @@ func NewH2muxConnection(
 	gracefulShutdownC <-chan struct{},
 ) (*h2muxConnection, error, bool) {
 	h := &h2muxConnection{
-		config:            config,
+		orchestrator:      orchestrator,
+		gracePeriod:       gracePeriod,
 		muxerConfig:       muxerConfig,
 		connIndexStr:      uint8ToString(connIndex),
 		connIndex:         connIndex,
@@ -88,7 +91,7 @@ func NewH2muxConnection(
 	return h, nil, false
 }
 
-func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelConfig, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
+func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelProperties, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {
 		return h.serveMuxer(serveCtx)
@@ -117,7 +120,7 @@ func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *Nam
 	return err
 }
 
-func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel *ClassicTunnelConfig, credentialManager CredentialManager, registrationOptions *tunnelpogs.RegistrationOptions, connectedFuse ConnectedFuse) error {
+func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel *ClassicTunnelProperties, credentialManager CredentialManager, registrationOptions *tunnelpogs.RegistrationOptions, connectedFuse ConnectedFuse) error {
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {
 		return h.serveMuxer(serveCtx)
@@ -224,7 +227,13 @@ func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
 		sourceConnectionType = TypeWebsocket
 	}
 
-	err := h.config.OriginProxy.ProxyHTTP(respWriter, req, sourceConnectionType == TypeWebsocket)
+	originProxy, err := h.orchestrator.GetOriginProxy()
+	if err != nil {
+		respWriter.WriteErrorResponse()
+		return err
+	}
+
+	err = originProxy.ProxyHTTP(respWriter, req, sourceConnectionType == TypeWebsocket)
 	if err != nil {
 		respWriter.WriteErrorResponse()
 	}

connection/h2mux_header.go

@@ -0,0 +1,128 @@
+package connection
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+)
+
+// H2RequestHeadersToH1Request converts the HTTP/2 headers coming from origintunneld
+// to an HTTP/1 Request object destined for the local origin web service.
+// This operation includes conversion of the pseudo-headers into their closest
+// HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3
+func H2RequestHeadersToH1Request(h2 []h2mux.Header, h1 *http.Request) error {
+	for _, header := range h2 {
+		name := strings.ToLower(header.Name)
+		if !IsH2muxControlRequestHeader(name) {
+			continue
+		}
+
+		switch name {
+		case ":method":
+			h1.Method = header.Value
+		case ":scheme":
+			// noop - use the preexisting scheme from h1.URL
+		case ":authority":
+			// Otherwise the host header will be based on the origin URL
+			h1.Host = header.Value
+		case ":path":
+			// We don't want to be an "opinionated" proxy, so ideally we would use :path as-is.
+			// However, this HTTP/1 Request object belongs to the Go standard library,
+			// whose URL package makes some opinionated decisions about the encoding of
+			// URL characters: see the docs of https://godoc.org/net/url#URL,
+			// in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath,
+			// which is always used when computing url.URL.String(), whether we'd like it or not.
+			//
+			// Well, not *always*. We could circumvent this by using url.URL.Opaque. But
+			// that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque
+			// is treated differently when HTTP_PROXY is set!
+			// See https://github.com/golang/go/issues/5684#issuecomment-66080888
+			//
+			// This means we are subject to the behavior of net/url's function `shouldEscape`
+			// (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101
+
+			if header.Value == "*" {
+				h1.URL.Path = "*"
+				continue
+			}
+			// Due to the behavior of validation.ValidateUrl, h1.URL may
+			// already have a partial value, with or without a trailing slash.
+			base := h1.URL.String()
+			base = strings.TrimRight(base, "/")
+			// But we know :path begins with '/', because we handled '*' above - see RFC7540
+			requestURL, err := url.Parse(base + header.Value)
+			if err != nil {
+				return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value))
+			}
+			h1.URL = requestURL
+		case "content-length":
+			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
+			if err != nil {
+				return fmt.Errorf("unparseable content length")
+			}
+			h1.ContentLength = contentLength
+		case RequestUserHeaders:
+			// Do not forward the serialized headers to the origin -- deserialize them, and ditch the serialized version
+			// Find and parse user headers serialized into a single one
+			userHeaders, err := DeserializeHeaders(header.Value)
+			if err != nil {
+				return errors.Wrap(err, "Unable to parse user headers")
+			}
+			for _, userHeader := range userHeaders {
+				h1.Header.Add(userHeader.Name, userHeader.Value)
+			}
+		default:
+			// All other control headers shall just be proxied transparently
+			h1.Header.Add(header.Name, header.Value)
+		}
+	}
+
+	return nil
+}
+
+func H1ResponseToH2ResponseHeaders(status int, h1 http.Header) (h2 []h2mux.Header) {
+	h2 = []h2mux.Header{
+		{Name: ":status", Value: strconv.Itoa(status)},
+	}
+	userHeaders := make(http.Header, len(h1))
+	for header, values := range h1 {
+		h2name := strings.ToLower(header)
+		if h2name == "content-length" {
+			// This header has meaning in HTTP/2 and will be used by the edge,
+			// so it should be sent as an HTTP/2 response header.
+
+			// Since these are http2 headers, they're required to be lowercase
+			h2 = append(h2, h2mux.Header{Name: "content-length", Value: values[0]})
+		} else if !IsH2muxControlResponseHeader(h2name) || IsWebsocketClientHeader(h2name) {
+			// User headers, on the other hand, must all be serialized so that
+			// HTTP/2 header validation won't be applied to HTTP/1 header values
+			userHeaders[header] = values
+		}
+	}
+
+	// Perform user header serialization and set them in the single header
+	h2 = append(h2, h2mux.Header{Name: ResponseUserHeaders, Value: SerializeHeaders(userHeaders)})
+	return h2
+}
+
+// IsH2muxControlRequestHeader is called in the direction of eyeball -> origin.
+func IsH2muxControlRequestHeader(headerName string) bool {
+	return headerName == "content-length" ||
+		headerName == "connection" || headerName == "upgrade" || // Websocket request headers
+		strings.HasPrefix(headerName, ":") ||
+		strings.HasPrefix(headerName, "cf-")
+}
+
+// IsH2muxControlResponseHeader is called in the direction of eyeball <- origin.
+func IsH2muxControlResponseHeader(headerName string) bool {
+	return headerName == "content-length" ||
+		strings.HasPrefix(headerName, ":") ||
+		strings.HasPrefix(headerName, "cf-int-") ||
+		strings.HasPrefix(headerName, "cf-cloudflared-")
+}

connection/h2mux_header_test.go

@@ -0,0 +1,642 @@
+package connection
+
+import (
+	"fmt"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"reflect"
+	"regexp"
+	"strings"
+	"testing"
+	"testing/quick"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+)
+
+type ByName []h2mux.Header
+
+func (a ByName) Len() int      { return len(a) }
+func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByName) Less(i, j int) bool {
+	if a[i].Name == a[j].Name {
+		return a[i].Value < a[j].Value
+	}
+
+	return a[i].Name < a[j].Name
+}
+
+func TestH2RequestHeadersToH1Request_RegularHeaders(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	mockHeaders := http.Header{
+		"Mock header 1": {"Mock value 1"},
+		"Mock header 2": {"Mock value 2"},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(createSerializedHeaders(RequestUserHeaders, mockHeaders), request)
+
+	assert.True(t, reflect.DeepEqual(mockHeaders, request.Header))
+	assert.NoError(t, headersConversionErr)
+}
+
+func createSerializedHeaders(headersField string, headers http.Header) []h2mux.Header {
+	return []h2mux.Header{{
+		Name:  headersField,
+		Value: SerializeHeaders(headers),
+	}}
+}
+
+func TestH2RequestHeadersToH1Request_NoHeaders(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	emptyHeaders := make(http.Header)
+	headersConversionErr := H2RequestHeadersToH1Request(
+		[]h2mux.Header{{
+			Name:  RequestUserHeaders,
+			Value: SerializeHeaders(emptyHeaders),
+		}},
+		request,
+	)
+
+	assert.True(t, reflect.DeepEqual(emptyHeaders, request.Header))
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_InvalidHostPath(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	mockRequestHeaders := []h2mux.Header{
+		{Name: ":path", Value: "//bad_path/"},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
+
+	assert.Equal(t, http.Header{
+		"Mock header": []string{"Mock value"},
+	}, request.Header)
+
+	assert.Equal(t, "http://example.com//bad_path/", request.URL.String())
+
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_HostPathWithQuery(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
+	assert.NoError(t, err)
+
+	mockRequestHeaders := []h2mux.Header{
+		{Name: ":path", Value: "/?query=mock%20value"},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
+
+	assert.Equal(t, http.Header{
+		"Mock header": []string{"Mock value"},
+	}, request.Header)
+
+	assert.Equal(t, "http://example.com/?query=mock%20value", request.URL.String())
+
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_HostPathWithURLEncoding(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
+	assert.NoError(t, err)
+
+	mockRequestHeaders := []h2mux.Header{
+		{Name: ":path", Value: "/mock%20path"},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
+
+	assert.Equal(t, http.Header{
+		"Mock header": []string{"Mock value"},
+	}, request.Header)
+
+	assert.Equal(t, "http://example.com/mock%20path", request.URL.String())
+
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_WeirdURLs(t *testing.T) {
+	type testCase struct {
+		path string
+		want string
+	}
+	testCases := []testCase{
+		{
+			path: "",
+			want: "",
+		},
+		{
+			path: "/",
+			want: "/",
+		},
+		{
+			path: "//",
+			want: "//",
+		},
+		{
+			path: "/test",
+			want: "/test",
+		},
+		{
+			path: "//test",
+			want: "//test",
+		},
+		{
+			// https://github.com/cloudflare/cloudflared/issues/81
+			path: "//test/",
+			want: "//test/",
+		},
+		{
+			path: "/%2Ftest",
+			want: "/%2Ftest",
+		},
+		{
+			path: "//%20test",
+			want: "//%20test",
+		},
+		{
+			// https://github.com/cloudflare/cloudflared/issues/124
+			path: "/test?get=somthing%20a",
+			want: "/test?get=somthing%20a",
+		},
+		{
+			path: "/%20",
+			want: "/%20",
+		},
+		{
+			// stdlib's EscapedPath() will always percent-encode ' '
+			path: "/ ",
+			want: "/%20",
+		},
+		{
+			path: "/ a ",
+			want: "/%20a%20",
+		},
+		{
+			path: "/a%20b",
+			want: "/a%20b",
+		},
+		{
+			path: "/foo/bar;param?query#frag",
+			want: "/foo/bar;param?query#frag",
+		},
+		{
+			// stdlib's EscapedPath() will always percent-encode non-ASCII chars
+			path: "/a␠b",
+			want: "/a%E2%90%A0b",
+		},
+		{
+			path: "/a-umlaut-ä",
+			want: "/a-umlaut-%C3%A4",
+		},
+		{
+			path: "/a-umlaut-%C3%A4",
+			want: "/a-umlaut-%C3%A4",
+		},
+		{
+			path: "/a-umlaut-%c3%a4",
+			want: "/a-umlaut-%c3%a4",
+		},
+		{
+			// here the second '#' is treated as part of the fragment
+			path: "/a#b#c",
+			want: "/a#b%23c",
+		},
+		{
+			path: "/a#b␠c",
+			want: "/a#b%E2%90%A0c",
+		},
+		{
+			path: "/a#b%20c",
+			want: "/a#b%20c",
+		},
+		{
+			path: "/a#b c",
+			want: "/a#b%20c",
+		},
+		{
+			// stdlib's EscapedPath() will always percent-encode '\'
+			path: "/\\",
+			want: "/%5C",
+		},
+		{
+			path: "/a\\",
+			want: "/a%5C",
+		},
+		{
+			path: "/a,b.c.",
+			want: "/a,b.c.",
+		},
+		{
+			path: "/.",
+			want: "/.",
+		},
+		{
+			// stdlib's EscapedPath() will always percent-encode '`'
+			path: "/a`",
+			want: "/a%60",
+		},
+		{
+			path: "/a[0]",
+			want: "/a[0]",
+		},
+		{
+			path: "/?a[0]=5 &b[]=",
+			want: "/?a[0]=5 &b[]=",
+		},
+		{
+			path: "/?a=%22b%20%22",
+			want: "/?a=%22b%20%22",
+		},
+	}
+
+	for index, testCase := range testCases {
+		requestURL := "https://example.com"
+
+		request, err := http.NewRequest(http.MethodGet, requestURL, nil)
+		assert.NoError(t, err)
+
+		mockRequestHeaders := []h2mux.Header{
+			{Name: ":path", Value: testCase.path},
+			{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		}
+
+		headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
+		assert.NoError(t, headersConversionErr)
+
+		assert.Equal(t,
+			http.Header{
+				"Mock header": []string{"Mock value"},
+			},
+			request.Header)
+
+		assert.Equal(t,
+			"https://example.com"+testCase.want,
+			request.URL.String(),
+			"Failed URL index: %v %#v", index, testCase)
+	}
+}
+
+func TestH2RequestHeadersToH1Request_QuickCheck(t *testing.T) {
+	config := &quick.Config{
+		Values: func(args []reflect.Value, rand *rand.Rand) {
+			args[0] = reflect.ValueOf(randomHTTP2Path(t, rand))
+		},
+	}
+
+	type testOrigin struct {
+		url string
+
+		expectedScheme   string
+		expectedBasePath string
+	}
+	testOrigins := []testOrigin{
+		{
+			url:              "http://origin.hostname.example.com:8080",
+			expectedScheme:   "http",
+			expectedBasePath: "http://origin.hostname.example.com:8080",
+		},
+		{
+			url:              "http://origin.hostname.example.com:8080/",
+			expectedScheme:   "http",
+			expectedBasePath: "http://origin.hostname.example.com:8080",
+		},
+		{
+			url:              "http://origin.hostname.example.com:8080/api",
+			expectedScheme:   "http",
+			expectedBasePath: "http://origin.hostname.example.com:8080/api",
+		},
+		{
+			url:              "http://origin.hostname.example.com:8080/api/",
+			expectedScheme:   "http",
+			expectedBasePath: "http://origin.hostname.example.com:8080/api",
+		},
+		{
+			url:              "https://origin.hostname.example.com:8080/api",
+			expectedScheme:   "https",
+			expectedBasePath: "https://origin.hostname.example.com:8080/api",
+		},
+	}
+
+	// use multiple schemes to demonstrate that the URL is based on the
+	// origin's scheme, not the :scheme header
+	for _, testScheme := range []string{"http", "https"} {
+		for _, testOrigin := range testOrigins {
+			assertion := func(testPath string) bool {
+				const expectedMethod = "POST"
+				const expectedHostname = "request.hostname.example.com"
+
+				h2 := []h2mux.Header{
+					{Name: ":method", Value: expectedMethod},
+					{Name: ":scheme", Value: testScheme},
+					{Name: ":authority", Value: expectedHostname},
+					{Name: ":path", Value: testPath},
+					{Name: RequestUserHeaders, Value: ""},
+				}
+				h1, err := http.NewRequest("GET", testOrigin.url, nil)
+				require.NoError(t, err)
+
+				err = H2RequestHeadersToH1Request(h2, h1)
+				return assert.NoError(t, err) &&
+					assert.Equal(t, expectedMethod, h1.Method) &&
+					assert.Equal(t, expectedHostname, h1.Host) &&
+					assert.Equal(t, testOrigin.expectedScheme, h1.URL.Scheme) &&
+					assert.Equal(t, testOrigin.expectedBasePath+testPath, h1.URL.String())
+			}
+			err := quick.Check(assertion, config)
+			assert.NoError(t, err)
+		}
+	}
+}
+
+func randomASCIIPrintableChar(rand *rand.Rand) int {
+	// smallest printable ASCII char is 32, largest is 126
+	const startPrintable = 32
+	const endPrintable = 127
+	return startPrintable + rand.Intn(endPrintable-startPrintable)
+}
+
+// randomASCIIText generates an ASCII string, some of whose characters may be
+// percent-encoded. Its "logical length" (ignoring percent-encoding) is
+// between 1 and `maxLength`.
+func randomASCIIText(rand *rand.Rand, minLength int, maxLength int) string {
+	length := minLength + rand.Intn(maxLength)
+	var result strings.Builder
+	for i := 0; i < length; i++ {
+		c := randomASCIIPrintableChar(rand)
+
+		// 1/4 chance of using percent encoding when not necessary
+		if c == '%' || rand.Intn(4) == 0 {
+			result.WriteString(fmt.Sprintf("%%%02X", c))
+		} else {
+			result.WriteByte(byte(c))
+		}
+	}
+	return result.String()
+}
+
+// Calls `randomASCIIText` and ensures the result is a valid URL path,
+// i.e. one that can pass unchanged through url.URL.String()
+func randomHTTP1Path(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
+	text := randomASCIIText(rand, minLength, maxLength)
+	re, err := regexp.Compile("[^/;,]*")
+	require.NoError(t, err)
+	return "/" + re.ReplaceAllStringFunc(text, url.PathEscape)
+}
+
+// Calls `randomASCIIText` and ensures the result is a valid URL query,
+// i.e. one that can pass unchanged through url.URL.String()
+func randomHTTP1Query(rand *rand.Rand, minLength int, maxLength int) string {
+	text := randomASCIIText(rand, minLength, maxLength)
+	return "?" + strings.ReplaceAll(text, "#", "%23")
+}
+
+// Calls `randomASCIIText` and ensures the result is a valid URL fragment,
+// i.e. one that can pass unchanged through url.URL.String()
+func randomHTTP1Fragment(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
+	text := randomASCIIText(rand, minLength, maxLength)
+	u, err := url.Parse("#" + text)
+	require.NoError(t, err)
+	return u.String()
+}
+
+// Assemble a random :path pseudoheader that is legal by Go stdlib standards
+// (i.e. all characters will satisfy "net/url".shouldEscape for their respective locations)
+func randomHTTP2Path(t *testing.T, rand *rand.Rand) string {
+	result := randomHTTP1Path(t, rand, 1, 64)
+	if rand.Intn(2) == 1 {
+		result += randomHTTP1Query(rand, 1, 32)
+	}
+	if rand.Intn(2) == 1 {
+		result += randomHTTP1Fragment(t, rand, 1, 16)
+	}
+	return result
+}
+
+func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []h2mux.Header) {
+	for name, values := range headers {
+		for _, value := range values {
+			h2muxHeaders = append(h2muxHeaders, h2mux.Header{Name: name, Value: value})
+		}
+	}
+
+	return h2muxHeaders
+}
+
+func TestParseRequestHeaders(t *testing.T) {
+	mockUserHeadersToSerialize := http.Header{
+		"Mock-Header-One":   {"1", "1.5"},
+		"Mock-Header-Two":   {"2"},
+		"Mock-Header-Three": {"3"},
+	}
+
+	mockHeaders := []h2mux.Header{
+		{Name: "One", Value: "1"}, // will be dropped
+		{Name: "Cf-Two", Value: "cf-value-1"},
+		{Name: "Cf-Two", Value: "cf-value-2"},
+		{Name: RequestUserHeaders, Value: SerializeHeaders(mockUserHeadersToSerialize)},
+	}
+
+	expectedHeaders := []h2mux.Header{
+		{Name: "Cf-Two", Value: "cf-value-1"},
+		{Name: "Cf-Two", Value: "cf-value-2"},
+		{Name: "Mock-Header-One", Value: "1"},
+		{Name: "Mock-Header-One", Value: "1.5"},
+		{Name: "Mock-Header-Two", Value: "2"},
+		{Name: "Mock-Header-Three", Value: "3"},
+	}
+	h1 := &http.Request{
+		Header: make(http.Header),
+	}
+	err := H2RequestHeadersToH1Request(mockHeaders, h1)
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, expectedHeaders, stdlibHeaderToH2muxHeader(h1.Header))
+}
+
+func TestIsH2muxControlRequestHeader(t *testing.T) {
+	controlRequestHeaders := []string{
+		// Anything that begins with cf-
+		"cf-sample-header",
+
+		// Any http2 pseudoheader
+		":sample-pseudo-header",
+
+		// content-length is a special case, it has to be there
+		// for some requests to work (per the HTTP2 spec)
+		"content-length",
+
+		// Websocket request headers
+		"connection",
+		"upgrade",
+	}
+
+	for _, header := range controlRequestHeaders {
+		assert.True(t, IsH2muxControlRequestHeader(header))
+	}
+}
+
+func TestIsH2muxControlResponseHeader(t *testing.T) {
+	controlResponseHeaders := []string{
+		// Anything that begins with cf-int- or cf-cloudflared-
+		"cf-int-sample-header",
+		"cf-cloudflared-sample-header",
+
+		// Any http2 pseudoheader
+		":sample-pseudo-header",
+
+		// content-length is a special case, it has to be there
+		// for some requests to work (per the HTTP2 spec)
+		"content-length",
+	}
+
+	for _, header := range controlResponseHeaders {
+		assert.True(t, IsH2muxControlResponseHeader(header))
+	}
+}
+
+func TestIsNotH2muxControlRequestHeader(t *testing.T) {
+	notControlRequestHeaders := []string{
+		"mock-header",
+		"another-sample-header",
+	}
+
+	for _, header := range notControlRequestHeaders {
+		assert.False(t, IsH2muxControlRequestHeader(header))
+	}
+}
+
+func TestIsNotH2muxControlResponseHeader(t *testing.T) {
+	notControlResponseHeaders := []string{
+		"mock-header",
+		"another-sample-header",
+		"upgrade",
+		"connection",
+		"cf-whatever", // On the response path, we only want to filter cf-int- and cf-cloudflared-
+	}
+
+	for _, header := range notControlResponseHeaders {
+		assert.False(t, IsH2muxControlResponseHeader(header))
+	}
+}
+
+func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
+	mockHeaders := http.Header{
+		"User-header-one":       {""},
+		"User-header-two":       {"1", "2"},
+		"cf-header":             {"cf-value"},
+		"cf-int-header":         {"cf-int-value"},
+		"cf-cloudflared-header": {"cf-cloudflared-value"},
+		"Content-Length":        {"123"},
+	}
+	mockResponse := http.Response{
+		StatusCode: 200,
+		Header:     mockHeaders,
+	}
+
+	headers := H1ResponseToH2ResponseHeaders(mockResponse.StatusCode, mockResponse.Header)
+
+	serializedHeadersIndex := -1
+	for i, header := range headers {
+		if header.Name == ResponseUserHeaders {
+			serializedHeadersIndex = i
+			break
+		}
+	}
+	assert.NotEqual(t, -1, serializedHeadersIndex)
+	actualControlHeaders := append(
+		headers[:serializedHeadersIndex],
+		headers[serializedHeadersIndex+1:]...,
+	)
+	expectedControlHeaders := []h2mux.Header{
+		{Name: ":status", Value: "200"},
+		{Name: "content-length", Value: "123"},
+	}
+
+	assert.ElementsMatch(t, expectedControlHeaders, actualControlHeaders)
+
+	actualUserHeaders, err := DeserializeHeaders(headers[serializedHeadersIndex].Value)
+	expectedUserHeaders := []h2mux.Header{
+		{Name: "User-header-one", Value: ""},
+		{Name: "User-header-two", Value: "1"},
+		{Name: "User-header-two", Value: "2"},
+		{Name: "cf-header", Value: "cf-value"},
+	}
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, expectedUserHeaders, actualUserHeaders)
+}
+
+// The purpose of this test is to check that our code and the http.Header
+// implementation don't throw validation errors about header size
+func TestHeaderSize(t *testing.T) {
+	largeValue := randSeq(5 * 1024 * 1024) // 5Mb
+	largeHeaders := http.Header{
+		"User-header": {largeValue},
+	}
+	mockResponse := http.Response{
+		StatusCode: 200,
+		Header:     largeHeaders,
+	}
+
+	serializedHeaders := H1ResponseToH2ResponseHeaders(mockResponse.StatusCode, mockResponse.Header)
+	request, err := http.NewRequest(http.MethodGet, "https://example.com/", nil)
+	assert.NoError(t, err)
+	for _, header := range serializedHeaders {
+		request.Header.Set(header.Name, header.Value)
+	}
+
+	for _, header := range serializedHeaders {
+		if header.Name != ResponseUserHeaders {
+			continue
+		}
+
+		deserializedHeaders, err := DeserializeHeaders(header.Value)
+		assert.NoError(t, err)
+		assert.Equal(t, largeValue, deserializedHeaders[0].Value)
+	}
+}
+
+func randSeq(n int) string {
+	randomizer := rand.New(rand.NewSource(17))
+	var letters = []rune(":;,+/=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letters[randomizer.Intn(len(letters))]
+	}
+	return string(b)
+}
+
+func BenchmarkH1ResponseToH2ResponseHeaders(b *testing.B) {
+	ser := "eC1mb3J3YXJkZWQtcHJvdG8:aHR0cHM;dXBncmFkZS1pbnNlY3VyZS1yZXF1ZXN0cw:MQ;YWNjZXB0LWxhbmd1YWdl:ZW4tVVMsZW47cT0wLjkscnU7cT0wLjg;YWNjZXB0LWVuY29kaW5n:Z3ppcA;eC1mb3J3YXJkZWQtZm9y:MTczLjI0NS42MC42;dXNlci1hZ2VudA:TW96aWxsYS81LjAgKE1hY2ludG9zaDsgSW50ZWwgTWFjIE9TIFggMTBfMTRfNikgQXBwbGVXZWJLaXQvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lLzg0LjAuNDE0Ny44OSBTYWZhcmkvNTM3LjM2;c2VjLWZldGNoLW1vZGU:bmF2aWdhdGU;Y2RuLWxvb3A:Y2xvdWRmbGFyZQ;c2VjLWZldGNoLWRlc3Q:ZG9jdW1lbnQ;c2VjLWZldGNoLXVzZXI:PzE;c2VjLWZldGNoLXNpdGU:bm9uZQ;Y29va2ll:X19jZmR1aWQ9ZGNkOWZjOGNjNWMxMzE0NTMyYTFkMjhlZDEyOWRhOTYwMTU2OTk1MTYzNDsgX19jZl9ibT1mYzY2MzMzYzAzZmM0MWFiZTZmOWEyYzI2ZDUwOTA0YzIxYzZhMTQ2LTE1OTU2MjIzNDEtMTgwMC1BZTVzS2pIU2NiWGVFM05mMUhrTlNQMG1tMHBLc2pQWkloVnM1Z2g1SkNHQkFhS1UxVDB2b003alBGN3FjMHVSR2NjZGcrWHdhL1EzbTJhQzdDVU4xZ2M9;YWNjZXB0:dGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksaW1hZ2Uvd2VicCxpbWFnZS9hcG5nLCovKjtxPTAuOCxhcHBsaWNhdGlvbi9zaWduZWQtZXhjaGFuZ2U7dj1iMztxPTAuOQ"
+	h2, _ := DeserializeHeaders(ser)
+	h1 := make(http.Header)
+	for _, header := range h2 {
+		h1.Add(header.Name, header.Value)
+	}
+	h1.Add("Content-Length", "200")
+	h1.Add("Cf-Something", "Else")
+	h1.Add("Upgrade", "websocket")
+
+	h1resp := &http.Response{
+		StatusCode: 200,
+		Header:     h1,
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_ = H1ResponseToH2ResponseHeaders(h1resp.StatusCode, h1resp.Header)
+	}
+}

connection/h2mux_test.go

@@ -48,7 +48,7 @@ func newH2MuxConnection(t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
 	}()
 	var connIndex = uint8(0)
 	testObserver := NewObserver(&log, &log, false)
-	h2muxConn, err, _ := NewH2muxConnection(testConfig, testMuxerConfig, originConn, connIndex, testObserver, nil)
+	h2muxConn, err, _ := NewH2muxConnection(testOrchestrator, testGracePeriod, testMuxerConfig, originConn, connIndex, testObserver, nil)
 	require.NoError(t, err)
 	return h2muxConn, <-edgeMuxChan
 }
@@ -147,7 +147,7 @@ func TestServeStreamWS(t *testing.T) {
 	headers := []h2mux.Header{
 		{
 			Name:  ":path",
-			Value: "/ws",
+			Value: "/ws/echo",
 		},
 		{
 			Name:  "connection",
@@ -167,10 +167,10 @@ func TestServeStreamWS(t *testing.T) {
 	assert.True(t, hasHeader(stream, ResponseMetaHeader, responseMetaHeaderOrigin))
 
 	data := []byte("test websocket")
-	err = wsutil.WriteClientText(writePipe, data)
+	err = wsutil.WriteClientBinary(writePipe, data)
 	require.NoError(t, err)
 
-	respBody, err := wsutil.ReadServerText(stream)
+	respBody, err := wsutil.ReadServerBinary(stream)
 	require.NoError(t, err)
 	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
 

connection/header.go

@@ -4,8 +4,6 @@ import (
 	"encoding/base64"
 	"fmt"
 	"net/http"
-	"net/url"
-	"strconv"
 	"strings"
 
 	"github.com/pkg/errors"
@@ -44,93 +42,9 @@ func mustInitRespMetaHeader(src string) string {
 
 var headerEncoding = base64.RawStdEncoding
 
-// note: all h2mux headers should be lower-case (http/2 style)
-const ()
-
-// H2RequestHeadersToH1Request converts the HTTP/2 headers coming from origintunneld
-// to an HTTP/1 Request object destined for the local origin web service.
-// This operation includes conversion of the pseudo-headers into their closest
-// HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3
-func H2RequestHeadersToH1Request(h2 []h2mux.Header, h1 *http.Request) error {
-	for _, header := range h2 {
-		name := strings.ToLower(header.Name)
-		if !IsControlRequestHeader(name) {
-			continue
-		}
-
-		switch name {
-		case ":method":
-			h1.Method = header.Value
-		case ":scheme":
-			// noop - use the preexisting scheme from h1.URL
-		case ":authority":
-			// Otherwise the host header will be based on the origin URL
-			h1.Host = header.Value
-		case ":path":
-			// We don't want to be an "opinionated" proxy, so ideally we would use :path as-is.
-			// However, this HTTP/1 Request object belongs to the Go standard library,
-			// whose URL package makes some opinionated decisions about the encoding of
-			// URL characters: see the docs of https://godoc.org/net/url#URL,
-			// in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath,
-			// which is always used when computing url.URL.String(), whether we'd like it or not.
-			//
-			// Well, not *always*. We could circumvent this by using url.URL.Opaque. But
-			// that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque
-			// is treated differently when HTTP_PROXY is set!
-			// See https://github.com/golang/go/issues/5684#issuecomment-66080888
-			//
-			// This means we are subject to the behavior of net/url's function `shouldEscape`
-			// (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101
-
-			if header.Value == "*" {
-				h1.URL.Path = "*"
-				continue
-			}
-			// Due to the behavior of validation.ValidateUrl, h1.URL may
-			// already have a partial value, with or without a trailing slash.
-			base := h1.URL.String()
-			base = strings.TrimRight(base, "/")
-			// But we know :path begins with '/', because we handled '*' above - see RFC7540
-			requestURL, err := url.Parse(base + header.Value)
-			if err != nil {
-				return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value))
-			}
-			h1.URL = requestURL
-		case "content-length":
-			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
-			if err != nil {
-				return fmt.Errorf("unparseable content length")
-			}
-			h1.ContentLength = contentLength
-		case RequestUserHeaders:
-			// Do not forward the serialized headers to the origin -- deserialize them, and ditch the serialized version
-			// Find and parse user headers serialized into a single one
-			userHeaders, err := DeserializeHeaders(header.Value)
-			if err != nil {
-				return errors.Wrap(err, "Unable to parse user headers")
-			}
-			for _, userHeader := range userHeaders {
-				h1.Header.Add(userHeader.Name, userHeader.Value)
-			}
-		default:
-			// All other control headers shall just be proxied transparently
-			h1.Header.Add(header.Name, header.Value)
-		}
-	}
-
-	return nil
-}
-
-func IsControlRequestHeader(headerName string) bool {
-	return headerName == "content-length" ||
-		headerName == "connection" || headerName == "upgrade" || // Websocket request headers
-		strings.HasPrefix(headerName, ":") ||
-		strings.HasPrefix(headerName, "cf-")
-}
-
+// IsControlResponseHeader is called in the direction of eyeball <- origin.
 func IsControlResponseHeader(headerName string) bool {
-	return headerName == "content-length" ||
-		strings.HasPrefix(headerName, ":") ||
+	return strings.HasPrefix(headerName, ":") ||
 		strings.HasPrefix(headerName, "cf-int-") ||
 		strings.HasPrefix(headerName, "cf-cloudflared-")
 }
@@ -142,31 +56,6 @@ func IsWebsocketClientHeader(headerName string) bool {
 		headerName == "upgrade"
 }
 
-func H1ResponseToH2ResponseHeaders(status int, h1 http.Header) (h2 []h2mux.Header) {
-	h2 = []h2mux.Header{
-		{Name: ":status", Value: strconv.Itoa(status)},
-	}
-	userHeaders := make(http.Header, len(h1))
-	for header, values := range h1 {
-		h2name := strings.ToLower(header)
-		if h2name == "content-length" {
-			// This header has meaning in HTTP/2 and will be used by the edge,
-			// so it should be sent as an HTTP/2 response header.
-
-			// Since these are http2 headers, they're required to be lowercase
-			h2 = append(h2, h2mux.Header{Name: "content-length", Value: values[0]})
-		} else if !IsControlResponseHeader(h2name) || IsWebsocketClientHeader(h2name) {
-			// User headers, on the other hand, must all be serialized so that
-			// HTTP/2 header validation won't be applied to HTTP/1 header values
-			userHeaders[header] = values
-		}
-	}
-
-	// Perform user header serialization and set them in the single header
-	h2 = append(h2, h2mux.Header{Name: ResponseUserHeaders, Value: SerializeHeaders(userHeaders)})
-	return h2
-}
-
 // Serialize HTTP1.x headers by base64-encoding each header name and value,
 // and then joining them in the format of [key:value;]
 func SerializeHeaders(h1Headers http.Header) string {

connection/header_test.go

@@ -2,441 +2,14 @@ package connection
 
 import (
 	"fmt"
-	"math/rand"
 	"net/http"
-	"net/url"
 	"reflect"
-	"regexp"
 	"sort"
-	"strings"
 	"testing"
-	"testing/quick"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/cloudflare/cloudflared/h2mux"
 )
 
-type ByName []h2mux.Header
-
-func (a ByName) Len() int      { return len(a) }
-func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
-func (a ByName) Less(i, j int) bool {
-	if a[i].Name == a[j].Name {
-		return a[i].Value < a[j].Value
-	}
-
-	return a[i].Name < a[j].Name
-}
-
-func TestH2RequestHeadersToH1Request_RegularHeaders(t *testing.T) {
-	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
-	assert.NoError(t, err)
-
-	mockHeaders := http.Header{
-		"Mock header 1": {"Mock value 1"},
-		"Mock header 2": {"Mock value 2"},
-	}
-
-	headersConversionErr := H2RequestHeadersToH1Request(createSerializedHeaders(RequestUserHeaders, mockHeaders), request)
-
-	assert.True(t, reflect.DeepEqual(mockHeaders, request.Header))
-	assert.NoError(t, headersConversionErr)
-}
-
-func createSerializedHeaders(headersField string, headers http.Header) []h2mux.Header {
-	return []h2mux.Header{{
-		Name:  headersField,
-		Value: SerializeHeaders(headers),
-	}}
-}
-
-func TestH2RequestHeadersToH1Request_NoHeaders(t *testing.T) {
-	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
-	assert.NoError(t, err)
-
-	emptyHeaders := make(http.Header)
-	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{{
-			Name:  RequestUserHeaders,
-			Value: SerializeHeaders(emptyHeaders),
-		}},
-		request,
-	)
-
-	assert.True(t, reflect.DeepEqual(emptyHeaders, request.Header))
-	assert.NoError(t, headersConversionErr)
-}
-
-func TestH2RequestHeadersToH1Request_InvalidHostPath(t *testing.T) {
-	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
-	assert.NoError(t, err)
-
-	mockRequestHeaders := []h2mux.Header{
-		{Name: ":path", Value: "//bad_path/"},
-		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
-	}
-
-	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
-
-	assert.Equal(t, http.Header{
-		"Mock header": []string{"Mock value"},
-	}, request.Header)
-
-	assert.Equal(t, "http://example.com//bad_path/", request.URL.String())
-
-	assert.NoError(t, headersConversionErr)
-}
-
-func TestH2RequestHeadersToH1Request_HostPathWithQuery(t *testing.T) {
-	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
-	assert.NoError(t, err)
-
-	mockRequestHeaders := []h2mux.Header{
-		{Name: ":path", Value: "/?query=mock%20value"},
-		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
-	}
-
-	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
-
-	assert.Equal(t, http.Header{
-		"Mock header": []string{"Mock value"},
-	}, request.Header)
-
-	assert.Equal(t, "http://example.com/?query=mock%20value", request.URL.String())
-
-	assert.NoError(t, headersConversionErr)
-}
-
-func TestH2RequestHeadersToH1Request_HostPathWithURLEncoding(t *testing.T) {
-	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
-	assert.NoError(t, err)
-
-	mockRequestHeaders := []h2mux.Header{
-		{Name: ":path", Value: "/mock%20path"},
-		{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
-	}
-
-	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
-
-	assert.Equal(t, http.Header{
-		"Mock header": []string{"Mock value"},
-	}, request.Header)
-
-	assert.Equal(t, "http://example.com/mock%20path", request.URL.String())
-
-	assert.NoError(t, headersConversionErr)
-}
-
-func TestH2RequestHeadersToH1Request_WeirdURLs(t *testing.T) {
-	type testCase struct {
-		path string
-		want string
-	}
-	testCases := []testCase{
-		{
-			path: "",
-			want: "",
-		},
-		{
-			path: "/",
-			want: "/",
-		},
-		{
-			path: "//",
-			want: "//",
-		},
-		{
-			path: "/test",
-			want: "/test",
-		},
-		{
-			path: "//test",
-			want: "//test",
-		},
-		{
-			// https://github.com/cloudflare/cloudflared/issues/81
-			path: "//test/",
-			want: "//test/",
-		},
-		{
-			path: "/%2Ftest",
-			want: "/%2Ftest",
-		},
-		{
-			path: "//%20test",
-			want: "//%20test",
-		},
-		{
-			// https://github.com/cloudflare/cloudflared/issues/124
-			path: "/test?get=somthing%20a",
-			want: "/test?get=somthing%20a",
-		},
-		{
-			path: "/%20",
-			want: "/%20",
-		},
-		{
-			// stdlib's EscapedPath() will always percent-encode ' '
-			path: "/ ",
-			want: "/%20",
-		},
-		{
-			path: "/ a ",
-			want: "/%20a%20",
-		},
-		{
-			path: "/a%20b",
-			want: "/a%20b",
-		},
-		{
-			path: "/foo/bar;param?query#frag",
-			want: "/foo/bar;param?query#frag",
-		},
-		{
-			// stdlib's EscapedPath() will always percent-encode non-ASCII chars
-			path: "/a␠b",
-			want: "/a%E2%90%A0b",
-		},
-		{
-			path: "/a-umlaut-ä",
-			want: "/a-umlaut-%C3%A4",
-		},
-		{
-			path: "/a-umlaut-%C3%A4",
-			want: "/a-umlaut-%C3%A4",
-		},
-		{
-			path: "/a-umlaut-%c3%a4",
-			want: "/a-umlaut-%c3%a4",
-		},
-		{
-			// here the second '#' is treated as part of the fragment
-			path: "/a#b#c",
-			want: "/a#b%23c",
-		},
-		{
-			path: "/a#b␠c",
-			want: "/a#b%E2%90%A0c",
-		},
-		{
-			path: "/a#b%20c",
-			want: "/a#b%20c",
-		},
-		{
-			path: "/a#b c",
-			want: "/a#b%20c",
-		},
-		{
-			// stdlib's EscapedPath() will always percent-encode '\'
-			path: "/\\",
-			want: "/%5C",
-		},
-		{
-			path: "/a\\",
-			want: "/a%5C",
-		},
-		{
-			path: "/a,b.c.",
-			want: "/a,b.c.",
-		},
-		{
-			path: "/.",
-			want: "/.",
-		},
-		{
-			// stdlib's EscapedPath() will always percent-encode '`'
-			path: "/a`",
-			want: "/a%60",
-		},
-		{
-			path: "/a[0]",
-			want: "/a[0]",
-		},
-		{
-			path: "/?a[0]=5 &b[]=",
-			want: "/?a[0]=5 &b[]=",
-		},
-		{
-			path: "/?a=%22b%20%22",
-			want: "/?a=%22b%20%22",
-		},
-	}
-
-	for index, testCase := range testCases {
-		requestURL := "https://example.com"
-
-		request, err := http.NewRequest(http.MethodGet, requestURL, nil)
-		assert.NoError(t, err)
-
-		mockRequestHeaders := []h2mux.Header{
-			{Name: ":path", Value: testCase.path},
-			{Name: RequestUserHeaders, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
-		}
-
-		headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
-		assert.NoError(t, headersConversionErr)
-
-		assert.Equal(t,
-			http.Header{
-				"Mock header": []string{"Mock value"},
-			},
-			request.Header)
-
-		assert.Equal(t,
-			"https://example.com"+testCase.want,
-			request.URL.String(),
-			"Failed URL index: %v %#v", index, testCase)
-	}
-}
-
-func TestH2RequestHeadersToH1Request_QuickCheck(t *testing.T) {
-	config := &quick.Config{
-		Values: func(args []reflect.Value, rand *rand.Rand) {
-			args[0] = reflect.ValueOf(randomHTTP2Path(t, rand))
-		},
-	}
-
-	type testOrigin struct {
-		url string
-
-		expectedScheme   string
-		expectedBasePath string
-	}
-	testOrigins := []testOrigin{
-		{
-			url:              "http://origin.hostname.example.com:8080",
-			expectedScheme:   "http",
-			expectedBasePath: "http://origin.hostname.example.com:8080",
-		},
-		{
-			url:              "http://origin.hostname.example.com:8080/",
-			expectedScheme:   "http",
-			expectedBasePath: "http://origin.hostname.example.com:8080",
-		},
-		{
-			url:              "http://origin.hostname.example.com:8080/api",
-			expectedScheme:   "http",
-			expectedBasePath: "http://origin.hostname.example.com:8080/api",
-		},
-		{
-			url:              "http://origin.hostname.example.com:8080/api/",
-			expectedScheme:   "http",
-			expectedBasePath: "http://origin.hostname.example.com:8080/api",
-		},
-		{
-			url:              "https://origin.hostname.example.com:8080/api",
-			expectedScheme:   "https",
-			expectedBasePath: "https://origin.hostname.example.com:8080/api",
-		},
-	}
-
-	// use multiple schemes to demonstrate that the URL is based on the
-	// origin's scheme, not the :scheme header
-	for _, testScheme := range []string{"http", "https"} {
-		for _, testOrigin := range testOrigins {
-			assertion := func(testPath string) bool {
-				const expectedMethod = "POST"
-				const expectedHostname = "request.hostname.example.com"
-
-				h2 := []h2mux.Header{
-					{Name: ":method", Value: expectedMethod},
-					{Name: ":scheme", Value: testScheme},
-					{Name: ":authority", Value: expectedHostname},
-					{Name: ":path", Value: testPath},
-					{Name: RequestUserHeaders, Value: ""},
-				}
-				h1, err := http.NewRequest("GET", testOrigin.url, nil)
-				require.NoError(t, err)
-
-				err = H2RequestHeadersToH1Request(h2, h1)
-				return assert.NoError(t, err) &&
-					assert.Equal(t, expectedMethod, h1.Method) &&
-					assert.Equal(t, expectedHostname, h1.Host) &&
-					assert.Equal(t, testOrigin.expectedScheme, h1.URL.Scheme) &&
-					assert.Equal(t, testOrigin.expectedBasePath+testPath, h1.URL.String())
-			}
-			err := quick.Check(assertion, config)
-			assert.NoError(t, err)
-		}
-	}
-}
-
-func randomASCIIPrintableChar(rand *rand.Rand) int {
-	// smallest printable ASCII char is 32, largest is 126
-	const startPrintable = 32
-	const endPrintable = 127
-	return startPrintable + rand.Intn(endPrintable-startPrintable)
-}
-
-// randomASCIIText generates an ASCII string, some of whose characters may be
-// percent-encoded. Its "logical length" (ignoring percent-encoding) is
-// between 1 and `maxLength`.
-func randomASCIIText(rand *rand.Rand, minLength int, maxLength int) string {
-	length := minLength + rand.Intn(maxLength)
-	var result strings.Builder
-	for i := 0; i < length; i++ {
-		c := randomASCIIPrintableChar(rand)
-
-		// 1/4 chance of using percent encoding when not necessary
-		if c == '%' || rand.Intn(4) == 0 {
-			result.WriteString(fmt.Sprintf("%%%02X", c))
-		} else {
-			result.WriteByte(byte(c))
-		}
-	}
-	return result.String()
-}
-
-// Calls `randomASCIIText` and ensures the result is a valid URL path,
-// i.e. one that can pass unchanged through url.URL.String()
-func randomHTTP1Path(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
-	text := randomASCIIText(rand, minLength, maxLength)
-	re, err := regexp.Compile("[^/;,]*")
-	require.NoError(t, err)
-	return "/" + re.ReplaceAllStringFunc(text, url.PathEscape)
-}
-
-// Calls `randomASCIIText` and ensures the result is a valid URL query,
-// i.e. one that can pass unchanged through url.URL.String()
-func randomHTTP1Query(rand *rand.Rand, minLength int, maxLength int) string {
-	text := randomASCIIText(rand, minLength, maxLength)
-	return "?" + strings.ReplaceAll(text, "#", "%23")
-}
-
-// Calls `randomASCIIText` and ensures the result is a valid URL fragment,
-// i.e. one that can pass unchanged through url.URL.String()
-func randomHTTP1Fragment(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
-	text := randomASCIIText(rand, minLength, maxLength)
-	u, err := url.Parse("#" + text)
-	require.NoError(t, err)
-	return u.String()
-}
-
-// Assemble a random :path pseudoheader that is legal by Go stdlib standards
-// (i.e. all characters will satisfy "net/url".shouldEscape for their respective locations)
-func randomHTTP2Path(t *testing.T, rand *rand.Rand) string {
-	result := randomHTTP1Path(t, rand, 1, 64)
-	if rand.Intn(2) == 1 {
-		result += randomHTTP1Query(rand, 1, 32)
-	}
-	if rand.Intn(2) == 1 {
-		result += randomHTTP1Fragment(t, rand, 1, 16)
-	}
-	return result
-}
-
-func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []h2mux.Header) {
-	for name, values := range headers {
-		for _, value := range values {
-			h2muxHeaders = append(h2muxHeaders, h2mux.Header{Name: name, Value: value})
-		}
-	}
-
-	return h2muxHeaders
-}
-
 func TestSerializeHeaders(t *testing.T) {
 	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
 	assert.NoError(t, err)
@@ -511,70 +84,13 @@ func TestDeserializeMalformed(t *testing.T) {
 	}
 }
 
-func TestParseRequestHeaders(t *testing.T) {
-	mockUserHeadersToSerialize := http.Header{
-		"Mock-Header-One":   {"1", "1.5"},
-		"Mock-Header-Two":   {"2"},
-		"Mock-Header-Three": {"3"},
-	}
-
-	mockHeaders := []h2mux.Header{
-		{Name: "One", Value: "1"}, // will be dropped
-		{Name: "Cf-Two", Value: "cf-value-1"},
-		{Name: "Cf-Two", Value: "cf-value-2"},
-		{Name: RequestUserHeaders, Value: SerializeHeaders(mockUserHeadersToSerialize)},
-	}
-
-	expectedHeaders := []h2mux.Header{
-		{Name: "Cf-Two", Value: "cf-value-1"},
-		{Name: "Cf-Two", Value: "cf-value-2"},
-		{Name: "Mock-Header-One", Value: "1"},
-		{Name: "Mock-Header-One", Value: "1.5"},
-		{Name: "Mock-Header-Two", Value: "2"},
-		{Name: "Mock-Header-Three", Value: "3"},
-	}
-	h1 := &http.Request{
-		Header: make(http.Header),
-	}
-	err := H2RequestHeadersToH1Request(mockHeaders, h1)
-	assert.NoError(t, err)
-	assert.ElementsMatch(t, expectedHeaders, stdlibHeaderToH2muxHeader(h1.Header))
-}
-
-func TestIsControlRequestHeader(t *testing.T) {
-	controlRequestHeaders := []string{
-		// Anything that begins with cf-
-		"cf-sample-header",
-
-		// Any http2 pseudoheader
-		":sample-pseudo-header",
-
-		// content-length is a special case, it has to be there
-		// for some requests to work (per the HTTP2 spec)
-		"content-length",
-
-		// Websocket request headers
-		"connection",
-		"upgrade",
-	}
-
-	for _, header := range controlRequestHeaders {
-		assert.True(t, IsControlRequestHeader(header))
-	}
-}
-
 func TestIsControlResponseHeader(t *testing.T) {
 	controlResponseHeaders := []string{
 		// Anything that begins with cf-int- or cf-cloudflared-
 		"cf-int-sample-header",
 		"cf-cloudflared-sample-header",
-
 		// Any http2 pseudoheader
 		":sample-pseudo-header",
-
-		// content-length is a special case, it has to be there
-		// for some requests to work (per the HTTP2 spec)
-		"content-length",
 	}
 
 	for _, header := range controlResponseHeaders {
@@ -582,17 +98,6 @@ func TestIsControlResponseHeader(t *testing.T) {
 	}
 }
 
-func TestIsNotControlRequestHeader(t *testing.T) {
-	notControlRequestHeaders := []string{
-		"mock-header",
-		"another-sample-header",
-	}
-
-	for _, header := range notControlRequestHeaders {
-		assert.False(t, IsControlRequestHeader(header))
-	}
-}
-
 func TestIsNotControlResponseHeader(t *testing.T) {
 	notControlResponseHeaders := []string{
 		"mock-header",
@@ -606,112 +111,3 @@ func TestIsNotControlResponseHeader(t *testing.T) {
 		assert.False(t, IsControlResponseHeader(header))
 	}
 }
-
-func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
-	mockHeaders := http.Header{
-		"User-header-one":       {""},
-		"User-header-two":       {"1", "2"},
-		"cf-header":             {"cf-value"},
-		"cf-int-header":         {"cf-int-value"},
-		"cf-cloudflared-header": {"cf-cloudflared-value"},
-		"Content-Length":        {"123"},
-	}
-	mockResponse := http.Response{
-		StatusCode: 200,
-		Header:     mockHeaders,
-	}
-
-	headers := H1ResponseToH2ResponseHeaders(mockResponse.StatusCode, mockResponse.Header)
-
-	serializedHeadersIndex := -1
-	for i, header := range headers {
-		if header.Name == ResponseUserHeaders {
-			serializedHeadersIndex = i
-			break
-		}
-	}
-	assert.NotEqual(t, -1, serializedHeadersIndex)
-	actualControlHeaders := append(
-		headers[:serializedHeadersIndex],
-		headers[serializedHeadersIndex+1:]...,
-	)
-	expectedControlHeaders := []h2mux.Header{
-		{Name: ":status", Value: "200"},
-		{Name: "content-length", Value: "123"},
-	}
-
-	assert.ElementsMatch(t, expectedControlHeaders, actualControlHeaders)
-
-	actualUserHeaders, err := DeserializeHeaders(headers[serializedHeadersIndex].Value)
-	expectedUserHeaders := []h2mux.Header{
-		{Name: "User-header-one", Value: ""},
-		{Name: "User-header-two", Value: "1"},
-		{Name: "User-header-two", Value: "2"},
-		{Name: "cf-header", Value: "cf-value"},
-	}
-	assert.NoError(t, err)
-	assert.ElementsMatch(t, expectedUserHeaders, actualUserHeaders)
-}
-
-// The purpose of this test is to check that our code and the http.Header
-// implementation don't throw validation errors about header size
-func TestHeaderSize(t *testing.T) {
-	largeValue := randSeq(5 * 1024 * 1024) // 5Mb
-	largeHeaders := http.Header{
-		"User-header": {largeValue},
-	}
-	mockResponse := http.Response{
-		StatusCode: 200,
-		Header:     largeHeaders,
-	}
-
-	serializedHeaders := H1ResponseToH2ResponseHeaders(mockResponse.StatusCode, mockResponse.Header)
-	request, err := http.NewRequest(http.MethodGet, "https://example.com/", nil)
-	assert.NoError(t, err)
-	for _, header := range serializedHeaders {
-		request.Header.Set(header.Name, header.Value)
-	}
-
-	for _, header := range serializedHeaders {
-		if header.Name != ResponseUserHeaders {
-			continue
-		}
-
-		deserializedHeaders, err := DeserializeHeaders(header.Value)
-		assert.NoError(t, err)
-		assert.Equal(t, largeValue, deserializedHeaders[0].Value)
-	}
-}
-
-func randSeq(n int) string {
-	randomizer := rand.New(rand.NewSource(17))
-	var letters = []rune(":;,+/=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
-	b := make([]rune, n)
-	for i := range b {
-		b[i] = letters[randomizer.Intn(len(letters))]
-	}
-	return string(b)
-}
-
-func BenchmarkH1ResponseToH2ResponseHeaders(b *testing.B) {
-	ser := "eC1mb3J3YXJkZWQtcHJvdG8:aHR0cHM;dXBncmFkZS1pbnNlY3VyZS1yZXF1ZXN0cw:MQ;YWNjZXB0LWxhbmd1YWdl:ZW4tVVMsZW47cT0wLjkscnU7cT0wLjg;YWNjZXB0LWVuY29kaW5n:Z3ppcA;eC1mb3J3YXJkZWQtZm9y:MTczLjI0NS42MC42;dXNlci1hZ2VudA:TW96aWxsYS81LjAgKE1hY2ludG9zaDsgSW50ZWwgTWFjIE9TIFggMTBfMTRfNikgQXBwbGVXZWJLaXQvNTM3LjM2IChLSFRNTCwgbGlrZSBHZWNrbykgQ2hyb21lLzg0LjAuNDE0Ny44OSBTYWZhcmkvNTM3LjM2;c2VjLWZldGNoLW1vZGU:bmF2aWdhdGU;Y2RuLWxvb3A:Y2xvdWRmbGFyZQ;c2VjLWZldGNoLWRlc3Q:ZG9jdW1lbnQ;c2VjLWZldGNoLXVzZXI:PzE;c2VjLWZldGNoLXNpdGU:bm9uZQ;Y29va2ll:X19jZmR1aWQ9ZGNkOWZjOGNjNWMxMzE0NTMyYTFkMjhlZDEyOWRhOTYwMTU2OTk1MTYzNDsgX19jZl9ibT1mYzY2MzMzYzAzZmM0MWFiZTZmOWEyYzI2ZDUwOTA0YzIxYzZhMTQ2LTE1OTU2MjIzNDEtMTgwMC1BZTVzS2pIU2NiWGVFM05mMUhrTlNQMG1tMHBLc2pQWkloVnM1Z2g1SkNHQkFhS1UxVDB2b003alBGN3FjMHVSR2NjZGcrWHdhL1EzbTJhQzdDVU4xZ2M9;YWNjZXB0:dGV4dC9odG1sLGFwcGxpY2F0aW9uL3hodG1sK3htbCxhcHBsaWNhdGlvbi94bWw7cT0wLjksaW1hZ2Uvd2VicCxpbWFnZS9hcG5nLCovKjtxPTAuOCxhcHBsaWNhdGlvbi9zaWduZWQtZXhjaGFuZ2U7dj1iMztxPTAuOQ"
-	h2, _ := DeserializeHeaders(ser)
-	h1 := make(http.Header)
-	for _, header := range h2 {
-		h1.Add(header.Name, header.Value)
-	}
-	h1.Add("Content-Length", "200")
-	h1.Add("Cf-Something", "Else")
-	h1.Add("Upgrade", "websocket")
-
-	h1resp := &http.Response{
-		StatusCode: 200,
-		Header:     h1,
-	}
-
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_ = H1ResponseToH2ResponseHeaders(h1resp.StatusCode, h1resp.Header)
-	}
-}

connection/http2.go

@@ -30,12 +30,12 @@ var errEdgeConnectionClosed = fmt.Errorf("connection with edge closed")
 // HTTP2Connection represents a net.Conn that uses HTTP2 frames to proxy traffic from the edge to cloudflared on the
 // origin.
 type HTTP2Connection struct {
-	conn        net.Conn
-	server      *http2.Server
-	config      *Config
-	connOptions *tunnelpogs.ConnectionOptions
-	observer    *Observer
-	connIndex   uint8
+	conn         net.Conn
+	server       *http2.Server
+	orchestrator Orchestrator
+	connOptions  *tunnelpogs.ConnectionOptions
+	observer     *Observer
+	connIndex    uint8
 	// newRPCClientFunc allows us to mock RPCs during testing
 	newRPCClientFunc func(context.Context, io.ReadWriteCloser, *zerolog.Logger) NamedTunnelRPCClient
 
@@ -49,7 +49,7 @@ type HTTP2Connection struct {
 // NewHTTP2Connection returns a new instance of HTTP2Connection.
 func NewHTTP2Connection(
 	conn net.Conn,
-	config *Config,
+	orchestrator Orchestrator,
 	connOptions *tunnelpogs.ConnectionOptions,
 	observer *Observer,
 	connIndex uint8,
@@ -61,7 +61,7 @@ func NewHTTP2Connection(
 		server: &http2.Server{
 			MaxConcurrentStreams: MaxConcurrentStreams,
 		},
-		config:               config,
+		orchestrator:         orchestrator,
 		connOptions:          connOptions,
 		observer:             observer,
 		connIndex:            connIndex,
@@ -106,9 +106,15 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	originProxy, err := c.orchestrator.GetOriginProxy()
+	if err != nil {
+		c.observer.log.Error().Msg(err.Error())
+		return
+	}
+
 	switch connType {
 	case TypeControlStream:
-		if err := c.controlStreamHandler.ServeControlStream(r.Context(), respWriter, c.connOptions, true); err != nil {
+		if err := c.controlStreamHandler.ServeControlStream(r.Context(), respWriter, c.connOptions); err != nil {
 			c.controlStreamErr = err
 			c.log.Error().Err(err)
 			respWriter.WriteErrorResponse()
@@ -116,7 +122,7 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	case TypeWebsocket, TypeHTTP:
 		stripWebsocketUpgradeHeader(r)
-		if err := c.config.OriginProxy.ProxyHTTP(respWriter, r, connType == TypeWebsocket); err != nil {
+		if err := originProxy.ProxyHTTP(respWriter, r, connType == TypeWebsocket); err != nil {
 			err := fmt.Errorf("Failed to proxy HTTP: %w", err)
 			c.log.Error().Err(err)
 			respWriter.WriteErrorResponse()
@@ -125,13 +131,13 @@ func (c *HTTP2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	case TypeTCP:
 		host, err := getRequestHost(r)
 		if err != nil {
-			err := fmt.Errorf(`cloudflared recieved a warp-routing request with an empty host value: %w`, err)
+			err := fmt.Errorf(`cloudflared received a warp-routing request with an empty host value: %w`, err)
 			c.log.Error().Err(err)
 			respWriter.WriteErrorResponse()
 		}
 
 		rws := NewHTTPResponseReadWriterAcker(respWriter, r)
-		if err := c.config.OriginProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
+		if err := originProxy.ProxyTCP(r.Context(), rws, &TCPRequest{
 			Dest:    host,
 			CFRay:   FindCfRayHeader(r),
 			LBProbe: IsLBProbeRequest(r),
@@ -184,12 +190,14 @@ func (rp *http2RespWriter) WriteRespHeaders(status int, header http.Header) erro
 	for name, values := range header {
 		// Since these are http2 headers, they're required to be lowercase
 		h2name := strings.ToLower(name)
+
 		if h2name == "content-length" {
 			// This header has meaning in HTTP/2 and will be used by the edge,
-			// so it should be sent as an HTTP/2 response header.
+			// so it should be sent *also* as an HTTP/2 response header.
 			dest[name] = values
-			// Since these are http2 headers, they're required to be lowercase
-		} else if !IsControlResponseHeader(h2name) || IsWebsocketClientHeader(h2name) {
+		}
+
+		if !IsControlResponseHeader(h2name) || IsWebsocketClientHeader(h2name) {
 			// User headers, on the other hand, must all be serialized so that
 			// HTTP/2 header validation won't be applied to HTTP/1 header values
 			userHeaders[name] = values

connection/http2_test.go

@@ -2,6 +2,7 @@ package connection
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -27,22 +28,23 @@ var (
 )
 
 func newTestHTTP2Connection() (*HTTP2Connection, net.Conn) {
-	edgeConn, originConn := net.Pipe()
+	edgeConn, cfdConn := net.Pipe()
 	var connIndex = uint8(0)
 	log := zerolog.Nop()
 	obs := NewObserver(&log, &log, false)
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelConfig{},
+		&NamedTunnelProperties{},
 		connIndex,
 		nil,
 		nil,
 		1*time.Second,
 	)
 	return NewHTTP2Connection(
-		originConn,
-		testConfig,
+		cfdConn,
+		// OriginProxy is set in testConfigManager
+		testOrchestrator,
 		&pogs.ConnectionOptions{},
 		obs,
 		connIndex,
@@ -130,7 +132,7 @@ type mockNamedTunnelRPCClient struct {
 
 func (mc mockNamedTunnelRPCClient) RegisterConnection(
 	c context.Context,
-	config *NamedTunnelConfig,
+	properties *NamedTunnelProperties,
 	options *tunnelpogs.ConnectionOptions,
 	connIndex uint8,
 	observer *Observer,
@@ -166,6 +168,8 @@ type wsRespWriter struct {
 	*httptest.ResponseRecorder
 	readPipe  *io.PipeReader
 	writePipe *io.PipeWriter
+	closed    bool
+	panicked  bool
 }
 
 func newWSRespWriter() *wsRespWriter {
@@ -174,46 +178,59 @@ func newWSRespWriter() *wsRespWriter {
 		httptest.NewRecorder(),
 		readPipe,
 		writePipe,
+		false,
+		false,
 	}
 }
 
+type nowriter struct {
+	io.Reader
+}
+
+func (nowriter) Write(_ []byte) (int, error) {
+	return 0, fmt.Errorf("writer not implemented")
+}
+
 func (w *wsRespWriter) RespBody() io.ReadWriter {
 	return nowriter{w.readPipe}
 }
 
 func (w *wsRespWriter) Write(data []byte) (n int, err error) {
+	if w.closed {
+		w.panicked = true
+		return 0, errors.New("wsRespWriter panicked")
+	}
 	return w.writePipe.Write(data)
 }
 
+func (w *wsRespWriter) close() {
+	w.closed = true
+}
+
 func TestServeWS(t *testing.T) {
 	http2Conn, _ := newTestHTTP2Connection()
 
 	ctx, cancel := context.WithCancel(context.Background())
-	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		http2Conn.Serve(ctx)
-	}()
 
 	respWriter := newWSRespWriter()
 	readPipe, writePipe := io.Pipe()
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/ws", readPipe)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/ws/echo", readPipe)
 	require.NoError(t, err)
 	req.Header.Set(InternalUpgradeHeader, WebsocketUpgrade)
 
-	wg.Add(1)
+	serveDone := make(chan struct{})
 	go func() {
-		defer wg.Done()
+		defer close(serveDone)
 		http2Conn.ServeHTTP(respWriter, req)
+		respWriter.close()
 	}()
 
 	data := []byte("test websocket")
-	err = wsutil.WriteClientText(writePipe, data)
+	err = wsutil.WriteClientBinary(writePipe, data)
 	require.NoError(t, err)
 
-	respBody, err := wsutil.ReadServerText(respWriter.RespBody())
+	respBody, err := wsutil.ReadServerBinary(respWriter.RespBody())
 	require.NoError(t, err)
 	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
 
@@ -223,7 +240,65 @@ func TestServeWS(t *testing.T) {
 	require.Equal(t, http.StatusOK, resp.StatusCode)
 	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(ResponseMetaHeader))
 
+	<-serveDone
+	require.False(t, respWriter.panicked)
+}
+
+// TestNoWriteAfterServeHTTPReturns is a regression test of https://jira.cfops.it/browse/TUN-5184
+// to make sure we don't write to the ResponseWriter after the ServeHTTP method returns
+func TestNoWriteAfterServeHTTPReturns(t *testing.T) {
+	cfdHTTP2Conn, edgeTCPConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+
+	serverDone := make(chan struct{})
+	go func() {
+		defer close(serverDone)
+		cfdHTTP2Conn.Serve(ctx)
+	}()
+
+	edgeTransport := http2.Transport{}
+	edgeHTTP2Conn, err := edgeTransport.NewClientConn(edgeTCPConn)
+	require.NoError(t, err)
+	message := []byte(t.Name())
+
+	for i := 0; i < 100; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			readPipe, writePipe := io.Pipe()
+			reqCtx, reqCancel := context.WithCancel(ctx)
+			req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, "http://localhost:8080/ws/flaky", readPipe)
+			require.NoError(t, err)
+			req.Header.Set(InternalUpgradeHeader, WebsocketUpgrade)
+
+			resp, err := edgeHTTP2Conn.RoundTrip(req)
+			require.NoError(t, err)
+			// http2RespWriter should rewrite status 101 to 200
+			require.Equal(t, http.StatusOK, resp.StatusCode)
+
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for {
+					select {
+					case <-reqCtx.Done():
+						return
+					default:
+					}
+					_ = wsutil.WriteClientBinary(writePipe, message)
+				}
+			}()
+
+			time.Sleep(time.Millisecond * 100)
+			reqCancel()
+		}()
+	}
+
 	wg.Wait()
+	cancel()
+	<-serverDone
 }
 
 func TestServeControlStream(t *testing.T) {
@@ -238,7 +313,7 @@ func TestServeControlStream(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelConfig{},
+		&NamedTunnelProperties{},
 		1,
 		rpcClientFactory.newMockRPCClient,
 		nil,
@@ -288,7 +363,7 @@ func TestFailRegistration(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelConfig{},
+		&NamedTunnelProperties{},
 		http2Conn.connIndex,
 		rpcClientFactory.newMockRPCClient,
 		nil,
@@ -334,7 +409,7 @@ func TestGracefulShutdownHTTP2(t *testing.T) {
 	controlStream := NewControlStream(
 		obs,
 		mockConnectedFuse{},
-		&NamedTunnelConfig{},
+		&NamedTunnelProperties{},
 		http2Conn.connIndex,
 		rpcClientFactory.newMockRPCClient,
 		shutdownC,

connection/protocol.go

@@ -12,16 +12,14 @@ import (
 )
 
 const (
-	AvailableProtocolFlagMessage = "Available protocols: http2 - Go's implementation, h2mux - Cloudflare's implementation of HTTP/2, and auto - automatically select between http2 and h2mux"
+	AvailableProtocolFlagMessage = "Available protocols: 'auto' - automatically chooses the best protocol over time (the default; and also the recommended one); 'quic' - based on QUIC, relying on UDP egress to Cloudflare edge; 'http2' - using Go's HTTP2 library, relying on TCP egress to Cloudflare edge; 'h2mux' - Cloudflare's implementation of HTTP/2, deprecated"
 	// edgeH2muxTLSServerName is the server name to establish h2mux connection with edge
 	edgeH2muxTLSServerName = "cftunnel.com"
 	// edgeH2TLSServerName is the server name to establish http2 connection with edge
 	edgeH2TLSServerName = "h2.cftunnel.com"
 	// edgeQUICServerName is the server name to establish quic connection with edge.
 	edgeQUICServerName = "quic.cftunnel.com"
-	// threshold to switch back to h2mux when the user intentionally pick --protocol http2
-	explicitHTTP2FallbackThreshold = -1
-	autoSelectFlag                 = "auto"
+	autoSelectFlag     = "auto"
 )
 
 var (
@@ -42,7 +40,7 @@ const (
 	// H2mux on HTTP2 failure to connect.
 	HTTP2Warp
 	//QUICWarp is used only with named tunnels. It's useful for warp-routing where we want to fallback to HTTP2 but
-	// dont' want HTTP2 to fallback to H2mux
+	// don't want HTTP2 to fallback to H2mux
 	QUICWarp
 )
 
@@ -197,7 +195,7 @@ type PercentageFetcher func() (edgediscovery.ProtocolPercents, error)
 func NewProtocolSelector(
 	protocolFlag string,
 	warpRoutingEnabled bool,
-	namedTunnel *NamedTunnelConfig,
+	namedTunnel *NamedTunnelProperties,
 	fetchFunc PercentageFetcher,
 	ttl time.Duration,
 	log *zerolog.Logger,
@@ -237,25 +235,23 @@ func selectNamedTunnelProtocols(
 	threshold int32,
 	protocol Protocol,
 ) (ProtocolSelector, error) {
-	if protocolFlag == H2mux.String() {
-		return &staticProtocolSelector{
-			current: H2mux,
-		}, nil
+	// If the user picks a protocol, then we stick to it no matter what.
+	switch protocolFlag {
+	case H2mux.String():
+		return &staticProtocolSelector{current: H2mux}, nil
+	case QUIC.String():
+		return &staticProtocolSelector{current: QUIC}, nil
+	case HTTP2.String():
+		return &staticProtocolSelector{current: HTTP2}, nil
 	}
 
-	if protocolFlag == QUIC.String() {
-		return newAutoProtocolSelector(QUIC, []Protocol{QUIC, HTTP2, H2mux}, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
+	// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
+	// fallback on failures.
+	if protocolFlag == autoSelectFlag {
+		return newAutoProtocolSelector(protocol, []Protocol{QUIC, HTTP2, H2mux}, threshold, fetchFunc, ttl, log), nil
 	}
 
-	if protocolFlag == HTTP2.String() {
-		return newAutoProtocolSelector(HTTP2, []Protocol{HTTP2, H2mux}, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
-	}
-
-	if protocolFlag != autoSelectFlag {
-		return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
-	}
-
-	return newAutoProtocolSelector(protocol, []Protocol{QUIC, HTTP2, H2mux}, threshold, fetchFunc, ttl, log), nil
+	return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
 }
 
 func selectWarpRoutingProtocols(
@@ -266,19 +262,21 @@ func selectWarpRoutingProtocols(
 	threshold int32,
 	protocol Protocol,
 ) (ProtocolSelector, error) {
-	if protocolFlag == QUIC.String() {
-		return newAutoProtocolSelector(QUICWarp, []Protocol{QUICWarp, HTTP2Warp}, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
+	// If the user picks a protocol, then we stick to it no matter what.
+	switch protocolFlag {
+	case QUIC.String():
+		return &staticProtocolSelector{current: QUICWarp}, nil
+	case HTTP2.String():
+		return &staticProtocolSelector{current: HTTP2Warp}, nil
 	}
 
-	if protocolFlag == HTTP2.String() {
-		return newAutoProtocolSelector(HTTP2Warp, []Protocol{HTTP2Warp}, explicitHTTP2FallbackThreshold, fetchFunc, ttl, log), nil
+	// If the user does not pick (hopefully the majority) then we use the one derived from the TXT DNS record and
+	// fallback on failures.
+	if protocolFlag == autoSelectFlag {
+		return newAutoProtocolSelector(protocol, []Protocol{QUICWarp, HTTP2Warp}, threshold, fetchFunc, ttl, log), nil
 	}
 
-	if protocolFlag != autoSelectFlag {
-		return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
-	}
-
-	return newAutoProtocolSelector(protocol, []Protocol{QUICWarp, HTTP2Warp}, threshold, fetchFunc, ttl, log), nil
+	return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
 }
 
 func switchThreshold(accountTag string) int32 {

connection/protocol_test.go

@@ -16,7 +16,7 @@ const (
 )
 
 var (
-	testNamedTunnelConfig = &NamedTunnelConfig{
+	testNamedTunnelProperties = &NamedTunnelProperties{
 		Credentials: Credentials{
 			AccountTag: "testAccountTag",
 		},
@@ -26,7 +26,7 @@ var (
 func mockFetcher(getError bool, protocolPercent ...edgediscovery.ProtocolPercent) PercentageFetcher {
 	return func() (edgediscovery.ProtocolPercents, error) {
 		if getError {
-			return nil, fmt.Errorf("failed to fetch precentage")
+			return nil, fmt.Errorf("failed to fetch percentage")
 		}
 		return protocolPercent, nil
 	}
@@ -51,7 +51,7 @@ func TestNewProtocolSelector(t *testing.T) {
 		hasFallback        bool
 		expectedFallback   Protocol
 		warpRoutingEnabled bool
-		namedTunnelConfig  *NamedTunnelConfig
+		namedTunnelConfig  *NamedTunnelProperties
 		fetchFunc          PercentageFetcher
 		wantErr            bool
 	}{
@@ -66,47 +66,59 @@ func TestNewProtocolSelector(t *testing.T) {
 			protocol:          "h2mux",
 			expectedProtocol:  H2mux,
 			fetchFunc:         func() (edgediscovery.ProtocolPercents, error) { return nil, nil },
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel over http2",
 			protocol:          "http2",
 			expectedProtocol:  HTTP2,
-			hasFallback:       true,
-			expectedFallback:  H2mux,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
-			name:              "named tunnel http2 disabled",
+			name:              "named tunnel http2 disabled still gets http2 because it is manually picked",
 			protocol:          "http2",
-			expectedProtocol:  H2mux,
+			expectedProtocol:  HTTP2,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
+		},
+		{
+			name:              "named tunnel quic disabled still gets quic because it is manually picked",
+			protocol:          "quic",
+			expectedProtocol:  QUIC,
+			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
+			namedTunnelConfig: testNamedTunnelProperties,
+		},
+		{
+			name:              "named tunnel quic and http2 disabled",
+			protocol:          "auto",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:             "named tunnel quic disabled",
-			protocol:         "quic",
+			protocol:         "auto",
 			expectedProtocol: HTTP2,
 			// Hasfallback true is because if http2 fails, then we further fallback to h2mux.
 			hasFallback:       true,
 			expectedFallback:  H2mux,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: -1}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel auto all http2 disabled",
 			protocol:          "auto",
 			expectedProtocol:  H2mux,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel auto to h2mux",
 			protocol:          "auto",
 			expectedProtocol:  H2mux,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel auto to http2",
@@ -115,7 +127,7 @@ func TestNewProtocolSelector(t *testing.T) {
 			hasFallback:       true,
 			expectedFallback:  H2mux,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:              "named tunnel auto to quic",
@@ -124,7 +136,7 @@ func TestNewProtocolSelector(t *testing.T) {
 			hasFallback:       true,
 			expectedFallback:  HTTP2,
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 		},
 		{
 			name:               "warp routing requesting h2mux",
@@ -133,7 +145,7 @@ func TestNewProtocolSelector(t *testing.T) {
 			hasFallback:        false,
 			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
 			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelConfig,
+			namedTunnelConfig:  testNamedTunnelProperties,
 		},
 		{
 			name:               "warp routing requesting h2mux picks HTTP2 even if http2 percent is -1",
@@ -142,7 +154,7 @@ func TestNewProtocolSelector(t *testing.T) {
 			hasFallback:        false,
 			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}),
 			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelConfig,
+			namedTunnelConfig:  testNamedTunnelProperties,
 		},
 		{
 			name:               "warp routing http2",
@@ -151,17 +163,17 @@ func TestNewProtocolSelector(t *testing.T) {
 			hasFallback:        false,
 			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
 			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelConfig,
+			namedTunnelConfig:  testNamedTunnelProperties,
 		},
 		{
 			name:               "warp routing quic",
-			protocol:           "quic",
+			protocol:           "auto",
 			expectedProtocol:   QUICWarp,
 			hasFallback:        true,
 			expectedFallback:   HTTP2Warp,
 			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
 			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelConfig,
+			namedTunnelConfig:  testNamedTunnelProperties,
 		},
 		{
 			name:               "warp routing auto",
@@ -170,7 +182,7 @@ func TestNewProtocolSelector(t *testing.T) {
 			hasFallback:        false,
 			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
 			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelConfig,
+			namedTunnelConfig:  testNamedTunnelProperties,
 		},
 		{
 			name:               "warp routing auto- quic",
@@ -180,7 +192,7 @@ func TestNewProtocolSelector(t *testing.T) {
 			expectedFallback:   HTTP2Warp,
 			fetchFunc:          mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}, edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}),
 			warpRoutingEnabled: true,
-			namedTunnelConfig:  testNamedTunnelConfig,
+			namedTunnelConfig:  testNamedTunnelProperties,
 		},
 		{
 			// None named tunnel can only use h2mux, so specifying an unknown protocol is not an error
@@ -192,14 +204,14 @@ func TestNewProtocolSelector(t *testing.T) {
 			name:              "named tunnel unknown protocol",
 			protocol:          "unknown",
 			fetchFunc:         mockFetcher(false, edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 100}),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 			wantErr:           true,
 		},
 		{
 			name:              "named tunnel fetch error",
 			protocol:          "auto",
 			fetchFunc:         mockFetcher(true),
-			namedTunnelConfig: testNamedTunnelConfig,
+			namedTunnelConfig: testNamedTunnelProperties,
 			expectedProtocol:  HTTP2,
 			wantErr:           false,
 		},
@@ -225,7 +237,7 @@ func TestNewProtocolSelector(t *testing.T) {
 
 func TestAutoProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
-	selector, err := NewProtocolSelector("auto", noWarpRoutingEnabled, testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
+	selector, err := NewProtocolSelector("auto", noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, H2mux, selector.Current())
 
@@ -254,7 +266,8 @@ func TestAutoProtocolSelectorRefresh(t *testing.T) {
 
 func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
-	selector, err := NewProtocolSelector("http2", noWarpRoutingEnabled, testNamedTunnelConfig, fetcher.fetch(), testNoTTL, &log)
+	// Since the user chooses http2 on purpose, we always stick to it.
+	selector, err := NewProtocolSelector("http2", noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), testNoTTL, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, HTTP2, selector.Current())
 
@@ -269,7 +282,7 @@ func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}}
 	fetcher.err = nil
-	assert.Equal(t, H2mux, selector.Current())
+	assert.Equal(t, HTTP2, selector.Current())
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: 0}}
 	assert.Equal(t, HTTP2, selector.Current())
@@ -278,13 +291,13 @@ func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
 	assert.Equal(t, HTTP2, selector.Current())
 
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "http2", Percentage: -1}}
-	assert.Equal(t, H2mux, selector.Current())
+	assert.Equal(t, HTTP2, selector.Current())
 }
 
 func TestProtocolSelectorRefreshTTL(t *testing.T) {
 	fetcher := dynamicMockFetcher{}
 	fetcher.protocolPercents = edgediscovery.ProtocolPercents{edgediscovery.ProtocolPercent{Protocol: "quic", Percentage: 100}}
-	selector, err := NewProtocolSelector("auto", noWarpRoutingEnabled, testNamedTunnelConfig, fetcher.fetch(), time.Hour, &log)
+	selector, err := NewProtocolSelector("auto", noWarpRoutingEnabled, testNamedTunnelProperties, fetcher.fetch(), time.Hour, &log)
 	assert.NoError(t, err)
 	assert.Equal(t, QUIC, selector.Current())
 

connection/quic.go

@@ -9,11 +9,16 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"time"
 
+	"github.com/google/uuid"
 	"github.com/lucas-clemente/quic-go"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
+	"golang.org/x/sync/errgroup"
 
+	"github.com/cloudflare/cloudflared/datagramsession"
+	"github.com/cloudflare/cloudflared/ingress"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
@@ -29,63 +34,106 @@ const (
 
 // QUICConnection represents the type that facilitates Proxying via QUIC streams.
 type QUICConnection struct {
-	session           quic.Session
-	logger            *zerolog.Logger
-	httpProxy         OriginProxy
-	gracefulShutdownC <-chan struct{}
-	stoppedGracefully bool
+	session              quic.Session
+	logger               *zerolog.Logger
+	orchestrator         Orchestrator
+	sessionManager       datagramsession.Manager
+	controlStreamHandler ControlStreamHandler
+	connOptions          *tunnelpogs.ConnectionOptions
 }
 
 // NewQUICConnection returns a new instance of QUICConnection.
 func NewQUICConnection(
-	ctx context.Context,
 	quicConfig *quic.Config,
 	edgeAddr net.Addr,
 	tlsConfig *tls.Config,
-	httpProxy OriginProxy,
+	orchestrator Orchestrator,
 	connOptions *tunnelpogs.ConnectionOptions,
 	controlStreamHandler ControlStreamHandler,
-	observer *Observer,
+	logger *zerolog.Logger,
 ) (*QUICConnection, error) {
 	session, err := quic.DialAddr(edgeAddr.String(), tlsConfig, quicConfig)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to dial to edge")
+		return nil, fmt.Errorf("failed to dial to edge: %w", err)
 	}
 
-	registrationStream, err := session.OpenStream()
+	datagramMuxer, err := quicpogs.NewDatagramMuxer(session)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to open a registration stream")
-	}
-
-	err = controlStreamHandler.ServeControlStream(ctx, registrationStream, connOptions, false)
-	if err != nil {
-		// Not wrapping error here to be consistent with the http2 message.
 		return nil, err
 	}
 
+	sessionManager := datagramsession.NewManager(datagramMuxer, logger)
+
 	return &QUICConnection{
-		session:   session,
-		httpProxy: httpProxy,
-		logger:    observer.log,
+		session:              session,
+		orchestrator:         orchestrator,
+		logger:               logger,
+		sessionManager:       sessionManager,
+		controlStreamHandler: controlStreamHandler,
+		connOptions:          connOptions,
 	}, nil
 }
 
 // Serve starts a QUIC session that begins accepting streams.
 func (q *QUICConnection) Serve(ctx context.Context) error {
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
+	// origintunneld assumes the first stream is used for the control plane
+	controlStream, err := q.session.OpenStream()
+	if err != nil {
+		return fmt.Errorf("failed to open a registration control stream: %w", err)
+	}
 
+	// If either goroutine returns nil error, we rely on this cancellation to make sure the other goroutine exits
+	// as fast as possible as well. Nil error means we want to exit for good (caller code won't retry serving this
+	// connection).
+	// If either goroutine returns a non nil error, then the error group cancels the context, thus also canceling the
+	// other goroutine as fast as possible.
+	ctx, cancel := context.WithCancel(ctx)
+	errGroup, ctx := errgroup.WithContext(ctx)
+
+	// In the future, if cloudflared can autonomously push traffic to the edge, we have to make sure the control
+	// stream is already fully registered before the other goroutines can proceed.
+	errGroup.Go(func() error {
+		defer cancel()
+		return q.serveControlStream(ctx, controlStream)
+	})
+	errGroup.Go(func() error {
+		defer cancel()
+		return q.acceptStream(ctx)
+	})
+	errGroup.Go(func() error {
+		defer cancel()
+		return q.sessionManager.Serve(ctx)
+	})
+
+	return errGroup.Wait()
+}
+
+func (q *QUICConnection) serveControlStream(ctx context.Context, controlStream quic.Stream) error {
+	// This blocks until the control plane is done.
+	err := q.controlStreamHandler.ServeControlStream(ctx, controlStream, q.connOptions)
+	if err != nil {
+		// Not wrapping error here to be consistent with the http2 message.
+		return err
+	}
+
+	return nil
+}
+
+func (q *QUICConnection) acceptStream(ctx context.Context) error {
+	defer q.Close()
 	for {
-		stream, err := q.session.AcceptStream(ctx)
+		quicStream, err := q.session.AcceptStream(ctx)
 		if err != nil {
 			// context.Canceled is usually a user ctrl+c. We don't want to log an error here as it's intentional.
-			if errors.Is(err, context.Canceled) {
+			if errors.Is(err, context.Canceled) || q.controlStreamHandler.IsStopped() {
 				return nil
 			}
-			return errors.Wrap(err, "failed to accept QUIC stream")
+			return fmt.Errorf("failed to accept QUIC stream: %w", err)
 		}
 		go func() {
+			stream := quicpogs.NewSafeStreamCloser(quicStream)
 			defer stream.Close()
+
 			if err = q.handleStream(stream); err != nil {
 				q.logger.Err(err).Msg("Failed to handle QUIC stream")
 			}
@@ -98,12 +146,39 @@ func (q *QUICConnection) Close() {
 	q.session.CloseWithError(0, "")
 }
 
-func (q *QUICConnection) handleStream(stream quic.Stream) error {
-	connectRequest, err := quicpogs.ReadConnectRequestData(stream)
+func (q *QUICConnection) handleStream(stream io.ReadWriteCloser) error {
+	signature, err := quicpogs.DetermineProtocol(stream)
+	if err != nil {
+		return err
+	}
+	switch signature {
+	case quicpogs.DataStreamProtocolSignature:
+		reqServerStream, err := quicpogs.NewRequestServerStream(stream, signature)
+		if err != nil {
+			return nil
+		}
+		return q.handleDataStream(reqServerStream)
+	case quicpogs.RPCStreamProtocolSignature:
+		rpcStream, err := quicpogs.NewRPCServerStream(stream, signature)
+		if err != nil {
+			return err
+		}
+		return q.handleRPCStream(rpcStream)
+	default:
+		return fmt.Errorf("unknown protocol %v", signature)
+	}
+}
+
+func (q *QUICConnection) handleDataStream(stream *quicpogs.RequestServerStream) error {
+	connectRequest, err := stream.ReadConnectRequestData()
 	if err != nil {
 		return err
 	}
 
+	originProxy, err := q.orchestrator.GetOriginProxy()
+	if err != nil {
+		return err
+	}
 	switch connectRequest.Type {
 	case quicpogs.ConnectionTypeHTTP, quicpogs.ConnectionTypeWebsocket:
 		req, err := buildHTTPRequest(connectRequest, stream)
@@ -112,34 +187,106 @@ func (q *QUICConnection) handleStream(stream quic.Stream) error {
 		}
 
 		w := newHTTPResponseAdapter(stream)
-		return q.httpProxy.ProxyHTTP(w, req, connectRequest.Type == quicpogs.ConnectionTypeWebsocket)
+		return originProxy.ProxyHTTP(w, req, connectRequest.Type == quicpogs.ConnectionTypeWebsocket)
 	case quicpogs.ConnectionTypeTCP:
-		rwa := &streamReadWriteAcker{
-			ReadWriter: stream,
-		}
-		return q.httpProxy.ProxyTCP(context.Background(), rwa, &TCPRequest{Dest: connectRequest.Dest})
+		rwa := &streamReadWriteAcker{stream}
+		return originProxy.ProxyTCP(context.Background(), rwa, &TCPRequest{Dest: connectRequest.Dest})
 	}
 	return nil
 }
 
+func (q *QUICConnection) handleRPCStream(rpcStream *quicpogs.RPCServerStream) error {
+	return rpcStream.Serve(q, q, q.logger)
+}
+
+// RegisterUdpSession is the RPC method invoked by edge to register and run a session
+func (q *QUICConnection) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeAfterIdleHint time.Duration) error {
+	// Each session is a series of datagram from an eyeball to a dstIP:dstPort.
+	// (src port, dst IP, dst port) uniquely identifies a session, so it needs a dedicated connected socket.
+	originProxy, err := ingress.DialUDP(dstIP, dstPort)
+	if err != nil {
+		q.logger.Err(err).Msgf("Failed to create udp proxy to %s:%d", dstIP, dstPort)
+		return err
+	}
+	session, err := q.sessionManager.RegisterSession(ctx, sessionID, originProxy)
+	if err != nil {
+		q.logger.Err(err).Str("sessionID", sessionID.String()).Msgf("Failed to register udp session")
+		return err
+	}
+
+	go q.serveUDPSession(session, closeAfterIdleHint)
+
+	q.logger.Debug().Msgf("Registered session %v, %v, %v", sessionID, dstIP, dstPort)
+	return nil
+}
+
+func (q *QUICConnection) serveUDPSession(session *datagramsession.Session, closeAfterIdleHint time.Duration) {
+	ctx := q.session.Context()
+	closedByRemote, err := session.Serve(ctx, closeAfterIdleHint)
+	// If session is terminated by remote, then we know it has been unregistered from session manager and edge
+	if !closedByRemote {
+		if err != nil {
+			q.closeUDPSession(ctx, session.ID, err.Error())
+		} else {
+			q.closeUDPSession(ctx, session.ID, "terminated without error")
+		}
+	}
+	q.logger.Debug().Err(err).Str("sessionID", session.ID.String()).Msg("Session terminated")
+}
+
+// closeUDPSession first unregisters the session from session manager, then it tries to unregister from edge
+func (q *QUICConnection) closeUDPSession(ctx context.Context, sessionID uuid.UUID, message string) {
+	q.sessionManager.UnregisterSession(ctx, sessionID, message, false)
+	stream, err := q.session.OpenStream()
+	if err != nil {
+		// Log this at debug because this is not an error if session was closed due to lost connection
+		// with edge
+		q.logger.Debug().Err(err).Str("sessionID", sessionID.String()).
+			Msgf("Failed to open quic stream to unregister udp session with edge")
+		return
+	}
+	rpcClientStream, err := quicpogs.NewRPCClientStream(ctx, stream, q.logger)
+	if err != nil {
+		// Log this at debug because this is not an error if session was closed due to lost connection
+		// with edge
+		q.logger.Err(err).Str("sessionID", sessionID.String()).
+			Msgf("Failed to open rpc stream to unregister udp session with edge")
+		return
+	}
+	if err := rpcClientStream.UnregisterUdpSession(ctx, sessionID, message); err != nil {
+		q.logger.Err(err).Str("sessionID", sessionID.String()).
+			Msgf("Failed to unregister udp session with edge")
+	}
+}
+
+// UnregisterUdpSession is the RPC method invoked by edge to unregister and terminate a sesssion
+func (q *QUICConnection) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, message string) error {
+	return q.sessionManager.UnregisterSession(ctx, sessionID, message, true)
+}
+
+// UpdateConfiguration is the RPC method invoked by edge when there is a new configuration
+func (q *QUICConnection) UpdateConfiguration(ctx context.Context, version int32, config []byte) *tunnelpogs.UpdateConfigurationResponse {
+	return q.orchestrator.UpdateConfig(version, config)
+}
+
 // streamReadWriteAcker is a light wrapper over QUIC streams with a callback to send response back to
 // the client.
 type streamReadWriteAcker struct {
-	io.ReadWriter
+	*quicpogs.RequestServerStream
 }
 
 // AckConnection acks response back to the proxy.
 func (s *streamReadWriteAcker) AckConnection() error {
-	return quicpogs.WriteConnectResponseData(s, nil)
+	return s.WriteConnectResponseData(nil)
 }
 
 // httpResponseAdapter translates responses written by the HTTP Proxy into ones that can be used in QUIC.
 type httpResponseAdapter struct {
-	io.Writer
+	*quicpogs.RequestServerStream
 }
 
-func newHTTPResponseAdapter(w io.Writer) httpResponseAdapter {
-	return httpResponseAdapter{w}
+func newHTTPResponseAdapter(s *quicpogs.RequestServerStream) httpResponseAdapter {
+	return httpResponseAdapter{s}
 }
 
 func (hrw httpResponseAdapter) WriteRespHeaders(status int, header http.Header) error {
@@ -151,11 +298,11 @@ func (hrw httpResponseAdapter) WriteRespHeaders(status int, header http.Header)
 			metadata = append(metadata, quicpogs.Metadata{Key: httpHeaderKey, Val: v})
 		}
 	}
-	return quicpogs.WriteConnectResponseData(hrw, nil, metadata...)
+	return hrw.WriteConnectResponseData(nil, metadata...)
 }
 
 func (hrw httpResponseAdapter) WriteErrorResponse(err error) {
-	quicpogs.WriteConnectResponseData(hrw, err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
+	hrw.WriteConnectResponseData(err, quicpogs.Metadata{Key: "HttpStatus", Val: strconv.Itoa(http.StatusBadGateway)})
 }
 
 func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadCloser) (*http.Request, error) {
@@ -176,7 +323,7 @@ func buildHTTPRequest(connectRequest *quicpogs.ConnectRequest, body io.ReadClose
 			// metadata.Key is off the format httpHeaderKey:<HTTPHeader>
 			httpHeaderKey := strings.Split(metadata.Key, ":")
 			if len(httpHeaderKey) != 2 {
-				return nil, fmt.Errorf("Header Key: %s malformed", metadata.Key)
+				return nil, fmt.Errorf("header Key: %s malformed", metadata.Key)
 			}
 			req.Header.Add(httpHeaderKey[1], metadata.Val)
 		}

connection/quic_test.go

@@ -3,43 +3,41 @@ package connection
 import (
 	"bytes"
 	"context"
-	"crypto/rand"
-	"crypto/rsa"
 	"crypto/tls"
-	"crypto/x509"
-	"encoding/pem"
 	"fmt"
 	"io"
-	"math/big"
 	"net"
 	"net/http"
 	"net/url"
 	"os"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/gobwas/ws/wsutil"
+	"github.com/google/uuid"
 	"github.com/lucas-clemente/quic-go"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"github.com/cloudflare/cloudflared/datagramsession"
 	quicpogs "github.com/cloudflare/cloudflared/quic"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
+var (
+	testTLSServerConfig = quicpogs.GenerateTLSConfig()
+	testQUICConfig      = &quic.Config{
+		KeepAlive:       true,
+		EnableDatagrams: true,
+	}
+)
+
 // TestQUICServer tests if a quic server accepts and responds to a quic client with the acceptance protocol.
 // It also serves as a demonstration for communication with the QUIC connection started by a cloudflared.
 func TestQUICServer(t *testing.T) {
-	quicConfig := &quic.Config{
-		KeepAlive: true,
-	}
-
-	// Setup test.
-	log := zerolog.New(os.Stdout)
-
 	// Start a UDP Listener for QUIC.
 	udpAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0")
 	require.NoError(t, err)
@@ -47,21 +45,9 @@ func TestQUICServer(t *testing.T) {
 	require.NoError(t, err)
 	defer udpListener.Close()
 
-	// Create a simple tls config.
-	tlsConfig := generateTLSConfig()
-
-	// Create a client config
-	tlsClientConfig := &tls.Config{
-		InsecureSkipVerify: true,
-		NextProtos:         []string{"argotunnel"},
-	}
-
-	// Start a mock httpProxy
-	originProxy := &mockOriginProxyWithRequest{}
-
 	// This is simply a sample websocket frame message.
 	wsBuf := &bytes.Buffer{}
-	wsutil.WriteClientText(wsBuf, []byte("Hello"))
+	wsutil.WriteClientBinary(wsBuf, []byte("Hello"))
 
 	var tests = []struct {
 		desc             string
@@ -76,15 +62,15 @@ func TestQUICServer(t *testing.T) {
 			dest:           "/ok",
 			connectionType: quicpogs.ConnectionTypeHTTP,
 			metadata: []quicpogs.Metadata{
-				quicpogs.Metadata{
+				{
 					Key: "HttpHeader:Cf-Ray",
 					Val: "123123123",
 				},
-				quicpogs.Metadata{
+				{
 					Key: "HttpHost",
 					Val: "cf.host",
 				},
-				quicpogs.Metadata{
+				{
 					Key: "HttpMethod",
 					Val: "GET",
 				},
@@ -93,22 +79,22 @@ func TestQUICServer(t *testing.T) {
 		},
 		{
 			desc:           "test http body request streaming",
-			dest:           "/echo_body",
+			dest:           "/slow_echo_body",
 			connectionType: quicpogs.ConnectionTypeHTTP,
 			metadata: []quicpogs.Metadata{
-				quicpogs.Metadata{
+				{
 					Key: "HttpHeader:Cf-Ray",
 					Val: "123123123",
 				},
-				quicpogs.Metadata{
+				{
 					Key: "HttpHost",
 					Val: "cf.host",
 				},
-				quicpogs.Metadata{
+				{
 					Key: "HttpMethod",
 					Val: "POST",
 				},
-				quicpogs.Metadata{
+				{
 					Key: "HttpHeader:Content-Length",
 					Val: "24",
 				},
@@ -118,28 +104,28 @@ func TestQUICServer(t *testing.T) {
 		},
 		{
 			desc:           "test ws proxy",
-			dest:           "/ok",
+			dest:           "/ws/echo",
 			connectionType: quicpogs.ConnectionTypeWebsocket,
 			metadata: []quicpogs.Metadata{
-				quicpogs.Metadata{
+				{
 					Key: "HttpHeader:Cf-Cloudflared-Proxy-Connection-Upgrade",
 					Val: "Websocket",
 				},
-				quicpogs.Metadata{
+				{
 					Key: "HttpHeader:Another-Header",
 					Val: "Misc",
 				},
-				quicpogs.Metadata{
+				{
 					Key: "HttpHost",
 					Val: "cf.host",
 				},
-				quicpogs.Metadata{
+				{
 					Key: "HttpMethod",
 					Val: "get",
 				},
 			},
 			message:          wsBuf.Bytes(),
-			expectedResponse: []byte{0x81, 0x5, 0x48, 0x65, 0x6c, 0x6c, 0x6f},
+			expectedResponse: []byte{0x82, 0x5, 0x48, 0x65, 0x6c, 0x6c, 0x6f},
 		},
 		{
 			desc:             "test tcp proxy",
@@ -154,29 +140,17 @@ func TestQUICServer(t *testing.T) {
 		t.Run(test.desc, func(t *testing.T) {
 			ctx, cancel := context.WithCancel(context.Background())
 			var wg sync.WaitGroup
+			wg.Add(1)
 			go func() {
-				wg.Add(1)
 				defer wg.Done()
 				quicServer(
-					t, udpListener, tlsConfig, quicConfig,
+					t, udpListener, testTLSServerConfig, testQUICConfig,
 					test.dest, test.connectionType, test.metadata, test.message, test.expectedResponse,
 				)
 			}()
 
-			controlStream := fakeControlStream{}
-
-			qC, err := NewQUICConnection(
-				ctx,
-				quicConfig,
-				udpListener.LocalAddr(),
-				tlsClientConfig,
-				originProxy,
-				&pogs.ConnectionOptions{},
-				controlStream,
-				NewObserver(&log, &log, false),
-			)
-			require.NoError(t, err)
-			go qC.Serve(ctx)
+			qc := testQUICConnection(udpListener.LocalAddr(), t)
+			go qc.Serve(ctx)
 
 			wg.Wait()
 			cancel()
@@ -188,7 +162,8 @@ type fakeControlStream struct {
 	ControlStreamHandler
 }
 
-func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions, shouldWaitForUnregister bool) error {
+func (fakeControlStream) ServeControlStream(ctx context.Context, rw io.ReadWriteCloser, connOptions *tunnelpogs.ConnectionOptions) error {
+	<-ctx.Done()
 	return nil
 }
 func (fakeControlStream) IsStopped() bool {
@@ -215,53 +190,33 @@ func quicServer(
 	session, err := earlyListener.Accept(ctx)
 	require.NoError(t, err)
 
-	stream, err := session.OpenStreamSync(context.Background())
+	quicStream, err := session.OpenStreamSync(context.Background())
+	require.NoError(t, err)
+	stream := quicpogs.NewSafeStreamCloser(quicStream)
+
+	reqClientStream := quicpogs.RequestClientStream{ReadWriteCloser: stream}
+	err = reqClientStream.WriteConnectRequestData(dest, connectionType, metadata...)
 	require.NoError(t, err)
 
-	err = quicpogs.WriteConnectRequestData(stream, dest, connectionType, metadata...)
-	require.NoError(t, err)
-
-	_, err = quicpogs.ReadConnectResponseData(stream)
+	_, err = reqClientStream.ReadConnectResponseData()
 	require.NoError(t, err)
 
 	if message != nil {
 		// ALPN successful. Write data.
-		_, err := stream.Write([]byte(message))
+		_, err := stream.Write(message)
 		require.NoError(t, err)
 	}
 
 	response := make([]byte, len(expectedResponse))
-	stream.Read(response)
-	require.NoError(t, err)
+	_, err = stream.Read(response)
+	if err != io.EOF {
+		require.NoError(t, err)
+	}
 
 	// For now it is an echo server. Verify if the same data is returned.
 	assert.Equal(t, expectedResponse, response)
 }
 
-// Setup a bare-bones TLS config for the server
-func generateTLSConfig() *tls.Config {
-	key, err := rsa.GenerateKey(rand.Reader, 1024)
-	if err != nil {
-		panic(err)
-	}
-	template := x509.Certificate{SerialNumber: big.NewInt(1)}
-	certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
-	if err != nil {
-		panic(err)
-	}
-	keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
-	certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
-
-	tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
-	if err != nil {
-		panic(err)
-	}
-	return &tls.Config{
-		Certificates: []tls.Certificate{tlsCert},
-		NextProtos:   []string{"argotunnel"},
-	}
-}
-
 type mockOriginProxyWithRequest struct{}
 
 func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, r *http.Request, isWebsocket bool) error {
@@ -278,11 +233,14 @@ func (moc *mockOriginProxyWithRequest) ProxyHTTP(w ResponseWriter, r *http.Reque
 	}
 
 	if isWebsocket {
-		return wsEndpoint(w, r)
+		return wsEchoEndpoint(w, r)
 	}
 	switch r.URL.Path {
 	case "/ok":
 		originRespEndpoint(w, http.StatusOK, []byte(http.StatusText(http.StatusOK)))
+	case "/slow_echo_body":
+		time.Sleep(5)
+		fallthrough
 	case "/echo_body":
 		resp := &http.Response{
 			StatusCode: http.StatusOK,
@@ -309,23 +267,23 @@ func TestBuildHTTPRequest(t *testing.T) {
 			connectRequest: &quicpogs.ConnectRequest{
 				Dest: "http://test.com",
 				Metadata: []quicpogs.Metadata{
-					quicpogs.Metadata{
+					{
 						Key: "HttpHeader:Cf-Cloudflared-Proxy-Connection-Upgrade",
 						Val: "Websocket",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpHeader:Content-Length",
 						Val: "514",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpHeader:Another-Header",
 						Val: "Misc",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpHost",
 						Val: "cf.host",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpMethod",
 						Val: "get",
 					},
@@ -355,19 +313,19 @@ func TestBuildHTTPRequest(t *testing.T) {
 			connectRequest: &quicpogs.ConnectRequest{
 				Dest: "http://test.com",
 				Metadata: []quicpogs.Metadata{
-					quicpogs.Metadata{
+					{
 						Key: "HttpHeader:Cf-Cloudflared-Proxy-Connection-Upgrade",
 						Val: "Websocket",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpHeader:Another-Header",
 						Val: "Misc",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpHost",
 						Val: "cf.host",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpMethod",
 						Val: "get",
 					},
@@ -396,19 +354,19 @@ func TestBuildHTTPRequest(t *testing.T) {
 			connectRequest: &quicpogs.ConnectRequest{
 				Dest: "http://test.com",
 				Metadata: []quicpogs.Metadata{
-					quicpogs.Metadata{
+					{
 						Key: "HttpHeader:Another-Header",
 						Val: "Misc",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpHeader:Transfer-Encoding",
 						Val: "chunked",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpHost",
 						Val: "cf.host",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpMethod",
 						Val: "get",
 					},
@@ -438,19 +396,19 @@ func TestBuildHTTPRequest(t *testing.T) {
 			connectRequest: &quicpogs.ConnectRequest{
 				Dest: "http://test.com",
 				Metadata: []quicpogs.Metadata{
-					quicpogs.Metadata{
+					{
 						Key: "HttpHeader:Another-Header",
 						Val: "Misc",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpHeader:Transfer-Encoding",
 						Val: "gzip,chunked",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpHost",
 						Val: "cf.host",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpMethod",
 						Val: "get",
 					},
@@ -481,15 +439,15 @@ func TestBuildHTTPRequest(t *testing.T) {
 				Type: quicpogs.ConnectionTypeWebsocket,
 				Dest: "http://test.com",
 				Metadata: []quicpogs.Metadata{
-					quicpogs.Metadata{
+					{
 						Key: "HttpHeader:Another-Header",
 						Val: "Misc",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpHost",
 						Val: "cf.host",
 					},
-					quicpogs.Metadata{
+					{
 						Key: "HttpMethod",
 						Val: "get",
 					},
@@ -530,3 +488,155 @@ func (moc *mockOriginProxyWithRequest) ProxyTCP(ctx context.Context, rwa ReadWri
 	io.Copy(rwa, rwa)
 	return nil
 }
+
+func TestServeUDPSession(t *testing.T) {
+	// Start a UDP Listener for QUIC.
+	udpAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0")
+	require.NoError(t, err)
+	udpListener, err := net.ListenUDP(udpAddr.Network(), udpAddr)
+	require.NoError(t, err)
+	defer udpListener.Close()
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	// Establish QUIC connection with edge
+	edgeQUICSessionChan := make(chan quic.Session)
+	go func() {
+		earlyListener, err := quic.Listen(udpListener, testTLSServerConfig, testQUICConfig)
+		require.NoError(t, err)
+
+		edgeQUICSession, err := earlyListener.Accept(ctx)
+		require.NoError(t, err)
+		edgeQUICSessionChan <- edgeQUICSession
+	}()
+
+	qc := testQUICConnection(udpListener.LocalAddr(), t)
+	go qc.Serve(ctx)
+
+	edgeQUICSession := <-edgeQUICSessionChan
+	serveSession(ctx, qc, edgeQUICSession, closedByOrigin, io.EOF.Error(), t)
+	serveSession(ctx, qc, edgeQUICSession, closedByTimeout, datagramsession.SessionIdleErr(time.Millisecond*50).Error(), t)
+	serveSession(ctx, qc, edgeQUICSession, closedByRemote, "eyeball closed connection", t)
+	cancel()
+}
+
+func serveSession(ctx context.Context, qc *QUICConnection, edgeQUICSession quic.Session, closeType closeReason, expectedReason string, t *testing.T) {
+	var (
+		payload = []byte(t.Name())
+	)
+	sessionID := uuid.New()
+	cfdConn, originConn := net.Pipe()
+	// Registers and run a new session
+	session, err := qc.sessionManager.RegisterSession(ctx, sessionID, cfdConn)
+	require.NoError(t, err)
+
+	sessionDone := make(chan struct{})
+	go func() {
+		qc.serveUDPSession(session, time.Millisecond*50)
+		close(sessionDone)
+	}()
+
+	// Send a message to the quic session on edge side, it should be deumx to this datagram session
+	muxedPayload := append(payload, sessionID[:]...)
+	err = edgeQUICSession.SendMessage(muxedPayload)
+	require.NoError(t, err)
+
+	readBuffer := make([]byte, len(payload)+1)
+	n, err := originConn.Read(readBuffer)
+	require.NoError(t, err)
+	require.Equal(t, len(payload), n)
+	require.True(t, bytes.Equal(payload, readBuffer[:n]))
+
+	// Close connection to terminate session
+	switch closeType {
+	case closedByOrigin:
+		originConn.Close()
+	case closedByRemote:
+		err = qc.UnregisterUdpSession(ctx, sessionID, expectedReason)
+		require.NoError(t, err)
+	case closedByTimeout:
+	}
+
+	if closeType != closedByRemote {
+		// Session was not closed by remote, so closeUDPSession should be invoked to unregister from remote
+		unregisterFromEdgeChan := make(chan struct{})
+		sessionRPCServer := &mockSessionRPCServer{
+			sessionID:            sessionID,
+			unregisterReason:     expectedReason,
+			calledUnregisterChan: unregisterFromEdgeChan,
+		}
+		go runRPCServer(ctx, edgeQUICSession, sessionRPCServer, nil, t)
+
+		<-unregisterFromEdgeChan
+	}
+
+	<-sessionDone
+}
+
+type closeReason uint8
+
+const (
+	closedByOrigin closeReason = iota
+	closedByRemote
+	closedByTimeout
+)
+
+func runRPCServer(ctx context.Context, session quic.Session, sessionRPCServer tunnelpogs.SessionManager, configRPCServer tunnelpogs.ConfigurationManager, t *testing.T) {
+	stream, err := session.AcceptStream(ctx)
+	require.NoError(t, err)
+
+	if stream.StreamID() == 0 {
+		// Skip the first stream, it's the control stream of the QUIC connection
+		stream, err = session.AcceptStream(ctx)
+		require.NoError(t, err)
+	}
+	protocol, err := quicpogs.DetermineProtocol(stream)
+	assert.NoError(t, err)
+	rpcServerStream, err := quicpogs.NewRPCServerStream(stream, protocol)
+	assert.NoError(t, err)
+
+	log := zerolog.New(os.Stdout)
+	err = rpcServerStream.Serve(sessionRPCServer, configRPCServer, &log)
+	assert.NoError(t, err)
+}
+
+type mockSessionRPCServer struct {
+	sessionID            uuid.UUID
+	unregisterReason     string
+	calledUnregisterChan chan struct{}
+}
+
+func (s mockSessionRPCServer) RegisterUdpSession(ctx context.Context, sessionID uuid.UUID, dstIP net.IP, dstPort uint16, closeIdleAfter time.Duration) error {
+	return fmt.Errorf("mockSessionRPCServer doesn't implement RegisterUdpSession")
+}
+
+func (s mockSessionRPCServer) UnregisterUdpSession(ctx context.Context, sessionID uuid.UUID, reason string) error {
+	if s.sessionID != sessionID {
+		return fmt.Errorf("expect session ID %s, got %s", s.sessionID, sessionID)
+	}
+	if s.unregisterReason != reason {
+		return fmt.Errorf("expect unregister reason %s, got %s", s.unregisterReason, reason)
+	}
+	close(s.calledUnregisterChan)
+	return nil
+}
+
+func testQUICConnection(udpListenerAddr net.Addr, t *testing.T) *QUICConnection {
+	tlsClientConfig := &tls.Config{
+		InsecureSkipVerify: true,
+		NextProtos:         []string{"argotunnel"},
+	}
+	// Start a mock httpProxy
+	log := zerolog.New(os.Stdout)
+	qc, err := NewQUICConnection(
+		testQUICConfig,
+		udpListenerAddr,
+		tlsClientConfig,
+		&mockOrchestrator{originProxy: &mockOriginProxyWithRequest{}},
+		&tunnelpogs.ConnectionOptions{},
+		fakeControlStream{},
+		&log,
+	)
+	require.NoError(t, err)
+	return qc
+}

connection/rpc.go

@@ -37,7 +37,7 @@ func NewTunnelServerClient(
 	}
 }
 
-func (tsc *tunnelServerClient) Authenticate(ctx context.Context, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) (tunnelpogs.AuthOutcome, error) {
+func (tsc *tunnelServerClient) Authenticate(ctx context.Context, classicTunnel *ClassicTunnelProperties, registrationOptions *tunnelpogs.RegistrationOptions) (tunnelpogs.AuthOutcome, error) {
 	authResp, err := tsc.client.Authenticate(ctx, classicTunnel.OriginCert, classicTunnel.Hostname, registrationOptions)
 	if err != nil {
 		return nil, err
@@ -54,7 +54,7 @@ func (tsc *tunnelServerClient) Close() {
 type NamedTunnelRPCClient interface {
 	RegisterConnection(
 		c context.Context,
-		config *NamedTunnelConfig,
+		config *NamedTunnelProperties,
 		options *tunnelpogs.ConnectionOptions,
 		connIndex uint8,
 		observer *Observer,
@@ -86,15 +86,15 @@ func newRegistrationRPCClient(
 
 func (rsc *registrationServerClient) RegisterConnection(
 	ctx context.Context,
-	config *NamedTunnelConfig,
+	properties *NamedTunnelProperties,
 	options *tunnelpogs.ConnectionOptions,
 	connIndex uint8,
 	observer *Observer,
 ) error {
 	conn, err := rsc.client.RegisterConnection(
 		ctx,
-		config.Credentials.Auth(),
-		config.Credentials.TunnelID,
+		properties.Credentials.Auth(),
+		properties.Credentials.TunnelID,
 		connIndex,
 		options,
 	)
@@ -137,7 +137,7 @@ const (
 	authenticate rpcName = " authenticate"
 )
 
-func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter CredentialManager, classicTunnel *ClassicTunnelProperties, registrationOptions *tunnelpogs.RegistrationOptions) error {
 	h.observer.sendRegisteringEvent(registrationOptions.ConnectionID)
 
 	stream, err := h.newRPCStream(ctx, register)
@@ -174,7 +174,7 @@ type CredentialManager interface {
 func (h *h2muxConnection) processRegistrationSuccess(
 	registration *tunnelpogs.TunnelRegistration,
 	name rpcName,
-	credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig,
+	credentialManager CredentialManager, classicTunnel *ClassicTunnelProperties,
 ) error {
 	for _, logLine := range registration.LogLines {
 		h.observer.log.Info().Msg(logLine)
@@ -205,7 +205,7 @@ func (h *h2muxConnection) processRegisterTunnelError(err tunnelpogs.TunnelRegist
 	}
 }
 
-func (h *h2muxConnection) reconnectTunnel(ctx context.Context, credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+func (h *h2muxConnection) reconnectTunnel(ctx context.Context, credentialManager CredentialManager, classicTunnel *ClassicTunnelProperties, registrationOptions *tunnelpogs.RegistrationOptions) error {
 	token, err := credentialManager.ReconnectToken()
 	if err != nil {
 		return err
@@ -264,7 +264,7 @@ func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelSe
 
 func (h *h2muxConnection) registerNamedTunnel(
 	ctx context.Context,
-	namedTunnel *NamedTunnelConfig,
+	namedTunnel *NamedTunnelProperties,
 	connOptions *tunnelpogs.ConnectionOptions,
 ) error {
 	stream, err := h.newRPCStream(ctx, register)
@@ -283,7 +283,7 @@ func (h *h2muxConnection) registerNamedTunnel(
 func (h *h2muxConnection) unregister(isNamedTunnel bool) {
 	h.observer.sendUnregisteringEvent(h.connIndex)
 
-	unregisterCtx, cancel := context.WithTimeout(context.Background(), h.config.GracePeriod)
+	unregisterCtx, cancel := context.WithTimeout(context.Background(), h.gracePeriod)
 	defer cancel()
 
 	stream, err := h.newRPCStream(unregisterCtx, unregister)
@@ -296,13 +296,13 @@ func (h *h2muxConnection) unregister(isNamedTunnel bool) {
 		rpcClient := h.newRPCClientFunc(unregisterCtx, stream, h.observer.log)
 		defer rpcClient.Close()
 
-		rpcClient.GracefulShutdown(unregisterCtx, h.config.GracePeriod)
+		rpcClient.GracefulShutdown(unregisterCtx, h.gracePeriod)
 	} else {
 		rpcClient := NewTunnelServerClient(unregisterCtx, stream, h.observer.log)
 		defer rpcClient.Close()
 
 		// gracePeriod is encoded in int64 using capnproto
-		_ = rpcClient.client.UnregisterTunnel(unregisterCtx, h.config.GracePeriod.Nanoseconds())
+		_ = rpcClient.client.UnregisterTunnel(unregisterCtx, h.gracePeriod.Nanoseconds())
 	}
 
 	h.observer.log.Info().Uint8(LogFieldConnIndex, h.connIndex).Msg("Unregistered tunnel connection")

datagramsession/event.go

@@ -0,0 +1,50 @@
+package datagramsession
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/google/uuid"
+)
+
+// registerSessionEvent is an event to start tracking a new session
+type registerSessionEvent struct {
+	sessionID   uuid.UUID
+	originProxy io.ReadWriteCloser
+	resultChan  chan *Session
+}
+
+func newRegisterSessionEvent(sessionID uuid.UUID, originProxy io.ReadWriteCloser) *registerSessionEvent {
+	return &registerSessionEvent{
+		sessionID:   sessionID,
+		originProxy: originProxy,
+		resultChan:  make(chan *Session, 1),
+	}
+}
+
+// unregisterSessionEvent is an event to stop tracking and terminate the session.
+type unregisterSessionEvent struct {
+	sessionID uuid.UUID
+	err       *errClosedSession
+}
+
+// ClosedSessionError represent a condition that closes the session other than I/O
+// I/O error is not included, because the side that closes the session is ambiguous.
+type errClosedSession struct {
+	message  string
+	byRemote bool
+}
+
+func (sc *errClosedSession) Error() string {
+	if sc.byRemote {
+		return fmt.Sprintf("session closed by remote due to %s", sc.message)
+	} else {
+		return fmt.Sprintf("session closed by local due to %s", sc.message)
+	}
+}
+
+// newDatagram is an event when transport receives new datagram
+type newDatagram struct {
+	sessionID uuid.UUID
+	payload   []byte
+}

datagramsession/manager.go

@@ -0,0 +1,144 @@
+package datagramsession
+
+import (
+	"context"
+	"io"
+
+	"github.com/google/uuid"
+	"github.com/lucas-clemente/quic-go"
+	"github.com/rs/zerolog"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	requestChanCapacity = 16
+)
+
+// Manager defines the APIs to manage sessions from the same transport.
+type Manager interface {
+	// Serve starts the event loop
+	Serve(ctx context.Context) error
+	// RegisterSession starts tracking a session. Caller is responsible for starting the session
+	RegisterSession(ctx context.Context, sessionID uuid.UUID, dstConn io.ReadWriteCloser) (*Session, error)
+	// UnregisterSession stops tracking the session and terminates it
+	UnregisterSession(ctx context.Context, sessionID uuid.UUID, message string, byRemote bool) error
+}
+
+type manager struct {
+	registrationChan   chan *registerSessionEvent
+	unregistrationChan chan *unregisterSessionEvent
+	datagramChan       chan *newDatagram
+	transport          transport
+	sessions           map[uuid.UUID]*Session
+	log                *zerolog.Logger
+}
+
+func NewManager(transport transport, log *zerolog.Logger) Manager {
+	return &manager{
+		registrationChan:   make(chan *registerSessionEvent),
+		unregistrationChan: make(chan *unregisterSessionEvent),
+		// datagramChan is buffered, so it can read more datagrams from transport while the event loop is processing other events
+		datagramChan: make(chan *newDatagram, requestChanCapacity),
+		transport:    transport,
+		sessions:     make(map[uuid.UUID]*Session),
+		log:          log,
+	}
+}
+
+func (m *manager) Serve(ctx context.Context) error {
+	errGroup, ctx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		for {
+			sessionID, payload, err := m.transport.ReceiveFrom()
+			if err != nil {
+				if aerr, ok := err.(*quic.ApplicationError); ok && uint64(aerr.ErrorCode) == uint64(quic.NoError) {
+					return nil
+				} else {
+					return err
+				}
+			}
+			datagram := &newDatagram{
+				sessionID: sessionID,
+				payload:   payload,
+			}
+			select {
+			case <-ctx.Done():
+				return ctx.Err()
+			// Only the event loop routine can update/lookup the sessions map to avoid concurrent access
+			// Send the datagram to the event loop. It will find the session to send to
+			case m.datagramChan <- datagram:
+			}
+		}
+	})
+	errGroup.Go(func() error {
+		for {
+			select {
+			case <-ctx.Done():
+				return nil
+			case datagram := <-m.datagramChan:
+				m.sendToSession(datagram)
+			case registration := <-m.registrationChan:
+				m.registerSession(ctx, registration)
+			// TODO: TUN-5422: Unregister inactive session upon timeout
+			case unregistration := <-m.unregistrationChan:
+				m.unregisterSession(unregistration)
+			}
+		}
+	})
+	return errGroup.Wait()
+}
+
+func (m *manager) RegisterSession(ctx context.Context, sessionID uuid.UUID, originProxy io.ReadWriteCloser) (*Session, error) {
+	event := newRegisterSessionEvent(sessionID, originProxy)
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	case m.registrationChan <- event:
+		session := <-event.resultChan
+		return session, nil
+	}
+}
+
+func (m *manager) registerSession(ctx context.Context, registration *registerSessionEvent) {
+	session := newSession(registration.sessionID, m.transport, registration.originProxy, m.log)
+	m.sessions[registration.sessionID] = session
+	registration.resultChan <- session
+}
+
+func (m *manager) UnregisterSession(ctx context.Context, sessionID uuid.UUID, message string, byRemote bool) error {
+	event := &unregisterSessionEvent{
+		sessionID: sessionID,
+		err: &errClosedSession{
+			message:  message,
+			byRemote: byRemote,
+		},
+	}
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case m.unregistrationChan <- event:
+		return nil
+	}
+}
+
+func (m *manager) unregisterSession(unregistration *unregisterSessionEvent) {
+	session, ok := m.sessions[unregistration.sessionID]
+	if ok {
+		delete(m.sessions, unregistration.sessionID)
+		session.close(unregistration.err)
+	}
+}
+
+func (m *manager) sendToSession(datagram *newDatagram) {
+	session, ok := m.sessions[datagram.sessionID]
+	if !ok {
+		m.log.Error().Str("sessionID", datagram.sessionID.String()).Msg("session not found")
+		return
+	}
+	// session writes to destination over a connected UDP socket, which should not be blocking, so this call doesn't
+	// need to run in another go routine
+	_, err := session.transportToDst(datagram.payload)
+	if err != nil {
+		m.log.Err(err).Str("sessionID", datagram.sessionID.String()).Msg("Failed to write payload to session")
+	}
+}

datagramsession/manager_test.go

@@ -0,0 +1,222 @@
+package datagramsession
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sync/errgroup"
+)
+
+func TestManagerServe(t *testing.T) {
+	const (
+		sessions            = 20
+		msgs                = 50
+		remoteUnregisterMsg = "eyeball closed connection"
+	)
+	log := zerolog.Nop()
+	transport := &mockQUICTransport{
+		reqChan:  newDatagramChannel(1),
+		respChan: newDatagramChannel(1),
+	}
+	mg := NewManager(transport, &log)
+
+	eyeballTracker := make(map[uuid.UUID]*datagramChannel)
+	for i := 0; i < sessions; i++ {
+		sessionID := uuid.New()
+		eyeballTracker[sessionID] = newDatagramChannel(1)
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	serveDone := make(chan struct{})
+	go func(ctx context.Context) {
+		mg.Serve(ctx)
+		close(serveDone)
+	}(ctx)
+
+	go func(ctx context.Context) {
+		for {
+			sessionID, payload, err := transport.respChan.Receive(ctx)
+			if err != nil {
+				require.Equal(t, context.Canceled, err)
+				return
+			}
+			respChan := eyeballTracker[sessionID]
+			require.NoError(t, respChan.Send(ctx, sessionID, payload))
+		}
+	}(ctx)
+
+	errGroup, ctx := errgroup.WithContext(ctx)
+	for sID, receiver := range eyeballTracker {
+		// Assign loop variables to local variables
+		sessionID := sID
+		eyeballRespReceiver := receiver
+		errGroup.Go(func() error {
+			payload := testPayload(sessionID)
+			expectResp := testResponse(payload)
+
+			cfdConn, originConn := net.Pipe()
+
+			origin := mockOrigin{
+				expectMsgCount: msgs,
+				expectedMsg:    payload,
+				expectedResp:   expectResp,
+				conn:           originConn,
+			}
+			eyeball := mockEyeball{
+				expectMsgCount:  msgs,
+				expectedMsg:     expectResp,
+				expectSessionID: sessionID,
+				respReceiver:    eyeballRespReceiver,
+			}
+
+			reqErrGroup, reqCtx := errgroup.WithContext(ctx)
+			reqErrGroup.Go(func() error {
+				return origin.serve()
+			})
+			reqErrGroup.Go(func() error {
+				return eyeball.serve(reqCtx)
+			})
+
+			session, err := mg.RegisterSession(ctx, sessionID, cfdConn)
+			require.NoError(t, err)
+
+			sessionDone := make(chan struct{})
+			go func() {
+				closedByRemote, err := session.Serve(ctx, time.Minute*2)
+				closeSession := &errClosedSession{
+					message:  remoteUnregisterMsg,
+					byRemote: true,
+				}
+				require.Equal(t, closeSession, err)
+				require.True(t, closedByRemote)
+				close(sessionDone)
+			}()
+
+			for i := 0; i < msgs; i++ {
+				require.NoError(t, transport.newRequest(ctx, sessionID, testPayload(sessionID)))
+			}
+
+			// Make sure eyeball and origin have received all messages before unregistering the session
+			require.NoError(t, reqErrGroup.Wait())
+
+			require.NoError(t, mg.UnregisterSession(ctx, sessionID, remoteUnregisterMsg, true))
+			<-sessionDone
+
+			return nil
+		})
+	}
+
+	require.NoError(t, errGroup.Wait())
+	cancel()
+	transport.close()
+	<-serveDone
+}
+
+type mockOrigin struct {
+	expectMsgCount int
+	expectedMsg    []byte
+	expectedResp   []byte
+	conn           io.ReadWriteCloser
+}
+
+func (mo *mockOrigin) serve() error {
+	expectedMsgLen := len(mo.expectedMsg)
+	readBuffer := make([]byte, expectedMsgLen+1)
+	for i := 0; i < mo.expectMsgCount; i++ {
+		n, err := mo.conn.Read(readBuffer)
+		if err != nil {
+			return err
+		}
+		if n != expectedMsgLen {
+			return fmt.Errorf("Expect to read %d bytes, read %d", expectedMsgLen, n)
+		}
+		if !bytes.Equal(readBuffer[:n], mo.expectedMsg) {
+			return fmt.Errorf("Expect %v, read %v", mo.expectedMsg, readBuffer[:n])
+		}
+
+		_, err = mo.conn.Write(mo.expectedResp)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func testPayload(sessionID uuid.UUID) []byte {
+	return []byte(fmt.Sprintf("Message from %s", sessionID))
+}
+
+func testResponse(msg []byte) []byte {
+	return []byte(fmt.Sprintf("Response to %v", msg))
+}
+
+type mockEyeball struct {
+	expectMsgCount  int
+	expectedMsg     []byte
+	expectSessionID uuid.UUID
+	respReceiver    *datagramChannel
+}
+
+func (me *mockEyeball) serve(ctx context.Context) error {
+	for i := 0; i < me.expectMsgCount; i++ {
+		sessionID, msg, err := me.respReceiver.Receive(ctx)
+		if err != nil {
+			return err
+		}
+		if sessionID != me.expectSessionID {
+			return fmt.Errorf("Expect session %s, got %s", me.expectSessionID, sessionID)
+		}
+		if !bytes.Equal(msg, me.expectedMsg) {
+			return fmt.Errorf("Expect %v, read %v", me.expectedMsg, msg)
+		}
+	}
+	return nil
+}
+
+// datagramChannel is a channel for Datagram with wrapper to send/receive with context
+type datagramChannel struct {
+	datagramChan chan *newDatagram
+	closedChan   chan struct{}
+}
+
+func newDatagramChannel(capacity uint) *datagramChannel {
+	return &datagramChannel{
+		datagramChan: make(chan *newDatagram, capacity),
+		closedChan:   make(chan struct{}),
+	}
+}
+
+func (rc *datagramChannel) Send(ctx context.Context, sessionID uuid.UUID, payload []byte) error {
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case <-rc.closedChan:
+		return fmt.Errorf("datagram channel closed")
+	case rc.datagramChan <- &newDatagram{sessionID: sessionID, payload: payload}:
+		return nil
+	}
+}
+
+func (rc *datagramChannel) Receive(ctx context.Context) (uuid.UUID, []byte, error) {
+	select {
+	case <-ctx.Done():
+		return uuid.Nil, nil, ctx.Err()
+	case <-rc.closedChan:
+		return uuid.Nil, nil, fmt.Errorf("datagram channel closed")
+	case msg := <-rc.datagramChan:
+		return msg.sessionID, msg.payload, nil
+	}
+}
+
+func (rc *datagramChannel) Close() {
+	// No need to close msgChan, it will be garbage collect once there is no reference to it
+	close(rc.closedChan)
+}

datagramsession/session.go

@@ -0,0 +1,145 @@
+package datagramsession
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/rs/zerolog"
+)
+
+const (
+	defaultCloseIdleAfter = time.Second * 210
+)
+
+func SessionIdleErr(timeout time.Duration) error {
+	return fmt.Errorf("session idle for %v", timeout)
+}
+
+// Session is a bidirectional pipe of datagrams between transport and dstConn
+// Currently the only implementation of transport is quic DatagramMuxer
+// Destination can be a connection with origin or with eyeball
+// When the destination is origin:
+// - Datagrams from edge are read by Manager from the transport. Manager finds the corresponding Session and calls the
+//   write method of the Session to send to origin
+// - Datagrams from origin are read from conn and SentTo transport. Transport will return them to eyeball
+// When the destination is eyeball:
+// - Datagrams from eyeball are read from conn and SentTo transport. Transport will send them to cloudflared
+// - Datagrams from cloudflared are read by Manager from the transport. Manager finds the corresponding Session and calls the
+//   write method of the Session to send to eyeball
+type Session struct {
+	ID        uuid.UUID
+	transport transport
+	dstConn   io.ReadWriteCloser
+	// activeAtChan is used to communicate the last read/write time
+	activeAtChan chan time.Time
+	closeChan    chan error
+	log          *zerolog.Logger
+}
+
+func newSession(id uuid.UUID, transport transport, dstConn io.ReadWriteCloser, log *zerolog.Logger) *Session {
+	return &Session{
+		ID:        id,
+		transport: transport,
+		dstConn:   dstConn,
+		// activeAtChan has low capacity. It can be full when there are many concurrent read/write. markActive() will
+		// drop instead of blocking because last active time only needs to be an approximation
+		activeAtChan: make(chan time.Time, 2),
+		// capacity is 2 because close() and dstToTransport routine in Serve() can write to this channel
+		closeChan: make(chan error, 2),
+		log:       log,
+	}
+}
+
+func (s *Session) Serve(ctx context.Context, closeAfterIdle time.Duration) (closedByRemote bool, err error) {
+	go func() {
+		// QUIC implementation copies data to another buffer before returning https://github.com/lucas-clemente/quic-go/blob/v0.24.0/session.go#L1967-L1975
+		// This makes it safe to share readBuffer between iterations
+		const maxPacketSize = 1500
+		readBuffer := make([]byte, maxPacketSize)
+		for {
+			if err := s.dstToTransport(readBuffer); err != nil {
+				s.closeChan <- err
+				return
+			}
+		}
+	}()
+	err = s.waitForCloseCondition(ctx, closeAfterIdle)
+	if closeSession, ok := err.(*errClosedSession); ok {
+		closedByRemote = closeSession.byRemote
+	}
+	return closedByRemote, err
+}
+
+func (s *Session) waitForCloseCondition(ctx context.Context, closeAfterIdle time.Duration) error {
+	// Closing dstConn cancels read so dstToTransport routine in Serve() can return
+	defer s.dstConn.Close()
+	if closeAfterIdle == 0 {
+		// provide deafult is caller doesn't specify one
+		closeAfterIdle = defaultCloseIdleAfter
+	}
+
+	checkIdleFreq := closeAfterIdle / 8
+	checkIdleTicker := time.NewTicker(checkIdleFreq)
+	defer checkIdleTicker.Stop()
+
+	activeAt := time.Now()
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case reason := <-s.closeChan:
+			return reason
+		// TODO: TUN-5423 evaluate if using atomic is more efficient
+		case now := <-checkIdleTicker.C:
+			// The session is considered inactive if current time is after (last active time + allowed idle time)
+			if now.After(activeAt.Add(closeAfterIdle)) {
+				return SessionIdleErr(closeAfterIdle)
+			}
+		case activeAt = <-s.activeAtChan: // Update last active time
+		}
+	}
+}
+
+func (s *Session) dstToTransport(buffer []byte) error {
+	n, err := s.dstConn.Read(buffer)
+	s.markActive()
+	// https://pkg.go.dev/io#Reader suggests caller should always process n > 0 bytes
+	if n > 0 {
+		if n <= int(s.transport.MTU()) {
+			err = s.transport.SendTo(s.ID, buffer[:n])
+		} else {
+			// drop packet for now, eventually reply with ICMP for PMTUD
+			s.log.Debug().
+				Str("session", s.ID.String()).
+				Int("len", n).
+				Int("mtu", s.transport.MTU()).
+				Msg("dropped packet exceeding MTU")
+		}
+	}
+	// Some UDP application might send 0-size payload.
+	if err == nil && n == 0 {
+		err = s.transport.SendTo(s.ID, []byte{})
+	}
+	return err
+}
+
+func (s *Session) transportToDst(payload []byte) (int, error) {
+	s.markActive()
+	return s.dstConn.Write(payload)
+}
+
+// Sends the last active time to the idle checker loop without blocking. activeAtChan will only be full when there
+// are many concurrent read/write. It is fine to lose some precision
+func (s *Session) markActive() {
+	select {
+	case s.activeAtChan <- time.Now():
+	default:
+	}
+}
+
+func (s *Session) close(err *errClosedSession) {
+	s.closeChan <- err
+}

datagramsession/session_test.go

@@ -0,0 +1,234 @@
+package datagramsession
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/rs/zerolog"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sync/errgroup"
+)
+
+// TestCloseSession makes sure a session will stop after context is done
+func TestSessionCtxDone(t *testing.T) {
+	testSessionReturns(t, closeByContext, time.Minute*2)
+}
+
+// TestCloseSession makes sure a session will stop after close method is called
+func TestCloseSession(t *testing.T) {
+	testSessionReturns(t, closeByCallingClose, time.Minute*2)
+}
+
+// TestCloseIdle makess sure a session will stop after there is no read/write for a period defined by closeAfterIdle
+func TestCloseIdle(t *testing.T) {
+	testSessionReturns(t, closeByTimeout, time.Millisecond*100)
+}
+
+func testSessionReturns(t *testing.T, closeBy closeMethod, closeAfterIdle time.Duration) {
+	var (
+		localCloseReason = &errClosedSession{
+			message:  "connection closed by origin",
+			byRemote: false,
+		}
+	)
+	sessionID := uuid.New()
+	cfdConn, originConn := net.Pipe()
+	payload := testPayload(sessionID)
+	transport := &mockQUICTransport{
+		reqChan:  newDatagramChannel(1),
+		respChan: newDatagramChannel(1),
+	}
+	log := zerolog.Nop()
+	session := newSession(sessionID, transport, cfdConn, &log)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	sessionDone := make(chan struct{})
+	go func() {
+		closedByRemote, err := session.Serve(ctx, closeAfterIdle)
+		switch closeBy {
+		case closeByContext:
+			require.Equal(t, context.Canceled, err)
+			require.False(t, closedByRemote)
+		case closeByCallingClose:
+			require.Equal(t, localCloseReason, err)
+			require.Equal(t, localCloseReason.byRemote, closedByRemote)
+		case closeByTimeout:
+			require.Equal(t, SessionIdleErr(closeAfterIdle), err)
+			require.False(t, closedByRemote)
+		}
+		close(sessionDone)
+	}()
+
+	go func() {
+		n, err := session.transportToDst(payload)
+		require.NoError(t, err)
+		require.Equal(t, len(payload), n)
+	}()
+
+	readBuffer := make([]byte, len(payload)+1)
+	n, err := originConn.Read(readBuffer)
+	require.NoError(t, err)
+	require.Equal(t, len(payload), n)
+
+	lastRead := time.Now()
+
+	switch closeBy {
+	case closeByContext:
+		cancel()
+	case closeByCallingClose:
+		session.close(localCloseReason)
+	}
+
+	<-sessionDone
+	if closeBy == closeByTimeout {
+		require.True(t, time.Now().After(lastRead.Add(closeAfterIdle)))
+	}
+	// call cancelled again otherwise the linter will warn about possible context leak
+	cancel()
+}
+
+type closeMethod int
+
+const (
+	closeByContext closeMethod = iota
+	closeByCallingClose
+	closeByTimeout
+)
+
+func TestWriteToDstSessionPreventClosed(t *testing.T) {
+	testActiveSessionNotClosed(t, false, true)
+}
+
+func TestReadFromDstSessionPreventClosed(t *testing.T) {
+	testActiveSessionNotClosed(t, true, false)
+}
+
+func testActiveSessionNotClosed(t *testing.T, readFromDst bool, writeToDst bool) {
+	const closeAfterIdle = time.Millisecond * 100
+	const activeTime = time.Millisecond * 500
+
+	sessionID := uuid.New()
+	cfdConn, originConn := net.Pipe()
+	payload := testPayload(sessionID)
+	transport := &mockQUICTransport{
+		reqChan:  newDatagramChannel(100),
+		respChan: newDatagramChannel(100),
+	}
+	log := zerolog.Nop()
+	session := newSession(sessionID, transport, cfdConn, &log)
+
+	startTime := time.Now()
+	activeUntil := startTime.Add(activeTime)
+	ctx, cancel := context.WithCancel(context.Background())
+	errGroup, ctx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		session.Serve(ctx, closeAfterIdle)
+		if time.Now().Before(startTime.Add(activeTime)) {
+			return fmt.Errorf("session closed while it's still active")
+		}
+		return nil
+	})
+
+	if readFromDst {
+		errGroup.Go(func() error {
+			for {
+				if time.Now().After(activeUntil) {
+					return nil
+				}
+				if _, err := originConn.Write(payload); err != nil {
+					return err
+				}
+				time.Sleep(closeAfterIdle / 2)
+			}
+		})
+	}
+	if writeToDst {
+		errGroup.Go(func() error {
+			readBuffer := make([]byte, len(payload))
+			for {
+				n, err := originConn.Read(readBuffer)
+				if err != nil {
+					if err == io.EOF || err == io.ErrClosedPipe {
+						return nil
+					}
+					return err
+				}
+				if !bytes.Equal(payload, readBuffer[:n]) {
+					return fmt.Errorf("payload %v is not equal to %v", readBuffer[:n], payload)
+				}
+			}
+		})
+		errGroup.Go(func() error {
+			for {
+				if time.Now().After(activeUntil) {
+					return nil
+				}
+				if _, err := session.transportToDst(payload); err != nil {
+					return err
+				}
+				time.Sleep(closeAfterIdle / 2)
+			}
+		})
+	}
+
+	require.NoError(t, errGroup.Wait())
+	cancel()
+}
+
+func TestMarkActiveNotBlocking(t *testing.T) {
+	const concurrentCalls = 50
+	session := newSession(uuid.New(), nil, nil, nil)
+	var wg sync.WaitGroup
+	wg.Add(concurrentCalls)
+	for i := 0; i < concurrentCalls; i++ {
+		go func() {
+			session.markActive()
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+}
+
+func TestZeroBytePayload(t *testing.T) {
+	sessionID := uuid.New()
+	cfdConn, originConn := net.Pipe()
+	transport := &mockQUICTransport{
+		reqChan:  newDatagramChannel(1),
+		respChan: newDatagramChannel(1),
+	}
+	log := zerolog.Nop()
+	session := newSession(sessionID, transport, cfdConn, &log)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	errGroup, ctx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		// Read from underlying conn and send to transport
+		closedByRemote, err := session.Serve(ctx, time.Minute*2)
+		require.Equal(t, context.Canceled, err)
+		require.False(t, closedByRemote)
+		return nil
+	})
+
+	errGroup.Go(func() error {
+		// Write to underlying connection
+		n, err := originConn.Write([]byte{})
+		require.NoError(t, err)
+		require.Equal(t, 0, n)
+		return nil
+	})
+
+	receivedSessionID, payload, err := transport.respChan.Receive(ctx)
+	require.NoError(t, err)
+	require.Len(t, payload, 0)
+	require.Equal(t, sessionID, receivedSessionID)
+
+	cancel()
+	require.NoError(t, errGroup.Wait())
+}

datagramsession/transport.go

@@ -0,0 +1,13 @@
+package datagramsession
+
+import "github.com/google/uuid"
+
+// Transport is a connection between cloudflared and edge that can multiplex datagrams from multiple sessions
+type transport interface {
+	// SendTo writes payload for a session to the transport
+	SendTo(sessionID uuid.UUID, payload []byte) error
+	// ReceiveFrom reads the next datagram from the transport
+	ReceiveFrom() (uuid.UUID, []byte, error)
+	// Max transmission unit to receive from the transport
+	MTU() int
+}

datagramsession/transport_test.go

@@ -0,0 +1,36 @@
+package datagramsession
+
+import (
+	"context"
+
+	"github.com/google/uuid"
+)
+
+type mockQUICTransport struct {
+	reqChan  *datagramChannel
+	respChan *datagramChannel
+}
+
+func (mt *mockQUICTransport) SendTo(sessionID uuid.UUID, payload []byte) error {
+	buf := make([]byte, len(payload))
+	// The QUIC implementation copies data to another buffer before returning https://github.com/lucas-clemente/quic-go/blob/v0.24.0/session.go#L1967-L1975
+	copy(buf, payload)
+	return mt.respChan.Send(context.Background(), sessionID, buf)
+}
+
+func (mt *mockQUICTransport) ReceiveFrom() (uuid.UUID, []byte, error) {
+	return mt.reqChan.Receive(context.Background())
+}
+
+func (mt *mockQUICTransport) MTU() int {
+	return 1280
+}
+
+func (mt *mockQUICTransport) newRequest(ctx context.Context, sessionID uuid.UUID, payload []byte) error {
+	return mt.reqChan.Send(ctx, sessionID, payload)
+}
+
+func (mt *mockQUICTransport) close() {
+	mt.reqChan.Close()
+	mt.respChan.Close()
+}

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.16.4 as builder
+FROM golang:1.17.5 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/

github_release.py

@@ -72,7 +72,7 @@ def get_or_create_release(repo, version, dry_run=False):
     except UnknownObjectException:
         logging.info("Release %s not found", version)
 
-    # We dont want to create a new release tag if one doesnt already exist
+    # We don't want to create a new release tag if one doesn't already exist
     assert_tag_exists(repo, version)
 
     if dry_run:
@@ -198,7 +198,7 @@ def upload_asset(release, filepath, filename, release_version, kv_account_id, na
         pass # the macOS release copy fails with being the same file (already in the artifacts directory)
 
 def main():
-    """ Attempts to upload Asset to Github Release. Creates Release if it doesnt exist """
+    """ Attempts to upload Asset to Github Release. Creates Release if it doesn't exist """
     try:
         args = parse_args()
         client = Github(args.api_key)

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
 	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
 	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
-	github.com/coredns/coredns v1.7.0
+	github.com/coredns/coredns v1.8.7
 	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
@@ -22,54 +22,53 @@ require (
 	github.com/gobwas/pool v0.2.1 // indirect
 	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
-	github.com/google/uuid v1.1.2
-	github.com/gorilla/mux v1.7.3
+	github.com/google/uuid v1.3.0
+	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/websocket v1.4.2
-	github.com/json-iterator/go v1.1.10
-	github.com/kr/text v0.2.0 // indirect
+	github.com/json-iterator/go v1.1.12
 	github.com/kylelemons/godebug v1.1.0 // indirect
-	github.com/lucas-clemente/quic-go v0.23.0
+	github.com/lucas-clemente/quic-go v0.24.0
 	github.com/mattn/go-colorable v0.1.8
-	github.com/miekg/dns v1.1.31
+	github.com/miekg/dns v1.1.45
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
-	github.com/prometheus/client_golang v1.7.1
+	github.com/prometheus/client_golang v1.11.0
 	github.com/prometheus/client_model v0.2.0
-	github.com/prometheus/common v0.13.0 // indirect
+	github.com/prometheus/common v0.32.1 // indirect
 	github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92
 	github.com/rs/zerolog v1.20.0
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/stretchr/testify v1.6.0
-	github.com/urfave/cli/v2 v2.2.0
+	github.com/stretchr/testify v1.7.0
+	github.com/urfave/cli/v2 v2.3.0
 	go.uber.org/automaxprocs v1.4.0
-	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
-	golang.org/x/net v0.0.0-20210428140749-89ef3d95e781
-	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
+	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
+	golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210510120138-977fb7262007
-	golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf
-	google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d // indirect
-	google.golang.org/grpc v1.32.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
+	golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b
+	google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb // indirect
+	google.golang.org/grpc v1.43.0 // indirect
 	gopkg.in/coreos/go-oidc.v2 v2.1.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/square/go-jose.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0
-	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 	zombiezen.com/go/capnproto2 v2.18.0+incompatible
 )
 
 require (
 	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/apparentlymart/go-cidr v1.1.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/caddyserver/caddy v1.0.5 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
 	github.com/cheekybits/genny v1.0.0 // indirect
+	github.com/coredns/caddy v1.1.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
+	github.com/francoispqt/gojay v1.2.13 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect
 	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
@@ -77,23 +76,26 @@ require (
 	github.com/lucasb-eyer/go-colorful v1.0.3 // indirect
 	github.com/marten-seemann/qtls-go1-16 v0.1.4 // indirect
 	github.com/marten-seemann/qtls-go1-17 v0.1.0 // indirect
+	github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
 	github.com/mattn/go-runewidth v0.0.8 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/onsi/ginkgo v1.16.4 // indirect
+	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/procfs v0.1.3 // indirect
+	github.com/prometheus/procfs v0.6.0 // indirect
 	github.com/rivo/uniseg v0.1.0 // indirect
 	golang.org/x/mod v0.4.2 // indirect
-	golang.org/x/text v0.3.6 // indirect
-	golang.org/x/tools v0.1.1 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/appengine v1.6.6 // indirect
-	google.golang.org/protobuf v1.26.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 )
 
 replace github.com/urfave/cli/v2 => github.com/ipostelnik/cli/v2 v2.3.1-0.20210324024421-b6ea8234fe3d
+
+replace github.com/lucas-clemente/quic-go => github.com/chungthuang/quic-go v0.24.1-0.20220110095058-981dc498cb62

h2mux/activestreammap.go

@@ -93,7 +93,7 @@ func (m *activeStreamMap) Set(newStream *MuxedStream) bool {
 	return true
 }
 
-// Delete stops tracking the stream. It should be called only after it is closed and resetted.
+// Delete stops tracking the stream. It should be called only after it is closed and reset.
 func (m *activeStreamMap) Delete(streamID uint32) {
 	m.Lock()
 	defer m.Unlock()

h2mux/h2_compressor_brotli.go

@@ -1,3 +1,4 @@
+//go:build cgo
 // +build cgo
 
 package h2mux

h2mux/h2_compressor_none.go

@@ -1,3 +1,4 @@
+//go:build !cgo
 // +build !cgo
 
 package h2mux

h2mux/h2_dictionaries.go

@@ -12,7 +12,7 @@ import (
 /* This is an implementation of https://github.com/vkrasnov/h2-compression-dictionaries
 but modified for tunnels in a few key ways:
 Since tunnels is a server-to-server service, some aspects of the spec would cause
-unnessasary head-of-line blocking on the CPU and on the network, hence this implementation
+unnecessary head-of-line blocking on the CPU and on the network, hence this implementation
 allows for parallel compression on the "client", and buffering on the "server" to solve
 this problem. */
 
@@ -67,7 +67,7 @@ var compressionPresets = map[CompressionSetting]CompressionPreset{
 }
 
 func compressionSettingVal(version, fmt, sz, nd uint8) uint32 {
-	// Currently the compression settings are inlcude:
+	// Currently the compression settings are include:
 	// * version: only 1 is supported
 	// * fmt: only 2 for brotli is supported
 	// * sz: log2 of the maximal allowed dictionary size
@@ -438,7 +438,7 @@ func assignDictToStream(s *MuxedStream, p []byte) bool {
 	h2d.dictLock.Lock()
 
 	if w.comp != nil {
-		// Check again with lock, in therory the inteface allows for unordered writes
+		// Check again with lock, in therory the interface allows for unordered writes
 		h2d.dictLock.Unlock()
 		return false
 	}
@@ -468,7 +468,7 @@ func assignDictToStream(s *MuxedStream, p []byte) bool {
 		}
 	} else {
 		// Use the overflow dictionary as last resort
-		// If slots are availabe generate new dictioanries for path and content-type
+		// If slots are available generate new dictionaries for path and content-type
 		useID, _ = h2d.getGenericDictID()
 		pathID, pathFound = h2d.getNextDictID()
 		if pathFound {

h2mux/h2mux.go

@@ -174,7 +174,7 @@ func Handshake(
 	pingTimestamp := NewPingTimestamp()
 	connActive := NewSignal()
 	idleDuration := config.HeartbeatInterval
-	// Sanity check to enusre idelDuration is sane
+	// Sanity check to ensure idelDuration is sane
 	if idleDuration == 0 || idleDuration < defaultTimeout {
 		idleDuration = defaultTimeout
 		config.Log.Info().Msgf("muxer: Minimum idle time has been adjusted to %d", defaultTimeout)
@@ -274,7 +274,7 @@ func (m *Muxer) readPeerSettings(magic uint32) error {
 		m.compressionQuality = compressionPresets[CompressionNone]
 		return nil
 	}
-	// Values used for compression are the mimimum between the two peers
+	// Values used for compression are the minimum between the two peers
 	if sz < m.compressionQuality.dictSize {
 		m.compressionQuality.dictSize = sz
 	}

h2mux/muxmetrics_test.go

@@ -130,7 +130,7 @@ func TestMuxMetricsUpdater(t *testing.T) {
 			m.updateReceiveWindow(uint32(j))
 			m.updateSendWindow(uint32(j))
 
-			// should always be disgarded since the send time is before readerSend
+			// should always be discarded since the send time is before readerSend
 			rm := &roundTripMeasurement{receiveTime: readerStart, sendTime: readerStart.Add(-time.Duration(j*dataPoints) * time.Millisecond)}
 			m.updateRTT(rm)
 

ingress/config.go

@@ -1,6 +1,7 @@
 package ingress
 
 import (
+	"encoding/json"
 	"time"
 
 	"github.com/urfave/cli/v2"
@@ -38,6 +39,34 @@ const (
 	socksProxy = "socks"
 )
 
+// RemoteConfig models ingress settings that can be managed remotely, for example through the dashboard.
+type RemoteConfig struct {
+	Ingress     Ingress
+	WarpRouting config.WarpRoutingConfig
+}
+
+type remoteConfigJSON struct {
+	GlobalOriginRequest config.OriginRequestConfig      `json:"originRequest"`
+	IngressRules        []config.UnvalidatedIngressRule `json:"ingress"`
+	WarpRouting         config.WarpRoutingConfig        `json:"warp-routing"`
+}
+
+func (rc *RemoteConfig) UnmarshalJSON(b []byte) error {
+	var rawConfig remoteConfigJSON
+	if err := json.Unmarshal(b, &rawConfig); err != nil {
+		return err
+	}
+	ingress, err := validateIngress(rawConfig.IngressRules, originRequestFromConfig(rawConfig.GlobalOriginRequest))
+	if err != nil {
+		return err
+	}
+
+	rc.Ingress = ingress
+	rc.WarpRouting = rawConfig.WarpRouting
+
+	return nil
+}
+
 func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 	var connectTimeout time.Duration = defaultConnectTimeout
 	var tlsTimeout time.Duration = defaultTLSTimeout
@@ -119,7 +148,7 @@ func originRequestFromSingeRule(c *cli.Context) OriginRequestConfig {
 	}
 }
 
-func originRequestFromYAML(y config.OriginRequestConfig) OriginRequestConfig {
+func originRequestFromConfig(c config.OriginRequestConfig) OriginRequestConfig {
 	out := OriginRequestConfig{
 		ConnectTimeout:       defaultConnectTimeout,
 		TLSTimeout:           defaultTLSTimeout,
@@ -128,50 +157,58 @@ func originRequestFromYAML(y config.OriginRequestConfig) OriginRequestConfig {
 		KeepAliveTimeout:     defaultKeepAliveTimeout,
 		ProxyAddress:         defaultProxyAddress,
 	}
-	if y.ConnectTimeout != nil {
-		out.ConnectTimeout = *y.ConnectTimeout
+	if c.ConnectTimeout != nil {
+		out.ConnectTimeout = *c.ConnectTimeout
 	}
-	if y.TLSTimeout != nil {
-		out.TLSTimeout = *y.TLSTimeout
+	if c.TLSTimeout != nil {
+		out.TLSTimeout = *c.TLSTimeout
 	}
-	if y.TCPKeepAlive != nil {
-		out.TCPKeepAlive = *y.TCPKeepAlive
+	if c.TCPKeepAlive != nil {
+		out.TCPKeepAlive = *c.TCPKeepAlive
 	}
-	if y.NoHappyEyeballs != nil {
-		out.NoHappyEyeballs = *y.NoHappyEyeballs
+	if c.NoHappyEyeballs != nil {
+		out.NoHappyEyeballs = *c.NoHappyEyeballs
 	}
-	if y.KeepAliveConnections != nil {
-		out.KeepAliveConnections = *y.KeepAliveConnections
+	if c.KeepAliveConnections != nil {
+		out.KeepAliveConnections = *c.KeepAliveConnections
 	}
-	if y.KeepAliveTimeout != nil {
-		out.KeepAliveTimeout = *y.KeepAliveTimeout
+	if c.KeepAliveTimeout != nil {
+		out.KeepAliveTimeout = *c.KeepAliveTimeout
 	}
-	if y.HTTPHostHeader != nil {
-		out.HTTPHostHeader = *y.HTTPHostHeader
+	if c.HTTPHostHeader != nil {
+		out.HTTPHostHeader = *c.HTTPHostHeader
 	}
-	if y.OriginServerName != nil {
-		out.OriginServerName = *y.OriginServerName
+	if c.OriginServerName != nil {
+		out.OriginServerName = *c.OriginServerName
 	}
-	if y.CAPool != nil {
-		out.CAPool = *y.CAPool
+	if c.CAPool != nil {
+		out.CAPool = *c.CAPool
 	}
-	if y.NoTLSVerify != nil {
-		out.NoTLSVerify = *y.NoTLSVerify
+	if c.NoTLSVerify != nil {
+		out.NoTLSVerify = *c.NoTLSVerify
 	}
-	if y.DisableChunkedEncoding != nil {
-		out.DisableChunkedEncoding = *y.DisableChunkedEncoding
+	if c.DisableChunkedEncoding != nil {
+		out.DisableChunkedEncoding = *c.DisableChunkedEncoding
 	}
-	if y.BastionMode != nil {
-		out.BastionMode = *y.BastionMode
+	if c.BastionMode != nil {
+		out.BastionMode = *c.BastionMode
 	}
-	if y.ProxyAddress != nil {
-		out.ProxyAddress = *y.ProxyAddress
+	if c.ProxyAddress != nil {
+		out.ProxyAddress = *c.ProxyAddress
 	}
-	if y.ProxyPort != nil {
-		out.ProxyPort = *y.ProxyPort
+	if c.ProxyPort != nil {
+		out.ProxyPort = *c.ProxyPort
 	}
-	if y.ProxyType != nil {
-		out.ProxyType = *y.ProxyType
+	if c.ProxyType != nil {
+		out.ProxyType = *c.ProxyType
+	}
+	if len(c.IPRules) > 0 {
+		for _, r := range c.IPRules {
+			rule, err := ipaccess.NewRuleByCIDR(r.Prefix, r.Ports, r.Allow)
+			if err == nil {
+				out.IPRules = append(out.IPRules, rule)
+			}
+		}
 	}
 	return out
 }
@@ -188,10 +225,10 @@ type OriginRequestConfig struct {
 	TCPKeepAlive time.Duration `yaml:"tcpKeepAlive"`
 	// HTTP proxy should disable "happy eyeballs" for IPv4/v6 fallback
 	NoHappyEyeballs bool `yaml:"noHappyEyeballs"`
-	// HTTP proxy maximum keepalive connection pool size
-	KeepAliveConnections int `yaml:"keepAliveConnections"`
 	// HTTP proxy timeout for closing an idle connection
 	KeepAliveTimeout time.Duration `yaml:"keepAliveTimeout"`
+	// HTTP proxy maximum keepalive connection pool size
+	KeepAliveConnections int `yaml:"keepAliveConnections"`
 	// Sets the HTTP Host header for the local webserver.
 	HTTPHostHeader string `yaml:"httpHostHeader"`
 	// Hostname on the origin server certificate.
@@ -308,6 +345,19 @@ func (defaults *OriginRequestConfig) setProxyType(overrides config.OriginRequest
 	}
 }
 
+func (defaults *OriginRequestConfig) setIPRules(overrides config.OriginRequestConfig) {
+	if val := overrides.IPRules; len(val) > 0 {
+		ipAccessRule := make([]ipaccess.Rule, len(overrides.IPRules))
+		for i, r := range overrides.IPRules {
+			rule, err := ipaccess.NewRuleByCIDR(r.Prefix, r.Ports, r.Allow)
+			if err == nil {
+				ipAccessRule[i] = rule
+			}
+		}
+		defaults.IPRules = ipAccessRule
+	}
+}
+
 // SetConfig gets config for the requests that cloudflared sends to origins.
 // Each field has a setter method which sets a value for the field by trying to find:
 //   1. The user config for this rule
@@ -332,5 +382,6 @@ func setConfig(defaults OriginRequestConfig, overrides config.OriginRequestConfi
 	cfg.setProxyPort(overrides)
 	cfg.setProxyAddress(overrides)
 	cfg.setProxyType(overrides)
+	cfg.setIPRules(overrides)
 	return cfg
 }

ingress/config_test.go

@@ -0,0 +1,422 @@
+package ingress
+
+import (
+	"encoding/json"
+	"flag"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"github.com/urfave/cli/v2"
+	yaml "gopkg.in/yaml.v2"
+
+	"github.com/cloudflare/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ipaccess"
+)
+
+// Ensure that the nullable config from `config` package and the
+// non-nullable config from `ingress` package have the same number of
+// fields.
+// This test ensures that programmers didn't add a new field to
+// one struct and forget to add it to the other ;)
+func TestCorrespondingFields(t *testing.T) {
+	require.Equal(
+		t,
+		CountFields(t, config.OriginRequestConfig{}),
+		CountFields(t, OriginRequestConfig{}),
+	)
+}
+
+func CountFields(t *testing.T, val interface{}) int {
+	b, err := yaml.Marshal(val)
+	require.NoError(t, err)
+	m := make(map[string]interface{}, 0)
+	err = yaml.Unmarshal(b, &m)
+	require.NoError(t, err)
+	return len(m)
+}
+
+func TestUnmarshalRemoteConfigOverridesGlobal(t *testing.T) {
+	rawConfig := []byte(`
+{
+    "originRequest": {
+        "connectTimeout": 90,
+		"noHappyEyeballs": true
+    },
+    "ingress": [
+        {
+            "hostname": "jira.cfops.com",
+            "service": "http://192.16.19.1:80",
+            "originRequest": {
+                "noTLSVerify": true,
+                "connectTimeout": 10
+            }
+        },
+        {
+            "service": "http_status:404"
+        }
+    ],
+    "warp-routing": {
+        "enabled": true
+    }
+}	
+`)
+	var remoteConfig RemoteConfig
+	err := json.Unmarshal(rawConfig, &remoteConfig)
+	require.NoError(t, err)
+	require.True(t, remoteConfig.Ingress.Rules[0].Config.NoTLSVerify)
+	require.True(t, remoteConfig.Ingress.defaults.NoHappyEyeballs)
+}
+
+func TestOriginRequestConfigOverrides(t *testing.T) {
+	validate := func(ing Ingress) {
+		// Rule 0 didn't override anything, so it inherits the user-specified
+		// root-level configuration.
+		actual0 := ing.Rules[0].Config
+		expected0 := OriginRequestConfig{
+			ConnectTimeout:         1 * time.Minute,
+			TLSTimeout:             1 * time.Second,
+			TCPKeepAlive:           1 * time.Second,
+			NoHappyEyeballs:        true,
+			KeepAliveTimeout:       1 * time.Second,
+			KeepAliveConnections:   1,
+			HTTPHostHeader:         "abc",
+			OriginServerName:       "a1",
+			CAPool:                 "/tmp/path0",
+			NoTLSVerify:            true,
+			DisableChunkedEncoding: true,
+			BastionMode:            true,
+			ProxyAddress:           "127.1.2.3",
+			ProxyPort:              uint(100),
+			ProxyType:              "socks5",
+			IPRules: []ipaccess.Rule{
+				newIPRule(t, "10.0.0.0/8", []int{80, 8080}, false),
+				newIPRule(t, "fc00::/7", []int{443, 4443}, true),
+			},
+		}
+		require.Equal(t, expected0, actual0)
+
+		// Rule 1 overrode all the root-level config.
+		actual1 := ing.Rules[1].Config
+		expected1 := OriginRequestConfig{
+			ConnectTimeout:         2 * time.Minute,
+			TLSTimeout:             2 * time.Second,
+			TCPKeepAlive:           2 * time.Second,
+			NoHappyEyeballs:        false,
+			KeepAliveTimeout:       2 * time.Second,
+			KeepAliveConnections:   2,
+			HTTPHostHeader:         "def",
+			OriginServerName:       "b2",
+			CAPool:                 "/tmp/path1",
+			NoTLSVerify:            false,
+			DisableChunkedEncoding: false,
+			BastionMode:            false,
+			ProxyAddress:           "interface",
+			ProxyPort:              uint(200),
+			ProxyType:              "",
+			IPRules: []ipaccess.Rule{
+				newIPRule(t, "10.0.0.0/16", []int{3000, 3030}, false),
+				newIPRule(t, "192.16.0.0/24", []int{5000, 5050}, true),
+			},
+		}
+		require.Equal(t, expected1, actual1)
+	}
+
+	rulesYAML := `
+originRequest:
+  connectTimeout: 1m
+  tlsTimeout: 1s
+  noHappyEyeballs: true
+  tcpKeepAlive: 1s
+  keepAliveConnections: 1
+  keepAliveTimeout: 1s
+  httpHostHeader: abc
+  originServerName: a1
+  caPool: /tmp/path0
+  noTLSVerify: true
+  disableChunkedEncoding: true
+  bastionMode: True
+  proxyAddress: 127.1.2.3
+  proxyPort: 100
+  proxyType: socks5
+  ipRules:
+  - prefix: "10.0.0.0/8"
+    ports:
+    - 80
+    - 8080
+    allow: false
+  - prefix: "fc00::/7"
+    ports:
+    - 443
+    - 4443
+    allow: true
+ingress:
+- hostname: tun.example.com
+  service: https://localhost:8000
+- hostname: "*"
+  service: https://localhost:8001
+  originRequest:
+    connectTimeout: 2m
+    tlsTimeout: 2s
+    noHappyEyeballs: false
+    tcpKeepAlive: 2s
+    keepAliveConnections: 2
+    keepAliveTimeout: 2s
+    httpHostHeader: def
+    originServerName: b2
+    caPool: /tmp/path1
+    noTLSVerify: false
+    disableChunkedEncoding: false
+    bastionMode: false
+    proxyAddress: interface
+    proxyPort: 200
+    proxyType: ""
+    ipRules:
+    - prefix: "10.0.0.0/16"
+      ports:
+      - 3000
+      - 3030
+      allow: false
+    - prefix: "192.16.0.0/24"
+      ports:
+      - 5000
+      - 5050
+      allow: true
+`
+
+	ing, err := ParseIngress(MustReadIngress(rulesYAML))
+	require.NoError(t, err)
+	validate(ing)
+
+	rawConfig := []byte(`
+{
+    "originRequest": {
+        "connectTimeout": 60000000000,
+		"tlsTimeout": 1000000000,
+		"noHappyEyeballs": true,
+		"tcpKeepAlive": 1000000000,
+		"keepAliveConnections": 1,
+		"keepAliveTimeout": 1000000000,
+		"httpHostHeader": "abc",
+		"originServerName": "a1",
+		"caPool": "/tmp/path0",
+		"noTLSVerify": true,
+		"disableChunkedEncoding": true,
+		"bastionMode": true,
+		"proxyAddress": "127.1.2.3",
+		"proxyPort": 100,
+		"proxyType": "socks5",
+		"ipRules": [
+			{
+				"prefix": "10.0.0.0/8",
+				"ports": [80, 8080],
+				"allow": false
+			},
+			{
+				"prefix": "fc00::/7",
+				"ports": [443, 4443],
+				"allow": true
+			}
+		]
+    },
+    "ingress": [
+        {
+            "hostname": "tun.example.com",
+            "service": "https://localhost:8000"
+        },
+        {
+			"hostname": "*",
+            "service": "https://localhost:8001",
+			"originRequest": {
+				"connectTimeout": 120000000000,
+				"tlsTimeout": 2000000000,
+				"noHappyEyeballs": false,
+				"tcpKeepAlive": 2000000000,
+				"keepAliveConnections": 2,
+				"keepAliveTimeout": 2000000000,
+				"httpHostHeader": "def",
+				"originServerName": "b2",
+				"caPool": "/tmp/path1",
+				"noTLSVerify": false,
+				"disableChunkedEncoding": false,
+				"bastionMode": false,
+				"proxyAddress": "interface",
+				"proxyPort": 200,
+				"proxyType": "",
+				"ipRules": [
+					{
+						"prefix": "10.0.0.0/16",
+						"ports": [3000, 3030],
+						"allow": false
+					},
+					{
+						"prefix": "192.16.0.0/24",
+						"ports": [5000, 5050],
+						"allow": true
+					}
+				]
+    		}
+        }
+    ],
+    "warp-routing": {
+        "enabled": true
+    }
+}	
+`)
+	var remoteConfig RemoteConfig
+	err = json.Unmarshal(rawConfig, &remoteConfig)
+	require.NoError(t, err)
+	validate(remoteConfig.Ingress)
+}
+
+func TestOriginRequestConfigDefaults(t *testing.T) {
+	validate := func(ing Ingress) {
+		// Rule 0 didn't override anything, so it inherits the cloudflared defaults
+		actual0 := ing.Rules[0].Config
+		expected0 := OriginRequestConfig{
+			ConnectTimeout:       defaultConnectTimeout,
+			TLSTimeout:           defaultTLSTimeout,
+			TCPKeepAlive:         defaultTCPKeepAlive,
+			KeepAliveConnections: defaultKeepAliveConnections,
+			KeepAliveTimeout:     defaultKeepAliveTimeout,
+			ProxyAddress:         defaultProxyAddress,
+		}
+		require.Equal(t, expected0, actual0)
+
+		// Rule 1 overrode all defaults.
+		actual1 := ing.Rules[1].Config
+		expected1 := OriginRequestConfig{
+			ConnectTimeout:         2 * time.Minute,
+			TLSTimeout:             2 * time.Second,
+			TCPKeepAlive:           2 * time.Second,
+			NoHappyEyeballs:        false,
+			KeepAliveTimeout:       2 * time.Second,
+			KeepAliveConnections:   2,
+			HTTPHostHeader:         "def",
+			OriginServerName:       "b2",
+			CAPool:                 "/tmp/path1",
+			NoTLSVerify:            false,
+			DisableChunkedEncoding: false,
+			BastionMode:            false,
+			ProxyAddress:           "interface",
+			ProxyPort:              uint(200),
+			ProxyType:              "",
+			IPRules: []ipaccess.Rule{
+				newIPRule(t, "10.0.0.0/16", []int{3000, 3030}, false),
+				newIPRule(t, "192.16.0.0/24", []int{5000, 5050}, true),
+			},
+		}
+		require.Equal(t, expected1, actual1)
+	}
+
+	rulesYAML := `
+ingress:
+- hostname: tun.example.com
+  service: https://localhost:8000
+- hostname: "*"
+  service: https://localhost:8001
+  originRequest:
+    connectTimeout: 2m
+    tlsTimeout: 2s
+    noHappyEyeballs: false
+    tcpKeepAlive: 2s
+    keepAliveConnections: 2
+    keepAliveTimeout: 2s
+    httpHostHeader: def
+    originServerName: b2
+    caPool: /tmp/path1
+    noTLSVerify: false
+    disableChunkedEncoding: false
+    bastionMode: false
+    proxyAddress: interface
+    proxyPort: 200
+    proxyType: ""
+    ipRules:
+    - prefix: "10.0.0.0/16"
+      ports:
+      - 3000
+      - 3030
+      allow: false
+    - prefix: "192.16.0.0/24"
+      ports:
+      - 5000
+      - 5050
+      allow: true
+`
+	ing, err := ParseIngress(MustReadIngress(rulesYAML))
+	if err != nil {
+		t.Error(err)
+	}
+	validate(ing)
+
+	rawConfig := []byte(`
+{
+	"ingress": [
+        {
+            "hostname": "tun.example.com",
+            "service": "https://localhost:8000"
+        },
+        {
+			"hostname": "*",
+            "service": "https://localhost:8001",
+			"originRequest": {
+				"connectTimeout": 120000000000,
+				"tlsTimeout": 2000000000,
+				"noHappyEyeballs": false,
+				"tcpKeepAlive": 2000000000,
+				"keepAliveConnections": 2,
+				"keepAliveTimeout": 2000000000,
+				"httpHostHeader": "def",
+				"originServerName": "b2",
+				"caPool": "/tmp/path1",
+				"noTLSVerify": false,
+				"disableChunkedEncoding": false,
+				"bastionMode": false,
+				"proxyAddress": "interface",
+				"proxyPort": 200,
+				"proxyType": "",
+				"ipRules": [
+					{
+						"prefix": "10.0.0.0/16",
+						"ports": [3000, 3030],
+						"allow": false
+					},
+					{
+						"prefix": "192.16.0.0/24",
+						"ports": [5000, 5050],
+						"allow": true
+					}
+				]
+			}
+		}
+	]
+}
+`)
+
+	var remoteConfig RemoteConfig
+	err = json.Unmarshal(rawConfig, &remoteConfig)
+	require.NoError(t, err)
+	validate(remoteConfig.Ingress)
+}
+
+func TestDefaultConfigFromCLI(t *testing.T) {
+	set := flag.NewFlagSet("contrive", 0)
+	c := cli.NewContext(nil, set, nil)
+
+	expected := OriginRequestConfig{
+		ConnectTimeout:       defaultConnectTimeout,
+		TLSTimeout:           defaultTLSTimeout,
+		TCPKeepAlive:         defaultTCPKeepAlive,
+		KeepAliveConnections: defaultKeepAliveConnections,
+		KeepAliveTimeout:     defaultKeepAliveTimeout,
+		ProxyAddress:         defaultProxyAddress,
+	}
+	actual := originRequestFromSingeRule(c)
+	require.Equal(t, expected, actual)
+}
+
+func newIPRule(t *testing.T, prefix string, ports []int, allow bool) ipaccess.Rule {
+	rule, err := ipaccess.NewRuleByCIDR(&prefix, ports, allow)
+	require.NoError(t, err)
+	return rule
+}

ingress/ingress.go

@@ -7,7 +7,6 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
-	"sync"
 
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog"
@@ -145,13 +144,11 @@ func (ing Ingress) IsSingleRule() bool {
 
 // StartOrigins will start any origin services managed by cloudflared, e.g. proxy servers or Hello World.
 func (ing Ingress) StartOrigins(
-	wg *sync.WaitGroup,
 	log *zerolog.Logger,
 	shutdownC <-chan struct{},
-	errC chan error,
 ) error {
 	for _, rule := range ing.Rules {
-		if err := rule.Service.start(wg, log, shutdownC, errC, rule.Config); err != nil {
+		if err := rule.Service.start(log, shutdownC, rule.Config); err != nil {
 			return errors.Wrapf(err, "Error starting local service %s", rule.Service)
 		}
 	}
@@ -163,7 +160,7 @@ func (ing Ingress) CatchAll() *Rule {
 	return &ing.Rules[len(ing.Rules)-1]
 }
 
-func validate(ingress []config.UnvalidatedIngressRule, defaults OriginRequestConfig) (Ingress, error) {
+func validateIngress(ingress []config.UnvalidatedIngressRule, defaults OriginRequestConfig) (Ingress, error) {
 	rules := make([]Rule, len(ingress))
 	for i, r := range ingress {
 		cfg := setConfig(defaults, r.OriginRequest)
@@ -290,7 +287,7 @@ func ParseIngress(conf *config.Configuration) (Ingress, error) {
 	if len(conf.Ingress) == 0 {
 		return Ingress{}, ErrNoIngressRules
 	}
-	return validate(conf.Ingress, originRequestFromYAML(conf.OriginRequest))
+	return validateIngress(conf.Ingress, originRequestFromConfig(conf.OriginRequest))
 }
 
 func isHTTPService(url *url.URL) bool {

ingress/ingress_test.go

@@ -34,7 +34,7 @@ func Test_parseIngress(t *testing.T) {
 	localhost8000 := MustParseURL(t, "https://localhost:8000")
 	localhost8001 := MustParseURL(t, "https://localhost:8001")
 	fourOhFour := newStatusCode(404)
-	defaultConfig := setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{})
+	defaultConfig := setConfig(originRequestFromConfig(config.OriginRequestConfig{}), config.OriginRequestConfig{})
 	require.Equal(t, defaultKeepAliveConnections, defaultConfig.KeepAliveConnections)
 	tr := true
 	type args struct {
@@ -324,7 +324,17 @@ ingress:
 				{
 					Hostname: "socks.foo.com",
 					Service:  newSocksProxyOverWSService(accessPolicy()),
-					Config:   defaultConfig,
+					Config: setConfig(originRequestFromConfig(config.OriginRequestConfig{}), config.OriginRequestConfig{IPRules: []config.IngressIPRule{
+						{
+							Prefix: ipRulePrefix("1.1.1.0/24"),
+							Ports:  []int{80, 443},
+							Allow:  true,
+						},
+						{
+							Prefix: ipRulePrefix("0.0.0.0/0"),
+							Allow:  false,
+						},
+					}}),
 				},
 				{
 					Service: &fourOhFour,
@@ -345,7 +355,7 @@ ingress:
 				{
 					Hostname: "bastion.foo.com",
 					Service:  newBastionService(),
-					Config:   setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
+					Config:   setConfig(originRequestFromConfig(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
 				},
 				{
 					Service: &fourOhFour,
@@ -365,7 +375,7 @@ ingress:
 				{
 					Hostname: "bastion.foo.com",
 					Service:  newBastionService(),
-					Config:   setConfig(originRequestFromYAML(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
+					Config:   setConfig(originRequestFromConfig(config.OriginRequestConfig{}), config.OriginRequestConfig{BastionMode: &tr}),
 				},
 				{
 					Service: &fourOhFour,
@@ -397,6 +407,10 @@ ingress:
 	}
 }
 
+func ipRulePrefix(s string) *string {
+	return &s
+}
+
 func TestSingleOriginSetsConfig(t *testing.T) {
 	flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
 	flagSet.Bool("hello-world", true, "")

ingress/origin_connection.go

@@ -53,7 +53,7 @@ func (wc *tcpOverWSConnection) Stream(ctx context.Context, tunnelConn io.ReadWri
 	wc.streamHandler(wsConn, wc.conn, log)
 	cancel()
 	// Makes sure wsConn stops sending ping before terminating the stream
-	wsConn.WaitForShutdown()
+	wsConn.Close()
 }
 
 func (wc *tcpOverWSConnection) Close() {
@@ -73,21 +73,8 @@ func (sp *socksProxyOverWSConnection) Stream(ctx context.Context, tunnelConn io.
 	socks.StreamNetHandler(wsConn, sp.accessPolicy, log)
 	cancel()
 	// Makes sure wsConn stops sending ping before terminating the stream
-	wsConn.WaitForShutdown()
+	wsConn.Close()
 }
 
 func (sp *socksProxyOverWSConnection) Close() {
 }
-
-// wsProxyConnection represents a bidirectional stream for a websocket connection to the origin
-type wsProxyConnection struct {
-	rwc io.ReadWriteCloser
-}
-
-func (conn *wsProxyConnection) Stream(ctx context.Context, tunnelConn io.ReadWriter, log *zerolog.Logger) {
-	websocket.Stream(tunnelConn, conn.rwc, log)
-}
-
-func (conn *wsProxyConnection) Close() {
-	conn.rwc.Close()
-}

ingress/origin_connection_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -19,7 +20,6 @@ import (
 	"golang.org/x/net/proxy"
 	"golang.org/x/sync/errgroup"
 
-	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/socks"
 	"github.com/cloudflare/cloudflared/websocket"
@@ -97,7 +97,7 @@ func TestDefaultStreamWSOverTCPConnection(t *testing.T) {
 }
 
 // TestSocksStreamWSOverTCPConnection simulates proxying in socks mode.
-// Eyeball side runs cloudflared accesss tcp with --url flag to start a websocket forwarder which
+// Eyeball side runs cloudflared access tcp with --url flag to start a websocket forwarder which
 // wraps SOCKS5 traffic in websocket
 // Origin side runs a tcpOverWSConnection with socks.StreamHandler
 func TestSocksStreamWSOverTCPConnection(t *testing.T) {
@@ -192,8 +192,10 @@ func TestSocksStreamWSOverTCPConnection(t *testing.T) {
 
 func TestWsConnReturnsBeforeStreamReturns(t *testing.T) {
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		eyeballConn, err := connection.NewHTTP2RespWriter(r, w, connection.TypeWebsocket)
-		assert.NoError(t, err)
+		eyeballConn := &readWriter{
+			w: w,
+			r: r.Body,
+		}
 
 		cfdConn, originConn := net.Pipe()
 		tcpOverWSConn := tcpOverWSConnection{
@@ -319,3 +321,16 @@ func echoTCPOrigin(t *testing.T, conn net.Conn) {
 	_, err = conn.Write(testResponse)
 	assert.NoError(t, err)
 }
+
+type readWriter struct {
+	w io.Writer
+	r io.Reader
+}
+
+func (r *readWriter) Read(p []byte) (n int, err error) {
+	return r.r.Read(p)
+}
+
+func (r *readWriter) Write(p []byte) (n int, err error) {
+	return r.w.Write(p)
+}

ingress/origin_proxy.go

@@ -23,6 +23,7 @@ type StreamBasedOriginProxy interface {
 }
 
 func (o *unixSocketPath) RoundTrip(req *http.Request) (*http.Response, error) {
+	req.URL.Scheme = "http"
 	return o.transport.RoundTrip(req)
 }
 
@@ -40,6 +41,8 @@ func (o *httpService) RoundTrip(req *http.Request) (*http.Response, error) {
 
 	if o.hostHeader != "" {
 		// For incoming requests, the Host header is promoted to the Request.Host field and removed from the Header map.
+		// Pass the original Host header as X-Forwarded-Host.
+		req.Header.Set("X-Forwarded-Host", req.Host)
 		req.Host = o.hostHeader
 	}
 	return o.transport.RoundTrip(req)

ingress/origin_proxy_test.go

@@ -3,11 +3,11 @@ package ingress
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
-	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -118,7 +118,9 @@ func TestHTTPServiceHostHeaderOverride(t *testing.T) {
 			w.WriteHeader(http.StatusSwitchingProtocols)
 			return
 		}
-		w.Write([]byte("ok"))
+		// return the X-Forwarded-Host header for assertions
+		// as the httptest Server URL isn't available here yet
+		w.Write([]byte(r.Header.Get("X-Forwarded-Host")))
 	}
 	origin := httptest.NewServer(http.HandlerFunc(handler))
 	defer origin.Close()
@@ -129,10 +131,8 @@ func TestHTTPServiceHostHeaderOverride(t *testing.T) {
 	httpService := &httpService{
 		url: originURL,
 	}
-	var wg sync.WaitGroup
 	shutdownC := make(chan struct{})
-	errC := make(chan error)
-	require.NoError(t, httpService.start(&wg, testLogger, shutdownC, errC, cfg))
+	require.NoError(t, httpService.start(testLogger, shutdownC, cfg))
 
 	req, err := http.NewRequest(http.MethodGet, originURL.String(), nil)
 	require.NoError(t, err)
@@ -141,6 +141,49 @@ func TestHTTPServiceHostHeaderOverride(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, http.StatusOK, resp.StatusCode)
 
+	respBody, err := ioutil.ReadAll(resp.Body)
+	require.NoError(t, err)
+	require.Equal(t, respBody, []byte(originURL.Host))
+}
+
+// TestHTTPServiceUsesIngressRuleScheme makes sure httpService uses scheme defined in ingress rule and not by eyeball request
+func TestHTTPServiceUsesIngressRuleScheme(t *testing.T) {
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		require.NotNil(t, r.TLS)
+		// Echo the X-Forwarded-Proto header for assertions
+		w.Write([]byte(r.Header.Get("X-Forwarded-Proto")))
+	}
+	origin := httptest.NewTLSServer(http.HandlerFunc(handler))
+	defer origin.Close()
+
+	originURL, err := url.Parse(origin.URL)
+	require.NoError(t, err)
+	require.Equal(t, "https", originURL.Scheme)
+
+	cfg := OriginRequestConfig{
+		NoTLSVerify: true,
+	}
+	httpService := &httpService{
+		url: originURL,
+	}
+	shutdownC := make(chan struct{})
+	require.NoError(t, httpService.start(testLogger, shutdownC, cfg))
+
+	// Tunnel uses scheme defined in the service field of the ingress rule, independent of the X-Forwarded-Proto header
+	protos := []string{"https", "http", "dne"}
+	for _, p := range protos {
+		req, err := http.NewRequest(http.MethodGet, originURL.String(), nil)
+		require.NoError(t, err)
+		req.Header.Add("X-Forwarded-Proto", p)
+
+		resp, err := httpService.RoundTrip(req)
+		require.NoError(t, err)
+		require.Equal(t, http.StatusOK, resp.StatusCode)
+
+		respBody, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+		require.Equal(t, respBody, []byte(p))
+	}
 }
 
 func tcpListenRoutine(listener net.Listener, closeChan chan struct{}) {

ingress/origin_request_config_test.go

@@ -1,203 +0,0 @@
-package ingress
-
-import (
-	"flag"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-	"github.com/urfave/cli/v2"
-	yaml "gopkg.in/yaml.v2"
-
-	"github.com/cloudflare/cloudflared/config"
-)
-
-// Ensure that the nullable config from `config` package and the
-// non-nullable config from `ingress` package have the same number of
-// fields.
-// This test ensures that programmers didn't add a new field to
-// one struct and forget to add it to the other ;)
-func TestCorrespondingFields(t *testing.T) {
-	require.Equal(
-		t,
-		CountFields(t, config.OriginRequestConfig{}),
-		CountFields(t, OriginRequestConfig{}),
-	)
-}
-
-func CountFields(t *testing.T, val interface{}) int {
-	b, err := yaml.Marshal(val)
-	require.NoError(t, err)
-	m := make(map[string]interface{}, 0)
-	err = yaml.Unmarshal(b, &m)
-	require.NoError(t, err)
-	return len(m)
-}
-
-func TestOriginRequestConfigOverrides(t *testing.T) {
-	rulesYAML := `
-originRequest:
-  connectTimeout: 1m
-  tlsTimeout: 1s
-  noHappyEyeballs: true
-  tcpKeepAlive: 1s
-  keepAliveConnections: 1
-  keepAliveTimeout: 1s
-  httpHostHeader: abc
-  originServerName: a1
-  caPool: /tmp/path0
-  noTLSVerify: true
-  disableChunkedEncoding: true
-  bastionMode: True
-  proxyAddress: 127.1.2.3
-  proxyPort: 100
-  proxyType: socks5
-ingress:
-- hostname: tun.example.com
-  service: https://localhost:8000
-- hostname: "*"
-  service: https://localhost:8001
-  originRequest:
-    connectTimeout: 2m
-    tlsTimeout: 2s
-    noHappyEyeballs: false
-    tcpKeepAlive: 2s
-    keepAliveConnections: 2
-    keepAliveTimeout: 2s
-    httpHostHeader: def
-    originServerName: b2
-    caPool: /tmp/path1
-    noTLSVerify: false
-    disableChunkedEncoding: false
-    bastionMode: false
-    proxyAddress: interface
-    proxyPort: 200
-    proxyType: ""
-`
-	ing, err := ParseIngress(MustReadIngress(rulesYAML))
-	if err != nil {
-		t.Error(err)
-	}
-
-	// Rule 0 didn't override anything, so it inherits the user-specified
-	// root-level configuration.
-	actual0 := ing.Rules[0].Config
-	expected0 := OriginRequestConfig{
-		ConnectTimeout:         1 * time.Minute,
-		TLSTimeout:             1 * time.Second,
-		NoHappyEyeballs:        true,
-		TCPKeepAlive:           1 * time.Second,
-		KeepAliveConnections:   1,
-		KeepAliveTimeout:       1 * time.Second,
-		HTTPHostHeader:         "abc",
-		OriginServerName:       "a1",
-		CAPool:                 "/tmp/path0",
-		NoTLSVerify:            true,
-		DisableChunkedEncoding: true,
-		BastionMode:            true,
-		ProxyAddress:           "127.1.2.3",
-		ProxyPort:              uint(100),
-		ProxyType:              "socks5",
-	}
-	require.Equal(t, expected0, actual0)
-
-	// Rule 1 overrode all the root-level config.
-	actual1 := ing.Rules[1].Config
-	expected1 := OriginRequestConfig{
-		ConnectTimeout:         2 * time.Minute,
-		TLSTimeout:             2 * time.Second,
-		NoHappyEyeballs:        false,
-		TCPKeepAlive:           2 * time.Second,
-		KeepAliveConnections:   2,
-		KeepAliveTimeout:       2 * time.Second,
-		HTTPHostHeader:         "def",
-		OriginServerName:       "b2",
-		CAPool:                 "/tmp/path1",
-		NoTLSVerify:            false,
-		DisableChunkedEncoding: false,
-		BastionMode:            false,
-		ProxyAddress:           "interface",
-		ProxyPort:              uint(200),
-		ProxyType:              "",
-	}
-	require.Equal(t, expected1, actual1)
-}
-
-func TestOriginRequestConfigDefaults(t *testing.T) {
-	rulesYAML := `
-ingress:
-- hostname: tun.example.com
-  service: https://localhost:8000
-- hostname: "*"
-  service: https://localhost:8001
-  originRequest:
-    connectTimeout: 2m
-    tlsTimeout: 2s
-    noHappyEyeballs: false
-    tcpKeepAlive: 2s
-    keepAliveConnections: 2
-    keepAliveTimeout: 2s
-    httpHostHeader: def
-    originServerName: b2
-    caPool: /tmp/path1
-    noTLSVerify: false
-    disableChunkedEncoding: false
-    bastionMode: false
-    proxyAddress: interface
-    proxyPort: 200
-    proxyType: ""
-`
-	ing, err := ParseIngress(MustReadIngress(rulesYAML))
-	if err != nil {
-		t.Error(err)
-	}
-
-	// Rule 0 didn't override anything, so it inherits the cloudflared defaults
-	actual0 := ing.Rules[0].Config
-	expected0 := OriginRequestConfig{
-		ConnectTimeout:       defaultConnectTimeout,
-		TLSTimeout:           defaultTLSTimeout,
-		TCPKeepAlive:         defaultTCPKeepAlive,
-		KeepAliveConnections: defaultKeepAliveConnections,
-		KeepAliveTimeout:     defaultKeepAliveTimeout,
-		ProxyAddress:         defaultProxyAddress,
-	}
-	require.Equal(t, expected0, actual0)
-
-	// Rule 1 overrode all defaults.
-	actual1 := ing.Rules[1].Config
-	expected1 := OriginRequestConfig{
-		ConnectTimeout:         2 * time.Minute,
-		TLSTimeout:             2 * time.Second,
-		NoHappyEyeballs:        false,
-		TCPKeepAlive:           2 * time.Second,
-		KeepAliveConnections:   2,
-		KeepAliveTimeout:       2 * time.Second,
-		HTTPHostHeader:         "def",
-		OriginServerName:       "b2",
-		CAPool:                 "/tmp/path1",
-		NoTLSVerify:            false,
-		DisableChunkedEncoding: false,
-		BastionMode:            false,
-		ProxyAddress:           "interface",
-		ProxyPort:              uint(200),
-		ProxyType:              "",
-	}
-	require.Equal(t, expected1, actual1)
-}
-
-func TestDefaultConfigFromCLI(t *testing.T) {
-	set := flag.NewFlagSet("contrive", 0)
-	c := cli.NewContext(nil, set, nil)
-
-	expected := OriginRequestConfig{
-		ConnectTimeout:       defaultConnectTimeout,
-		TLSTimeout:           defaultTLSTimeout,
-		TCPKeepAlive:         defaultTCPKeepAlive,
-		KeepAliveConnections: defaultKeepAliveConnections,
-		KeepAliveTimeout:     defaultKeepAliveTimeout,
-		ProxyAddress:         defaultProxyAddress,
-	}
-	actual := originRequestFromSingeRule(c)
-	require.Equal(t, expected, actual)
-}

ingress/origin_service.go

@@ -8,7 +8,6 @@ import (
 	"net"
 	"net/http"
 	"net/url"
-	"sync"
 	"time"
 
 	"github.com/pkg/errors"
@@ -20,13 +19,18 @@ import (
 	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
+const (
+	HelloWorldService = "Hello World test origin"
+)
+
 // OriginService is something a tunnel can proxy traffic to.
 type OriginService interface {
 	String() string
 	// Start the origin service if it's managed by cloudflared, e.g. proxy servers or Hello World.
 	// If it's not managed by cloudflared, this is a no-op because the user is responsible for
 	// starting the origin service.
-	start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error
+	// Implementor of services managed by cloudflared should terminate the service if shutdownC is closed
+	start(log *zerolog.Logger, shutdownC <-chan struct{}, cfg OriginRequestConfig) error
 }
 
 // unixSocketPath is an OriginService representing a unix socket (which accepts HTTP)
@@ -39,7 +43,7 @@ func (o *unixSocketPath) String() string {
 	return "unix socket: " + o.path
 }
 
-func (o *unixSocketPath) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (o *unixSocketPath) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
 	transport, err := newHTTPTransport(o, cfg, log)
 	if err != nil {
 		return err
@@ -54,7 +58,7 @@ type httpService struct {
 	transport  *http.Transport
 }
 
-func (o *httpService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (o *httpService) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
 	transport, err := newHTTPTransport(o, cfg, log)
 	if err != nil {
 		return err
@@ -78,7 +82,7 @@ func (o *rawTCPService) String() string {
 	return o.name
 }
 
-func (o *rawTCPService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (o *rawTCPService) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
 	return nil
 }
 
@@ -139,7 +143,7 @@ func (o *tcpOverWSService) String() string {
 	return o.dest
 }
 
-func (o *tcpOverWSService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (o *tcpOverWSService) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
 	if cfg.ProxyType == socksProxy {
 		o.streamHandler = socks.StreamHandler
 	} else {
@@ -148,7 +152,7 @@ func (o *tcpOverWSService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdo
 	return nil
 }
 
-func (o *socksProxyOverWSService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (o *socksProxyOverWSService) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
 	return nil
 }
 
@@ -164,18 +168,16 @@ type helloWorld struct {
 }
 
 func (o *helloWorld) String() string {
-	return "Hello World test origin"
+	return HelloWorldService
 }
 
 // Start starts a HelloWorld server and stores its address in the Service receiver.
 func (o *helloWorld) start(
-	wg *sync.WaitGroup,
 	log *zerolog.Logger,
 	shutdownC <-chan struct{},
-	errC chan error,
 	cfg OriginRequestConfig,
 ) error {
-	if err := o.httpService.start(wg, log, shutdownC, errC, cfg); err != nil {
+	if err := o.httpService.start(log, shutdownC, cfg); err != nil {
 		return err
 	}
 
@@ -183,11 +185,7 @@ func (o *helloWorld) start(
 	if err != nil {
 		return errors.Wrap(err, "Cannot start Hello World Server")
 	}
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		_ = hello.StartHelloWorldServer(log, helloListener, shutdownC)
-	}()
+	go hello.StartHelloWorldServer(log, helloListener, shutdownC)
 	o.server = helloListener
 
 	o.httpService.url = &url.URL{
@@ -218,10 +216,8 @@ func (o *statusCode) String() string {
 }
 
 func (o *statusCode) start(
-	wg *sync.WaitGroup,
 	log *zerolog.Logger,
-	shutdownC <-chan struct{},
-	errC chan error,
+	_ <-chan struct{},
 	cfg OriginRequestConfig,
 ) error {
 	return nil
@@ -296,6 +292,6 @@ func (mos MockOriginHTTPService) String() string {
 	return "MockOriginService"
 }
 
-func (mos MockOriginHTTPService) start(wg *sync.WaitGroup, log *zerolog.Logger, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+func (mos MockOriginHTTPService) start(log *zerolog.Logger, _ <-chan struct{}, cfg OriginRequestConfig) error {
 	return nil
 }

ingress/origin_udp_proxy.go

@@ -0,0 +1,27 @@
+package ingress
+
+import (
+	"fmt"
+	"io"
+	"net"
+)
+
+type UDPProxy struct {
+	io.ReadWriteCloser
+}
+
+func DialUDP(dstIP net.IP, dstPort uint16) (*UDPProxy, error) {
+	dstAddr := &net.UDPAddr{
+		IP:   dstIP,
+		Port: int(dstPort),
+	}
+
+	// We use nil as local addr to force runtime to find the best suitable local address IP given the destination
+	// address as context.
+	udpConn, err := net.DialUDP("udp", nil, dstAddr)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create UDP proxy to origin (%v:%v): %w", dstIP, dstPort, err)
+	}
+
+	return &UDPProxy{udpConn}, nil
+}