Release 2020.2.0

This reverts commit 053b2c17f1.

.gitignore

@@ -7,3 +7,10 @@ guide/public
 .idea
 .vscode
 \#*\#
+cscope.*
+cloudflared
+cloudflared.exe
+!cmd/cloudflared/
+.DS_Store
+*-session.log
+ssh_server_tests/.env

Dockerfile

@@ -0,0 +1,28 @@
+# use a builder image for building cloudflare
+FROM golang:1.13.3 as builder
+ENV GO111MODULE=on
+ENV CGO_ENABLED=0
+ENV GOOS=linux
+
+WORKDIR /go/src/github.com/cloudflare/cloudflared/
+
+# copy our sources into the builder image
+COPY . .
+
+# compile cloudflared
+RUN make cloudflared
+
+# ---
+
+# use a distroless base image with glibc
+FROM gcr.io/distroless/base-debian10:nonroot
+
+# copy our compiled binary
+COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+
+# run as non-privileged user
+USER nonroot
+
+# command / entrypoint of container
+ENTRYPOINT ["cloudflared", "--no-autoupdate"]
+CMD ["version"]

Gopkg.lock

@@ -1,432 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  name = "github.com/BurntSushi/toml"
-  packages = ["."]
-  revision = "b26d9c308763d68093482582cea63d69be07a0f0"
-  version = "v0.3.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/beorn7/perks"
-  packages = ["quantile"]
-  revision = "3a771d992973f24aa725d07868b467d1ddfceafb"
-
-[[projects]]
-  name = "github.com/certifi/gocertifi"
-  packages = ["."]
-  revision = "deb3ae2ef2610fde3330947281941c562861188b"
-  version = "2018.01.18"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/cloudflare/brotli-go"
-  packages = ["."]
-  revision = "18c9f6c67e3dfc12e0ddaca748d2887f97a7ac28"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/coredns/coredns"
-  packages = [
-    "core/dnsserver",
-    "coremain",
-    "pb",
-    "plugin",
-    "plugin/cache",
-    "plugin/cache/freq",
-    "plugin/etcd/msg",
-    "plugin/metrics",
-    "plugin/metrics/vars",
-    "plugin/pkg/cache",
-    "plugin/pkg/dnstest",
-    "plugin/pkg/dnsutil",
-    "plugin/pkg/doh",
-    "plugin/pkg/edns",
-    "plugin/pkg/fuzz",
-    "plugin/pkg/log",
-    "plugin/pkg/nonwriter",
-    "plugin/pkg/rcode",
-    "plugin/pkg/response",
-    "plugin/pkg/trace",
-    "plugin/pkg/uniq",
-    "plugin/pkg/watch",
-    "plugin/test",
-    "request"
-  ]
-  revision = "992e7928c7c258628d2b13b769acc86781b9faea"
-
-[[projects]]
-  name = "github.com/coreos/go-systemd"
-  packages = ["daemon"]
-  revision = "39ca1b05acc7ad1220e09f133283b8859a8b71ab"
-  version = "v17"
-
-[[projects]]
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  revision = "346938d642f2ec3594ed81d874461961cd0faa76"
-  version = "v1.1.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/elgs/gosqljson"
-  packages = ["."]
-  revision = "027aa4915315a0b2825c0f025cea347829b974fa"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/equinox-io/equinox"
-  packages = [
-    ".",
-    "internal/go-update",
-    "internal/go-update/internal/binarydist",
-    "internal/go-update/internal/osext",
-    "internal/osext",
-    "proto"
-  ]
-  revision = "f24972fa72facf59d05c91c848b65eac38815915"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/facebookgo/grace"
-  packages = ["gracenet"]
-  revision = "75cf19382434e82df4dd84953f566b8ad23d6e9e"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/flynn/go-shlex"
-  packages = ["."]
-  revision = "3f9db97f856818214da2e1057f8ad84803971cff"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/getsentry/raven-go"
-  packages = ["."]
-  revision = "ed7bcb39ff10f39ab08e317ce16df282845852fa"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/golang-collections/collections"
-  packages = ["queue"]
-  revision = "604e922904d35e97f98a774db7881f049cd8d970"
-
-[[projects]]
-  name = "github.com/golang/protobuf"
-  packages = [
-    "proto",
-    "ptypes",
-    "ptypes/any",
-    "ptypes/duration",
-    "ptypes/timestamp"
-  ]
-  revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
-  version = "v1.1.0"
-
-[[projects]]
-  name = "github.com/google/uuid"
-  packages = ["."]
-  revision = "064e2069ce9c359c118179501254f67d7d37ba24"
-  version = "0.2"
-
-[[projects]]
-  name = "github.com/gorilla/context"
-  packages = ["."]
-  revision = "08b5f424b9271eedf6f9f0ce86cb9396ed337a42"
-  version = "v1.1.1"
-
-[[projects]]
-  name = "github.com/gorilla/mux"
-  packages = ["."]
-  revision = "e3702bed27f0d39777b0b37b664b6280e8ef8fbf"
-  version = "v1.6.2"
-
-[[projects]]
-  name = "github.com/gorilla/websocket"
-  packages = ["."]
-  revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/grpc-ecosystem/grpc-opentracing"
-  packages = ["go/otgrpc"]
-  revision = "8e809c8a86450a29b90dcc9efbf062d0fe6d9746"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/lib/pq"
-  packages = [
-    ".",
-    "oid"
-  ]
-  revision = "90697d60dd844d5ef6ff15135d0203f65d2f53b8"
-
-[[projects]]
-  name = "github.com/mattn/go-colorable"
-  packages = ["."]
-  revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072"
-  version = "v0.0.9"
-
-[[projects]]
-  name = "github.com/mattn/go-isatty"
-  packages = ["."]
-  revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
-  version = "v0.0.3"
-
-[[projects]]
-  name = "github.com/matttproud/golang_protobuf_extensions"
-  packages = ["pbutil"]
-  revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c"
-  version = "v1.0.1"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mholt/caddy"
-  packages = [
-    ".",
-    "caddyfile",
-    "telemetry"
-  ]
-  revision = "d3b731e9255b72d4571a5aac125634cf1b6031dc"
-
-[[projects]]
-  name = "github.com/miekg/dns"
-  packages = ["."]
-  revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1"
-  version = "v1.0.8"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/mitchellh/go-homedir"
-  packages = ["."]
-  revision = "3864e76763d94a6df2f9960b16a20a33da9f9a66"
-
-[[projects]]
-  name = "github.com/opentracing/opentracing-go"
-  packages = [
-    ".",
-    "ext",
-    "log"
-  ]
-  revision = "1949ddbfd147afd4d964a9f00b24eb291e0e7c38"
-  version = "v1.0.2"
-
-[[projects]]
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  name = "github.com/prometheus/client_golang"
-  packages = [
-    "prometheus",
-    "prometheus/promhttp"
-  ]
-  revision = "967789050ba94deca04a5e84cce8ad472ce313c1"
-  version = "v0.9.0-pre1"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/prometheus/client_model"
-  packages = ["go"]
-  revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/prometheus/common"
-  packages = [
-    "expfmt",
-    "internal/bitbucket.org/ww/goautoneg",
-    "model"
-  ]
-  revision = "7600349dcfe1abd18d72d3a1770870d9800a7801"
-
-[[projects]]
-  branch = "master"
-  name = "github.com/prometheus/procfs"
-  packages = [
-    ".",
-    "internal/util",
-    "nfs",
-    "xfs"
-  ]
-  revision = "ae68e2d4c00fed4943b5f6698d504a5fe083da8a"
-
-[[projects]]
-  name = "github.com/rifflock/lfshook"
-  packages = ["."]
-  revision = "bf539943797a1f34c1f502d07de419b5238ae6c6"
-  version = "v2.3"
-
-[[projects]]
-  name = "github.com/sirupsen/logrus"
-  packages = ["."]
-  revision = "c155da19408a8799da419ed3eeb0cb5db0ad5dbc"
-  version = "v1.0.5"
-
-[[projects]]
-  name = "github.com/stretchr/testify"
-  packages = ["assert"]
-  revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
-  version = "v1.2.2"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-  packages = [
-    "ed25519",
-    "ed25519/internal/edwards25519",
-    "ssh/terminal"
-  ]
-  revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/net"
-  packages = [
-    "bpf",
-    "context",
-    "http/httpguts",
-    "http2",
-    "http2/hpack",
-    "idna",
-    "internal/iana",
-    "internal/socket",
-    "internal/timeseries",
-    "ipv4",
-    "ipv6",
-    "trace",
-    "websocket"
-  ]
-  revision = "32a936f46389aa10549d60bd7833e54b01685d09"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sync"
-  packages = ["errgroup"]
-  revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
-
-[[projects]]
-  branch = "master"
-  name = "golang.org/x/sys"
-  packages = [
-    "unix",
-    "windows",
-    "windows/registry",
-    "windows/svc",
-    "windows/svc/eventlog",
-    "windows/svc/mgr"
-  ]
-  revision = "ce36f3865eeb42541ce3f87f32f8462c5687befa"
-
-[[projects]]
-  name = "golang.org/x/text"
-  packages = [
-    "collate",
-    "collate/build",
-    "internal/colltab",
-    "internal/gen",
-    "internal/tag",
-    "internal/triegen",
-    "internal/ucd",
-    "language",
-    "secure/bidirule",
-    "transform",
-    "unicode/bidi",
-    "unicode/cldr",
-    "unicode/norm",
-    "unicode/rangetable"
-  ]
-  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
-  version = "v0.3.0"
-
-[[projects]]
-  branch = "master"
-  name = "google.golang.org/genproto"
-  packages = ["googleapis/rpc/status"]
-  revision = "ff3583edef7de132f219f0efc00e097cabcc0ec0"
-
-[[projects]]
-  name = "google.golang.org/grpc"
-  packages = [
-    ".",
-    "balancer",
-    "balancer/base",
-    "balancer/roundrobin",
-    "codes",
-    "connectivity",
-    "credentials",
-    "encoding",
-    "encoding/proto",
-    "grpclog",
-    "internal",
-    "internal/backoff",
-    "internal/channelz",
-    "internal/grpcrand",
-    "keepalive",
-    "metadata",
-    "naming",
-    "peer",
-    "resolver",
-    "resolver/dns",
-    "resolver/passthrough",
-    "stats",
-    "status",
-    "tap",
-    "transport"
-  ]
-  revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8"
-  version = "v1.13.0"
-
-[[projects]]
-  branch = "altsrc-parse-durations"
-  name = "gopkg.in/urfave/cli.v2"
-  packages = [
-    ".",
-    "altsrc"
-  ]
-  revision = "d604b6ffeee878fbf084fd2761466b6649989cee"
-  source = "https://github.com/cbranch/cli"
-
-[[projects]]
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
-  version = "v2.2.1"
-
-[[projects]]
-  name = "zombiezen.com/go/capnproto2"
-  packages = [
-    ".",
-    "encoding/text",
-    "internal/fulfiller",
-    "internal/nodemap",
-    "internal/packed",
-    "internal/queue",
-    "internal/schema",
-    "internal/strquote",
-    "pogs",
-    "rpc",
-    "rpc/internal/refcount",
-    "schemas",
-    "server",
-    "std/capnp/rpc"
-  ]
-  revision = "7cfd211c19c7f5783c695f3654efa46f0df259c3"
-  source = "https://github.com/zombiezen/go-capnproto2"
-  version = "v2.17.1"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  inputs-digest = "9aac6a0260101988582bfa55d39559e53cf8ccd7e37caad3350aeea768f5bdb2"
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,66 +0,0 @@
-[prune]
-  go-tests = true
-  unused-packages = true
-
-  [[prune.project]]
-    name = "github.com/cloudflare/brotli-go"
-    unused-packages = false
-
-
-
-[[constraint]]
-  name = "github.com/facebookgo/grace"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/getsentry/raven-go"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/pkg/errors"
-  version = "0.8.0"
-
-[[constraint]]
-  name = "github.com/prometheus/client_golang"
-  version = "0.9.0-pre1"
-
-[[constraint]]
-  name = "github.com/sirupsen/logrus"
-  version = "1.0.3"
-
-[[constraint]]
-  name = "github.com/stretchr/testify"
-  version = "1.2.1"
-
-[[constraint]]
-  name = "golang.org/x/net"
-  branch = "master"
-
-[[constraint]]
-  name = "golang.org/x/sync"
-  branch = "master"
-
-[[constraint]]
-  name = "gopkg.in/urfave/cli.v2"
-  source = "https://github.com/cbranch/cli"
-  branch = "altsrc-parse-durations"
-
-[[constraint]]
-  name = "zombiezen.com/go/capnproto2"
-  source = "https://github.com/zombiezen/go-capnproto2"
-
-[[constraint]]
-  name = "github.com/gorilla/websocket"
-  version = "1.2.0"
-
-[[constraint]]
-  name = "github.com/coredns/coredns"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/cloudflare/brotli-go"
-  branch = "master"
-
-[[override]]
-  name = "github.com/mholt/caddy"
-  branch = "master"

Makefile

@@ -16,23 +16,39 @@ ifeq ($(EQUINOX_IS_DRAFT), true)
 	EQUINOX_FLAGS := --draft $(EQUINOX_FLAGS)
 endif
 
+ifeq ($(GOARCH),)
+	GOARCH := amd64
+endif
+
 .PHONY: all
 all: cloudflared test
 
+.PHONY: clean
+clean:
+	go clean
+
 .PHONY: cloudflared
-cloudflared:
-	go build -v $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
+cloudflared: tunnel-deps
+	go build -v -mod=vendor $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
+
+.PHONY: container
+container:
+	docker build -t cloudflare/cloudflared:"$(VERSION)" .
 
 .PHONY: test
-test:
-	go test -v -race $(VERSION_FLAGS) ./...
+test: vet
+	go test -v -mod=vendor -race $(VERSION_FLAGS) ./...
+
+.PHONY: test-ssh-server
+test-ssh-server:
+	docker-compose -f ssh_server_tests/docker-compose.yml up
 
 .PHONY: cloudflared-deb
 cloudflared-deb: cloudflared
 	mkdir -p $(PACKAGE_DIR)
 	cp cloudflared $(PACKAGE_DIR)/cloudflared
 	fakeroot fpm -C $(PACKAGE_DIR) -s dir -t deb --deb-compression bzip2 \
-		-a $(GOARCH) -v $(VERSION) -n cloudflared
+		-a $(GOARCH) -v $(VERSION) -n cloudflared cloudflared=/usr/local/bin/
 
 .PHONY: cloudflared-darwin-amd64.tgz
 cloudflared-darwin-amd64.tgz: cloudflared
@@ -43,8 +59,8 @@ cloudflared-darwin-amd64.tgz: cloudflared
 homebrew-upload: cloudflared-darwin-amd64.tgz
 	aws s3 --endpoint-url $(S3_ENDPOINT) cp --acl public-read $$^ $(S3_URI)/cloudflared-$$(VERSION)-$1.tgz
 	aws s3 --endpoint-url $(S3_ENDPOINT) cp --acl public-read $(S3_URI)/cloudflared-$$(VERSION)-$1.tgz  $(S3_URI)/cloudflared-stable-$1.tgz
-		
-.PHONY: homebrew-release 
+
+.PHONY: homebrew-release
 homebrew-release: homebrew-upload
 	./publish-homebrew-formula.sh cloudflared-darwin-amd64.tgz $(VERSION) homebrew-cloudflare
 
@@ -55,3 +71,17 @@ release: bin/equinox
 bin/equinox:
 	mkdir -p bin
 	curl -s https://bin.equinox.io/c/75JtLRTsJ3n/release-tool-beta-$(EQUINOX_PLATFORM).tgz | tar xz -C bin/
+
+.PHONY: tunnel-deps
+tunnel-deps: tunnelrpc/tunnelrpc.capnp.go
+
+tunnelrpc/tunnelrpc.capnp.go: tunnelrpc/tunnelrpc.capnp
+	which capnp  # https://capnproto.org/install.html
+	which capnpc-go  # go get zombiezen.com/go/capnproto2/capnpc-go
+	capnp compile -ogo tunnelrpc/tunnelrpc.capnp
+
+.PHONY: vet
+vet:
+	go vet -mod=vendor ./...
+	which go-sumtype  # go get github.com/BurntSushi/go-sumtype
+	go-sumtype $$(go list -mod=vendor ./...)

RELEASE_NOTES

@@ -1,3 +1,278 @@
+2020.2.0
+- 2020-01-30 TUN-2651: Fix panic in h2mux reader when a stream error is encountered
+- 2020-01-27 TUN-2645: Revert "TUN-2645: Turn on reconnect tokens"
+- 2020-01-28 TUN-2693: Metrics for ReconnectTunnel
+- 2020-01-28 TUN-2696: Add unknown registerRPCName
+- 2020-01-28 TUN-2699: Metrics for Authenticate RPCs
+- 2020-01-28 TUN-2690: cloudflared reconnect uses wrong context
+- 2020-01-29 TUN-2707: Inconsistent cardinality in tunnel error metrics
+- 2020-01-13 TUN-2645: Turn on reconnect tokens
+- 2019-12-23 TUN-2646: Make --edge flag work again for local development
+
+2019.12.0
+- 2019-12-11 TUN-2631: only notify that activeStreamMap is closed if ignoreNewStreams=true
+- 2019-12-17 bug(cloudflared): Set the MaxIdleConnsPerHost of http.Transport to proxy-keepalive-connections (#155)
+- 2019-12-17 refactor(docker): optimize Dockerfile (#126)
+- 2019-12-19 Fix timer scheduling for systemd update service (#159)
+- 2019-12-13 TUN-2637: Manage edge IPs in a region-aware manner
+- 2019-12-03 bug(cloudflared): nil pointer deference on h2DictWriter Close() (#154)
+- 2019-12-03 TUN-2608: h2mux.Muxer.Shutdown always returns a non-nil channel
+- 2019-12-04 TUN-2555: origin/supervisor.go calls Authenticate
+- 2019-12-06 TUN-2554: cloudflared calls ReconnectTunnel
+- 2019-11-20 TUN-2575: Constructors + simpler conversions for AuthOutcome
+- 2019-11-22 Fix Docker build failure (#149)
+- 2019-11-21 TUN-2573: Refactor TunnelRegistration into PermanentRegistrationError, RetryableRegistrationError and SuccessfulTunnelRegistration
+- 2019-11-22 TUN-2582: EventDigest field in tunnelrpc
+- 2019-11-22 Fix "happy eyeballs" not being disabled since Golang 1.12 upgrade * The Dialer.DualStack setting is now ignored and deprecated; RFC 6555 Fast Fallback ("Happy Eyeballs") is now enabled by default. To disable, set Dialer.FallbackDelay to a negative value.
+- 2019-11-25 TUN-2591: ReconnectTunnel now sends EventDigest
+- 2019-11-21 TUN-2606: add DialEdge helpers
+- 2019-11-21 TUN-2607: add RPC stream helpers
+
+2019.11.3
+- 2019-11-20 TUN-2562: Update Cloudflare Origin CA RSA root
+
+2019.11.2
+- 2019-11-18 TUN-2567: AuthOutcome can be turned back into AuthResponse
+- 2019-11-18 TUN-2563: Exposes config_version metrics
+
+2019.11.1
+- 2019-11-12 Add db-connect, a SQL over HTTPS server
+- 2019-11-12 TUN-2053: Add a /healthcheck endpoint to the metrics server
+- 2019-11-13 TUN-2178: public API to create new h2mux.MuxedStreamRequest
+- 2019-11-13 TUN-2490: respect original representation of HTTP request path
+- 2019-11-18 TUN-2547: TunnelRPC definitions for Authenticate flow
+- 2019-11-18 TUN-2551: TunnelRPC definitions for ReconnectTunnel flow
+- 2019-11-05 TUN-2506: Expose active streams metrics
+
+2019.11.0
+- 2019-11-04 TUN-2502: Switch to go modules
+- 2019-11-04 TUN-2500: Don't send client registration errors to Sentry
+- 2019-11-04 TUN-2489: Delete stream from activestreammap when read and write are both closed
+- 2019-11-05 TUN-2505: Terminate stream on receipt of RST_STREAM; MuxedStream.CloseWrite() should terminate the MuxedStream.Write() loop
+- 2019-10-30 TUN-2451: Log inavlid path
+- 2019-10-22 TUN-2425: Enable cloudflared to serve multiple Hello World servers by having each of them create its own ServeMux
+- 2019-10-22 AUTH-2173: Prepends access login url with scheme if one doesnt exist
+- 2019-10-23 TUN-2460: Configure according to the ClientConfig recevied from a successful Connect
+- 2019-10-23 AUTH-2177: Reads and writes error streams
+
+2019.10.4
+- 2019-10-21 TUN-2450: Remove Brew publishing formula
+
+2019.10.3
+- 2019-10-18 Fix #129: Excessive memory usage streaming large files (#142)
+
+2019.10.2
+- 2019-10-17 AUTH-2167: Adds CLI option for host key directory
+
+2019.10.1
+- 2019-10-17 Adds variable to fix windows build
+
+2019.10.0
+- 2019-10-11 AUTH-2105: Dont require --destination arg
+- 2019-10-14 TUN-2344: log more details: http2.Framer.ErrorDetail() if available, connectionID
+- 2019-10-16 AUTH-2159: Moves shutdownC close into error handling AUTH-2161: Lowers size of preamble length AUTH-2160: Fixes url parsing logic
+- 2019-10-16 AUTH-2135: Adds support for IPv6 and tests
+- 2019-10-02 AUTH-2105: Adds support for local forwarding. Refactor auditlogger creation. AUTH-2088: Adds dynamic destination routing
+- 2019-10-09 AUTH-2114: Uses short lived cert auth for outgoing client connection
+- 2019-09-30 AUTH-2089: Revise ssh server to function as a proxy
+
+2019.9.2
+- 2019-09-26 TUN-2355: Roll back TUN-2276
+
+2019.9.1
+- 2019-09-23 TUN-2334: remove tlsConfig.ServerName special case
+- 2019-09-23 AUTH-2077: Quotes open browser command in windows
+- 2019-09-11 AUTH-2050: Adds time.sleep to temporarily avoid hitting tunnel muxer dealock issue
+- 2019-09-10 AUTH-2056: Writes stderr to its own stream for non-pty connections
+- 2019-09-16 TUN-2307: Capnp is the only serialization format used in tunnelpogs
+- 2019-09-18 TUN-2315: Replace Scope with IntentLabel
+- 2019-09-17 TUN-2309: Split ConnectResult into ConnectError and ConnectSuccess, each implementing its own capnp serialization logic
+- 2019-09-18 AUTH-2052: Adds tests for SSH server
+- 2019-09-18 AUTH-2067: Log commands correctly
+- 2019-09-19 AUTH-2055: Verifies token at edge on access login
+- 2019-09-04 TUN-2201: change SRV records used by cloudflared
+- 2019-09-06 TUN-2280: Revert "TUN-2260: add name/group to CapnpConnectParameters, remove Scope"
+- 2019-09-03 AUTH-1943 hooked up uploader to logger, added timestamp to session logs, add tests
+- 2019-09-04 AUTH-2036: Refactor user retrieval, shutdown after ssh server stops, add custom version string
+- 2019-09-06 AUTH-1942 added event log to ssh server
+- 2019-09-04 AUTH-2037: Adds support for ssh port forwarding
+- 2019-09-05 TUN-2276: Path encoding broken
+
+2019.9.0
+- 2019-09-05 TUN-2279: Revert path encoding fix
+- 2019-08-30 AUTH-2021 - check error for failing tests
+- 2019-08-29 AUTH-2030: Support both authorized_key and short lived cert authentication simultaniously without specifiying at start time
+- 2019-08-29 AUTH-2026: Adds support for non-pty sessions and inline command exec
+- 2019-08-26 AUTH-1943: Adds session logging
+- 2019-08-26 TUN-2162: Decomplect OpenStream to allow finer-grained timeouts
+- 2019-08-29 TUN-2260: add name/group to CapnpConnectParameters, remove Scope
+
+2019.8.4
+- 2019-08-30 Fix #111: Add support for specifying a specific HTTP Host: header on the origin. (#114)
+- 2019-08-22 TUN-2165: Add ClientConfig to tunnelrpc.ConnectResult
+- 2019-08-20 AUTH-2014: Checks users login shell
+- 2019-08-26 TUN-2243: Revert "STOR-519: Add db-connect, a SQL over HTTPS server"
+- 2019-08-27 TUN-2244: Add NO_AUTOUPDATE env var
+- 2019-08-22 AUTH-2018: Adds support for authorized keys and short lived certs
+- 2019-08-28 AUTH-2022: Adds ssh timeout configuration
+- 2019-08-28 TUN-1968: Gracefully diff StreamHandler.UpdateConfig
+- 2019-08-26 AUTH-2021 - s3 bucket uploading for SSH logs
+- 2019-08-19 AUTH-2004: Adds static host key support
+- 2019-07-18 AUTH-1941: Adds initial SSH server implementation
+
+2019.8.3
+- 2019-08-20 STOR-519: Add db-connect, a SQL over HTTPS server
+- 2019-08-20 Release 2019.8.2
+- 2019-08-20 Revert "AUTH-1941: Adds initial SSH server implementation"
+- 2019-08-11 TUN-2163: Add GrapQLType method to Scope interface
+- 2019-08-06 TUN-2152: Requests with a query in the URL are erroneously escaped
+- 2019-07-18 AUTH-1941: Adds initial SSH server implementation
+
+2019.8.1
+- 2019-08-05 TUN-2111: Implement custom serialization logic for FallibleConfig and OriginConfig
+- 2019-08-06 Revert "TUN-1736: Missing headers when passing an invalid path"
+
+2019.8.0
+- 2019-07-11 TUN-1956: Go 1.12 update
+- 2019-07-24 TUN-1736: Missing headers when passing an invalid path
+- 2019-07-30 TUN-2117: read group/system-name from CLI, send it to edge
+- 2019-08-02 TUN-2125: Add PostgresType() to Scope
+- 2019-08-05 TUN-2147: Implemented ScopeUnmarshaler
+- 2019-07-31 TUN-2110: Implement custom deserialization logic for OriginConfig
+- 2019-07-31 AUTH-1972: Deletes token lock file if backoff retry attempts exceeded and intercepts signals until lock is released
+
+2019.7.0
+- 2019-05-28 TUN-1913: Define OriginService for each type of origin
+- 2019-04-29 Build a docker container
+- 2019-06-12 TUN-1952: Group ClientConfig fields by the component that uses the config, and return the part of the config that failed to be applied
+- 2019-06-05 TUN-1893: Proxy requests to the origin based on tunnel hostname
+- 2019-06-17 TUN-1961: Create EdgeConnectionManager to maintain outbound connections to the edge
+- 2019-06-18 TUN-1885: Reconfigure cloudflared on receiving new ClientConfig
+- 2019-06-19 TUN-1976: Pass tunnel hostname through header
+- 2019-06-20 TUN-1982: Load custom origin CA when OriginCAPool is specified
+- 2019-06-26 TUN-2005: Upgrade logrus
+- 2019-06-20 TUN-1981: Write response header & body on proxy error to notify eyeballs of failure category
+- 2019-06-20 TUN-1977: Validate OriginConfig has valid URL, and use scheme to determine if a HTTPOriginService is expecting HTTP or Unix
+- 2019-06-13 DoH: change the media type to application/dns-message
+- 2019-06-26 AUTH-1736: Better handling of token revocation
+
+2019.6.0
+- 2019-05-17 TUN-1828: Update declarative tunnel config struct
+- 2019-05-29 Handle exit code on err
+- 2019-05-29 AUTH-1802: Fixed ssh-config templating
+- 2019-05-30 TUN-1914: Conflate HTTP and Unix OriginConfig, and add TLS config to WebSocketOriginConfig
+- 2019-06-03 AUTH-1811: ssh-gen config fixes
+
+2019.5.0
+- 2019-04-25 TUN-1781: ServeStream should return early on error
+- 2019-04-30 TUN-1786: Remove low-level Windows service logging
+- 2019-05-03 TUN-1807: Send cloudflared version in Connect RPC
+- 2019-01-23 AUTH-1557: Short Lived Certs
+- 2019-05-13 TUN-1847: Log a distinct message when OpenStream fails while waiting for response headers
+- 2019-05-13 AUTH-1706: fixes and testing
+- 2019-05-22 TUN-1880: Save debug and warn level log to logfile
+- 2019-05-22 AUTH-1781: fixed race condition for short lived certs, doc required config
+
+2019.4.1
+- 2019-03-18 TUN-1626: Create new supervisor to establish connection with origintunneld
+- 2019-04-04 TUN-1619: Add flag to test declarative tunnels.
+- 2019-04-05 TUN-1577: decompose carrier.StartServer to make TestStartServer less flappy
+- 2019-03-29 TUN-1606: Define CloudflaredConfig RPC structure, interface for cloudflared's RPC server
+- 2019-04-02 TUN-1682: Add context to OpenStream to prevent it from blocking indefinitely.
+- 2019-04-16 TUN-1732: cloudflared metrics should track userHostnames
+- 2019-04-17 TUN-1734: Pin packages at exact versions
+- 2019-04-18 TUN-1669: Update license message in help text. Also fix test
+
+2019.4.0
+- 2019-03-28 TUN-1648: ConnectionID is now a UUID
+- 2019-04-01 TUN-1673: Conflate Hello and Connect RPCs
+
+2019.3.2
+- 2019-03-22 TUN-1637: Free tunnels shouldn't require cert.pem
+- 2019-03-18 TUN-1604: Define Connect RPC call
+
+2019.3.1
+- 2019-03-09 Add rdp as a supported protocol in URL validation (#76)
+- 2019-03-15 TUN-1613: improved cloudflared RegisterTunnel fail metrics
+- 2019-03-17 TUN-1615: revert miekg/dns to last known working revision
+
+2019.3.0
+- 2018-12-28 make http transport aware of proxy from envvar
+- 2019-02-28 TUN-1559: fix nil dereference in TunnelConfig.CloseConnOnce
+- 2019-03-04 TUN-1451: Make non-interactive, non-service execution possible on Windows
+- 2019-03-04 TUN-1562: Refactor connectedSignal to be safe to close multiple times
+- 2019-02-27 TUN-1550: Add validation timeout for non-responsive origins
+- 2019-03-06 AUTH-1531: Named flags for ssh service tokens
+- 2019-02-14 Support unix sockets.
+- 2019-03-08 TUN-1389: Non-scalar flags in a cloudflared config.yml don't get logged
+- 2019-03-07 TUN-1522: If we can't get SRV from default resolver, get them from 1.1.1.1 DoT
+
+2019.2.1
+- 2019-02-14 TUN-1381:  should tell you if you're on the latest version rather than just exiting silently
+- 2019-02-15 TUN-1467: build with Go 1.11
+- 2019-02-15 AUTH-1519: Added logging
+- 2019-02-19 TUN-1525: cloudflared metrics for registration success/fail
+- 2019-02-19 TUN-1510: Wrap the close() in sync.Once.Do
+
+2019.2.0
+- 2019-01-24 AUTH-1462: better curl arg parsing
+- 2019-02-01 TUN-1456: Only make one UUID
+- 2019-01-30 cloudflared/linux_service: Add missing /etc/init.d shebang
+- 2019-02-07 AUTH-1511: Add custom headers for ssh command
+- 2019-02-01 AUTH-1503: Added RDP support
+- 2019-02-01 AUTH-1403: Print the paths in the ssh-config instructions
+
+2019.1.0
+- 2018-12-10 TUN-1231: Horizontal overflow wrapping on the Hello page
+- 2018-12-17 TUN-1140: Show usage if invoked with no args or config
+- 2018-11-06 TUN-632 Filter out common network exceptions from going to Sentry on StartServer
+- 2019-01-07 TUN-1138: Install cloudflared service directory with 755 permissions
+- 2019-01-07 TUN-1265: Silent exit when failing to parse config
+- 2019-01-10 TUN-1350: Enhance error messages with cloudflarestatus.com link, if relevant
+- 2019-01-16 TUN-1358: Close readyList after Muxer.Serve() has stopped running
+- 2019-01-24 AUTH-1423: move from stdout to stderr
+- 2019-01-24 AUTH-1404: reauth if the token is about to expire within 15 minutes
+- 2019-01-24 AUTH-1459: improved ssh streaming error message
+- 2019-01-24 AUTH-1211: print all the versions
+- 2019-01-24 AUTH-1337: fix url path
+- 2019-01-28 TUN-1418: Rename ProtocolLogger to TransportLogger, and use TransportLogger to log RPC events.
+- 2019-01-28 TUN-1419: Identify request/response headers/content length with ray ID
+
+2018.12.1
+- 2018-12-11 TUN-1270: cloudflared panic (HA metrics missing label)
+
+2018.12.0
+- 2018-11-15 TUN-1196: Allow TLS config client CA and root CA to be constructed from multiple certificates
+- 2018-11-20 TUN-1209: TLS Config Certificates and GetCertificate can both be set
+- 2018-11-26 TUN-1212: Expose tunnel_id in metrics
+- 2018-11-30 TUN-1204: remove 'cloudflared hello' command
+- 2018-12-04 Fix license URL typo
+- 2018-12-07 TUN-1250: ValidateHTTPService shouldn't follow 302s
+
+2018.11.0
+- 2018-10-31 AUTH-1282: Fixed an issue where we were receiving as opposed sending on the channel.
+- 2018-11-06 TUN-1179: Fix log message in cmd/cloudflared/transfer.Run
+- 2018-11-13 AUTH-1308: get jwt even when you are already logged in
+- 2018-11-12 TUN-1190: check URL parse error when starting SSH proxy server
+- 2018-11-15 AUTH-1320: Fixed request issue and unhide the ssh command
+
+2018.10.5
+- 2018-10-18 TUN-968: Flow control for large requests/responses
+- 2018-10-26 TUN-1158: Windows: use process arguments rather than trivial service arguments
+- 2018-10-20 #30: Fix the Content-Length header for HTTP2->HTTP1
+- 2018-10-29 TUN-1160: pass Host header during origin url validation
+
+2018.10.4
+- 2018-09-21 AUTH-1070: added SSH/protocol forwarding
+- 2018-10-19 AUTH-1235: fixed packaging of deb dev file
+- 2018-10-19 TUN-1097: Host missing from WebSocket request
+- 2018-10-19 AUTH-1188: UX Review and Changes for CLI SSH Access
+
+2018.10.3
+- 2018-10-08 TUN-1099: Bring back changes in 2018.10.1
+- 2018-10-08 TUN-1098: removed deprecation error
+- 2018-10-08 TUN-1101: False negatives in Cloudflared error reporting
+
 2018.10.2
 - 2018-10-06 TUN-1093: Revert cloudflared to 2018.8.0
 

awsuploader/directory_upload_manager.go

@@ -0,0 +1,106 @@
+package awsuploader
+
+import (
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// DirectoryUploadManager is used to manage file uploads on an interval from a directory
+type DirectoryUploadManager struct {
+	logger        *logrus.Logger
+	uploader      Uploader
+	rootDirectory string
+	sweepInterval time.Duration
+	ticker        *time.Ticker
+	shutdownC     chan struct{}
+	workQueue     chan string
+}
+
+// NewDirectoryUploadManager create a new DirectoryUploadManager
+// uploader is an Uploader to use as an actual uploading engine
+// directory is the directory to sweep for files to upload
+// sweepInterval is how often to iterate the directory and upload the files within
+func NewDirectoryUploadManager(logger *logrus.Logger, uploader Uploader, directory string, sweepInterval time.Duration, shutdownC chan struct{}) *DirectoryUploadManager {
+	workerCount := 10
+	manager := &DirectoryUploadManager{
+		logger:        logger,
+		uploader:      uploader,
+		rootDirectory: directory,
+		sweepInterval: sweepInterval,
+		shutdownC:     shutdownC,
+		workQueue:     make(chan string, workerCount),
+	}
+
+	//start workers
+	for i := 0; i < workerCount; i++ {
+		go manager.worker()
+	}
+
+	return manager
+}
+
+// Upload a file using the uploader
+// This is useful for "out of band" uploads that need to be triggered immediately instead of waiting for the sweep
+func (m *DirectoryUploadManager) Upload(filepath string) error {
+	return m.uploader.Upload(filepath)
+}
+
+// Start the upload ticker to walk the directories
+func (m *DirectoryUploadManager) Start() {
+	m.ticker = time.NewTicker(m.sweepInterval)
+	go m.run()
+}
+
+func (m *DirectoryUploadManager) run() {
+	for {
+		select {
+		case <-m.shutdownC:
+			m.ticker.Stop()
+			return
+		case <-m.ticker.C:
+			m.sweep()
+		}
+	}
+}
+
+// sweep the directory and kick off uploads
+func (m *DirectoryUploadManager) sweep() {
+	filepath.Walk(m.rootDirectory, func(path string, info os.FileInfo, err error) error {
+		if err != nil || info.IsDir() {
+			return nil
+		}
+		//30 days ago
+		retentionTime := 30 * (time.Hour * 24)
+		checkTime := time.Now().Add(-time.Duration(retentionTime))
+
+		//delete the file it is stale
+		if info.ModTime().Before(checkTime) {
+			os.Remove(path)
+			return nil
+		}
+		//add the upload to the work queue
+		go func() {
+			m.workQueue <- path
+		}()
+		return nil
+	})
+}
+
+// worker handles upload requests
+func (m *DirectoryUploadManager) worker() {
+	for {
+		select {
+		case <-m.shutdownC:
+			return
+		case filepath := <-m.workQueue:
+			if err := m.Upload(filepath); err != nil {
+				m.logger.WithError(err).Error("Cannot upload file to s3 bucket")
+			} else {
+				os.Remove(filepath)
+			}
+		}
+	}
+}

awsuploader/directory_upload_manager_test.go

@@ -0,0 +1,137 @@
+package awsuploader
+
+import (
+	"errors"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+type MockUploader struct {
+	shouldFail bool
+}
+
+func (m *MockUploader) Upload(filepath string) error {
+	if m.shouldFail {
+		return errors.New("upload set to fail")
+	}
+	return nil
+}
+
+func NewMockUploader(shouldFail bool) Uploader {
+	return &MockUploader{shouldFail: shouldFail}
+}
+
+func getDirectoryPath(t *testing.T) string {
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatal("couldn't create the test directory!", err)
+	}
+	return filepath.Join(dir, "uploads")
+}
+
+func setupTestDirectory(t *testing.T) string {
+	path := getDirectoryPath(t)
+	os.RemoveAll(path)
+	time.Sleep(100 * time.Millisecond) //short way to wait for the OS to delete the folder
+	err := os.MkdirAll(path, os.ModePerm)
+	if err != nil {
+		t.Fatal("couldn't create the test directory!", err)
+	}
+	return path
+}
+
+func createUploadManager(t *testing.T, shouldFailUpload bool) *DirectoryUploadManager {
+	rootDirectory := setupTestDirectory(t)
+	uploader := NewMockUploader(shouldFailUpload)
+	logger := logrus.New()
+	shutdownC := make(chan struct{})
+	return NewDirectoryUploadManager(logger, uploader, rootDirectory, 1*time.Second, shutdownC)
+}
+
+func createFile(t *testing.T, fileName string) (*os.File, string) {
+	path := filepath.Join(getDirectoryPath(t), fileName)
+	f, err := os.Create(path)
+	if err != nil {
+		t.Fatal("upload to create file for sweep test", err)
+	}
+	return f, path
+}
+
+func TestUploadSuccess(t *testing.T) {
+	manager := createUploadManager(t, false)
+	path := filepath.Join(getDirectoryPath(t), "test_file")
+	if err := manager.Upload(path); err != nil {
+		t.Fatal("the upload request method failed", err)
+	}
+}
+
+func TestUploadFailure(t *testing.T) {
+	manager := createUploadManager(t, true)
+	path := filepath.Join(getDirectoryPath(t), "test_file")
+	if err := manager.Upload(path); err == nil {
+		t.Fatal("the upload request method should have failed and didn't", err)
+	}
+}
+
+func TestSweepSuccess(t *testing.T) {
+	manager := createUploadManager(t, false)
+	f, path := createFile(t, "test_file")
+	defer f.Close()
+
+	manager.Start()
+	time.Sleep(2 * time.Second)
+	if _, err := os.Stat(path); os.IsExist(err) {
+		//the file should have been deleted
+		t.Fatal("the manager failed to delete the file", err)
+	}
+}
+
+func TestSweepFailure(t *testing.T) {
+	manager := createUploadManager(t, true)
+	f, path := createFile(t, "test_file")
+	defer f.Close()
+
+	manager.Start()
+	time.Sleep(2 * time.Second)
+	_, serr := f.Stat()
+	if serr != nil {
+		//the file should still exist
+		os.Remove(path)
+		t.Fatal("the manager failed to delete the file", serr)
+	}
+}
+
+func TestHighLoad(t *testing.T) {
+	manager := createUploadManager(t, false)
+	for i := 0; i < 30; i++ {
+		f, _ := createFile(t, randomString(6))
+		defer f.Close()
+	}
+	manager.Start()
+	time.Sleep(4 * time.Second)
+
+	directory := getDirectoryPath(t)
+	files, err := ioutil.ReadDir(directory)
+	if err != nil || len(files) > 0 {
+		t.Fatalf("the manager failed to upload all the files: %s files left: %d", err, len(files))
+	}
+}
+
+// LowerCase [a-z]
+const randSet = "abcdefghijklmnopqrstuvwxyz"
+
+// String returns a string of length 'n' from a set of letters 'lset'
+func randomString(n int) string {
+	b := make([]byte, n)
+	lsetLen := len(randSet)
+	for i := range b {
+		b[i] = randSet[rand.Intn(lsetLen)]
+	}
+	return string(b)
+}

awsuploader/file_uploader.go

@@ -0,0 +1,62 @@
+package awsuploader
+
+import (
+	"context"
+	"os"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+//FileUploader aws compliant bucket upload
+type FileUploader struct {
+	storage    *s3.S3
+	bucketName string
+	clientID   string
+	secretID   string
+}
+
+// NewFileUploader creates a new S3 compliant bucket uploader
+func NewFileUploader(bucketName, region, accessKeyID, secretID, token, s3Host string) (*FileUploader, error) {
+	sess, err := session.NewSession(&aws.Config{
+		Region:      aws.String(region),
+		Credentials: credentials.NewStaticCredentials(accessKeyID, secretID, token),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	var storage *s3.S3
+	if s3Host != "" {
+		storage = s3.New(sess, &aws.Config{Endpoint: aws.String(s3Host)})
+	} else {
+		storage = s3.New(sess)
+	}
+
+	return &FileUploader{
+		storage:    storage,
+		bucketName: bucketName,
+	}, nil
+}
+
+// Upload a file to the bucket
+func (u *FileUploader) Upload(filepath string) error {
+	info, err := os.Stat(filepath)
+	if err != nil {
+		return err
+	}
+	file, err := os.Open(filepath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, serr := u.storage.PutObjectWithContext(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(u.bucketName),
+		Key:    aws.String(info.Name()),
+		Body:   file,
+	})
+	return serr
+}

awsuploader/upload_manager.go

@@ -0,0 +1,7 @@
+package awsuploader
+
+// UploadManager is used to manage file uploads on an interval
+type UploadManager interface {
+	Upload(string) error
+	Start()
+}

awsuploader/uploader.go

@@ -0,0 +1,7 @@
+package awsuploader
+
+// Uploader the functions required to upload to a bucket
+type Uploader interface {
+	//Upload a file to the bucket
+	Upload(string) error
+}

carrier/carrier.go

@@ -0,0 +1,216 @@
+//Package carrier provides a WebSocket proxy to carry or proxy a connection
+//from the local client to the edge. See it as a wrapper around any protocol
+//that it packages up in a WebSocket connection to the edge.
+package carrier
+
+import (
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
+	cloudflaredWebsocket "github.com/cloudflare/cloudflared/websocket"
+	"github.com/gorilla/websocket"
+	"github.com/sirupsen/logrus"
+)
+
+type StartOptions struct {
+	OriginURL string
+	Headers   http.Header
+}
+
+// StdinoutStream is empty struct for wrapping stdin/stdout
+// into a single ReadWriter
+type StdinoutStream struct {
+}
+
+// Read will read from Stdin
+func (c *StdinoutStream) Read(p []byte) (int, error) {
+	return os.Stdin.Read(p)
+
+}
+
+// Write will write to Stdout
+func (c *StdinoutStream) Write(p []byte) (int, error) {
+	return os.Stdout.Write(p)
+}
+
+// Helper to allow defering the response close with a check that the resp is not nil
+func closeRespBody(resp *http.Response) {
+	if resp != nil {
+		resp.Body.Close()
+	}
+}
+
+// StartClient will copy the data from stdin/stdout over a WebSocket connection
+// to the edge (originURL)
+func StartClient(logger *logrus.Logger, stream io.ReadWriter, options *StartOptions) error {
+	return serveStream(logger, stream, options)
+}
+
+// StartServer will setup a listener on a specified address/port and then
+// forward connections to the origin by calling `Serve()`.
+func StartServer(logger *logrus.Logger, address string, shutdownC <-chan struct{}, options *StartOptions) error {
+	listener, err := net.Listen("tcp", address)
+	if err != nil {
+		logger.WithError(err).Error("failed to start forwarding server")
+		return err
+	}
+	logger.Info("Started listening on ", address)
+	return Serve(logger, listener, shutdownC, options)
+}
+
+// Serve accepts incoming connections on the specified net.Listener.
+// Each connection is handled in a new goroutine: its data is copied over a
+// WebSocket connection to the edge (originURL).
+// `Serve` always closes `listener`.
+func Serve(logger *logrus.Logger, listener net.Listener, shutdownC <-chan struct{}, options *StartOptions) error {
+	defer listener.Close()
+	for {
+		select {
+		case <-shutdownC:
+			return nil
+		default:
+			conn, err := listener.Accept()
+			if err != nil {
+				return err
+			}
+			go serveConnection(logger, conn, options)
+		}
+	}
+}
+
+// serveConnection handles connections for the Serve() call
+func serveConnection(logger *logrus.Logger, c net.Conn, options *StartOptions) {
+	defer c.Close()
+	serveStream(logger, c, options)
+}
+
+// serveStream will serve the data over the WebSocket stream
+func serveStream(logger *logrus.Logger, conn io.ReadWriter, options *StartOptions) error {
+	wsConn, err := createWebsocketStream(options)
+	if err != nil {
+		logger.WithError(err).Errorf("failed to connect to %s\n", options.OriginURL)
+		return err
+	}
+	defer wsConn.Close()
+
+	cloudflaredWebsocket.Stream(wsConn, conn)
+
+	return nil
+}
+
+// createWebsocketStream will create a WebSocket connection to stream data over
+// It also handles redirects from Access and will present that flow if
+// the token is not present on the request
+func createWebsocketStream(options *StartOptions) (*cloudflaredWebsocket.Conn, error) {
+	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header = options.Headers
+
+	wsConn, resp, err := cloudflaredWebsocket.ClientConnect(req, nil)
+	defer closeRespBody(resp)
+	if err != nil && IsAccessResponse(resp) {
+		wsConn, err = createAccessAuthenticatedStream(options)
+		if err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+
+	return &cloudflaredWebsocket.Conn{Conn: wsConn}, nil
+}
+
+// IsAccessResponse checks the http Response to see if the url location
+// contains the Access structure.
+func IsAccessResponse(resp *http.Response) bool {
+	if resp == nil || resp.StatusCode != http.StatusFound {
+		return false
+	}
+
+	location, err := resp.Location()
+	if err != nil || location == nil {
+		return false
+	}
+	if strings.HasPrefix(location.Path, "/cdn-cgi/access/login") {
+		return true
+	}
+
+	return false
+}
+
+// createAccessAuthenticatedStream will try load a token from storage and make
+// a connection with the token set on the request. If it still get redirect,
+// this probably means the token in storage is invalid (expired/revoked). If that
+// happens it deletes the token and runs the connection again, so the user can
+// login again and generate a new one.
+func createAccessAuthenticatedStream(options *StartOptions) (*websocket.Conn, error) {
+	wsConn, resp, err := createAccessWebSocketStream(options)
+	defer closeRespBody(resp)
+	if err == nil {
+		return wsConn, nil
+	}
+
+	if !IsAccessResponse(resp) {
+		return nil, err
+	}
+
+	// Access Token is invalid for some reason. Go through regen flow
+	originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	if err := token.RemoveTokenIfExists(originReq.URL); err != nil {
+		return nil, err
+	}
+	wsConn, resp, err = createAccessWebSocketStream(options)
+	defer closeRespBody(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	return wsConn, nil
+}
+
+// createAccessWebSocketStream builds an Access request and makes a connection
+func createAccessWebSocketStream(options *StartOptions) (*websocket.Conn, *http.Response, error) {
+	req, err := BuildAccessRequest(options)
+	if err != nil {
+		return nil, nil, err
+	}
+	return cloudflaredWebsocket.ClientConnect(req, nil)
+}
+
+// buildAccessRequest builds an HTTP request with the Access token set
+func BuildAccessRequest(options *StartOptions) (*http.Request, error) {
+	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	token, err := token.FetchToken(req.URL)
+	if err != nil {
+		return nil, err
+	}
+
+	// We need to create a new request as FetchToken will modify req (boo mutable)
+	// as it has to follow redirect on the API and such, so here we init a new one
+	originRequest, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	originRequest.Header.Set("cf-access-token", token)
+
+	for k, v := range options.Headers {
+		if len(v) >= 1 {
+			originRequest.Header.Set(k, v[0])
+		}
+	}
+
+	return originRequest, nil
+}

carrier/carrier_test.go

@@ -0,0 +1,155 @@
+package carrier
+
+import (
+	"bytes"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+
+	ws "github.com/gorilla/websocket"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	// example in Sec-Websocket-Key in rfc6455
+	testSecWebsocketKey = "dGhlIHNhbXBsZSBub25jZQ=="
+)
+
+type testStreamer struct {
+	buf *bytes.Buffer
+	l   sync.RWMutex
+}
+
+func newTestStream() *testStreamer {
+	return &testStreamer{buf: new(bytes.Buffer)}
+}
+
+func (s *testStreamer) Read(p []byte) (int, error) {
+	s.l.RLock()
+	defer s.l.RUnlock()
+	return s.buf.Read(p)
+
+}
+
+func (s *testStreamer) Write(p []byte) (int, error) {
+	s.l.Lock()
+	defer s.l.Unlock()
+	return s.buf.Write(p)
+}
+
+func TestStartClient(t *testing.T) {
+	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
+	logger := logrus.New()
+	ts := newTestWebSocketServer()
+	defer ts.Close()
+
+	buf := newTestStream()
+	options := &StartOptions{
+		OriginURL: "http://" + ts.Listener.Addr().String(),
+		Headers:   nil,
+	}
+	err := StartClient(logger, buf, options)
+	assert.NoError(t, err)
+	buf.Write([]byte(message))
+
+	readBuffer := make([]byte, len(message))
+	buf.Read(readBuffer)
+	assert.Equal(t, message, string(readBuffer))
+}
+
+func TestStartServer(t *testing.T) {
+	listener, err := net.Listen("tcp", "localhost:")
+	if err != nil {
+		t.Fatalf("Error starting listener: %v", err)
+	}
+	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
+	logger := logrus.New()
+	shutdownC := make(chan struct{})
+	ts := newTestWebSocketServer()
+	defer ts.Close()
+	options := &StartOptions{
+		OriginURL: "http://" + ts.Listener.Addr().String(),
+		Headers:   nil,
+	}
+
+	go func() {
+		err := Serve(logger, listener, shutdownC, options)
+		if err != nil {
+			t.Fatalf("Error running server: %v", err)
+		}
+	}()
+
+	conn, err := net.Dial("tcp", listener.Addr().String())
+	conn.Write([]byte(message))
+
+	readBuffer := make([]byte, len(message))
+	conn.Read(readBuffer)
+	assert.Equal(t, string(readBuffer), message)
+}
+
+func TestIsAccessResponse(t *testing.T) {
+	validLocationHeader := http.Header{}
+	validLocationHeader.Add("location", "https://test.cloudflareaccess.com/cdn-cgi/access/login/blahblah")
+	invalidLocationHeader := http.Header{}
+	invalidLocationHeader.Add("location", "https://google.com")
+	testCases := []struct {
+		Description string
+		In          *http.Response
+		ExpectedOut bool
+	}{
+		{"nil response", nil, false},
+		{"redirect with no location", &http.Response{StatusCode: http.StatusFound}, false},
+		{"200 ok", &http.Response{StatusCode: http.StatusOK}, false},
+		{"redirect with location", &http.Response{StatusCode: http.StatusFound, Header: validLocationHeader}, true},
+		{"redirect with invalid location", &http.Response{StatusCode: http.StatusFound, Header: invalidLocationHeader}, false},
+	}
+
+	for i, tc := range testCases {
+		if IsAccessResponse(tc.In) != tc.ExpectedOut {
+			t.Fatalf("Failed case %d -- %s", i, tc.Description)
+		}
+	}
+
+}
+
+func newTestWebSocketServer() *httptest.Server {
+	upgrader := ws.Upgrader{
+		ReadBufferSize:  1024,
+		WriteBufferSize: 1024,
+	}
+
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		conn, _ := upgrader.Upgrade(w, r, nil)
+		defer conn.Close()
+		for {
+			mt, message, err := conn.ReadMessage()
+			if err != nil {
+				break
+			}
+
+			if err := conn.WriteMessage(mt, []byte(message)); err != nil {
+				break
+			}
+		}
+	}))
+}
+
+func testRequest(t *testing.T, url string, stream io.ReadWriter) *http.Request {
+	req, err := http.NewRequest("GET", url, stream)
+	if err != nil {
+		t.Fatalf("testRequestHeader error")
+	}
+
+	req.Header.Add("Connection", "Upgrade")
+	req.Header.Add("Upgrade", "WebSocket")
+	req.Header.Add("Sec-Websocket-Key", testSecWebsocketKey)
+	req.Header.Add("Sec-Websocket-Protocol", "tunnel-protocol")
+	req.Header.Add("Sec-Websocket-Version", "13")
+	req.Header.Add("User-Agent", "curl/7.59.0")
+
+	return req
+}

cfsetup.yaml

@@ -1,13 +1,13 @@
-pinned_go: &pinned_go go=1.9.3-1
-build_dir: &build_dir /cfsetup_build/src/github.com/cloudflare/cloudflared/
-stretch:
+pinned_go: &pinned_go go=1.12.7-1
+build_dir: &build_dir /cfsetup_build
+default-flavor: stretch
+stretch: &stretch
   build:
     build_dir: *build_dir
     builddeps:
       - *pinned_go
       - build-essential
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared
@@ -19,7 +19,6 @@ stretch:
       - fakeroot
       - rubygem-fpm
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared-deb
@@ -29,7 +28,6 @@ stretch:
       - *pinned_go
       - build-essential
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=amd64
       - make release
@@ -40,28 +38,25 @@ stretch:
       - crossbuild-essential-armhf
       - gcc-arm-linux-gnueabihf
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=arm
       - export CC=arm-linux-gnueabihf-gcc
       - make release
   release-linux-386:
-      build_dir: *build_dir
-      builddeps:
-        - *pinned_go
-        - gcc-multilib
-      post-cache:
-        - export GOPATH=/cfsetup_build/
-        - export GOOS=linux
-        - export GOARCH=386
-        - make release
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - gcc-multilib
+    post-cache:
+      - export GOOS=linux
+      - export GOARCH=386
+      - make release
   release-windows-amd64:
     build_dir: *build_dir
     builddeps:
       - *pinned_go
       - gcc-mingw-w64
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=windows
       - export GOARCH=amd64
       - export CC=x86_64-w64-mingw32-gcc
@@ -72,7 +67,6 @@ stretch:
       - *pinned_go
       - gcc-mingw-w64
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=windows
       - export GOARCH=386
       - export CC=i686-w64-mingw32-gcc-win32
@@ -83,7 +77,22 @@ stretch:
       - *pinned_go
       - build-essential
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=amd64
+      # cd to a non-module directory: https://github.com/golang/go/issues/24250
+      - (cd / && go get github.com/BurntSushi/go-sumtype)
+      - export PATH="$HOME/go/bin:$PATH"
       - make test
+
+jessie: *stretch
+
+# cfsetup compose
+default-stack: test_dbconnect
+test_dbconnect:
+  compose:
+    up-args:
+      - --renew-anon-volumes
+      - --abort-on-container-exit
+      - --exit-code-from=cloudflared
+    files:
+      - dbconnect_tests/dbconnect.yaml

cmd/cloudflared/access/carrier.go

@@ -0,0 +1,73 @@
+package access
+
+import (
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/carrier"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/validation"
+	"github.com/pkg/errors"
+	cli "gopkg.in/urfave/cli.v2"
+)
+
+// ssh will start a WS proxy server for server mode
+// or copy from stdin/stdout for client mode
+// useful for proxying other protocols (like ssh) over websockets
+// (which you can put Access in front of)
+func ssh(c *cli.Context) error {
+	// get the hostname from the cmdline and error out if its not provided
+	rawHostName := c.String(sshHostnameFlag)
+	hostname, err := validation.ValidateHostname(rawHostName)
+	if err != nil || rawHostName == "" {
+		return cli.ShowCommandHelp(c, "ssh")
+	}
+	originURL := ensureURLScheme(hostname)
+
+	// get the headers from the cmdline and add them
+	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
+	if c.IsSet(sshTokenIDFlag) {
+		headers.Add("CF-Access-Client-Id", c.String(sshTokenIDFlag))
+	}
+	if c.IsSet(sshTokenSecretFlag) {
+		headers.Add("CF-Access-Client-Secret", c.String(sshTokenSecretFlag))
+	}
+
+	destination := c.String(sshDestinationFlag)
+	if destination != "" {
+		headers.Add("CF-Access-SSH-Destination", destination)
+	}
+
+	options := &carrier.StartOptions{
+		OriginURL: originURL,
+		Headers:   headers,
+	}
+
+	if c.NArg() > 0 || c.IsSet(sshURLFlag) {
+		localForwarder, err := config.ValidateUrl(c)
+		if err != nil {
+			logger.WithError(err).Error("Error validating origin URL")
+			return errors.Wrap(err, "error validating origin URL")
+		}
+		forwarder, err := url.Parse(localForwarder)
+		if err != nil {
+			logger.WithError(err).Error("Error validating origin URL")
+			return errors.Wrap(err, "error validating origin URL")
+		}
+		return carrier.StartServer(logger, forwarder.Host, shutdownC, options)
+	}
+
+	return carrier.StartClient(logger, &carrier.StdinoutStream{}, options)
+}
+
+func buildRequestHeaders(values []string) http.Header {
+	headers := make(http.Header)
+	for _, valuePair := range values {
+		split := strings.Split(valuePair, ":")
+		if len(split) > 1 {
+			headers.Add(strings.TrimSpace(split[0]), strings.TrimSpace(split[1]))
+		}
+	}
+	return headers
+}

cmd/cloudflared/access/carrier_test.go

@@ -0,0 +1,18 @@
+package access
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestBuildRequestHeaders(t *testing.T) {
+	headers := make(http.Header)
+	headers.Add("client", "value")
+	headers.Add("secret", "safe-value")
+
+	values := buildRequestHeaders([]string{"client: value", "secret: safe-value", "trash"})
+	assert.Equal(t, headers.Get("client"), values.Get("client"))
+	assert.Equal(t, headers.Get("secret"), values.Get("secret"))
+}

cmd/cloudflared/access/cmd.go

@@ -0,0 +1,454 @@
+package access
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"text/template"
+	"time"
+
+	"github.com/cloudflare/cloudflared/carrier"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
+	"github.com/cloudflare/cloudflared/sshgen"
+	"github.com/cloudflare/cloudflared/validation"
+	"github.com/pkg/errors"
+	"golang.org/x/net/idna"
+
+	"github.com/cloudflare/cloudflared/log"
+	raven "github.com/getsentry/raven-go"
+	cli "gopkg.in/urfave/cli.v2"
+)
+
+const (
+	sshHostnameFlag    = "hostname"
+	sshDestinationFlag = "destination"
+	sshURLFlag         = "url"
+	sshHeaderFlag      = "header"
+	sshTokenIDFlag     = "service-token-id"
+	sshTokenSecretFlag = "service-token-secret"
+	sshGenCertFlag     = "short-lived-cert"
+	sshConfigTemplate  = `
+Add to your {{.Home}}/.ssh/config:
+
+Host {{.Hostname}}
+{{- if .ShortLivedCerts}}
+  ProxyCommand bash -c '{{.Cloudflared}} access ssh-gen --hostname %h; ssh -tt %r@cfpipe-{{.Hostname}} >&2 <&1' 
+
+Host cfpipe-{{.Hostname}}
+  HostName {{.Hostname}}
+  ProxyCommand {{.Cloudflared}} access ssh --hostname %h
+  IdentityFile ~/.cloudflared/{{.Hostname}}-cf_key
+  CertificateFile ~/.cloudflared/{{.Hostname}}-cf_key-cert.pub
+{{- else}}
+  ProxyCommand {{.Cloudflared}} access ssh --hostname %h
+{{end}}
+`
+)
+
+const sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b@sentry.io/189878"
+
+var (
+	logger         = log.CreateLogger()
+	shutdownC      chan struct{}
+	graceShutdownC chan struct{}
+)
+
+// Init will initialize and store vars from the main program
+func Init(s, g chan struct{}) {
+	shutdownC, graceShutdownC = s, g
+}
+
+// Flags return the global flags for Access related commands (hopefully none)
+func Flags() []cli.Flag {
+	return []cli.Flag{} // no flags yet.
+}
+
+// Commands returns all the Access related subcommands
+func Commands() []*cli.Command {
+	return []*cli.Command{
+		{
+			Name:     "access",
+			Category: "Access (BETA)",
+			Usage:    "access <subcommand>",
+			Description: `(BETA) Cloudflare Access protects internal resources by securing, authenticating and monitoring access 
+			per-user and by application. With Cloudflare Access, only authenticated users with the required permissions are 
+			able to reach sensitive resources. The commands provided here allow you to interact with Access protected 
+			applications from the command line. This feature is considered beta. Your feedback is greatly appreciated!
+			https://cfl.re/CLIAuthBeta`,
+			Subcommands: []*cli.Command{
+				{
+					Name:   "login",
+					Action: login,
+					Usage:  "login <url of access application>",
+					Description: `The login subcommand initiates an authentication flow with your identity provider.
+					The subcommand will launch a browser. For headless systems, a url is provided.
+					Once authenticated with your identity provider, the login command will generate a JSON Web Token (JWT)
+					scoped to your identity, the application you intend to reach, and valid for a session duration set by your 
+					administrator. cloudflared stores the token in local storage.`,
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:   "url",
+							Hidden: true,
+						},
+					},
+				},
+				{
+					Name:   "curl",
+					Action: curl,
+					Usage:  "curl [--allow-request, -ar] <url> [<curl args>...]",
+					Description: `The curl subcommand wraps curl and automatically injects the JWT into a cf-access-token
+					header when using curl to reach an application behind Access.`,
+					ArgsUsage:       "allow-request will allow the curl request to continue even if the jwt is not present.",
+					SkipFlagParsing: true,
+				},
+				{
+					Name:        "token",
+					Action:      generateToken,
+					Usage:       "token -app=<url of access application>",
+					ArgsUsage:   "url of Access application",
+					Description: `The token subcommand produces a JWT which can be used to authenticate requests.`,
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name: "app",
+						},
+					},
+				},
+				{
+					Name:        "ssh",
+					Action:      ssh,
+					Aliases:     []string{"rdp"},
+					Usage:       "",
+					ArgsUsage:   "",
+					Description: `The ssh subcommand sends data over a proxy to the Cloudflare edge.`,
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:  sshHostnameFlag,
+							Usage: "specify the hostname of your application.",
+						},
+						&cli.StringFlag{
+							Name:  sshDestinationFlag,
+							Usage: "specify the destination address of your SSH server.",
+						},
+						&cli.StringFlag{
+							Name:  sshURLFlag,
+							Usage: "specify the host:port to forward data to Cloudflare edge.",
+						},
+						&cli.StringSliceFlag{
+							Name:    sshHeaderFlag,
+							Aliases: []string{"H"},
+							Usage:   "specify additional headers you wish to send.",
+						},
+						&cli.StringSliceFlag{
+							Name:    sshTokenIDFlag,
+							Aliases: []string{"id"},
+							Usage:   "specify an Access service token ID you wish to use.",
+						},
+						&cli.StringSliceFlag{
+							Name:    sshTokenSecretFlag,
+							Aliases: []string{"secret"},
+							Usage:   "specify an Access service token secret you wish to use.",
+						},
+					},
+				},
+				{
+					Name:        "ssh-config",
+					Action:      sshConfig,
+					Usage:       "",
+					Description: `Prints an example configuration ~/.ssh/config`,
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:  sshHostnameFlag,
+							Usage: "specify the hostname of your application.",
+						},
+						&cli.BoolFlag{
+							Name:  sshGenCertFlag,
+							Usage: "specify if you wish to generate short lived certs.",
+						},
+					},
+				},
+				{
+					Name:        "ssh-gen",
+					Action:      sshGen,
+					Usage:       "",
+					Description: `Generates a short lived certificate for given hostname`,
+					Flags: []cli.Flag{
+						&cli.StringFlag{
+							Name:  sshHostnameFlag,
+							Usage: "specify the hostname of your application.",
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+// login pops up the browser window to do the actual login and JWT generation
+func login(c *cli.Context) error {
+	raven.SetDSN(sentryDSN)
+	logger := log.CreateLogger()
+	args := c.Args()
+	rawURL := ensureURLScheme(args.First())
+	appURL, err := url.Parse(rawURL)
+	if args.Len() < 1 || err != nil {
+		logger.Errorf("Please provide the url of the Access application\n")
+		return err
+	}
+	if err := verifyTokenAtEdge(appURL, c); err != nil {
+		logger.WithError(err).Error("Could not verify token")
+		return err
+	}
+
+	token, err := token.GetTokenIfExists(appURL)
+	if err != nil || token == "" {
+		fmt.Fprintln(os.Stderr, "Unable to find token for provided application.")
+		return err
+	}
+	fmt.Fprintf(os.Stdout, "Successfully fetched your token:\n\n%s\n\n", string(token))
+
+	return nil
+}
+
+// ensureURLScheme prepends a URL with https:// if it doesnt have a scheme. http:// URLs will not be converted.
+func ensureURLScheme(url string) string {
+	url = strings.Replace(strings.ToLower(url), "http://", "https://", 1)
+	if !strings.HasPrefix(url, "https://") {
+		url = fmt.Sprintf("https://%s", url)
+
+	}
+	return url
+}
+
+// curl provides a wrapper around curl, passing Access JWT along in request
+func curl(c *cli.Context) error {
+	raven.SetDSN(sentryDSN)
+	logger := log.CreateLogger()
+	args := c.Args()
+	if args.Len() < 1 {
+		logger.Error("Please provide the access app and command you wish to run.")
+		return errors.New("incorrect args")
+	}
+
+	cmdArgs, allowRequest := parseAllowRequest(args.Slice())
+	appURL, err := getAppURL(cmdArgs)
+	if err != nil {
+		return err
+	}
+
+	tok, err := token.GetTokenIfExists(appURL)
+	if err != nil || tok == "" {
+		if allowRequest {
+			logger.Warn("You don't have an Access token set. Please run access token <access application> to fetch one.")
+			return shell.Run("curl", cmdArgs...)
+		}
+		tok, err = token.FetchToken(appURL)
+		if err != nil {
+			logger.Error("Failed to refresh token: ", err)
+			return err
+		}
+	}
+
+	cmdArgs = append(cmdArgs, "-H")
+	cmdArgs = append(cmdArgs, fmt.Sprintf("cf-access-token: %s", tok))
+	return shell.Run("curl", cmdArgs...)
+}
+
+// token dumps provided token to stdout
+func generateToken(c *cli.Context) error {
+	raven.SetDSN(sentryDSN)
+	appURL, err := url.Parse(c.String("app"))
+	if err != nil || c.NumFlags() < 1 {
+		fmt.Fprintln(os.Stderr, "Please provide a url.")
+		return err
+	}
+	tok, err := token.GetTokenIfExists(appURL)
+	if err != nil || tok == "" {
+		fmt.Fprintln(os.Stderr, "Unable to find token for provided application. Please run token command to generate token.")
+		return err
+	}
+
+	if _, err := fmt.Fprint(os.Stdout, tok); err != nil {
+		fmt.Fprintln(os.Stderr, "Failed to write token to stdout.")
+		return err
+	}
+	return nil
+}
+
+// sshConfig prints an example SSH config to stdout
+func sshConfig(c *cli.Context) error {
+	genCertBool := c.Bool(sshGenCertFlag)
+	hostname := c.String(sshHostnameFlag)
+	if hostname == "" {
+		hostname = "[your hostname]"
+	}
+
+	type config struct {
+		Home            string
+		ShortLivedCerts bool
+		Hostname        string
+		Cloudflared     string
+	}
+
+	t := template.Must(template.New("sshConfig").Parse(sshConfigTemplate))
+	return t.Execute(os.Stdout, config{Home: os.Getenv("HOME"), ShortLivedCerts: genCertBool, Hostname: hostname, Cloudflared: cloudflaredPath()})
+}
+
+// sshGen generates a short lived certificate for provided hostname
+func sshGen(c *cli.Context) error {
+	// get the hostname from the cmdline and error out if its not provided
+	rawHostName := c.String(sshHostnameFlag)
+	hostname, err := validation.ValidateHostname(rawHostName)
+	if err != nil || rawHostName == "" {
+		return cli.ShowCommandHelp(c, "ssh-gen")
+	}
+
+	originURL, err := url.Parse(ensureURLScheme(hostname))
+	if err != nil {
+		return err
+	}
+
+	// this fetchToken function mutates the appURL param. We should refactor that
+	fetchTokenURL := &url.URL{}
+	*fetchTokenURL = *originURL
+	token, err := token.FetchToken(fetchTokenURL)
+	if err != nil {
+		return err
+	}
+
+	if err := sshgen.GenerateShortLivedCertificate(originURL, token); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// getAppURL will pull the appURL needed for fetching a user's Access token
+func getAppURL(cmdArgs []string) (*url.URL, error) {
+	if len(cmdArgs) < 1 {
+		logger.Error("Please provide a valid URL as the first argument to curl.")
+		return nil, errors.New("not a valid url")
+	}
+
+	u, err := processURL(cmdArgs[0])
+	if err != nil {
+		logger.Error("Please provide a valid URL as the first argument to curl.")
+		return nil, err
+	}
+
+	return u, err
+}
+
+// parseAllowRequest will parse cmdArgs and return a copy of the args and result
+// of the allow request was present
+func parseAllowRequest(cmdArgs []string) ([]string, bool) {
+	if len(cmdArgs) > 1 {
+		if cmdArgs[0] == "--allow-request" || cmdArgs[0] == "-ar" {
+			return cmdArgs[1:], true
+		}
+	}
+
+	return cmdArgs, false
+}
+
+// processURL will preprocess the string (parse to a url, convert to punycode, etc).
+func processURL(s string) (*url.URL, error) {
+	u, err := url.ParseRequestURI(s)
+	if err != nil {
+		return nil, err
+	}
+
+	if u.Host == "" {
+		return nil, errors.New("not a valid host")
+	}
+
+	host, err := idna.ToASCII(u.Hostname())
+	if err != nil { // we fail to convert to punycode, just return the url we parsed.
+		return u, nil
+	}
+	if u.Port() != "" {
+		u.Host = fmt.Sprintf("%s:%s", host, u.Port())
+	} else {
+		u.Host = host
+	}
+
+	return u, nil
+}
+
+// cloudflaredPath pulls the full path of cloudflared on disk
+func cloudflaredPath() string {
+	for _, p := range strings.Split(os.Getenv("PATH"), ":") {
+		path := fmt.Sprintf("%s/%s", p, "cloudflared")
+		if isFileThere(path) {
+			return path
+		}
+	}
+	return "cloudflared"
+}
+
+// isFileThere will check for the presence of candidate path
+func isFileThere(candidate string) bool {
+	fi, err := os.Stat(candidate)
+	if err != nil || fi.IsDir() || !fi.Mode().IsRegular() {
+		return false
+	}
+	return true
+}
+
+// verifyTokenAtEdge checks for a token on disk, or generates a new one.
+// Then makes a request to to the origin with the token to ensure it is valid.
+// Returns nil if token is valid.
+func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context) error {
+	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
+	if c.IsSet(sshTokenIDFlag) {
+		headers.Add("CF-Access-Client-Id", c.String(sshTokenIDFlag))
+	}
+	if c.IsSet(sshTokenSecretFlag) {
+		headers.Add("CF-Access-Client-Secret", c.String(sshTokenSecretFlag))
+	}
+	options := &carrier.StartOptions{OriginURL: appUrl.String(), Headers: headers}
+
+	if valid, err := isTokenValid(options); err != nil {
+		return err
+	} else if valid {
+		return nil
+	}
+
+	if err := token.RemoveTokenIfExists(appUrl); err != nil {
+		return err
+	}
+
+	if valid, err := isTokenValid(options); err != nil {
+		return err
+	} else if !valid {
+		return errors.New("failed to verify token")
+	}
+
+	return nil
+}
+
+// isTokenValid makes a request to the origin and returns true if the response was not a 302.
+func isTokenValid(options *carrier.StartOptions) (bool, error) {
+	req, err := carrier.BuildAccessRequest(options)
+	if err != nil {
+		return false, errors.Wrap(err, "Could not create access request")
+	}
+
+	// Do not follow redirects
+	client := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+		Timeout: time.Second * 5,
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer resp.Body.Close()
+
+	// A redirect to login means the token was invalid.
+	return !carrier.IsAccessResponse(resp), nil
+}

cmd/cloudflared/access/cmd_test.go

@@ -0,0 +1,25 @@
+package access
+
+import "testing"
+
+func Test_ensureURLScheme(t *testing.T) {
+	type args struct {
+		url string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{"no scheme", args{"localhost:123"}, "https://localhost:123"},
+		{"http scheme", args{"http://test"}, "https://test"},
+		{"https scheme", args{"https://test"}, "https://test"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := ensureURLScheme(tt.args.url); got != tt.want {
+				t.Errorf("ensureURLScheme() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/cloudflared/buildinfo/build_info.go

@@ -0,0 +1,28 @@
+package buildinfo
+
+import (
+	"runtime"
+
+	"github.com/sirupsen/logrus"
+)
+
+type BuildInfo struct {
+	GoOS               string `json:"go_os"`
+	GoVersion          string `json:"go_version"`
+	GoArch             string `json:"go_arch"`
+	CloudflaredVersion string `json:"cloudflared_version"`
+}
+
+func GetBuildInfo(cloudflaredVersion string) *BuildInfo {
+	return &BuildInfo{
+		GoOS:               runtime.GOOS,
+		GoVersion:          runtime.Version(),
+		GoArch:             runtime.GOARCH,
+		CloudflaredVersion: cloudflaredVersion,
+	}
+}
+
+func (bi *BuildInfo) Log(logger *logrus.Logger) {
+	logger.Infof("Version %s", bi.CloudflaredVersion)
+	logger.Infof("GOOS: %s, GOVersion: %s, GoArch: %s", bi.GoOS, bi.GoVersion, bi.GoArch)
+}

cmd/cloudflared/config/configuration.go

@@ -0,0 +1,87 @@
+package config
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+
+	"github.com/cloudflare/cloudflared/validation"
+	homedir "github.com/mitchellh/go-homedir"
+	"gopkg.in/urfave/cli.v2"
+	"gopkg.in/urfave/cli.v2/altsrc"
+)
+
+var (
+	// File names from which we attempt to read configuration.
+	DefaultConfigFiles = []string{"config.yml", "config.yaml"}
+
+	// Launchd doesn't set root env variables, so there is default
+	// Windows default config dir was ~/cloudflare-warp in documentation; let's keep it compatible
+	DefaultConfigDirs = []string{"~/.cloudflared", "~/.cloudflare-warp", "~/cloudflare-warp", "/usr/local/etc/cloudflared", "/etc/cloudflared"}
+)
+
+const DefaultCredentialFile = "cert.pem"
+
+// FileExists checks to see if a file exist at the provided path.
+func FileExists(path string) (bool, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// ignore missing files
+			return false, nil
+		}
+		return false, err
+	}
+	f.Close()
+	return true, nil
+}
+
+// FindInputSourceContext pulls the input source from the config flag.
+func FindInputSourceContext(context *cli.Context) (altsrc.InputSourceContext, error) {
+	if context.String("config") != "" {
+		return altsrc.NewYamlSourceFromFile(context.String("config"))
+	}
+	return nil, nil
+}
+
+// FindDefaultConfigPath returns the first path that contains a config file.
+// If none of the combination of DefaultConfigDirs and DefaultConfigFiles
+// contains a config file, return empty string.
+func FindDefaultConfigPath() string {
+	for _, configDir := range DefaultConfigDirs {
+		for _, configFile := range DefaultConfigFiles {
+			dirPath, err := homedir.Expand(configDir)
+			if err != nil {
+				continue
+			}
+			path := filepath.Join(dirPath, configFile)
+			if ok, _ := FileExists(path); ok {
+				return path
+			}
+		}
+	}
+	return ""
+}
+
+// ValidateUnixSocket ensures --unix-socket param is used exclusively
+// i.e. it fails if a user specifies both --url and --unix-socket
+func ValidateUnixSocket(c *cli.Context) (string, error) {
+	if c.IsSet("unix-socket") && (c.IsSet("url") || c.NArg() > 0) {
+		return "", errors.New("--unix-socket must be used exclusivly.")
+	}
+	return c.String("unix-socket"), nil
+}
+
+// ValidateUrl will validate url flag correctness. It can be either from --url or argument
+// Notice ValidateUnixSocket, it will enforce --unix-socket is not used with --url or argument
+func ValidateUrl(c *cli.Context) (string, error) {
+	var url = c.String("url")
+	if c.NArg() > 0 {
+		if c.IsSet("url") {
+			return "", errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
+		}
+		url = c.Args().Get(0)
+	}
+	validUrl, err := validation.ValidateUrl(url)
+	return validUrl, err
+}

cmd/cloudflared/configuration.go

@@ -1,318 +0,0 @@
-package main
-
-import (
-	"crypto/tls"
-	"crypto/x509"
-	"encoding/hex"
-	"fmt"
-	"io/ioutil"
-	"math/rand"
-	"net"
-	"net/http"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strings"
-	"time"
-
-	"github.com/cloudflare/cloudflared/origin"
-	"github.com/cloudflare/cloudflared/tlsconfig"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/cloudflare/cloudflared/validation"
-
-	"github.com/sirupsen/logrus"
-	"gopkg.in/urfave/cli.v2"
-	"gopkg.in/urfave/cli.v2/altsrc"
-
-	"github.com/mitchellh/go-homedir"
-	"github.com/pkg/errors"
-)
-
-var (
-	defaultConfigFiles = []string{"config.yml", "config.yaml"}
-
-	// Launchd doesn't set root env variables, so there is default
-	// Windows default config dir was ~/cloudflare-warp in documentation; let's keep it compatible
-	defaultConfigDirs = []string{"~/.cloudflared", "~/.cloudflare-warp", "~/cloudflare-warp", "/usr/local/etc/cloudflared", "/etc/cloudflared"}
-)
-
-const defaultCredentialFile = "cert.pem"
-
-func fileExists(path string) (bool, error) {
-	f, err := os.Open(path)
-	if err != nil {
-		if os.IsNotExist(err) {
-			// ignore missing files
-			return false, nil
-		}
-		return false, err
-	}
-	f.Close()
-	return true, nil
-}
-
-// returns the first path that contains a cert.pem file. If none of the defaultConfigDirs
-// (differs by OS for legacy reasons) contains a cert.pem file, return empty string
-func findDefaultOriginCertPath() string {
-	for _, defaultConfigDir := range defaultConfigDirs {
-		originCertPath, _ := homedir.Expand(filepath.Join(defaultConfigDir, defaultCredentialFile))
-		if ok, _ := fileExists(originCertPath); ok {
-			return originCertPath
-		}
-	}
-	return ""
-}
-
-// returns the first path that contains a config file. If none of the combination of
-// defaultConfigDirs (differs by OS for legacy reasons) and defaultConfigFiles
-// contains a config file, return empty string
-func findDefaultConfigPath() string {
-	for _, configDir := range defaultConfigDirs {
-		for _, configFile := range defaultConfigFiles {
-			dirPath, err := homedir.Expand(configDir)
-			if err != nil {
-				continue
-			}
-			path := filepath.Join(dirPath, configFile)
-			if ok, _ := fileExists(path); ok {
-				return path
-			}
-		}
-	}
-	return ""
-}
-
-func findInputSourceContext(context *cli.Context) (altsrc.InputSourceContext, error) {
-	if context.String("config") != "" {
-		return altsrc.NewYamlSourceFromFile(context.String("config"))
-	}
-	return nil, nil
-}
-
-func generateRandomClientID() string {
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	id := make([]byte, 32)
-	r.Read(id)
-	return hex.EncodeToString(id)
-}
-
-func enoughOptionsSet(c *cli.Context) bool {
-	// For cloudflared to work, the user needs to at least provide a hostname,
-	// or runs as stand alone DNS proxy .
-	// When using sudo, use -E flag to preserve env vars
-	if c.NumFlags() == 0 && c.NArg() == 0 && os.Getenv("TUNNEL_HOSTNAME") == "" && os.Getenv("TUNNEL_DNS") == "" {
-		if isRunningFromTerminal() {
-			logger.Errorf("No arguments were provided. You need to at least specify the hostname for this tunnel. See %s", quickStartUrl)
-			logger.Infof("If you want to run Argo Tunnel client as a stand alone DNS proxy, run with --proxy-dns option or set TUNNEL_DNS environment variable.")
-		} else {
-			logger.Errorf("You need to specify all the options in a configuration file, or use environment variables. See %s and %s", serviceUrl, argumentsUrl)
-			logger.Infof("If you want to run Argo Tunnel client as a stand alone DNS proxy, specify proxy-dns option in the configuration file, or set TUNNEL_DNS environment variable.")
-		}
-		cli.ShowAppHelp(c)
-		return false
-	}
-	return true
-}
-
-func handleDeprecatedOptions(c *cli.Context) error {
-	// Fail if the user provided an old authentication method
-	if c.IsSet("api-key") || c.IsSet("api-email") || c.IsSet("api-ca-key") {
-		logger.Error("You don't need to give us your api-key anymore. Please use the new login method. Just run cloudflared login")
-		return fmt.Errorf("Client provided deprecated options")
-	}
-	return nil
-}
-
-func validateUrl(c *cli.Context) (string, error) {
-	var url = c.String("url")
-	if c.NArg() > 0 {
-		if !c.IsSet("url") {
-			return "", errors.New("Please specify an origin URL.")
-		}
-	}
-	validUrl, err := validation.ValidateUrl(url)
-	return validUrl, err
-}
-
-func logClientOptions(c *cli.Context) {
-	flags := make(map[string]interface{})
-	for _, flag := range c.LocalFlagNames() {
-		flags[flag] = c.Generic(flag)
-	}
-	if len(flags) > 0 {
-		logger.Infof("Flags %v", flags)
-	}
-
-	envs := make(map[string]string)
-	// Find env variables for Argo Tunnel
-	for _, env := range os.Environ() {
-		// All Argo Tunnel env variables start with TUNNEL_
-		if strings.Contains(env, "TUNNEL_") {
-			vars := strings.Split(env, "=")
-			if len(vars) == 2 {
-				envs[vars[0]] = vars[1]
-			}
-		}
-	}
-	if len(envs) > 0 {
-		logger.Infof("Environmental variables %v", envs)
-	}
-}
-
-func dnsProxyStandAlone(c *cli.Context) bool {
-	return c.IsSet("proxy-dns") && (!c.IsSet("hostname") && !c.IsSet("tag") && !c.IsSet("hello-world"))
-}
-
-func getOriginCert(c *cli.Context) ([]byte, error) {
-	if c.String("origincert") == "" {
-		logger.Warnf("Cannot determine default origin certificate path. No file %s in %v", defaultCredentialFile, defaultConfigDirs)
-		if isRunningFromTerminal() {
-			logger.Errorf("You need to specify the origin certificate path with --origincert option, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", argumentsUrl)
-			return nil, fmt.Errorf("Client didn't specify origincert path when running from terminal")
-		} else {
-			logger.Errorf("You need to specify the origin certificate path by specifying the origincert option in the configuration file, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", serviceUrl)
-			return nil, fmt.Errorf("Client didn't specify origincert path")
-		}
-	}
-	// Check that the user has acquired a certificate using the login command
-	originCertPath, err := homedir.Expand(c.String("origincert"))
-	if err != nil {
-		logger.WithError(err).Errorf("Cannot resolve path %s", c.String("origincert"))
-		return nil, fmt.Errorf("Cannot resolve path %s", c.String("origincert"))
-	}
-	ok, err := fileExists(originCertPath)
-	if err != nil {
-		logger.Errorf("Cannot check if origin cert exists at path %s", c.String("origincert"))
-		return nil, fmt.Errorf("Cannot check if origin cert exists at path %s", c.String("origincert"))
-	}
-	if !ok {
-		logger.Errorf(`Cannot find a valid certificate for your origin at the path:
-
-    %s
-
-If the path above is wrong, specify the path with the -origincert option.
-If you don't have a certificate signed by Cloudflare, run the command:
-
-	%s login
-`, originCertPath, os.Args[0])
-		return nil, fmt.Errorf("Cannot find a valid certificate at the path %s", originCertPath)
-	}
-	// Easier to send the certificate as []byte via RPC than decoding it at this point
-	originCert, err := ioutil.ReadFile(originCertPath)
-	if err != nil {
-		logger.WithError(err).Errorf("Cannot read %s to load origin certificate", originCertPath)
-		return nil, fmt.Errorf("Cannot read %s to load origin certificate", originCertPath)
-	}
-	return originCert, nil
-}
-
-func prepareTunnelConfig(c *cli.Context, buildInfo *origin.BuildInfo, logger, protoLogger *logrus.Logger) (*origin.TunnelConfig, error) {
-	hostname, err := validation.ValidateHostname(c.String("hostname"))
-	if err != nil {
-		logger.WithError(err).Error("Invalid hostname")
-		return nil, errors.Wrap(err, "Invalid hostname")
-	}
-	clientID := c.String("id")
-	if !c.IsSet("id") {
-		clientID = generateRandomClientID()
-	}
-
-	tags, err := NewTagSliceFromCLI(c.StringSlice("tag"))
-	if err != nil {
-		logger.WithError(err).Error("Tag parse failure")
-		return nil, errors.Wrap(err, "Tag parse failure")
-	}
-
-	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
-
-	url, err := validateUrl(c)
-	if err != nil {
-		logger.WithError(err).Error("Error validating url")
-		return nil, errors.Wrap(err, "Error validating url")
-	}
-	logger.Infof("Proxying tunnel requests to %s", url)
-
-	originCert, err := getOriginCert(c)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error getting origin cert")
-	}
-
-	originCertPool, err := loadCertPool(c, logger)
-	if err != nil {
-		logger.WithError(err).Error("Error loading cert pool")
-		return nil, errors.Wrap(err, "Error loading cert pool")
-	}
-
-	tunnelMetrics := origin.NewTunnelMetrics()
-	httpTransport := &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		DialContext: (&net.Dialer{
-			Timeout:   c.Duration("proxy-connect-timeout"),
-			KeepAlive: c.Duration("proxy-tcp-keepalive"),
-			DualStack: !c.Bool("proxy-no-happy-eyeballs"),
-		}).DialContext,
-		MaxIdleConns:          c.Int("proxy-keepalive-connections"),
-		IdleConnTimeout:       c.Duration("proxy-keepalive-timeout"),
-		TLSHandshakeTimeout:   c.Duration("proxy-tls-timeout"),
-		ExpectContinueTimeout: 1 * time.Second,
-		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: c.IsSet("no-tls-verify")},
-	}
-
-	if !c.IsSet("hello-world") && c.IsSet("origin-server-name") {
-		httpTransport.TLSClientConfig.ServerName = c.String("origin-server-name")
-	}
-
-	return &origin.TunnelConfig{
-		EdgeAddrs:          c.StringSlice("edge"),
-		OriginUrl:          url,
-		Hostname:           hostname,
-		OriginCert:         originCert,
-		TlsConfig:          tlsconfig.CreateTunnelConfig(c, c.StringSlice("edge")),
-		ClientTlsConfig:    httpTransport.TLSClientConfig,
-		Retries:            c.Uint("retries"),
-		HeartbeatInterval:  c.Duration("heartbeat-interval"),
-		MaxHeartbeats:      c.Uint64("heartbeat-count"),
-		ClientID:           clientID,
-		BuildInfo:          buildInfo,
-		ReportedVersion:    Version,
-		LBPool:             c.String("lb-pool"),
-		Tags:               tags,
-		HAConnections:      c.Int("ha-connections"),
-		HTTPTransport:      httpTransport,
-		Metrics:            tunnelMetrics,
-		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
-		ProtocolLogger:     protoLogger,
-		Logger:             logger,
-		IsAutoupdated:      c.Bool("is-autoupdated"),
-		GracePeriod:        c.Duration("grace-period"),
-		RunFromTerminal:    isRunningFromTerminal(),
-		NoChunkedEncoding:  c.Bool("no-chunked-encoding"),
-		CompressionQuality: c.Uint64("compression-quality"),
-	}, nil
-}
-
-func loadCertPool(c *cli.Context, logger *logrus.Logger) (*x509.CertPool, error) {
-	const originCAPoolFlag = "origin-ca-pool"
-	originCAPoolFilename := c.String(originCAPoolFlag)
-	var originCustomCAPool []byte
-
-	if originCAPoolFilename != "" {
-		var err error
-		originCustomCAPool, err = ioutil.ReadFile(originCAPoolFilename)
-		if err != nil {
-			return nil, errors.Wrap(err, fmt.Sprintf("unable to read the file %s for --%s", originCAPoolFilename, originCAPoolFlag))
-		}
-	}
-
-	originCertPool, err := tlsconfig.LoadOriginCertPool(originCustomCAPool)
-	if err != nil {
-		return nil, errors.Wrap(err, "error loading the certificate pool")
-	}
-
-	// Windows users should be notified that they can use the flag
-	if runtime.GOOS == "windows" && originCAPoolFilename == "" {
-		logger.Infof("cloudflared does not support loading the system root certificate pool on Windows. Please use the --%s to specify it", originCAPoolFlag)
-	}
-
-	return originCertPool, nil
-}

cmd/cloudflared/encrypter/encrypt.go

@@ -0,0 +1,176 @@
+// Package encrypter is suitable for encrypting messages you would like to securely share between two points.
+// Useful for providing end to end encryption (E2EE). It uses Box (NaCl) for encrypting the messages.
+// tldr is it uses Elliptic Curves (Curve25519) for the keys, XSalsa20 and Poly1305 for encryption.
+// You can read more here https://godoc.org/golang.org/x/crypto/nacl/box.
+//
+//		msg := []byte("super safe message.")
+//		alice, err := New("alice_priv_key.pem", "alice_pub_key.pem")
+//		if err != nil {
+//			log.Fatal(err)
+//		}
+//
+//		bob, err := New("bob_priv_key.pem", "bob_pub_key.pem")
+//		if err != nil {
+//			log.Fatal(err)
+//		}
+//		encrypted, err := alice.Encrypt(msg, bob.PublicKey())
+//		if err != nil {
+//			log.Fatal(err)
+//		}
+//
+//		data, err := bob.Decrypt(encrypted, alice.PublicKey())
+//		if err != nil {
+//			log.Fatal(err)
+//		}
+//		fmt.Println(string(data))
+package encrypter
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/pem"
+	"errors"
+	"io"
+	"os"
+
+	"golang.org/x/crypto/nacl/box"
+)
+
+// Encrypter represents a keypair value with auxiliary functions to make
+// doing encryption and decryption easier
+type Encrypter struct {
+	privateKey *[32]byte
+	publicKey  *[32]byte
+}
+
+// New returns a new encrypter with initialized keypair
+func New(privateKey, publicKey string) (*Encrypter, error) {
+	e := &Encrypter{}
+	pubKey, key, err := e.fetchOrGenerateKeys(privateKey, publicKey)
+	if err != nil {
+		return nil, err
+	}
+	e.privateKey, e.publicKey = key, pubKey
+	return e, nil
+}
+
+// PublicKey returns a base64 encoded public key. Useful for transport (like in HTTP requests)
+func (e *Encrypter) PublicKey() string {
+	return base64.URLEncoding.EncodeToString(e.publicKey[:])
+}
+
+// Decrypt data that was encrypted using our publicKey. It will use our privateKey and the sender's publicKey to decrypt
+// data is an encrypted buffer of data, mostly like from the Encrypt function. Messages contain the nonce data on the front
+// of the message.
+// senderPublicKey is a base64 encoded version of the sender's public key (most likely from the PublicKey function).
+// The return value is the decrypted buffer or an error.
+func (e *Encrypter) Decrypt(data []byte, senderPublicKey string) ([]byte, error) {
+	var decryptNonce [24]byte
+	copy(decryptNonce[:], data[:24]) // we pull the nonce from the front of the actual message.
+	pubKey, err := e.decodePublicKey(senderPublicKey)
+	if err != nil {
+		return nil, err
+	}
+	decrypted, ok := box.Open(nil, data[24:], &decryptNonce, pubKey, e.privateKey)
+	if !ok {
+		return nil, errors.New("failed to decrypt message")
+	}
+	return decrypted, nil
+}
+
+// Encrypt data using our privateKey and the recipient publicKey
+// data is a buffer of data that we would like to encrypt. Messages will have the nonce added to front
+// as they have to unique for each message shared.
+// recipientPublicKey is a base64 encoded version of the sender's public key (most likely from the PublicKey function).
+// The return value is the encrypted buffer or an error.
+func (e *Encrypter) Encrypt(data []byte, recipientPublicKey string) ([]byte, error) {
+	var nonce [24]byte
+	if _, err := io.ReadFull(rand.Reader, nonce[:]); err != nil {
+		return nil, err
+	}
+
+	pubKey, err := e.decodePublicKey(recipientPublicKey)
+	if err != nil {
+		return nil, err
+	}
+	// This encrypts msg and adds the nonce to the front of the message, since the nonce has to be
+	// the same for encrypting and decrypting
+	return box.Seal(nonce[:], data, &nonce, pubKey, e.privateKey), nil
+}
+
+// WriteKeys keys will take the currently initialized keypair and write them to provided filenames
+func (e *Encrypter) WriteKeys(privateKey, publicKey string) error {
+	if err := e.writeKey(e.privateKey[:], "BOX PRIVATE KEY", privateKey); err != nil {
+		return err
+	}
+	return e.writeKey(e.publicKey[:], "PUBLIC KEY", publicKey)
+}
+
+// fetchOrGenerateKeys will either load or create a keypair if it doesn't exist
+func (e *Encrypter) fetchOrGenerateKeys(privateKey, publicKey string) (*[32]byte, *[32]byte, error) {
+	key, err := e.fetchKey(privateKey)
+	if os.IsNotExist(err) {
+		return box.GenerateKey(rand.Reader)
+	} else if err != nil {
+		return nil, nil, err
+	}
+
+	pub, err := e.fetchKey(publicKey)
+	if os.IsNotExist(err) {
+		return box.GenerateKey(rand.Reader)
+	} else if err != nil {
+		return nil, nil, err
+	}
+	return pub, key, nil
+}
+
+// writeKey will write a key to disk in DER format (it's a standard pem key)
+func (e *Encrypter) writeKey(key []byte, pemType, filename string) error {
+	data := pem.EncodeToMemory(&pem.Block{
+		Type:  pemType,
+		Bytes: key,
+	})
+
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+
+	_, err = f.Write(data)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// fetchKey will load a a DER formatted key from disk
+func (e *Encrypter) fetchKey(filename string) (*[32]byte, error) {
+	f, err := os.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	buf := new(bytes.Buffer)
+	io.Copy(buf, f)
+
+	p, _ := pem.Decode(buf.Bytes())
+	if p == nil {
+		return nil, errors.New("Failed to decode key")
+	}
+	var newKey [32]byte
+	copy(newKey[:], p.Bytes)
+
+	return &newKey, nil
+}
+
+// decodePublicKey will base64 decode the provided key to the box representation
+func (e *Encrypter) decodePublicKey(key string) (*[32]byte, error) {
+	pub, err := base64.URLEncoding.DecodeString(key)
+	if err != nil {
+		return nil, err
+	}
+	var newKey [32]byte
+	copy(newKey[:], pub)
+	return &newKey, nil
+}

cmd/cloudflared/hello.go

@@ -1,21 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"gopkg.in/urfave/cli.v2"
-
-	"github.com/cloudflare/cloudflared/hello"
-)
-
-
-func helloWorld(c *cli.Context) error {
-	address := fmt.Sprintf(":%d", c.Int("port"))
-	listener, err := hello.CreateTLSListener(address)
-	if err != nil {
-		return err
-	}
-	defer listener.Close()
-	err = hello.StartHelloWorldServer(logger, listener, nil)
-	return err
-}

cmd/cloudflared/linux_service.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	cli "gopkg.in/urfave/cli.v2"
 )
 
@@ -30,7 +31,13 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 	app.Run(os.Args)
 }
 
-const serviceConfigDir = "/etc/cloudflared"
+// The directory and files that are used by the service.
+// These are hard-coded in the templates below.
+const (
+	serviceConfigDir      = "/etc/cloudflared"
+	serviceConfigFile     = "config.yml"
+	serviceCredentialFile = "cert.pem"
+)
 
 var systemdTemplates = []ServiceTemplate{
 	{
@@ -66,7 +73,7 @@ ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 64 ]; then s
 Description=Update Argo Tunnel
 
 [Timer]
-OnUnitActiveSec=1d
+OnCalendar=daily
 
 [Install]
 WantedBy=timers.target
@@ -77,7 +84,8 @@ WantedBy=timers.target
 var sysvTemplate = ServiceTemplate{
 	Path:     "/etc/init.d/cloudflared",
 	FileMode: 0755,
-	Content: `# For RedHat and cousins:
+	Content: `#!/bin/sh
+# For RedHat and cousins:
 # chkconfig: 2345 99 01
 # description: Argo Tunnel agent
 # processname: {{.Path}}
@@ -174,6 +182,23 @@ func isSystemd() bool {
 	return false
 }
 
+func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile string) error {
+	if err := ensureConfigDirExists(serviceConfigDir); err != nil {
+		return err
+	}
+	srcCredentialPath := filepath.Join(userConfigDir, userCredentialFile)
+	destCredentialPath := filepath.Join(serviceConfigDir, serviceCredentialFile)
+	if err := copyCredential(srcCredentialPath, destCredentialPath); err != nil {
+		return err
+	}
+	srcConfigPath := filepath.Join(userConfigDir, userConfigFile)
+	destConfigPath := filepath.Join(serviceConfigDir, serviceConfigFile)
+	if err := copyConfig(srcConfigPath, destConfigPath); err != nil {
+		return err
+	}
+	return nil
+}
+
 func installLinuxService(c *cli.Context) error {
 	etPath, err := os.Executable()
 	if err != nil {
@@ -181,11 +206,12 @@ func installLinuxService(c *cli.Context) error {
 	}
 	templateArgs := ServiceTemplateArgs{Path: etPath}
 
-	defaultConfigDir := filepath.Dir(c.String("config"))
-	defaultConfigFile := filepath.Base(c.String("config"))
-	if err = copyCredentials(serviceConfigDir, defaultConfigDir, defaultConfigFile, defaultCredentialFile); err != nil {
+	userConfigDir := filepath.Dir(c.String("config"))
+	userConfigFile := filepath.Base(c.String("config"))
+	userCredentialFile := config.DefaultCredentialFile
+	if err = copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile); err != nil {
 		logger.WithError(err).Infof("Failed to copy user configuration. Before running the service, ensure that %s contains two files, %s and %s",
-			serviceConfigDir, defaultCredentialFile, defaultConfigFiles[0])
+			serviceConfigDir, serviceCredentialFile, serviceConfigFile)
 		return err
 	}
 

cmd/cloudflared/login.go

@@ -1,194 +0,0 @@
-package main
-
-import (
-	"crypto/rand"
-	"encoding/base32"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"runtime"
-	"syscall"
-	"time"
-
-	homedir "github.com/mitchellh/go-homedir"
-	cli "gopkg.in/urfave/cli.v2"
-)
-
-const baseLoginURL = "https://dash.cloudflare.com/warp"
-const baseCertStoreURL = "https://login.cloudflarewarp.com"
-const clientTimeout = time.Minute * 20
-
-func login(c *cli.Context) error {
-	configPath, err := homedir.Expand(defaultConfigDirs[0])
-	if err != nil {
-		return err
-	}
-	ok, err := fileExists(configPath)
-	if !ok && err == nil {
-		// create config directory if doesn't already exist
-		err = os.Mkdir(configPath, 0700)
-	}
-	if err != nil {
-		return err
-	}
-	path := filepath.Join(configPath, defaultCredentialFile)
-	fileInfo, err := os.Stat(path)
-	if err == nil && fileInfo.Size() > 0 {
-		fmt.Fprintf(os.Stderr, `You have an existing certificate at %s which login would overwrite.
-If this is intentional, please move or delete that file then run this command again.
-`, path)
-		return nil
-	}
-	if err != nil && err.(*os.PathError).Err != syscall.ENOENT {
-		return err
-	}
-
-	// for local debugging
-	baseURL := baseCertStoreURL
-	if c.IsSet("url") {
-		baseURL = c.String("url")
-	}
-	// Generate a random post URL
-	certURL := baseURL + generateRandomPath()
-	loginURL, err := url.Parse(baseLoginURL)
-	if err != nil {
-		// shouldn't happen, URL is hardcoded
-		return err
-	}
-	loginURL.RawQuery = "callback=" + url.QueryEscape(certURL)
-
-	err = open(loginURL.String())
-	if err != nil {
-		fmt.Fprintf(os.Stderr, `Please open the following URL and log in with your Cloudflare account:
-
-%s
-
-Leave cloudflared running to install the certificate automatically.
-`, loginURL.String())
-	} else {
-		fmt.Fprintf(os.Stderr, `A browser window should have opened at the following URL:
-
-%s
-
-If the browser failed to open, open it yourself and visit the URL above.
-
-`, loginURL.String())
-	}
-
-	if download(certURL, path) {
-		fmt.Fprintf(os.Stderr, `You have successfully logged in.
-If you wish to copy your credentials to a server, they have been saved to:
-%s
-`, path)
-	} else {
-		fmt.Fprintf(os.Stderr, `Failed to write the certificate due to the following error:
-%v
-
-Your browser will download the certificate instead. You will have to manually
-copy it to the following path:
-
-%s
-
-`, err, path)
-	}
-	return nil
-}
-
-// generateRandomPath generates a random URL to associate with the certificate.
-func generateRandomPath() string {
-	randomBytes := make([]byte, 40)
-	_, err := rand.Read(randomBytes)
-	if err != nil {
-		panic(err)
-	}
-	return "/" + base32.StdEncoding.EncodeToString(randomBytes)
-}
-
-// open opens the specified URL in the default browser of the user.
-func open(url string) error {
-	var cmd string
-	var args []string
-
-	switch runtime.GOOS {
-	case "windows":
-		cmd = "cmd"
-		args = []string{"/c", "start"}
-	case "darwin":
-		cmd = "open"
-	default: // "linux", "freebsd", "openbsd", "netbsd"
-		cmd = "xdg-open"
-	}
-	args = append(args, url)
-	return exec.Command(cmd, args...).Start()
-}
-
-func download(certURL, filePath string) bool {
-	client := &http.Client{Timeout: clientTimeout}
-	// attempt a (long-running) certificate get
-	for i := 0; i < 20; i++ {
-		ok, err := tryDownload(client, certURL, filePath)
-		if ok {
-			putSuccess(client, certURL)
-			return true
-		}
-		if err != nil {
-			logger.WithError(err).Error("Error fetching certificate")
-			return false
-		}
-	}
-	return false
-}
-
-func tryDownload(client *http.Client, certURL, filePath string) (ok bool, err error) {
-	resp, err := client.Get(certURL)
-	if err != nil {
-		return false, err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode == 404 {
-		return false, nil
-	}
-	if resp.StatusCode != 200 {
-		return false, fmt.Errorf("Unexpected HTTP error code %d", resp.StatusCode)
-	}
-	if resp.Header.Get("Content-Type") != "application/x-pem-file" {
-		return false, fmt.Errorf("Unexpected content type %s", resp.Header.Get("Content-Type"))
-	}
-	// write response
-	file, err := os.Create(filePath)
-	if err != nil {
-		return false, err
-	}
-	defer file.Close()
-	written, err := io.Copy(file, resp.Body)
-	switch {
-	case err != nil:
-		return false, err
-	case resp.ContentLength != written && resp.ContentLength != -1:
-		return false, fmt.Errorf("Short read (%d bytes) from server while writing certificate", written)
-	default:
-		return true, nil
-	}
-}
-
-func putSuccess(client *http.Client, certURL string) {
-	// indicate success to the relay server
-	req, err := http.NewRequest("PUT", certURL+"/ok", nil)
-	if err != nil {
-		logger.WithError(err).Error("HTTP request error")
-		return
-	}
-	resp, err := client.Do(req)
-	if err != nil {
-		logger.WithError(err).Error("HTTP error")
-		return
-	}
-	resp.Body.Close()
-	if resp.StatusCode != 200 {
-		logger.Errorf("Unexpected HTTP error code %d", resp.StatusCode)
-	}
-}

cmd/cloudflared/main.go

@@ -2,48 +2,45 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
-	"os"
-	"runtime/trace"
-	"sync"
-	"syscall"
+	"strings"
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/sqlgateway"
-	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/log"
 	"github.com/cloudflare/cloudflared/metrics"
-	"github.com/cloudflare/cloudflared/origin"
-	"github.com/cloudflare/cloudflared/tunneldns"
 
-	"golang.org/x/crypto/ssh/terminal"
+	raven "github.com/getsentry/raven-go"
+	homedir "github.com/mitchellh/go-homedir"
+	cli "gopkg.in/urfave/cli.v2"
 
-	"github.com/getsentry/raven-go"
-	"github.com/mitchellh/go-homedir"
-	"gopkg.in/urfave/cli.v2"
-	"gopkg.in/urfave/cli.v2/altsrc"
-
-	"github.com/coreos/go-systemd/daemon"
-	"github.com/facebookgo/grace/gracenet"
 	"github.com/pkg/errors"
 )
 
 const (
-	sentryDSN       = "https://56a9c9fa5c364ab28f34b14f35ea0f1b:3e8827f6f9f740738eb11138f7bebb68@sentry.io/189878"
-	developerPortal = "https://developers.cloudflare.com/argo-tunnel"
-	quickStartUrl   = developerPortal + "/quickstart/quickstart/"
-	serviceUrl      = developerPortal + "/reference/service/"
-	argumentsUrl    = developerPortal + "/reference/arguments/"
-	licenseUrl      = developerPortal + "/licence/"
+	versionText = "Print the version"
 )
 
 var (
 	Version   = "DEV"
 	BuildTime = "unknown"
+	logger    = log.CreateLogger()
+	// Mostly network errors that we don't want reported back to Sentry, this is done by substring match.
+	ignoredErrors = []string{
+		"connection reset by peer",
+		"An existing connection was forcibly closed by the remote host.",
+		"use of closed connection",
+		"You need to enable Argo Smart Routing",
+		"3001 connection closed",
+		"3002 connection dropped",
+		"rpc exception: dial tcp",
+		"rpc exception: EOF",
+	}
 )
 
 func main() {
 	metrics.RegisterBuildInfo(BuildTime, Version)
-	raven.SetDSN(sentryDSN)
 	raven.SetRelease(Version)
 
 	// Force shutdown channel used by the app. When closed, app must terminate.
@@ -53,590 +50,92 @@ func main() {
 	// Windows service manager closes this channel when it receives stop command.
 	graceShutdownC := make(chan struct{})
 
+	cli.VersionFlag = &cli.BoolFlag{
+		Name:    "version",
+		Aliases: []string{"v", "V"},
+		Usage:   versionText,
+	}
+
 	app := &cli.App{}
 	app.Name = "cloudflared"
-	app.Copyright = fmt.Sprintf(`(c) %d Cloudflare Inc.
-   Use is subject to the license agreement at %s`, time.Now().Year(), licenseUrl)
-	app.Usage = "Cloudflare reverse tunnelling proxy agent"
+	app.Usage = "Cloudflare's command-line tool and agent"
 	app.ArgsUsage = "origin-url"
+	app.Copyright = fmt.Sprintf(
+		`(c) %d Cloudflare Inc.
+   Your installation of cloudflared software constitutes a symbol of your signature indicating that you accept
+   the terms of the Cloudflare License (https://developers.cloudflare.com/argo-tunnel/license/),
+   Terms (https://www.cloudflare.com/terms/) and Privacy Policy (https://www.cloudflare.com/privacypolicy/).`,
+		time.Now().Year(),
+	)
 	app.Version = fmt.Sprintf("%s (built %s)", Version, BuildTime)
-	app.Description = `A reverse tunnel proxy agent that connects to Cloudflare's infrastructure.
-   Upon connecting, you are assigned a unique subdomain on cftunnel.com.
-   You need to specify a hostname on a zone you control.
-   A DNS record will be created to CNAME your hostname to the unique subdomain on cftunnel.com.
+	app.Description = `cloudflared connects your machine or user identity to Cloudflare's global network.
+	You can use it to authenticate a session to reach an API behind Access, route web traffic to this machine,
+	and configure access control.`
+	app.Flags = flags()
+	app.Action = action(Version, shutdownC, graceShutdownC)
+	app.Before = tunnel.Before
+	app.Commands = commands(cli.ShowVersion)
 
-   Requests made to Cloudflare's servers for your hostname will be proxied
-   through the tunnel to your local webserver.`
-	app.Flags = []cli.Flag{
-		&cli.StringFlag{
-			Name:  "config",
-			Usage: "Specifies a config file in YAML format.",
-			Value: findDefaultConfigPath(),
-		},
-		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:  "autoupdate-freq",
-			Usage: "Autoupdate frequency. Default is 24h.",
-			Value: time.Hour * 24,
-		}),
-		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:  "no-autoupdate",
-			Usage: "Disable periodic check for updates, restarting the server with the new version.",
-			Value: false,
-		}),
-		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:   "is-autoupdated",
-			Usage:  "Signal the new process that Argo Tunnel client has been autoupdated",
-			Value:  false,
-			Hidden: true,
-		}),
-		altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
-			Name:    "edge",
-			Usage:   "Address of the Cloudflare tunnel server.",
-			EnvVars: []string{"TUNNEL_EDGE"},
-			Hidden:  true,
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "cacert",
-			Usage:   "Certificate Authority authenticating the Cloudflare tunnel connection.",
-			EnvVars: []string{"TUNNEL_CACERT"},
-			Hidden:  true,
-		}),
-		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "no-tls-verify",
-			Usage:   "Disables TLS verification of the certificate presented by your origin. Will allow any certificate from the origin to be accepted. Note: The connection from your machine to Cloudflare's Edge is still encrypted.",
-			EnvVars: []string{"NO_TLS_VERIFY"},
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "origincert",
-			Usage:   "Path to the certificate generated for your origin when you run cloudflared login.",
-			EnvVars: []string{"TUNNEL_ORIGIN_CERT"},
-			Value:   findDefaultOriginCertPath(),
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "origin-ca-pool",
-			Usage:   "Path to the CA for the certificate of your origin. This option should be used only if your certificate is not signed by Cloudflare.",
-			EnvVars: []string{"TUNNEL_ORIGIN_CA_POOL"},
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "url",
-			Value:   "https://localhost:8080",
-			Usage:   "Connect to the local webserver at `URL`.",
-			EnvVars: []string{"TUNNEL_URL"},
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "hostname",
-			Usage:   "Set a hostname on a Cloudflare zone to route traffic through this tunnel.",
-			EnvVars: []string{"TUNNEL_HOSTNAME"},
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "origin-server-name",
-			Usage:   "Hostname on the origin server certificate.",
-			EnvVars: []string{"TUNNEL_ORIGIN_SERVER_NAME"},
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "id",
-			Usage:   "A unique identifier used to tie connections to this tunnel instance.",
-			EnvVars: []string{"TUNNEL_ID"},
-			Hidden:  true,
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "lb-pool",
-			Usage:   "The name of a (new/existing) load balancing pool to add this origin to.",
-			EnvVars: []string{"TUNNEL_LB_POOL"},
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "api-key",
-			Usage:   "This parameter has been deprecated since version 2017.10.1.",
-			EnvVars: []string{"TUNNEL_API_KEY"},
-			Hidden:  true,
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "api-email",
-			Usage:   "This parameter has been deprecated since version 2017.10.1.",
-			EnvVars: []string{"TUNNEL_API_EMAIL"},
-			Hidden:  true,
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "api-ca-key",
-			Usage:   "This parameter has been deprecated since version 2017.10.1.",
-			EnvVars: []string{"TUNNEL_API_CA_KEY"},
-			Hidden:  true,
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "metrics",
-			Value:   "localhost:",
-			Usage:   "Listen address for metrics reporting.",
-			EnvVars: []string{"TUNNEL_METRICS"},
-		}),
-		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:    "metrics-update-freq",
-			Usage:   "Frequency to update tunnel metrics",
-			Value:   time.Second * 5,
-			EnvVars: []string{"TUNNEL_METRICS_UPDATE_FREQ"},
-		}),
-		altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
-			Name:    "tag",
-			Usage:   "Custom tags used to identify this tunnel, in format `KEY=VALUE`. Multiple tags may be specified",
-			EnvVars: []string{"TUNNEL_TAG"},
-		}),
-		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:   "heartbeat-interval",
-			Usage:  "Minimum idle time before sending a heartbeat.",
-			Value:  time.Second * 5,
-			Hidden: true,
-		}),
-		altsrc.NewUint64Flag(&cli.Uint64Flag{
-			Name:   "heartbeat-count",
-			Usage:  "Minimum number of unacked heartbeats to send before closing the connection.",
-			Value:  5,
-			Hidden: true,
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "loglevel",
-			Value:   "info",
-			Usage:   "Application logging level {panic, fatal, error, warn, info, debug}",
-			EnvVars: []string{"TUNNEL_LOGLEVEL"},
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "proto-loglevel",
-			Value:   "warn",
-			Usage:   "Protocol logging level {panic, fatal, error, warn, info, debug}",
-			EnvVars: []string{"TUNNEL_PROTO_LOGLEVEL"},
-		}),
-		altsrc.NewUintFlag(&cli.UintFlag{
-			Name:    "retries",
-			Value:   5,
-			Usage:   "Maximum number of retries for connection/protocol errors.",
-			EnvVars: []string{"TUNNEL_RETRIES"},
-		}),
-		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "hello-world",
-			Value:   false,
-			Usage:   "Run Hello World Server",
-			EnvVars: []string{"TUNNEL_HELLO_WORLD"},
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "pidfile",
-			Usage:   "Write the application's PID to this file after first successful connection.",
-			EnvVars: []string{"TUNNEL_PIDFILE"},
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "logfile",
-			Usage:   "Save application log to this file for reporting issues.",
-			EnvVars: []string{"TUNNEL_LOGFILE"},
-		}),
-		altsrc.NewIntFlag(&cli.IntFlag{
-			Name:   "ha-connections",
-			Value:  4,
-			Hidden: true,
-		}),
-		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:  "proxy-connect-timeout",
-			Usage: "HTTP proxy timeout for establishing a new connection",
-			Value: time.Second * 30,
-		}),
-		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:  "proxy-tls-timeout",
-			Usage: "HTTP proxy timeout for completing a TLS handshake",
-			Value: time.Second * 10,
-		}),
-		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:  "proxy-tcp-keepalive",
-			Usage: "HTTP proxy TCP keepalive duration",
-			Value: time.Second * 30,
-		}),
-		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:  "proxy-no-happy-eyeballs",
-			Usage: "HTTP proxy should disable \"happy eyeballs\" for IPv4/v6 fallback",
-		}),
-		altsrc.NewIntFlag(&cli.IntFlag{
-			Name:  "proxy-keepalive-connections",
-			Usage: "HTTP proxy maximum keepalive connection pool size",
-			Value: 100,
-		}),
-		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:  "proxy-keepalive-timeout",
-			Usage: "HTTP proxy timeout for closing an idle connection",
-			Value: time.Second * 90,
-		}),
-		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "proxy-dns",
-			Usage:   "Run a DNS over HTTPS proxy server.",
-			EnvVars: []string{"TUNNEL_DNS"},
-		}),
-		altsrc.NewIntFlag(&cli.IntFlag{
-			Name:    "proxy-dns-port",
-			Value:   53,
-			Usage:   "Listen on given port for the DNS over HTTPS proxy server.",
-			EnvVars: []string{"TUNNEL_DNS_PORT"},
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "proxy-dns-address",
-			Usage:   "Listen address for the DNS over HTTPS proxy server.",
-			Value:   "localhost",
-			EnvVars: []string{"TUNNEL_DNS_ADDRESS"},
-		}),
-		altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
-			Name:    "proxy-dns-upstream",
-			Usage:   "Upstream endpoint URL, you can specify multiple endpoints for redundancy.",
-			Value:   cli.NewStringSlice("https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"),
-			EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
-		}),
-		altsrc.NewDurationFlag(&cli.DurationFlag{
-			Name:    "grace-period",
-			Usage:   "Duration to accept new requests after cloudflared receives first SIGINT/SIGTERM. A second SIGINT/SIGTERM will force cloudflared to shutdown immediately.",
-			Value:   time.Second * 30,
-			EnvVars: []string{"TUNNEL_GRACE_PERIOD"},
-			Hidden:  true,
-		}),
-		altsrc.NewUintFlag(&cli.UintFlag{
-			Name:    "compression-quality",
-			Value:   0,
-			Usage:   "Use cross-stream compression instead HTTP compression. 0-off, 1-low, 2-medium, >=3-high",
-			EnvVars: []string{"TUNNEL_COMPRESSION_LEVEL"},
-			Hidden:  true,
-		}),
-		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "no-chunked-encoding",
-			Usage:   "Disables chunked transfer encoding; useful if you are running a WSGI server.",
-			EnvVars: []string{"TUNNEL_NO_CHUNKED_ENCODING"},
-		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "trace-output",
-			Usage:   "Name of trace output file, generated when cloudflared stops.",
-			EnvVars: []string{"TUNNEL_TRACE_OUTPUT"},
-		}),
-	}
-	app.Action = func(c *cli.Context) (err error) {
-		tags := make(map[string]string)
-		tags["hostname"] = c.String("hostname")
-		raven.SetTagsContext(tags)
-		raven.CapturePanic(func() { err = startServer(c, shutdownC, graceShutdownC) }, nil)
-		if err != nil {
-			raven.CaptureError(err, nil)
-		}
-		return err
-	}
-	app.Before = func(context *cli.Context) error {
-		if context.String("config") == "" {
-			logger.Warnf("Cannot determine default configuration path. No file %v in %v", defaultConfigFiles, defaultConfigDirs)
-		}
-		inputSource, err := findInputSourceContext(context)
-		if err != nil {
-			logger.WithError(err).Infof("Cannot load configuration from %s", context.String("config"))
-			return err
-		} else if inputSource != nil {
-			err := altsrc.ApplyInputSourceValues(context, inputSource, app.Flags)
-			if err != nil {
-				logger.WithError(err).Infof("Cannot apply configuration from %s", context.String("config"))
-				return err
-			}
-			logger.Infof("Applied configuration from %s", context.String("config"))
-		}
-		return nil
-	}
-	app.Commands = []*cli.Command{
-		{
-			Name:      "update",
-			Action:    update,
-			Usage:     "Update the agent if a new version exists",
-			ArgsUsage: " ",
-			Description: `Looks for a new version on the offical download server.
-   If a new version exists, updates the agent binary and quits.
-   Otherwise, does nothing.
-
-   To determine if an update happened in a script, check for error code 64.`,
-		},
-		{
-			Name:      "login",
-			Action:    login,
-			Usage:     "Generate a configuration file with your login details",
-			ArgsUsage: " ",
-			Flags: []cli.Flag{
-				&cli.StringFlag{
-					Name:   "url",
-					Hidden: true,
-				},
-			},
-		},
-		{
-			Name:   "hello",
-			Action: helloWorld,
-			Usage:  "Run a simple \"Hello World\" server for testing Argo Tunnel.",
-			Flags: []cli.Flag{
-				&cli.IntFlag{
-					Name:  "port",
-					Usage: "Listen on the selected port.",
-					Value: 8080,
-				},
-			},
-			ArgsUsage: " ", // can't be the empty string or we get the default output
-		},
-		{
-			Name:   "proxy-dns",
-			Action: tunneldns.Run,
-			Usage:  "Run a DNS over HTTPS proxy server.",
-			Flags: []cli.Flag{
-				&cli.StringFlag{
-					Name:    "metrics",
-					Value:   "localhost:",
-					Usage:   "Listen address for metrics reporting.",
-					EnvVars: []string{"TUNNEL_METRICS"},
-				},
-				&cli.StringFlag{
-					Name:    "address",
-					Usage:   "Listen address for the DNS over HTTPS proxy server.",
-					Value:   "localhost",
-					EnvVars: []string{"TUNNEL_DNS_ADDRESS"},
-				},
-				&cli.IntFlag{
-					Name:    "port",
-					Usage:   "Listen on given port for the DNS over HTTPS proxy server.",
-					Value:   53,
-					EnvVars: []string{"TUNNEL_DNS_PORT"},
-				},
-				&cli.StringSliceFlag{
-					Name:    "upstream",
-					Usage:   "Upstream endpoint URL, you can specify multiple endpoints for redundancy.",
-					Value:   cli.NewStringSlice("https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"),
-					EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
-				},
-			},
-			ArgsUsage: " ", // can't be the empty string or we get the default output
-		},
-		{
-			Name: "db",
-			Action: func(c *cli.Context) error {
-				tags := make(map[string]string)
-				tags["hostname"] = c.String("hostname")
-				raven.SetTagsContext(tags)
-
-				fmt.Printf("\nSQL Database Password: ")
-				pass, err := terminal.ReadPassword(int(syscall.Stdin))
-				if err != nil {
-					logger.Error(err)
-				}
-
-				go sqlgateway.StartProxy(c, logger, string(pass))
-
-				raven.CapturePanic(func() { err = startServer(c, shutdownC, graceShutdownC) }, nil)
-				if err != nil {
-					raven.CaptureError(err, nil)
-				}
-				return err
-			},
-			Before: func(c *cli.Context) error {
-				if c.String("config") == "" {
-					logger.Warnf("Cannot determine default configuration path. No file %v in %v", defaultConfigFiles, defaultConfigDirs)
-				}
-				inputSource, err := findInputSourceContext(c)
-				if err != nil {
-					logger.WithError(err).Infof("Cannot load configuration from %s", c.String("config"))
-					return err
-				} else if inputSource != nil {
-					err := altsrc.ApplyInputSourceValues(c, inputSource, app.Flags)
-					if err != nil {
-						logger.WithError(err).Infof("Cannot apply configuration from %s", c.String("config"))
-						return err
-					}
-					logger.Infof("Applied configuration from %s", c.String("config"))
-				}
-				return nil
-			},
-			Usage: "SQL Gateway is an SQL over HTTP reverse proxy",
-			Flags: []cli.Flag{
-				&cli.BoolFlag{
-					Name:  "db",
-					Value: true,
-					Usage: "Enable the SQL Gateway Proxy",
-				},
-				&cli.StringFlag{
-					Name:  "address",
-					Value: "",
-					Usage: "Database connection string: db://user:pass",
-				},
-			},
-		},
-	}
+	tunnel.Init(Version, shutdownC, graceShutdownC) // we need this to support the tunnel sub command...
+	access.Init(shutdownC, graceShutdownC)
 	runApp(app, shutdownC, graceShutdownC)
 }
 
-func startServer(c *cli.Context, shutdownC, graceShutdownC chan struct{}) error {
-	var wg sync.WaitGroup
-	listeners := gracenet.Net{}
-	errC := make(chan error)
-	connectedSignal := make(chan struct{})
-	dnsReadySignal := make(chan struct{})
+func commands(version func(c *cli.Context)) []*cli.Command {
+	cmds := []*cli.Command{
+		{
+			Name:      "update",
+			Action:    updater.Update,
+			Usage:     "Update the agent if a new version exists",
+			ArgsUsage: " ",
+			Description: `Looks for a new version on the official download server.
+If a new version exists, updates the agent binary and quits.
+Otherwise, does nothing.
 
-	// check whether client provides enough flags or env variables. If not, print help.
-	if ok := enoughOptionsSet(c); !ok {
-		return nil
+To determine if an update happened in a script, check for error code 64.`,
+		},
+		{
+			Name: "version",
+			Action: func(c *cli.Context) (err error) {
+				version(c)
+				return nil
+			},
+			Usage:       versionText,
+			Description: versionText,
+		},
 	}
+	cmds = append(cmds, tunnel.Commands()...)
+	cmds = append(cmds, access.Commands()...)
+	return cmds
+}
 
-	if err := configMainLogger(c); err != nil {
-		return errors.Wrap(err, "Error configuring logger")
-	}
+func flags() []cli.Flag {
+	flags := tunnel.Flags()
+	return append(flags, access.Flags()...)
+}
 
-	protoLogger, err := configProtoLogger(c)
-	if err != nil {
-		return errors.Wrap(err, "Error configuring protocol logger")
-	}
+func isEmptyInvocation(c *cli.Context) bool {
+	return c.NArg() == 0 && c.NumFlags() == 0
+}
 
-	if c.IsSet("trace-output") {
-		tmpTraceFile, err := ioutil.TempFile("", "trace")
+func action(version string, shutdownC, graceShutdownC chan struct{}) cli.ActionFunc {
+	return func(c *cli.Context) (err error) {
+		if isEmptyInvocation(c) {
+			cli.ShowAppHelpAndExit(c, 1)
+		}
+		tags := make(map[string]string)
+		tags["hostname"] = c.String("hostname")
+		raven.SetTagsContext(tags)
+		raven.CapturePanic(func() { err = tunnel.StartServer(c, version, shutdownC, graceShutdownC) }, nil)
+		exitCode := 0
 		if err != nil {
-			logger.WithError(err).Error("Failed to create new temporary file to save trace output")
+			handleError(err)
+			exitCode = 1
 		}
-
-		defer func() {
-			if err := tmpTraceFile.Close(); err != nil {
-				logger.WithError(err).Error("Failed to close trace output file %s", tmpTraceFile.Name())
-			}
-			if err := os.Rename(tmpTraceFile.Name(), c.String("trace-output")); err != nil {
-				logger.WithError(err).Errorf("Failed to rename temporary trace output file %s to %s", tmpTraceFile.Name(), c.String("trace-output"))
-			} else {
-				os.Remove(tmpTraceFile.Name())
-			}
-		}()
-
-		if err := trace.Start(tmpTraceFile); err != nil {
-			logger.WithError(err).Error("Failed to start trace")
-			return errors.Wrap(err, "Error starting tracing")
-		}
-		defer trace.Stop()
+		// we already handle error printing, so we pass an empty string so we
+		// don't have to print again.
+		return cli.Exit("", exitCode)
 	}
-
-	if c.String("logfile") != "" {
-		if err := initLogFile(c, logger, protoLogger); err != nil {
-			logger.Error(err)
-		}
-	}
-
-	if err := handleDeprecatedOptions(c); err != nil {
-		return err
-	}
-
-	buildInfo := origin.GetBuildInfo()
-	logger.Infof("Build info: %+v", *buildInfo)
-	logger.Infof("Version %s", Version)
-	logClientOptions(c)
-
-	if c.IsSet("proxy-dns") {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			errC <- runDNSProxyServer(c, dnsReadySignal, shutdownC)
-		}()
-	} else {
-		close(dnsReadySignal)
-	}
-
-	// Wait for proxy-dns to come up (if used)
-	<-dnsReadySignal
-
-	// update needs to be after DNS proxy is up to resolve equinox server address
-	if isAutoupdateEnabled(c) {
-		logger.Infof("Autoupdate frequency is set to %v", c.Duration("autoupdate-freq"))
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			errC <- autoupdate(c.Duration("autoupdate-freq"), &listeners, shutdownC)
-		}()
-	}
-
-	metricsListener, err := listeners.Listen("tcp", c.String("metrics"))
-	if err != nil {
-		logger.WithError(err).Error("Error opening metrics server listener")
-		return errors.Wrap(err, "Error opening metrics server listener")
-	}
-	defer metricsListener.Close()
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		errC <- metrics.ServeMetrics(metricsListener, shutdownC, logger)
-	}()
-
-	go notifySystemd(connectedSignal)
-	if c.IsSet("pidfile") {
-		go writePidFile(connectedSignal, c.String("pidfile"))
-	}
-
-	// Serve DNS proxy stand-alone if no hostname or tag or app is going to run
-	if dnsProxyStandAlone(c) {
-		close(connectedSignal)
-		// no grace period, handle SIGINT/SIGTERM immediately
-		return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, 0)
-	}
-
-	if c.IsSet("hello-world") {
-		helloListener, err := hello.CreateTLSListener("127.0.0.1:")
-		if err != nil {
-			logger.WithError(err).Error("Cannot start Hello World Server")
-			return errors.Wrap(err, "Cannot start Hello World Server")
-		}
-		defer helloListener.Close()
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			hello.StartHelloWorldServer(logger, helloListener, shutdownC)
-		}()
-		c.Set("url", "https://"+helloListener.Addr().String())
-	}
-
-	tunnelConfig, err := prepareTunnelConfig(c, buildInfo, logger, protoLogger)
-	if err != nil {
-		return err
-	}
-
-	wg.Add(1)
-	go func() {
-		defer wg.Done()
-		errC <- origin.StartTunnelDaemon(tunnelConfig, graceShutdownC, connectedSignal)
-	}()
-
-	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"))
-}
-
-func waitToShutdown(wg *sync.WaitGroup,
-	errC chan error,
-	shutdownC, graceShutdownC chan struct{},
-	gracePeriod time.Duration,
-) error {
-	var err error
-	if gracePeriod > 0 {
-		err = waitForSignalWithGraceShutdown(errC, shutdownC, graceShutdownC, gracePeriod)
-	} else {
-		err = waitForSignal(errC, shutdownC)
-		close(graceShutdownC)
-	}
-
-	if err != nil {
-		logger.WithError(err).Error("Quitting due to error")
-	} else {
-		logger.Info("Quitting...")
-	}
-	// Wait for clean exit, discarding all errors
-	go func() {
-		for range errC {
-		}
-	}()
-	wg.Wait()
-	return err
-}
-
-func notifySystemd(waitForSignal chan struct{}) {
-	<-waitForSignal
-	daemon.SdNotify(false, "READY=1")
-}
-
-func writePidFile(waitForSignal chan struct{}, pidFile string) {
-	<-waitForSignal
-	file, err := os.Create(pidFile)
-	if err != nil {
-		logger.WithError(err).Errorf("Unable to write pid to %s", pidFile)
-	}
-	defer file.Close()
-	fmt.Fprintf(file, "%d", os.Getpid())
 }
 
 func userHomeDir() (string, error) {
@@ -651,3 +150,14 @@ func userHomeDir() (string, error) {
 	}
 	return homeDir, nil
 }
+
+// In order to keep the amount of noise sent to Sentry low, typical network errors can be filtered out here by a substring match.
+func handleError(err error) {
+	errorMessage := err.Error()
+	for _, ignoredErrorMessage := range ignoredErrors {
+		if strings.Contains(errorMessage, ignoredErrorMessage) {
+			return
+		}
+	}
+	raven.CaptureError(err, nil)
+}

cmd/cloudflared/path/path.go

@@ -0,0 +1,30 @@
+package path
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/mitchellh/go-homedir"
+)
+
+// GenerateFilePathFromURL will return a filepath for given access application url
+func GenerateFilePathFromURL(url *url.URL, suffix string) (string, error) {
+	configPath, err := homedir.Expand(config.DefaultConfigDirs[0])
+	if err != nil {
+		return "", err
+	}
+	ok, err := config.FileExists(configPath)
+	if !ok && err == nil {
+		// create config directory if doesn't already exist
+		err = os.Mkdir(configPath, 0700)
+	}
+	if err != nil {
+		return "", err
+	}
+	name := strings.Replace(fmt.Sprintf("%s%s-%s", url.Hostname(), url.EscapedPath(), suffix), "/", "-", -1)
+	return filepath.Join(configPath, name), nil
+}

cmd/cloudflared/service_template.go

@@ -8,9 +8,9 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"path/filepath"
 	"text/template"
 
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/mitchellh/go-homedir"
 )
 
@@ -94,9 +94,9 @@ func runCommand(command string, args ...string) error {
 }
 
 func ensureConfigDirExists(configDir string) error {
-	ok, err := fileExists(configDir)
+	ok, err := config.FileExists(configDir)
 	if !ok && err == nil {
-		err = os.Mkdir(configDir, 0700)
+		err = os.Mkdir(configDir, 0755)
 	}
 	return err
 }
@@ -119,8 +119,7 @@ func openFile(path string, create bool) (file *os.File, exists bool, err error)
 	return file, false, err
 }
 
-func copyCertificate(srcConfigDir, destConfigDir, credentialFile string) error {
-	destCredentialPath := filepath.Join(destConfigDir, credentialFile)
+func copyCredential(srcCredentialPath, destCredentialPath string) error {
 	destFile, exists, err := openFile(destCredentialPath, true)
 	if err != nil {
 		return err
@@ -130,7 +129,6 @@ func copyCertificate(srcConfigDir, destConfigDir, credentialFile string) error {
 	}
 	defer destFile.Close()
 
-	srcCredentialPath := filepath.Join(srcConfigDir, credentialFile)
 	srcFile, _, err := openFile(srcCredentialPath, false)
 	if err != nil {
 		return err
@@ -146,17 +144,8 @@ func copyCertificate(srcConfigDir, destConfigDir, credentialFile string) error {
 	return nil
 }
 
-func copyCredentials(serviceConfigDir, defaultConfigDir, defaultConfigFile, defaultCredentialFile string) error {
-	if err := ensureConfigDirExists(serviceConfigDir); err != nil {
-		return err
-	}
-
-	if err := copyCertificate(defaultConfigDir, serviceConfigDir, defaultCredentialFile); err != nil {
-		return err
-	}
-
+func copyConfig(srcConfigPath, destConfigPath string) error {
 	// Copy or create config
-	destConfigPath := filepath.Join(serviceConfigDir, defaultConfigFile)
 	destFile, exists, err := openFile(destConfigPath, true)
 	if err != nil {
 		logger.WithError(err).Infof("cannot open %s", destConfigPath)
@@ -167,7 +156,6 @@ func copyCredentials(serviceConfigDir, defaultConfigDir, defaultConfigFile, defa
 	}
 	defer destFile.Close()
 
-	srcConfigPath := filepath.Join(defaultConfigDir, defaultConfigFile)
 	srcFile, _, err := openFile(srcConfigPath, false)
 	if err != nil {
 		fmt.Println("Your service needs a config file that at least specifies the hostname option.")

cmd/cloudflared/shell/launch_browser_darwin.go

@@ -0,0 +1,11 @@
+//+build darwin
+
+package shell
+
+import (
+	"os/exec"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	return exec.Command("open", url)
+}

cmd/cloudflared/shell/launch_browser_other.go

@@ -0,0 +1,11 @@
+//+build !windows,!darwin,!linux,!netbsd,!freebsd,!openbsd
+
+package shell
+
+import (
+	"os/exec"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	return nil
+}

cmd/cloudflared/shell/launch_browser_unix.go

@@ -0,0 +1,11 @@
+//+build linux freebsd openbsd netbsd
+
+package shell
+
+import (
+	"os/exec"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	return exec.Command("xdg-open", url)
+}

cmd/cloudflared/shell/launch_browser_windows.go

@@ -0,0 +1,18 @@
+//+build windows
+
+package shell
+
+import (
+	"fmt"
+	"os/exec"
+	"syscall"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	cmd := exec.Command("cmd")
+	// CmdLine is only defined when compiling for windows.
+	// Empty string is the cmd proc "Title". Needs to be included because the start command will interpret the first
+	// quoted string as that field and we want to quote the URL.
+	cmd.SysProcAttr = &syscall.SysProcAttr{CmdLine: fmt.Sprintf(`/c start "" "%s"`, url)}
+	return cmd
+}

cmd/cloudflared/shell/shell.go

@@ -0,0 +1,33 @@
+package shell
+
+import (
+	"io"
+	"os"
+	"os/exec"
+)
+
+// OpenBrowser opens the specified URL in the default browser of the user
+func OpenBrowser(url string) error {
+	return getBrowserCmd(url).Start()
+}
+
+// Run will kick off a shell task and pipe the results to the respective std pipes
+func Run(cmd string, args ...string) error {
+	c := exec.Command(cmd, args...)
+	stderr, err := c.StderrPipe()
+	if err != nil {
+		return err
+	}
+	go func() {
+		io.Copy(os.Stderr, stderr)
+	}()
+
+	stdout, err := c.StdoutPipe()
+	if err != nil {
+		return err
+	}
+	go func() {
+		io.Copy(os.Stdout, stdout)
+	}()
+	return c.Run()
+}

cmd/cloudflared/token/token.go

@@ -0,0 +1,179 @@
+package token
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/path"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
+	"github.com/cloudflare/cloudflared/log"
+	"github.com/cloudflare/cloudflared/origin"
+	"github.com/coreos/go-oidc/jose"
+)
+
+const (
+	keyName = "token"
+)
+
+var logger = log.CreateLogger()
+
+type lock struct {
+	lockFilePath string
+	backoff      *origin.BackoffHandler
+	sigHandler   *signalHandler
+}
+
+type signalHandler struct {
+	sigChannel chan os.Signal
+	signals    []os.Signal
+}
+
+func (s *signalHandler) register(handler func()) {
+	s.sigChannel = make(chan os.Signal, 1)
+	signal.Notify(s.sigChannel, s.signals...)
+	go func(s *signalHandler) {
+		for range s.sigChannel {
+			handler()
+		}
+	}(s)
+}
+
+func (s *signalHandler) deregister() {
+	signal.Stop(s.sigChannel)
+	close(s.sigChannel)
+}
+
+func errDeleteTokenFailed(lockFilePath string) error {
+	return fmt.Errorf("failed to acquire a new Access token. Please try to delete %s", lockFilePath)
+}
+
+// newLock will get a new file lock
+func newLock(path string) *lock {
+	lockPath := path + ".lock"
+	return &lock{
+		lockFilePath: lockPath,
+		backoff:      &origin.BackoffHandler{MaxRetries: 7},
+		sigHandler: &signalHandler{
+			signals: []os.Signal{syscall.SIGINT, syscall.SIGTERM},
+		},
+	}
+}
+
+func (l *lock) Acquire() error {
+	// Intercept SIGINT and SIGTERM to release lock before exiting
+	l.sigHandler.register(func() {
+		l.deleteLockFile()
+		os.Exit(0)
+	})
+
+	// Check for a path.lock file
+	// if the lock file exists; start polling
+	// if not, create the lock file and go through the normal flow.
+	// See AUTH-1736 for the reason why we do all this
+	for isTokenLocked(l.lockFilePath) {
+		if l.backoff.Backoff(context.Background()) {
+			continue
+		}
+
+		if err := l.deleteLockFile(); err != nil {
+			return err
+		}
+	}
+
+	// Create a lock file so other processes won't also try to get the token at
+	// the same time
+	if err := ioutil.WriteFile(l.lockFilePath, []byte{}, 0600); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (l *lock) deleteLockFile() error {
+	if err := os.Remove(l.lockFilePath); err != nil && !os.IsNotExist(err) {
+		return errDeleteTokenFailed(l.lockFilePath)
+	}
+	return nil
+}
+
+func (l *lock) Release() error {
+	defer l.sigHandler.deregister()
+	return l.deleteLockFile()
+}
+
+// isTokenLocked checks to see if there is another process attempting to get the token already
+func isTokenLocked(lockFilePath string) bool {
+	exists, err := config.FileExists(lockFilePath)
+	return exists && err == nil
+}
+
+// FetchToken will either load a stored token or generate a new one
+func FetchToken(appURL *url.URL) (string, error) {
+	if token, err := GetTokenIfExists(appURL); token != "" && err == nil {
+		return token, nil
+	}
+
+	path, err := path.GenerateFilePathFromURL(appURL, keyName)
+	if err != nil {
+		return "", err
+	}
+
+	fileLock := newLock(path)
+
+	err = fileLock.Acquire()
+	if err != nil {
+		return "", err
+	}
+	defer fileLock.Release()
+
+	// check to see if another process has gotten a token while we waited for the lock
+	if token, err := GetTokenIfExists(appURL); token != "" && err == nil {
+		return token, nil
+	}
+
+	// this weird parameter is the resource name (token) and the key/value
+	// we want to send to the transfer service. the key is token and the value
+	// is blank (basically just the id generated in the transfer service)
+	token, err := transfer.Run(appURL, keyName, keyName, "", path, true)
+	if err != nil {
+		return "", err
+	}
+
+	return string(token), nil
+}
+
+// GetTokenIfExists will return the token from local storage if it exists
+func GetTokenIfExists(url *url.URL) (string, error) {
+	path, err := path.GenerateFilePathFromURL(url, keyName)
+	if err != nil {
+		return "", err
+	}
+	content, err := ioutil.ReadFile(path)
+	if err != nil {
+		return "", err
+	}
+	token, err := jose.ParseJWT(string(content))
+	if err != nil {
+		return "", err
+	}
+
+	return token.Encode(), nil
+}
+
+// RemoveTokenIfExists removes the a token from local storage if it exists
+func RemoveTokenIfExists(url *url.URL) error {
+	path, err := path.GenerateFilePathFromURL(url, keyName)
+	if err != nil {
+		return err
+	}
+	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	return nil
+}

cmd/cloudflared/token/token_test.go

@@ -0,0 +1,52 @@
+package token
+
+import (
+	"os"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSignalHandler(t *testing.T) {
+	sigHandler := signalHandler{signals: []os.Signal{syscall.SIGUSR1}}
+	handlerRan := false
+	done := make(chan struct{})
+	timer := time.NewTimer(time.Second)
+	sigHandler.register(func(){
+		handlerRan = true
+		done <- struct{}{}
+	})
+
+	p, err := os.FindProcess(os.Getpid())
+	require.Nil(t, err)
+	p.Signal(syscall.SIGUSR1)
+
+	// Blocks for up to one second to make sure the handler callback runs before the assert.
+	select {
+		case <- done:
+			assert.True(t, handlerRan)
+		case <- timer.C:
+			t.Fail()
+	}
+	sigHandler.deregister()
+}
+
+func TestSignalHandlerClose(t *testing.T) {
+	sigHandler := signalHandler{signals: []os.Signal{syscall.SIGUSR1}}
+	done := make(chan struct{})
+	timer := time.NewTimer(time.Second)
+	sigHandler.register(func(){done <- struct{}{}})
+	sigHandler.deregister()
+
+	p, err := os.FindProcess(os.Getpid())
+	require.Nil(t, err)
+	p.Signal(syscall.SIGUSR1)
+	select {
+	case <- done:
+		t.Fail()
+	case <- timer.C:
+	}
+}

cmd/cloudflared/transfer/transfer.go

@@ -0,0 +1,162 @@
+package transfer
+
+import (
+	"bytes"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/encrypter"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
+	"github.com/cloudflare/cloudflared/log"
+)
+
+const (
+	baseStoreURL  = "https://login.argotunnel.com/"
+	clientTimeout = time.Second * 60
+)
+
+var logger = log.CreateLogger()
+
+// Run does the transfer "dance" with the end result downloading the supported resource.
+// The expanded description is run is encapsulation of shared business logic needed
+// to request a resource (token/cert/etc) from the transfer service (loginhelper).
+// The "dance" we refer to is building a HTTP request, opening that in a browser waiting for
+// the user to complete an action, while it long polls in the background waiting for an
+// action to be completed to download the resource.
+func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncrypt bool) ([]byte, error) {
+	encrypterClient, err := encrypter.New("cloudflared_priv.pem", "cloudflared_pub.pem")
+	if err != nil {
+		return nil, err
+	}
+	requestURL, err := buildRequestURL(transferURL, key, value+encrypterClient.PublicKey(), shouldEncrypt)
+	if err != nil {
+		return nil, err
+	}
+
+	// See AUTH-1423 for why we use stderr (the way git wraps ssh)
+	err = shell.OpenBrowser(requestURL)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Please open the following URL and log in with your Cloudflare account:\n\n%s\n\nLeave cloudflared running to download the %s automatically.\n", requestURL, resourceName)
+	} else {
+		fmt.Fprintf(os.Stderr, "A browser window should have opened at the following URL:\n\n%s\n\nIf the browser failed to open, please visit the URL above directly in your browser.\n", requestURL)
+	}
+
+	var resourceData []byte
+
+	if shouldEncrypt {
+		buf, key, err := transferRequest(baseStoreURL + "transfer/" + encrypterClient.PublicKey())
+		if err != nil {
+			return nil, err
+		}
+
+		decodedBuf, err := base64.StdEncoding.DecodeString(string(buf))
+		if err != nil {
+			return nil, err
+		}
+		decrypted, err := encrypterClient.Decrypt(decodedBuf, key)
+		if err != nil {
+			return nil, err
+		}
+
+		resourceData = decrypted
+	} else {
+		buf, _, err := transferRequest(baseStoreURL + encrypterClient.PublicKey())
+		if err != nil {
+			return nil, err
+		}
+		resourceData = buf
+	}
+
+	if err := ioutil.WriteFile(path, resourceData, 0600); err != nil {
+		return nil, err
+	}
+
+	return resourceData, nil
+}
+
+// BuildRequestURL creates a request suitable for a resource transfer.
+// it will return a constructed url based off the base url and query key/value provided.
+// cli will build a url for cli transfer request.
+func buildRequestURL(baseURL *url.URL, key, value string, cli bool) (string, error) {
+	q := baseURL.Query()
+	q.Set(key, value)
+	baseURL.RawQuery = q.Encode()
+	if !cli {
+		return baseURL.String(), nil
+	}
+
+	q.Set("redirect_url", baseURL.String()) // we add the token as a query param on both the redirect_url
+	baseURL.RawQuery = q.Encode()           // and this actual baseURL.
+	baseURL.Path = "cdn-cgi/access/cli"
+	return baseURL.String(), nil
+}
+
+// transferRequest downloads the requested resource from the request URL
+func transferRequest(requestURL string) ([]byte, string, error) {
+	client := &http.Client{Timeout: clientTimeout}
+	const pollAttempts = 10
+	// we do "long polling" on the endpoint to get the resource.
+	for i := 0; i < pollAttempts; i++ {
+		buf, key, err := poll(client, requestURL)
+		if err != nil {
+			return nil, "", err
+		} else if len(buf) > 0 {
+			if err := putSuccess(client, requestURL); err != nil {
+				logger.WithError(err).Error("Failed to update resource success")
+			}
+			return buf, key, nil
+		}
+	}
+	return nil, "", errors.New("Failed to fetch resource")
+}
+
+// poll the endpoint for the request resource, waiting for the user interaction
+func poll(client *http.Client, requestURL string) ([]byte, string, error) {
+	resp, err := client.Get(requestURL)
+	if err != nil {
+		return nil, "", err
+	}
+	defer resp.Body.Close()
+
+	// ignore everything other than server errors as the resource
+	// may not exist until the user does the interaction
+	if resp.StatusCode >= 500 {
+		return nil, "", fmt.Errorf("error on request %d", resp.StatusCode)
+	}
+	if resp.StatusCode != 200 {
+		logger.Info("Waiting for login...")
+		return nil, "", nil
+	}
+
+	buf := new(bytes.Buffer)
+	if _, err := io.Copy(buf, resp.Body); err != nil {
+		return nil, "", err
+	}
+	return buf.Bytes(), resp.Header.Get("service-public-key"), nil
+}
+
+// putSuccess tells the server we successfully downloaded the resource
+func putSuccess(client *http.Client, requestURL string) error {
+	req, err := http.NewRequest("PUT", requestURL+"/ok", nil)
+	if err != nil {
+		return err
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	resp.Body.Close()
+	if resp.StatusCode != 200 {
+		return fmt.Errorf("HTTP Response Status Code: %d", resp.StatusCode)
+	}
+	return nil
+}

cmd/cloudflared/tunnel/cmd_test.go

@@ -0,0 +1,17 @@
+package tunnel
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHostnameFromURI(t *testing.T) {
+	assert.Equal(t, "awesome.warptunnels.horse:22", hostnameFromURI("ssh://awesome.warptunnels.horse:22"))
+	assert.Equal(t, "awesome.warptunnels.horse:22", hostnameFromURI("ssh://awesome.warptunnels.horse"))
+	assert.Equal(t, "awesome.warptunnels.horse:2222", hostnameFromURI("ssh://awesome.warptunnels.horse:2222"))
+	assert.Equal(t, "localhost:3389", hostnameFromURI("rdp://localhost"))
+	assert.Equal(t, "localhost:3390", hostnameFromURI("rdp://localhost:3390"))
+	assert.Equal(t, "", hostnameFromURI("trash"))
+	assert.Equal(t, "", hostnameFromURI("https://awesomesauce.com"))
+}

cmd/cloudflared/tunnel/configuration.go

@@ -0,0 +1,294 @@
+package tunnel
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/origin"
+	"github.com/cloudflare/cloudflared/tlsconfig"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/validation"
+
+	"github.com/google/uuid"
+	"github.com/mitchellh/go-homedir"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/crypto/ssh/terminal"
+	"gopkg.in/urfave/cli.v2"
+)
+
+var (
+	developerPortal = "https://developers.cloudflare.com/argo-tunnel"
+	quickStartUrl   = developerPortal + "/quickstart/quickstart/"
+	serviceUrl      = developerPortal + "/reference/service/"
+	argumentsUrl    = developerPortal + "/reference/arguments/"
+)
+
+// returns the first path that contains a cert.pem file. If none of the DefaultConfigDirs
+// contains a cert.pem file, return empty string
+func findDefaultOriginCertPath() string {
+	for _, defaultConfigDir := range config.DefaultConfigDirs {
+		originCertPath, _ := homedir.Expand(filepath.Join(defaultConfigDir, config.DefaultCredentialFile))
+		if ok, _ := config.FileExists(originCertPath); ok {
+			return originCertPath
+		}
+	}
+	return ""
+}
+
+func generateRandomClientID(logger *logrus.Logger) (string, error) {
+	u, err := uuid.NewRandom()
+	if err != nil {
+		logger.WithError(err).Error("couldn't create UUID for client ID")
+		return "", err
+	}
+	return u.String(), nil
+}
+
+func handleDeprecatedOptions(c *cli.Context) error {
+	// Fail if the user provided an old authentication method
+	if c.IsSet("api-key") || c.IsSet("api-email") || c.IsSet("api-ca-key") {
+		logger.Error("You don't need to give us your api-key anymore. Please use the new login method. Just run cloudflared login")
+		return fmt.Errorf("Client provided deprecated options")
+	}
+	return nil
+}
+
+func logClientOptions(c *cli.Context) {
+	flags := make(map[string]interface{})
+	for _, flag := range c.LocalFlagNames() {
+		flags[flag] = c.Generic(flag)
+	}
+
+	sliceFlags := []string{"header", "tag", "proxy-dns-upstream", "upstream", "edge"}
+	for _, sliceFlag := range sliceFlags {
+		if len(c.StringSlice(sliceFlag)) > 0 {
+			flags[sliceFlag] = strings.Join(c.StringSlice(sliceFlag), ", ")
+		}
+	}
+
+	if len(flags) > 0 {
+		logger.WithFields(flags).Info("Flags")
+	}
+
+	envs := make(map[string]string)
+	// Find env variables for Argo Tunnel
+	for _, env := range os.Environ() {
+		// All Argo Tunnel env variables start with TUNNEL_
+		if strings.Contains(env, "TUNNEL_") {
+			vars := strings.Split(env, "=")
+			if len(vars) == 2 {
+				envs[vars[0]] = vars[1]
+			}
+		}
+	}
+	if len(envs) > 0 {
+		logger.Infof("Environmental variables %v", envs)
+	}
+}
+
+func dnsProxyStandAlone(c *cli.Context) bool {
+	return c.IsSet("proxy-dns") && (!c.IsSet("hostname") && !c.IsSet("tag") && !c.IsSet("hello-world"))
+}
+
+func getOriginCert(c *cli.Context) ([]byte, error) {
+	if c.String("origincert") == "" {
+		logger.Warnf("Cannot determine default origin certificate path. No file %s in %v", config.DefaultCredentialFile, config.DefaultConfigDirs)
+		if isRunningFromTerminal() {
+			logger.Errorf("You need to specify the origin certificate path with --origincert option, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", argumentsUrl)
+			return nil, fmt.Errorf("Client didn't specify origincert path when running from terminal")
+		} else {
+			logger.Errorf("You need to specify the origin certificate path by specifying the origincert option in the configuration file, or set TUNNEL_ORIGIN_CERT environment variable. See %s for more information.", serviceUrl)
+			return nil, fmt.Errorf("Client didn't specify origincert path")
+		}
+	}
+	// Check that the user has acquired a certificate using the login command
+	originCertPath, err := homedir.Expand(c.String("origincert"))
+	if err != nil {
+		logger.WithError(err).Errorf("Cannot resolve path %s", c.String("origincert"))
+		return nil, fmt.Errorf("Cannot resolve path %s", c.String("origincert"))
+	}
+	ok, err := config.FileExists(originCertPath)
+	if err != nil {
+		logger.Errorf("Cannot check if origin cert exists at path %s", c.String("origincert"))
+		return nil, fmt.Errorf("Cannot check if origin cert exists at path %s", c.String("origincert"))
+	}
+	if !ok {
+		logger.Errorf(`Cannot find a valid certificate for your origin at the path:
+
+    %s
+
+If the path above is wrong, specify the path with the -origincert option.
+If you don't have a certificate signed by Cloudflare, run the command:
+
+	%s login
+`, originCertPath, os.Args[0])
+		return nil, fmt.Errorf("Cannot find a valid certificate at the path %s", originCertPath)
+	}
+	// Easier to send the certificate as []byte via RPC than decoding it at this point
+	originCert, err := ioutil.ReadFile(originCertPath)
+	if err != nil {
+		logger.WithError(err).Errorf("Cannot read %s to load origin certificate", originCertPath)
+		return nil, fmt.Errorf("Cannot read %s to load origin certificate", originCertPath)
+	}
+	return originCert, nil
+}
+
+func prepareTunnelConfig(
+	c *cli.Context,
+	buildInfo *buildinfo.BuildInfo,
+	version string, logger,
+	transportLogger *logrus.Logger,
+) (*origin.TunnelConfig, error) {
+	hostname, err := validation.ValidateHostname(c.String("hostname"))
+	if err != nil {
+		logger.WithError(err).Error("Invalid hostname")
+		return nil, errors.Wrap(err, "Invalid hostname")
+	}
+	isFreeTunnel := hostname == ""
+	clientID := c.String("id")
+	if !c.IsSet("id") {
+		clientID, err = generateRandomClientID(logger)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	tags, err := NewTagSliceFromCLI(c.StringSlice("tag"))
+	if err != nil {
+		logger.WithError(err).Error("Tag parse failure")
+		return nil, errors.Wrap(err, "Tag parse failure")
+	}
+
+	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
+
+	originURL, err := config.ValidateUrl(c)
+	if err != nil {
+		logger.WithError(err).Error("Error validating origin URL")
+		return nil, errors.Wrap(err, "Error validating origin URL")
+	}
+
+	var originCert []byte
+	if !isFreeTunnel {
+		originCert, err = getOriginCert(c)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error getting origin cert")
+		}
+	}
+
+	originCertPool, err := tlsconfig.LoadOriginCA(c, logger)
+	if err != nil {
+		logger.WithError(err).Error("Error loading cert pool")
+		return nil, errors.Wrap(err, "Error loading cert pool")
+	}
+
+	tunnelMetrics := origin.NewTunnelMetrics()
+	httpTransport := &http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		MaxIdleConns:          c.Int("proxy-keepalive-connections"),
+		MaxIdleConnsPerHost:   c.Int("proxy-keepalive-connections"),
+		IdleConnTimeout:       c.Duration("proxy-keepalive-timeout"),
+		TLSHandshakeTimeout:   c.Duration("proxy-tls-timeout"),
+		ExpectContinueTimeout: 1 * time.Second,
+		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: c.IsSet("no-tls-verify")},
+	}
+
+	dialer := &net.Dialer{
+		Timeout:   c.Duration("proxy-connect-timeout"),
+		KeepAlive: c.Duration("proxy-tcp-keepalive"),
+	}
+	if c.Bool("proxy-no-happy-eyeballs") {
+		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
+	}
+	dialContext := dialer.DialContext
+
+	if c.IsSet("unix-socket") {
+		unixSocket, err := config.ValidateUnixSocket(c)
+		if err != nil {
+			logger.WithError(err).Error("Error validating --unix-socket")
+			return nil, errors.Wrap(err, "Error validating --unix-socket")
+		}
+
+		logger.Infof("Proxying tunnel requests to unix:%s", unixSocket)
+		httpTransport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+			// if --unix-socket specified, enforce network type "unix"
+			return dialContext(ctx, "unix", unixSocket)
+		}
+	} else {
+		logger.Infof("Proxying tunnel requests to %s", originURL)
+		httpTransport.DialContext = dialContext
+	}
+
+	if !c.IsSet("hello-world") && c.IsSet("origin-server-name") {
+		httpTransport.TLSClientConfig.ServerName = c.String("origin-server-name")
+	}
+
+	err = validation.ValidateHTTPService(originURL, hostname, httpTransport)
+	if err != nil {
+		logger.WithError(err).Error("unable to connect to the origin")
+		return nil, errors.Wrap(err, "unable to connect to the origin")
+	}
+
+	toEdgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c)
+	if err != nil {
+		logger.WithError(err).Error("unable to create TLS config to connect with edge")
+		return nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
+	}
+
+	return &origin.TunnelConfig{
+		BuildInfo:            buildInfo,
+		ClientID:             clientID,
+		ClientTlsConfig:      httpTransport.TLSClientConfig,
+		CompressionQuality:   c.Uint64("compression-quality"),
+		EdgeAddrs:            c.StringSlice("edge"),
+		GracePeriod:          c.Duration("grace-period"),
+		HAConnections:        c.Int("ha-connections"),
+		HTTPTransport:        httpTransport,
+		HeartbeatInterval:    c.Duration("heartbeat-interval"),
+		Hostname:             hostname,
+		HTTPHostHeader:       c.String("http-host-header"),
+		IncidentLookup:       origin.NewIncidentLookup(),
+		IsAutoupdated:        c.Bool("is-autoupdated"),
+		IsFreeTunnel:         isFreeTunnel,
+		LBPool:               c.String("lb-pool"),
+		Logger:               logger,
+		MaxHeartbeats:        c.Uint64("heartbeat-count"),
+		Metrics:              tunnelMetrics,
+		MetricsUpdateFreq:    c.Duration("metrics-update-freq"),
+		NoChunkedEncoding:    c.Bool("no-chunked-encoding"),
+		OriginCert:           originCert,
+		OriginUrl:            originURL,
+		ReportedVersion:      version,
+		Retries:              c.Uint("retries"),
+		RunFromTerminal:      isRunningFromTerminal(),
+		Tags:                 tags,
+		TlsConfig:            toEdgeTLSConfig,
+		TransportLogger:      transportLogger,
+		UseDeclarativeTunnel: c.Bool("use-declarative-tunnels"),
+		UseReconnectToken:    c.Bool("use-reconnect-token"),
+	}, nil
+}
+
+func serviceDiscoverer(c *cli.Context, logger *logrus.Logger) (connection.EdgeServiceDiscoverer, error) {
+	// If --edge is specfied, resolve edge server addresses
+	if len(c.StringSlice("edge")) > 0 {
+		return connection.NewEdgeHostnameResolver(c.StringSlice("edge"))
+	}
+	// Otherwise lookup edge server addresses through service discovery
+	return connection.NewEdgeAddrResolver(logger)
+}
+
+func isRunningFromTerminal() bool {
+	return terminal.IsTerminal(int(os.Stdout.Fd()))
+}

cmd/cloudflared/tunnel/configuration_test.go

@@ -0,0 +1,214 @@
+// +build ignore
+// TODO: Remove the above build tag and include this test when we start compiling with Golang 1.10.0+
+
+package tunnel
+
+import (
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/asn1"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Generated using `openssl req -newkey rsa:512 -nodes -x509 -days 3650`
+var samplePEM = []byte(`
+-----BEGIN CERTIFICATE-----
+MIIB4DCCAYoCCQCb/H0EUrdXEjANBgkqhkiG9w0BAQsFADB3MQswCQYDVQQGEwJV
+UzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcGA1UECgwQQ2xv
+dWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVneTERMA8GA1UE
+AwwIVGVzdCBPbmUwHhcNMTgwNDI2MTYxMDUxWhcNMjgwNDIzMTYxMDUxWjB3MQsw
+CQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcG
+A1UECgwQQ2xvdWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVn
+eTERMA8GA1UEAwwIVGVzdCBPbmUwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAwVQD
+K0SJ25UFLznm2pU3zhzMEvpDEofHVNnCjk4mlDrtVop7PkKZ8pDEmuQANltUrxC8
+yHBE2wXMv+GlH+bDtwIDAQABMA0GCSqGSIb3DQEBCwUAA0EAjVYQzozIFPkt/HRY
+uUoZ8zEHIDICb0syFf5VAjm9AgTwIPzUmD+c5vl6LWDnxq7L45nLCzhhQ6YmiwDz
+X7Wcyg==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIB4DCCAYoCCQDZfCdAJ+mwzDANBgkqhkiG9w0BAQsFADB3MQswCQYDVQQGEwJV
+UzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcGA1UECgwQQ2xv
+dWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVneTERMA8GA1UE
+AwwIVGVzdCBUd28wHhcNMTgwNDI2MTYxMTIwWhcNMjgwNDIzMTYxMTIwWjB3MQsw
+CQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcG
+A1UECgwQQ2xvdWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVn
+eTERMA8GA1UEAwwIVGVzdCBUd28wXDANBgkqhkiG9w0BAQEFAANLADBIAkEAoHKp
+ROVK3zCSsH7ocYeyRAML4V7SFAbZcb4WIwDnE08oMBVRkQVcW5tqEkvG3RiClfzV
+wZIJ3CfqKIeSNSDU9wIDAQABMA0GCSqGSIb3DQEBCwUAA0EAJw2gUbnPiq4C2p5b
+iWzlA9Q7aKo+VQ4H7IZS7tTccr59nVjvH/TG3eWujpnocr4TOqW9M3CK1DF9mUGP
+3pQ3Jg==
+-----END CERTIFICATE-----
+`)
+
+var systemCertPoolSubjects []*pkix.Name
+
+type certificateFixture struct {
+	ou string
+	cn string
+}
+
+func TestMain(m *testing.M) {
+	systemCertPool, err := x509.SystemCertPool()
+	if isUnrecoverableError(err) {
+		os.Exit(1)
+	}
+
+	if systemCertPool == nil {
+		// On Windows, let's just assume the system cert pool was empty
+		systemCertPool = x509.NewCertPool()
+	}
+
+	systemCertPoolSubjects, err = getCertPoolSubjects(systemCertPool)
+	if err != nil {
+		os.Exit(1)
+	}
+
+	os.Exit(m.Run())
+}
+
+func TestLoadOriginCertPoolJustSystemPool(t *testing.T) {
+	certPoolSubjects := loadCertPoolSubjects(t, nil)
+	extraSubjects := subjectSubtract(systemCertPoolSubjects, certPoolSubjects)
+
+	// Remove extra subjects from the cert pool
+	var filteredSystemCertPoolSubjects []*pkix.Name
+
+	t.Log(extraSubjects)
+
+OUTER:
+	for _, subject := range certPoolSubjects {
+		for _, extraSubject := range extraSubjects {
+			if subject == extraSubject {
+				t.Log(extraSubject)
+				continue OUTER
+			}
+		}
+
+		filteredSystemCertPoolSubjects = append(filteredSystemCertPoolSubjects, subject)
+	}
+
+	assert.Equal(t, len(filteredSystemCertPoolSubjects), len(systemCertPoolSubjects))
+
+	difference := subjectSubtract(systemCertPoolSubjects, filteredSystemCertPoolSubjects)
+	assert.Equal(t, 0, len(difference))
+}
+
+func TestLoadOriginCertPoolCFCertificates(t *testing.T) {
+	certPoolSubjects := loadCertPoolSubjects(t, nil)
+
+	extraSubjects := subjectSubtract(systemCertPoolSubjects, certPoolSubjects)
+
+	expected := []*certificateFixture{
+		{ou: "CloudFlare Origin SSL ECC Certificate Authority"},
+		{ou: "CloudFlare Origin SSL Certificate Authority"},
+		{cn: "origin-pull.cloudflare.net"},
+		{cn: "Argo Tunnel Sample Hello Server Certificate"},
+	}
+
+	assertFixturesMatchSubjects(t, expected, extraSubjects)
+}
+
+func TestLoadOriginCertPoolWithExtraPEMs(t *testing.T) {
+	certPoolWithoutPEMSubjects := loadCertPoolSubjects(t, nil)
+	certPoolWithPEMSubjects := loadCertPoolSubjects(t, samplePEM)
+
+	difference := subjectSubtract(certPoolWithoutPEMSubjects, certPoolWithPEMSubjects)
+
+	assert.Equal(t, 2, len(difference))
+
+	expected := []*certificateFixture{
+		{cn: "Test One"},
+		{cn: "Test Two"},
+	}
+
+	assertFixturesMatchSubjects(t, expected, difference)
+}
+
+func loadCertPoolSubjects(t *testing.T, originCAPoolPEM []byte) []*pkix.Name {
+	certPool, err := loadOriginCertPool(originCAPoolPEM)
+	if isUnrecoverableError(err) {
+		t.Fatal(err)
+	}
+	assert.NotEmpty(t, certPool.Subjects())
+	certPoolSubjects, err := getCertPoolSubjects(certPool)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return certPoolSubjects
+}
+
+func assertFixturesMatchSubjects(t *testing.T, fixtures []*certificateFixture, subjects []*pkix.Name) {
+	assert.Equal(t, len(fixtures), len(subjects))
+
+	for _, fixture := range fixtures {
+		found := false
+		for _, subject := range subjects {
+			found = found || fixtureMatchesSubjectPredicate(fixture, subject)
+		}
+
+		if !found {
+			t.Fail()
+		}
+	}
+}
+
+func fixtureMatchesSubjectPredicate(fixture *certificateFixture, subject *pkix.Name) bool {
+	cnMatch := true
+	if fixture.cn != "" {
+		cnMatch = fixture.cn == subject.CommonName
+	}
+
+	ouMatch := true
+	if fixture.ou != "" {
+		ouMatch = len(subject.OrganizationalUnit) > 0 && fixture.ou == subject.OrganizationalUnit[0]
+	}
+
+	return cnMatch && ouMatch
+}
+
+func subjectSubtract(left []*pkix.Name, right []*pkix.Name) []*pkix.Name {
+	var difference []*pkix.Name
+
+	var found bool
+	for _, r := range right {
+		found = false
+		for _, l := range left {
+			if (*l).String() == (*r).String() {
+				found = true
+			}
+		}
+
+		if !found {
+			difference = append(difference, r)
+		}
+	}
+
+	return difference
+}
+
+func getCertPoolSubjects(certPool *x509.CertPool) ([]*pkix.Name, error) {
+	var subjects []*pkix.Name
+
+	for _, subject := range certPool.Subjects() {
+		var sequence pkix.RDNSequence
+		_, err := asn1.Unmarshal(subject, &sequence)
+		if err != nil {
+			return nil, err
+		}
+
+		name := pkix.Name{}
+		name.FillFromRDNSequence(&sequence)
+
+		subjects = append(subjects, &name)
+	}
+
+	return subjects, nil
+}
+
+func isUnrecoverableError(err error) bool {
+	return err != nil && err.Error() != "crypto/x509: system root pool is not available on Windows"
+}

cmd/cloudflared/tunnel/logger.go

@@ -1,11 +1,10 @@
-package main
+package tunnel
 
 import (
 	"fmt"
 	"os"
 
 	"github.com/cloudflare/cloudflared/log"
-
 	"github.com/rifflock/lfshook"
 	"github.com/sirupsen/logrus"
 	"gopkg.in/urfave/cli.v2"
@@ -14,6 +13,9 @@ import (
 	"github.com/pkg/errors"
 )
 
+const debugLevelWarning = "At debug level, request URL, method, protocol, content legnth and header will be logged. " +
+	"Response status, content length and header will also be logged in debug level."
+
 var logger = log.CreateLogger()
 
 func configMainLogger(c *cli.Context) error {
@@ -23,18 +25,21 @@ func configMainLogger(c *cli.Context) error {
 		return errors.Wrap(err, "Unknown logging level specified")
 	}
 	logger.SetLevel(logLevel)
+	if logLevel == logrus.DebugLevel {
+		logger.Warn(debugLevelWarning)
+	}
 	return nil
 }
 
-func configProtoLogger(c *cli.Context) (*logrus.Logger, error) {
-	protoLogLevel, err := logrus.ParseLevel(c.String("proto-loglevel"))
+func configTransportLogger(c *cli.Context) (*logrus.Logger, error) {
+	transportLogLevel, err := logrus.ParseLevel(c.String("transport-loglevel"))
 	if err != nil {
-		logger.WithError(err).Fatal("Unknown protocol logging level specified")
-		return nil, errors.Wrap(err, "Unknown protocol logging level specified")
+		logger.WithError(err).Fatal("Unknown transport logging level specified")
+		return nil, errors.Wrap(err, "Unknown transport logging level specified")
 	}
-	protoLogger := logrus.New()
-	protoLogger.Level = protoLogLevel
-	return protoLogger, nil
+	transportLogger := logrus.New()
+	transportLogger.Level = transportLogLevel
+	return transportLogger, nil
 }
 
 func initLogFile(c *cli.Context, loggers ...*logrus.Logger) error {
@@ -54,6 +59,8 @@ func initLogFile(c *cli.Context, loggers ...*logrus.Logger) error {
 	}
 	defer f.Close()
 	pathMap := lfshook.PathMap{
+		logrus.DebugLevel: filePath,
+		logrus.WarnLevel:  filePath,
 		logrus.InfoLevel:  filePath,
 		logrus.ErrorLevel: filePath,
 		logrus.FatalLevel: filePath,

cmd/cloudflared/tunnel/login.go

@@ -0,0 +1,69 @@
+package tunnel
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
+	homedir "github.com/mitchellh/go-homedir"
+	cli "gopkg.in/urfave/cli.v2"
+)
+
+const (
+	baseLoginURL     = "https://dash.cloudflare.com/argotunnel"
+	callbackStoreURL = "https://login.argotunnel.com/"
+)
+
+func login(c *cli.Context) error {
+	path, ok, err := checkForExistingCert()
+	if ok {
+		fmt.Fprintf(os.Stdout, "You have an existing certificate at %s which login would overwrite.\nIf this is intentional, please move or delete that file then run this command again.\n", path)
+		return nil
+	} else if err != nil {
+		return err
+	}
+
+	loginURL, err := url.Parse(baseLoginURL)
+	if err != nil {
+		// shouldn't happen, URL is hardcoded
+		return err
+	}
+
+	_, err = transfer.Run(loginURL, "cert", "callback", callbackStoreURL, path, false)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "Failed to write the certificate due to the following error:\n%v\n\nYour browser will download the certificate instead. You will have to manually\ncopy it to the following path:\n\n%s\n", err, path)
+		return err
+	}
+
+	fmt.Fprintf(os.Stdout, "You have successfully logged in.\nIf you wish to copy your credentials to a server, they have been saved to:\n%s\n", path)
+	return nil
+}
+
+func checkForExistingCert() (string, bool, error) {
+	configPath, err := homedir.Expand(config.DefaultConfigDirs[0])
+	if err != nil {
+		return "", false, err
+	}
+	ok, err := config.FileExists(configPath)
+	if !ok && err == nil {
+		// create config directory if doesn't already exist
+		err = os.Mkdir(configPath, 0700)
+	}
+	if err != nil {
+		return "", false, err
+	}
+	path := filepath.Join(configPath, config.DefaultCredentialFile)
+	fileInfo, err := os.Stat(path)
+	if err == nil && fileInfo.Size() > 0 {
+		return path, true, nil
+	}
+	if err != nil && err.(*os.PathError).Err != syscall.ENOENT {
+		return path, false, err
+	}
+
+	return path, false, nil
+}

cmd/cloudflared/tunnel/server.go

@@ -1,4 +1,4 @@
-package main
+package tunnel
 
 import (
 	"github.com/cloudflare/cloudflared/tunneldns"

cmd/cloudflared/tunnel/signal.go

@@ -1,4 +1,4 @@
-package main
+package tunnel
 
 import (
 	"os"

cmd/cloudflared/tunnel/signal_test.go

@@ -1,4 +1,4 @@
-package main
+package tunnel
 
 import (
 	"fmt"

cmd/cloudflared/tunnel/tag.go

@@ -1,4 +1,4 @@
-package main
+package tunnel
 
 import (
 	"fmt"

cmd/cloudflared/tunnel/tag_test.go

@@ -1,4 +1,4 @@
-package main
+package tunnel
 
 import (
 	"testing"

cmd/cloudflared/update.go

@@ -1,115 +0,0 @@
-package main
-
-import (
-	"os"
-	"runtime"
-	"time"
-
-	"golang.org/x/crypto/ssh/terminal"
-	"gopkg.in/urfave/cli.v2"
-
-	"github.com/equinox-io/equinox"
-	"github.com/facebookgo/grace/gracenet"
-)
-
-const (
-	appID                    = "app_idCzgxYerVD"
-	noUpdateInShellMessage   = "cloudflared will not automatically update when run from the shell. To enable auto-updates, run cloudflared as a service: https://developers.cloudflare.com/argo-tunnel/reference/service/"
-	noUpdateOnWindowsMessage = "cloudflared will not automatically update on Windows systems."
-)
-
-var publicKey = []byte(`
------BEGIN ECDSA PUBLIC KEY-----
-MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE4OWZocTVZ8Do/L6ScLdkV+9A0IYMHoOf
-dsCmJ/QZ6aw0w9qkkwEpne1Lmo6+0pGexZzFZOH6w5amShn+RXt7qkSid9iWlzGq
-EKx0BZogHSor9Wy5VztdFaAaVbsJiCbO
------END ECDSA PUBLIC KEY-----
-`)
-
-type ReleaseInfo struct {
-	Updated bool
-	Version string
-	Error   error
-}
-
-func checkForUpdates() ReleaseInfo {
-	var opts equinox.Options
-	if err := opts.SetPublicKeyPEM(publicKey); err != nil {
-		return ReleaseInfo{Error: err}
-	}
-
-	resp, err := equinox.Check(appID, opts)
-	switch {
-	case err == equinox.NotAvailableErr:
-		return ReleaseInfo{}
-	case err != nil:
-		return ReleaseInfo{Error: err}
-	}
-
-	err = resp.Apply()
-	if err != nil {
-		return ReleaseInfo{Error: err}
-	}
-
-	return ReleaseInfo{Updated: true, Version: resp.ReleaseVersion}
-}
-
-func update(_ *cli.Context) error {
-	if updateApplied() {
-		os.Exit(64)
-	}
-	return nil
-}
-
-func autoupdate(freq time.Duration, listeners *gracenet.Net, shutdownC chan struct{}) error {
-	tickC := time.Tick(freq)
-	for {
-		if updateApplied() {
-			os.Args = append(os.Args, "--is-autoupdated=true")
-			pid, err := listeners.StartProcess()
-			if err != nil {
-				logger.WithError(err).Error("Unable to restart server automatically")
-				return err
-			}
-			// stop old process after autoupdate. Otherwise we create a new process
-			// after each update
-			logger.Infof("PID of the new process is %d", pid)
-			return nil
-		}
-		select {
-		case <-tickC:
-		case <-shutdownC:
-			return nil
-		}
-	}
-}
-
-func updateApplied() bool {
-	releaseInfo := checkForUpdates()
-	if releaseInfo.Updated {
-		logger.Infof("Updated to version %s", releaseInfo.Version)
-		return true
-	}
-	if releaseInfo.Error != nil {
-		logger.WithError(releaseInfo.Error).Error("Update check failed")
-	}
-	return false
-}
-
-func isAutoupdateEnabled(c *cli.Context) bool {
-	if runtime.GOOS == "windows" {
-		logger.Info(noUpdateOnWindowsMessage)
-		return false
-	}
-
-	if isRunningFromTerminal() {
-		logger.Info(noUpdateInShellMessage)
-		return false
-	}
-
-	return !c.Bool("no-autoupdate") && c.Duration("autoupdate-freq") != 0
-}
-
-func isRunningFromTerminal() bool {
-	return terminal.IsTerminal(int(os.Stdout.Fd()))
-}

cmd/cloudflared/updater/update.go

@@ -0,0 +1,189 @@
+package updater
+
+import (
+	"context"
+	"os"
+	"runtime"
+	"time"
+
+	"golang.org/x/crypto/ssh/terminal"
+	"gopkg.in/urfave/cli.v2"
+
+	"github.com/cloudflare/cloudflared/log"
+	"github.com/equinox-io/equinox"
+	"github.com/facebookgo/grace/gracenet"
+)
+
+const (
+	DefaultCheckUpdateFreq   = time.Hour * 24
+	appID                    = "app_idCzgxYerVD"
+	noUpdateInShellMessage   = "cloudflared will not automatically update when run from the shell. To enable auto-updates, run cloudflared as a service: https://developers.cloudflare.com/argo-tunnel/reference/service/"
+	noUpdateOnWindowsMessage = "cloudflared will not automatically update on Windows systems."
+)
+
+var (
+	publicKey = []byte(`
+-----BEGIN ECDSA PUBLIC KEY-----
+MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE4OWZocTVZ8Do/L6ScLdkV+9A0IYMHoOf
+dsCmJ/QZ6aw0w9qkkwEpne1Lmo6+0pGexZzFZOH6w5amShn+RXt7qkSid9iWlzGq
+EKx0BZogHSor9Wy5VztdFaAaVbsJiCbO
+-----END ECDSA PUBLIC KEY-----
+`)
+	logger = log.CreateLogger()
+)
+
+type UpdateOutcome struct {
+	Updated bool
+	Version string
+	Error   error
+}
+
+func (uo *UpdateOutcome) noUpdate() bool {
+	return uo.Error != nil && uo.Updated == false
+}
+
+func checkForUpdateAndApply() UpdateOutcome {
+	var opts equinox.Options
+	if err := opts.SetPublicKeyPEM(publicKey); err != nil {
+		return UpdateOutcome{Error: err}
+	}
+
+	resp, err := equinox.Check(appID, opts)
+	switch {
+	case err == equinox.NotAvailableErr:
+		return UpdateOutcome{}
+	case err != nil:
+		return UpdateOutcome{Error: err}
+	}
+
+	err = resp.Apply()
+	if err != nil {
+		return UpdateOutcome{Error: err}
+	}
+
+	return UpdateOutcome{Updated: true, Version: resp.ReleaseVersion}
+}
+
+func Update(_ *cli.Context) error {
+	updateOutcome := loggedUpdate()
+	if updateOutcome.Error != nil {
+		os.Exit(10)
+	}
+
+	if updateOutcome.noUpdate() {
+		logger.Infof("cloudflared is up to date (%s)", updateOutcome.Version)
+	}
+
+	return updateOutcome.Error
+}
+
+// Checks for an update and applies it if one is available
+func loggedUpdate() UpdateOutcome {
+	updateOutcome := checkForUpdateAndApply()
+	if updateOutcome.Updated {
+		logger.Infof("cloudflared has been updated to version %s", updateOutcome.Version)
+	}
+	if updateOutcome.Error != nil {
+		logger.WithError(updateOutcome.Error).Error("update check failed")
+	}
+
+	return updateOutcome
+}
+
+// AutoUpdater periodically checks for new version of cloudflared.
+type AutoUpdater struct {
+	configurable     *configurable
+	listeners        *gracenet.Net
+	updateConfigChan chan *configurable
+}
+
+// AutoUpdaterConfigurable is the attributes of AutoUpdater that can be reconfigured during runtime
+type configurable struct {
+	enabled bool
+	freq    time.Duration
+}
+
+func NewAutoUpdater(freq time.Duration, listeners *gracenet.Net) *AutoUpdater {
+	updaterConfigurable := &configurable{
+		enabled: true,
+		freq:    freq,
+	}
+	if freq == 0 {
+		updaterConfigurable.enabled = false
+		updaterConfigurable.freq = DefaultCheckUpdateFreq
+	}
+	return &AutoUpdater{
+		configurable:     updaterConfigurable,
+		listeners:        listeners,
+		updateConfigChan: make(chan *configurable),
+	}
+}
+
+func (a *AutoUpdater) Run(ctx context.Context) error {
+	ticker := time.NewTicker(a.configurable.freq)
+	for {
+		if a.configurable.enabled {
+			updateOutcome := loggedUpdate()
+			if updateOutcome.Updated {
+				os.Args = append(os.Args, "--is-autoupdated=true")
+				pid, err := a.listeners.StartProcess()
+				if err != nil {
+					logger.WithError(err).Error("Unable to restart server automatically")
+					return err
+				}
+				// stop old process after autoupdate. Otherwise we create a new process
+				// after each update
+				logger.Infof("PID of the new process is %d", pid)
+				return nil
+			}
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case newConfigurable := <-a.updateConfigChan:
+			ticker.Stop()
+			a.configurable = newConfigurable
+			ticker = time.NewTicker(a.configurable.freq)
+			// Check if there is new version of cloudflared after receiving new AutoUpdaterConfigurable
+		case <-ticker.C:
+		}
+	}
+}
+
+// Update is the method to pass new AutoUpdaterConfigurable to a running AutoUpdater. It is safe to be called concurrently
+func (a *AutoUpdater) Update(newFreq time.Duration) {
+	newConfigurable := &configurable{
+		enabled: true,
+		freq:    newFreq,
+	}
+	// A ero duration means autoupdate is disabled
+	if newFreq == 0 {
+		newConfigurable.enabled = false
+		newConfigurable.freq = DefaultCheckUpdateFreq
+	}
+	a.updateConfigChan <- newConfigurable
+}
+
+func IsAutoupdateEnabled(c *cli.Context) bool {
+	if !SupportAutoUpdate() {
+		return false
+	}
+	return !c.Bool("no-autoupdate") && c.Duration("autoupdate-freq") != 0
+}
+
+func SupportAutoUpdate() bool {
+	if runtime.GOOS == "windows" {
+		logger.Info(noUpdateOnWindowsMessage)
+		return false
+	}
+
+	if isRunningFromTerminal() {
+		logger.Info(noUpdateInShellMessage)
+		return false
+	}
+	return true
+}
+
+func isRunningFromTerminal() bool {
+	return terminal.IsTerminal(int(os.Stdout.Fd()))
+}

cmd/cloudflared/updater/update_test.go

@@ -0,0 +1,26 @@
+package updater
+
+import (
+	"context"
+	"testing"
+
+	"github.com/facebookgo/grace/gracenet"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDisabledAutoUpdater(t *testing.T) {
+	listeners := &gracenet.Net{}
+	autoupdater := NewAutoUpdater(0, listeners)
+	ctx, cancel := context.WithCancel(context.Background())
+	errC := make(chan error)
+	go func() {
+		errC <- autoupdater.Run(ctx)
+	}()
+
+	assert.False(t, autoupdater.configurable.enabled)
+	assert.Equal(t, DefaultCheckUpdateFreq, autoupdater.configurable.freq)
+
+	cancel()
+	// Make sure that autoupdater terminates after canceling the context
+	assert.Equal(t, context.Canceled, <-errC)
+}

cmd/cloudflared/windows_service.go

@@ -8,6 +8,7 @@ package main
 import (
 	"fmt"
 	"os"
+	"syscall"
 	"time"
 	"unsafe"
 
@@ -22,6 +23,7 @@ import (
 const (
 	windowsServiceName        = "Cloudflared"
 	windowsServiceDescription = "Argo Tunnel agent"
+	windowsServiceUrl         = "https://developers.cloudflare.com/argo-tunnel/reference/service/"
 
 	recoverActionDelay      = time.Second * 20
 	failureCountResetPeriod = time.Hour * 24
@@ -29,6 +31,10 @@ const (
 	// not defined in golang.org/x/sys/windows package
 	// https://msdn.microsoft.com/en-us/library/windows/desktop/ms681988(v=vs.85).aspx
 	serviceConfigFailureActionsFlag = 4
+
+	// ERROR_FAILED_SERVICE_CONTROLLER_CONNECT
+	// https://docs.microsoft.com/en-us/windows/desktop/debug/system-error-codes--1000-1299-
+	serviceControllerConnectionFailure = 1063
 )
 
 func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
@@ -49,16 +55,48 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 		},
 	})
 
+	// `IsAnInteractiveSession()` isn't exactly equivalent to "should the
+	// process run as a normal EXE?" There are legitimate non-service cases,
+	// like running cloudflared in a GCP startup script, for which
+	// `IsAnInteractiveSession()` returns false. For more context, see:
+	//     https://github.com/judwhite/go-svc/issues/6
+	// It seems that the "correct way" to check "is this a normal EXE?" is:
+	//     1. attempt to connect to the Service Control Manager
+	//     2. get ERROR_FAILED_SERVICE_CONTROLLER_CONNECT
+	// This involves actually trying to start the service.
+
 	isIntSess, err := svc.IsAnInteractiveSession()
 	if err != nil {
 		logger.Fatalf("failed to determine if we are running in an interactive session: %v", err)
 	}
-
 	if isIntSess {
 		app.Run(os.Args)
 		return
 	}
 
+	// Run executes service name by calling windowsService which is a Handler
+	// interface that implements Execute method.
+	// It will set service status to stop after Execute returns
+	err = svc.Run(windowsServiceName, &windowsService{app: app, shutdownC: shutdownC, graceShutdownC: graceShutdownC})
+	if err != nil {
+		if errno, ok := err.(syscall.Errno); ok && int(errno) == serviceControllerConnectionFailure {
+			// Hack: assume this is a false negative from the IsAnInteractiveSession() check above.
+			// Run the app in "interactive" mode anyway.
+			app.Run(os.Args)
+			return
+		}
+		logger.Fatalf("%s service failed: %v", windowsServiceName, err)
+	}
+}
+
+type windowsService struct {
+	app            *cli.App
+	shutdownC      chan struct{}
+	graceShutdownC chan struct{}
+}
+
+// called by the package code at the start of the service
+func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeRequest, statusChan chan<- svc.Status) (ssec bool, errno uint32) {
 	elog, err := eventlog.Open(windowsServiceName)
 	if err != nil {
 		logger.WithError(err).Errorf("Cannot open event log for %s", windowsServiceName)
@@ -67,26 +105,23 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 	defer elog.Close()
 
 	elog.Info(1, fmt.Sprintf("%s service starting", windowsServiceName))
-	// Run executes service name by calling windowsService which is a Handler
-	// interface that implements Execute method.
-	// It will set service status to stop after Execute returns
-	err = svc.Run(windowsServiceName, &windowsService{app: app, elog: elog, shutdownC: shutdownC, graceShutdownC: graceShutdownC})
-	if err != nil {
-		elog.Error(1, fmt.Sprintf("%s service failed: %v", windowsServiceName, err))
-		return
+	defer func() {
+		elog.Info(1, fmt.Sprintf("%s service stopped", windowsServiceName))
+	}()
+
+	// the arguments passed here are only meaningful if they were manually
+	// specified by the user, e.g. using the Services console or `sc start`.
+	// https://docs.microsoft.com/en-us/windows/desktop/services/service-entry-point
+	// https://stackoverflow.com/a/6235139
+	var args []string
+	if len(serviceArgs) > 1 {
+		args = serviceArgs
+	} else {
+		// fall back to the arguments from ImagePath (or, as sc calls it, binPath)
+		args = os.Args
 	}
-	elog.Info(1, fmt.Sprintf("%s service stopped", windowsServiceName))
-}
+	elog.Info(1, fmt.Sprintf("%s service arguments: %v", windowsServiceName, args))
 
-type windowsService struct {
-	app            *cli.App
-	elog           *eventlog.Log
-	shutdownC      chan struct{}
-	graceShutdownC chan struct{}
-}
-
-// called by the package code at the start of the service
-func (s *windowsService) Execute(args []string, r <-chan svc.ChangeRequest, statusChan chan<- svc.Status) (ssec bool, errno uint32) {
 	const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown
 	statusChan <- svc.Status{State: svc.StartPending}
 	errC := make(chan error)
@@ -100,26 +135,23 @@ func (s *windowsService) Execute(args []string, r <-chan svc.ChangeRequest, stat
 		case c := <-r:
 			switch c.Cmd {
 			case svc.Interrogate:
-				s.elog.Info(1, fmt.Sprintf("control request 1 #%d", c))
 				statusChan <- c.CurrentStatus
 			case svc.Stop:
-				s.elog.Info(1, "received stop control request")
 				close(s.graceShutdownC)
 				statusChan <- svc.Status{State: svc.StopPending}
 			case svc.Shutdown:
-				s.elog.Info(1, "received shutdown control request")
 				close(s.shutdownC)
 				statusChan <- svc.Status{State: svc.StopPending}
 			default:
-				s.elog.Error(1, fmt.Sprintf("unexpected control request #%d", c))
+				elog.Error(1, fmt.Sprintf("unexpected control request #%d", c))
 			}
 		case err := <-errC:
 			ssec = true
 			if err != nil {
-				s.elog.Error(1, fmt.Sprintf("cloudflared terminated with error %v", err))
+				elog.Error(1, fmt.Sprintf("cloudflared terminated with error %v", err))
 				errno = 1
 			} else {
-				s.elog.Info(1, "cloudflared terminated without error")
+				elog.Info(1, "cloudflared terminated without error")
 				errno = 0
 			}
 			return
@@ -163,7 +195,7 @@ func installWindowsService(c *cli.Context) error {
 	err = configRecoveryOption(s.Handle)
 	if err != nil {
 		logger.WithError(err).Errorf("Cannot set service recovery actions")
-		logger.Infof("See %s to manually configure service recovery actions", serviceUrl)
+		logger.Infof("See %s to manually configure service recovery actions", windowsServiceUrl)
 	}
 	return nil
 }

cmd/sqlgateway/sqlgateway.go

@@ -1,148 +0,0 @@
-package sqlgateway
-
-import (
-	"database/sql"
-	"encoding/json"
-	"fmt"
-	"math/rand"
-	"net/http"
-	"strings"
-	"time"
-
-	_ "github.com/lib/pq"
-	cli "gopkg.in/urfave/cli.v2"
-
-	"github.com/elgs/gosqljson"
-
-	"github.com/gorilla/mux"
-	"github.com/sirupsen/logrus"
-)
-
-type Message struct {
-	Connection Connection    `json:"connection"`
-	Command    string        `json:"command"`
-	Params     []interface{} `json:"params"`
-}
-
-type Connection struct {
-	SSLMode string `json:"sslmode"`
-	Token   string `json:"token"`
-}
-
-type Response struct {
-	Columns []string   `json:"columns"`
-	Rows    [][]string `json:"rows"`
-	Error   string     `json:"error"`
-}
-
-type Proxy struct {
-	Context  *cli.Context
-	Router   *mux.Router
-	Token    string
-	User     string
-	Password string
-	Driver   string
-	Database string
-	Logger   *logrus.Logger
-}
-
-func StartProxy(c *cli.Context, logger *logrus.Logger, password string) error {
-	proxy := NewProxy(c, logger, password)
-
-	logger.Infof("Starting SQL Gateway Proxy on port %s", strings.Split(c.String("url"), ":")[1])
-
-	err := http.ListenAndServe(":"+strings.Split(c.String("url"), ":")[1], proxy.Router)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func randID(n int, c *cli.Context) string {
-	charBytes := []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890")
-	b := make([]byte, n)
-	for i := range b {
-		b[i] = charBytes[rand.Intn(len(charBytes))]
-	}
-	return fmt.Sprintf("%s&%s", c.String("hostname"), b)
-}
-
-// db://user@dbname
-func parseInfo(input string) (string, string, string) {
-	p1 := strings.Split(input, "://")
-	p2 := strings.Split(p1[1], "@")
-	return p1[0], p2[0], p2[1]
-}
-
-func NewProxy(c *cli.Context, logger *logrus.Logger, pass string) *Proxy {
-	rand.Seed(time.Now().UnixNano())
-	driver, user, dbname := parseInfo(c.String("address"))
-	proxy := Proxy{
-		Context:  c,
-		Router:   mux.NewRouter(),
-		Token:    randID(64, c),
-		Logger:   logger,
-		User:     user,
-		Password: pass,
-		Database: dbname,
-		Driver:   driver,
-	}
-
-	logger.Info(fmt.Sprintf(`
-
-	--------------------
-	SQL Gateway Proxy
-	Token: %s
-	--------------------
-
-	`, proxy.Token))
-
-	proxy.Router.HandleFunc("/", proxy.proxyRequest).Methods("POST")
-	return &proxy
-}
-
-func (proxy *Proxy) proxyRequest(rw http.ResponseWriter, req *http.Request) {
-	var message Message
-	response := Response{}
-
-	err := json.NewDecoder(req.Body).Decode(&message)
-	if err != nil {
-		proxy.Logger.Error(err)
-		http.Error(rw, fmt.Sprintf("400 - %s", err.Error()), http.StatusBadRequest)
-		return
-	}
-
-	if message.Connection.Token != proxy.Token {
-		proxy.Logger.Error("Invalid token")
-		http.Error(rw, "400 - Invalid token", http.StatusBadRequest)
-		return
-	}
-
-	connStr := fmt.Sprintf("user=%s password=%s dbname=%s sslmode=%s", proxy.User, proxy.Password, proxy.Database, message.Connection.SSLMode)
-
-	db, err := sql.Open(proxy.Driver, connStr)
-	defer db.Close()
-
-	if err != nil {
-		proxy.Logger.Error(err)
-		http.Error(rw, fmt.Sprintf("400 - %s", err.Error()), http.StatusBadRequest)
-		return
-
-	} else {
-		proxy.Logger.Info("Forwarding SQL: ", message.Command)
-		rw.Header().Set("Content-Type", "application/json")
-
-		headers, data, err := gosqljson.QueryDbToArray(db, "lower", message.Command, message.Params...)
-
-		if err != nil {
-			proxy.Logger.Error(err)
-			http.Error(rw, fmt.Sprintf("400 - %s", err.Error()), http.StatusBadRequest)
-			return
-
-		} else {
-			response = Response{headers, data, ""}
-		}
-	}
-	json.NewEncoder(rw).Encode(response)
-}

connection/connection.go

@@ -0,0 +1,55 @@
+package connection
+
+import (
+	"context"
+	"net"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+const (
+	openStreamTimeout = 30 * time.Second
+)
+
+type Connection struct {
+	id    uuid.UUID
+	muxer *h2mux.Muxer
+	addr  *net.TCPAddr
+}
+
+func newConnection(muxer *h2mux.Muxer, addr *net.TCPAddr) (*Connection, error) {
+	id, err := uuid.NewRandom()
+	if err != nil {
+		return nil, err
+	}
+	return &Connection{
+		id:    id,
+		muxer: muxer,
+		addr:  addr,
+	}, nil
+}
+
+func (c *Connection) Serve(ctx context.Context) error {
+	// Serve doesn't return until h2mux is shutdown
+	return c.muxer.Serve(ctx)
+}
+
+// Connect is used to establish connections with cloudflare's edge network
+func (c *Connection) Connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters, logger *logrus.Entry) (tunnelpogs.ConnectResult, error) {
+	tsClient, err := NewRPCClient(ctx, c.muxer, logger.WithField("rpc", "connect"), openStreamTimeout)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot create new RPC connection")
+	}
+	defer tsClient.Close()
+	return tsClient.Connect(ctx, parameters)
+}
+
+func (c *Connection) Shutdown() {
+	c.muxer.Shutdown()
+}

connection/dial.go

@@ -0,0 +1,54 @@
+package connection
+
+import (
+	"context"
+	"crypto/tls"
+	"net"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// DialEdge makes a TLS connection to a Cloudflare edge node
+func DialEdge(
+	ctx context.Context,
+	timeout time.Duration,
+	tlsConfig *tls.Config,
+	edgeTCPAddr *net.TCPAddr,
+) (net.Conn, error) {
+	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
+	dialCtx, dialCancel := context.WithTimeout(ctx, timeout)
+	defer dialCancel()
+
+	dialer := net.Dialer{}
+	edgeConn, err := dialer.DialContext(dialCtx, "tcp", edgeTCPAddr.String())
+	if err != nil {
+		return nil, newDialError(err, "DialContext error")
+	}
+	tlsEdgeConn := tls.Client(edgeConn, tlsConfig)
+	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
+
+	if err = tlsEdgeConn.Handshake(); err != nil {
+		return nil, newDialError(err, "TLS handshake with edge error")
+	}
+	// clear the deadline on the conn; h2mux has its own timeouts
+	tlsEdgeConn.SetDeadline(time.Time{})
+	return tlsEdgeConn, nil
+}
+
+// DialError is an error returned from DialEdge
+type DialError struct {
+	cause error
+}
+
+func newDialError(err error, message string) error {
+	return DialError{cause: errors.Wrap(err, message)}
+}
+
+func (e DialError) Error() string {
+	return e.cause.Error()
+}
+
+func (e DialError) Cause() error {
+	return e.cause
+}

connection/discovery.go

@@ -0,0 +1,420 @@
+package connection
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"math/rand"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	// Used to discover HA origintunneld servers
+	srvService = "origintunneld"
+	srvProto   = "tcp"
+	srvName    = "argotunnel.com"
+
+	// Used to fallback to DoT when we can't use the default resolver to
+	// discover HA origintunneld servers (GitHub issue #75).
+	dotServerName = "cloudflare-dns.com"
+	dotServerAddr = "1.1.1.1:853"
+	dotTimeout    = time.Duration(15 * time.Second)
+
+	// SRV record resolution TTL
+	resolveEdgeAddrTTL = 1 * time.Hour
+
+	subsystemEdgeAddrResolver = "edgeAddrResolver"
+)
+
+// Redeclare network functions so they can be overridden in tests.
+var (
+	netLookupSRV = net.LookupSRV
+	netLookupIP  = net.LookupIP
+)
+
+// If the call to net.LookupSRV fails, try to fall back to DoT from Cloudflare directly.
+//
+// Note: Instead of DoT, we could also have used DoH. Either of these:
+//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
+//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
+// But both of these cases miss out on a key feature from the stdlib:
+//     "The returned records are sorted by priority and randomized by weight within a priority."
+//     (https://golang.org/pkg/net/#Resolver.LookupSRV)
+// Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it.
+// See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552
+var fallbackLookupSRV = lookupSRVWithDOT
+
+var friendlyDNSErrorLines = []string{
+	`Please try the following things to diagnose this issue:`,
+	`  1. ensure that argotunnel.com is returning "origintunneld" service records.`,
+	`     Run your system's equivalent of: dig srv _origintunneld._tcp.argotunnel.com`,
+	`  2. ensure that your DNS resolver is not returning compressed SRV records.`,
+	`     See GitHub issue https://github.com/golang/go/issues/27546`,
+	`     For example, you could use Cloudflare's 1.1.1.1 as your resolver:`,
+	`     https://developers.cloudflare.com/1.1.1.1/setting-up-1.1.1.1/`,
+}
+
+// EdgeServiceDiscoverer is an interface for looking up Cloudflare's edge network addresses
+type EdgeServiceDiscoverer interface {
+	// Addr returns an unused address to connect to cloudflare's edge network.
+	// Before this method returns, the address will be removed from the pool of available addresses,
+	// so the caller can assume they have exclusive access to the address for tunneling purposes.
+	// The caller should remember to put it back via ReplaceAddr or MarkAddrBad.
+	Addr() (*net.TCPAddr, error)
+	// AnyAddr returns an address to connect to cloudflare's edge network.
+	// It may or may not be in active use for a tunnel.
+	// The caller should NOT return it via ReplaceAddr or MarkAddrBad!
+	AnyAddr() (*net.TCPAddr, error)
+	// ReplaceAddr is called when the address is no longer needed, e.g. due to a scaling-down of numHAConnections.
+	// It returns the address to the pool of available addresses.
+	ReplaceAddr(addr *net.TCPAddr)
+	// MarkAddrBad is called when there was a connectivity error for the address.
+	// It marks the address as unused but doesn't return it to the pool of available addresses.
+	MarkAddrBad(addr *net.TCPAddr)
+	// AvailableAddrs returns the number of addresses available for use
+	// (less those that have been marked bad).
+	AvailableAddrs() int
+	// Refresh rediscovers Cloudflare's edge network addresses.
+	// It resets the state of "bad" addresses but not those in active use.
+	Refresh() error
+}
+
+// EdgeAddrResolver discovers the addresses of Cloudflare's edge network through SRV record.
+// It implements EdgeServiceDiscoverer interface
+type EdgeAddrResolver struct {
+	sync.Mutex
+	// HA regions
+	regions []*region
+	// Logger for noteworthy events
+	logger *logrus.Entry
+}
+
+type region struct {
+	// Addresses that we expect will be in active use
+	addrs []*net.TCPAddr
+	// Addresses that are in active use.
+	// This is actually a set of net.TCPAddr's, but we can't make a map like
+	//     map[net.TCPAddr]bool
+	// since net.TCPAddr contains a field of type net.IP and therefore it cannot be used as a map key.
+	// So instead we use map[string]*net.TCPAddr, where the keys are obtained by net.TCPAddr.String().
+	// (We keep the "raw" *net.TCPAddr values for the convenience of AnyAddr(). If that method didn't
+	// exist, we wouldn't strictly need the values, and this could be a map[string]bool.)
+	inUse map[string]*net.TCPAddr
+	// Addresses that were discarded due to a network error.
+	// Not sure what we'll do with these, but it feels good to keep them around for now.
+	bad []*net.TCPAddr
+}
+
+func NewEdgeAddrResolver(logger *logrus.Logger) (EdgeServiceDiscoverer, error) {
+	r := &EdgeAddrResolver{
+		logger: logger.WithField("subsystem", subsystemEdgeAddrResolver),
+	}
+	if err := r.Refresh(); err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+func (r *EdgeAddrResolver) Addr() (*net.TCPAddr, error) {
+	r.Lock()
+	defer r.Unlock()
+
+	// compute the largest region based on len(addrs)
+	var largestRegion *region
+	{
+		if len(r.regions) == 0 {
+			return nil, errors.New("No HA regions")
+		}
+		largestRegion = r.regions[0]
+		for _, region := range r.regions[1:] {
+			if len(region.addrs) > len(largestRegion.addrs) {
+				largestRegion = region
+			}
+		}
+		if len(largestRegion.addrs) == 0 {
+			return nil, errors.New("No IP address to claim")
+		}
+	}
+
+	var addr *net.TCPAddr
+	addr, largestRegion.addrs = popAddr(largestRegion.addrs)
+	largestRegion.inUse[addr.String()] = addr
+	return addr, nil
+}
+
+func (r *EdgeAddrResolver) AnyAddr() (*net.TCPAddr, error) {
+	r.Lock()
+	defer r.Unlock()
+	for _, region := range r.regions {
+		// return an unused addr
+		if len(region.addrs) > 0 {
+			return region.addrs[rand.Intn(len(region.addrs))], nil
+		}
+		// return an addr that's in use
+		for _, addr := range region.inUse {
+			return addr, nil
+		}
+	}
+	return nil, fmt.Errorf("No IP addresses")
+}
+
+func (r *EdgeAddrResolver) ReplaceAddr(addr *net.TCPAddr) {
+	r.Lock()
+	defer r.Unlock()
+	addrString := addr.String()
+	for _, region := range r.regions {
+		if _, ok := region.inUse[addrString]; ok {
+			delete(region.inUse, addrString)
+			region.addrs = append(region.addrs, addr)
+			break
+		}
+	}
+}
+
+func (r *EdgeAddrResolver) MarkAddrBad(addr *net.TCPAddr) {
+	r.Lock()
+	defer r.Unlock()
+	addrString := addr.String()
+	for _, region := range r.regions {
+		if _, ok := region.inUse[addrString]; ok {
+			delete(region.inUse, addrString)
+			region.bad = append(region.bad, addr)
+			break
+		}
+	}
+}
+
+func (r *EdgeAddrResolver) AvailableAddrs() int {
+	r.Lock()
+	defer r.Unlock()
+	result := 0
+	for _, region := range r.regions {
+		result += len(region.addrs)
+	}
+	return result
+}
+
+func (r *EdgeAddrResolver) Refresh() error {
+	addrLists, err := EdgeDiscovery(r.logger)
+	if err != nil {
+		return err
+	}
+
+	r.Lock()
+	defer r.Unlock()
+	inUse := allInUse(r.regions)
+	r.regions = makeHARegions(addrLists, inUse)
+	return nil
+}
+
+// EdgeDiscovery implements HA service discovery lookup.
+func EdgeDiscovery(logger *logrus.Entry) ([][]*net.TCPAddr, error) {
+	_, addrs, err := netLookupSRV(srvService, srvProto, srvName)
+	if err != nil {
+		_, fallbackAddrs, fallbackErr := fallbackLookupSRV(srvService, srvProto, srvName)
+		if fallbackErr != nil || len(fallbackAddrs) == 0 {
+			// use the original DNS error `err` in messages, not `fallbackErr`
+			logger.Errorln("Error looking up Cloudflare edge IPs: the DNS query failed:", err)
+			for _, s := range friendlyDNSErrorLines {
+				logger.Errorln(s)
+			}
+			return nil, errors.Wrapf(err, "Could not lookup srv records on _%v._%v.%v", srvService, srvProto, srvName)
+		}
+		// Accept the fallback results and keep going
+		addrs = fallbackAddrs
+	}
+
+	var resolvedIPsPerCNAME [][]*net.TCPAddr
+	for _, addr := range addrs {
+		ips, err := resolveSRVToTCP(addr)
+		if err != nil {
+			return nil, err
+		}
+		resolvedIPsPerCNAME = append(resolvedIPsPerCNAME, ips)
+	}
+
+	return resolvedIPsPerCNAME, nil
+}
+
+func lookupSRVWithDOT(service, proto, name string) (cname string, addrs []*net.SRV, err error) {
+	// Inspiration: https://github.com/artyom/dot/blob/master/dot.go
+	r := &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, _ string, _ string) (net.Conn, error) {
+			var dialer net.Dialer
+			conn, err := dialer.DialContext(ctx, "tcp", dotServerAddr)
+			if err != nil {
+				return nil, err
+			}
+			tlsConfig := &tls.Config{ServerName: dotServerName}
+			return tls.Client(conn, tlsConfig), nil
+		},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), dotTimeout)
+	defer cancel()
+	return r.LookupSRV(ctx, srvService, srvProto, srvName)
+}
+
+func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
+	ips, err := netLookupIP(srv.Target)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Couldn't resolve SRV record %v", srv)
+	}
+	if len(ips) == 0 {
+		return nil, fmt.Errorf("SRV record %v had no IPs", srv)
+	}
+	addrs := make([]*net.TCPAddr, len(ips))
+	for i, ip := range ips {
+		addrs[i] = &net.TCPAddr{IP: ip, Port: int(srv.Port)}
+	}
+	return addrs, nil
+}
+
+// EdgeHostnameResolver discovers the addresses of Cloudflare's edge network via a list of server hostnames.
+// It implements EdgeServiceDiscoverer interface, and is used mainly for testing connectivity.
+type EdgeHostnameResolver struct {
+	sync.Mutex
+	// hostnames of edge servers
+	hostnames []string
+	// Addrs to connect to cloudflare's edge network
+	addrs []*net.TCPAddr
+	// Addresses that are in active use.
+	// This is actually a set of net.TCPAddr's. We have to encode the keys
+	// with .String(), since net.TCPAddr contains a field of type net.IP and
+	// therefore it cannot be used as a map key
+	inUse map[string]*net.TCPAddr
+	// Addresses that were discarded due to a network error.
+	// Not sure what we'll do with these, but it feels good to keep them around for now.
+	bad []*net.TCPAddr
+}
+
+func NewEdgeHostnameResolver(edgeHostnames []string) (EdgeServiceDiscoverer, error) {
+	r := &EdgeHostnameResolver{
+		hostnames: edgeHostnames,
+		inUse:     map[string]*net.TCPAddr{},
+	}
+	if err := r.Refresh(); err != nil {
+		return nil, err
+	}
+	return r, nil
+}
+
+func (r *EdgeHostnameResolver) Addr() (*net.TCPAddr, error) {
+	r.Lock()
+	defer r.Unlock()
+	if len(r.addrs) == 0 {
+		return nil, errors.New("No IP address to claim")
+	}
+	var addr *net.TCPAddr
+	addr, r.addrs = popAddr(r.addrs)
+	r.inUse[addr.String()] = addr
+	return addr, nil
+}
+
+func (r *EdgeHostnameResolver) AnyAddr() (*net.TCPAddr, error) {
+	r.Lock()
+	defer r.Unlock()
+	// return an unused addr
+	if len(r.addrs) > 0 {
+		return r.addrs[rand.Intn(len(r.addrs))], nil
+	}
+	// return an addr that's in use
+	for _, addr := range r.inUse {
+		return addr, nil
+	}
+	return nil, errors.New("No IP addresses")
+}
+
+func (r *EdgeHostnameResolver) ReplaceAddr(addr *net.TCPAddr) {
+	r.Lock()
+	defer r.Unlock()
+	delete(r.inUse, addr.String())
+	r.addrs = append(r.addrs, addr)
+}
+func (r *EdgeHostnameResolver) MarkAddrBad(addr *net.TCPAddr) {
+	r.Lock()
+	defer r.Unlock()
+	delete(r.inUse, addr.String())
+	r.bad = append(r.bad, addr)
+}
+
+func (r *EdgeHostnameResolver) AvailableAddrs() int {
+	r.Lock()
+	defer r.Unlock()
+	return len(r.addrs)
+}
+
+func (r *EdgeHostnameResolver) Refresh() error {
+	newAddrs, err := ResolveAddrs(r.hostnames)
+	if err != nil {
+		return err
+	}
+	r.Lock()
+	defer r.Unlock()
+	var notInUse []*net.TCPAddr
+	for _, newAddr := range newAddrs {
+		if _, ok := r.inUse[newAddr.String()]; !ok {
+			notInUse = append(notInUse, newAddr)
+		}
+	}
+	r.addrs = notInUse
+	r.bad = nil
+	return nil
+}
+
+// Resolve TCP address given a list of addresses. Address can be a hostname, however, it will return at most one
+// of the hostname's IP addresses
+func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) {
+	var tcpAddrs []*net.TCPAddr
+	for _, addr := range addrs {
+		tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
+		if err != nil {
+			return nil, err
+		}
+		tcpAddrs = append(tcpAddrs, tcpAddr)
+	}
+	return tcpAddrs, nil
+}
+
+// Compute total set of IP addresses in use. This is useful if the regions
+// are returned in a different order, or if an IP address is assigned to
+// a different region for some reasion.
+func allInUse(regions []*region) map[string]*net.TCPAddr {
+	result := make(map[string]*net.TCPAddr)
+	for _, region := range regions {
+		for k, v := range region.inUse {
+			result[k] = v
+		}
+	}
+	return result
+}
+
+func makeHARegions(addrLists [][]*net.TCPAddr, inUse map[string]*net.TCPAddr) (regions []*region) {
+	for _, addrList := range addrLists {
+		region := &region{inUse: map[string]*net.TCPAddr{}}
+		for _, addr := range addrList {
+			addrString := addr.String()
+			// No matter what region `addr` used to belong to, it's now a part
+			// of this region, so add it to this region's `inUse` map.
+			if _, ok := inUse[addrString]; ok {
+				region.inUse[addrString] = addr
+			} else {
+				region.addrs = append(region.addrs, addr)
+			}
+		}
+		regions = append(regions, region)
+	}
+	return
+}
+
+func popAddr(addrs []*net.TCPAddr) (*net.TCPAddr, []*net.TCPAddr) {
+	first := addrs[0]
+	addrs[0] = nil // prevent memory leak
+	addrs = addrs[1:]
+	return first, addrs
+}

connection/discovery_test.go

@@ -0,0 +1,317 @@
+package connection
+
+import (
+	"net"
+	"sync"
+	"testing"
+	"testing/quick"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEdgeDiscovery(t *testing.T) {
+	mockAddrs := newMockAddrs(19, 2, 5)
+	netLookupSRV = mockNetLookupSRV(mockAddrs)
+	netLookupIP = mockNetLookupIP(mockAddrs)
+
+	expectedAddrSet := map[string]bool{}
+	for _, addrs := range mockAddrs.addrMap {
+		for _, addr := range addrs {
+			expectedAddrSet[addr.String()] = true
+		}
+	}
+
+	addrLists, err := EdgeDiscovery(logrus.New().WithFields(logrus.Fields{}))
+	assert.NoError(t, err)
+	actualAddrSet := map[string]bool{}
+	for _, addrs := range addrLists {
+		for _, addr := range addrs {
+			actualAddrSet[addr.String()] = true
+		}
+	}
+
+	assert.Equal(t, expectedAddrSet, actualAddrSet)
+}
+
+func TestAllInUse(t *testing.T) {
+	for _, testCase := range []struct {
+		regions  []*region
+		expected map[string]*net.TCPAddr
+	}{
+		{
+			regions:  nil,
+			expected: map[string]*net.TCPAddr{},
+		},
+		{
+			regions: []*region{
+				&region{inUse: map[string]*net.TCPAddr{}},
+				&region{inUse: map[string]*net.TCPAddr{}},
+			},
+			expected: map[string]*net.TCPAddr{},
+		},
+		{
+			regions: []*region{
+				&region{inUse: map[string]*net.TCPAddr{":1": &net.TCPAddr{Port: 1}}},
+				&region{inUse: map[string]*net.TCPAddr{":4": &net.TCPAddr{Port: 4}}},
+			},
+			expected: map[string]*net.TCPAddr{":1": &net.TCPAddr{Port: 1}, ":4": &net.TCPAddr{Port: 4}},
+		},
+	} {
+		actual := allInUse(testCase.regions)
+		assert.Equal(t, testCase.expected, actual)
+	}
+}
+
+func TestMakeRegions(t *testing.T) {
+	for _, testCase := range []struct {
+		addrList [][]*net.TCPAddr
+		inUse    map[string]*net.TCPAddr
+		expected []*region
+	}{
+		{
+			addrList: [][]*net.TCPAddr{},
+			expected: nil,
+		},
+		{
+			addrList: [][]*net.TCPAddr{
+				[]*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}},
+			},
+			expected: []*region{
+				&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}}, inUse: map[string]*net.TCPAddr{}},
+			},
+		},
+		{
+			addrList: [][]*net.TCPAddr{
+				[]*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}},
+				[]*net.TCPAddr{&net.TCPAddr{Port: 3}, &net.TCPAddr{Port: 4}},
+			},
+			expected: []*region{
+				&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}}, inUse: map[string]*net.TCPAddr{}},
+				&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 3}, &net.TCPAddr{Port: 4}}, inUse: map[string]*net.TCPAddr{}},
+			},
+		},
+		{
+			addrList: [][]*net.TCPAddr{
+				[]*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}},
+				[]*net.TCPAddr{&net.TCPAddr{Port: 3}, &net.TCPAddr{Port: 4}},
+			},
+			inUse: map[string]*net.TCPAddr{
+				":1": &net.TCPAddr{Port: 1},
+				":4": &net.TCPAddr{Port: 4},
+			},
+			expected: []*region{
+				&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 2}}, inUse: map[string]*net.TCPAddr{":1": &net.TCPAddr{Port: 1}}},
+				&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 3}}, inUse: map[string]*net.TCPAddr{":4": &net.TCPAddr{Port: 4}}},
+			},
+		},
+	} {
+		actual := makeHARegions(testCase.addrList, testCase.inUse)
+		assert.Equal(t, testCase.expected, actual)
+	}
+}
+
+func assertIsBalanced(t *testing.T, regions []*region) bool {
+	// Compute max(len(region.addrs) for region in regions)
+	// No region should have significantly fewer addresses than this
+	var longestAddrs int
+	{
+		longestAddrs = 0
+		for _, region := range regions {
+			if l := len(region.addrs); l > longestAddrs {
+				longestAddrs = l
+			}
+		}
+	}
+	for _, region := range regions {
+		if len(region.addrs) == longestAddrs || len(region.addrs) == longestAddrs-1 {
+			continue
+		}
+		return assert.Fail(t,
+			"found a region with %v free addrs, while the longest addrs list is %v",
+			len(region.addrs), longestAddrs)
+	}
+	return true
+}
+
+// Various end-to-end tests, run with quickcheck (i.e. the testing/quick package)
+func TestEdgeAddrResolver(t *testing.T) {
+	concurrentReplacement := func(mockAddrs mockAddrs) bool {
+		netLookupSRV = mockNetLookupSRV(mockAddrs)
+		netLookupIP = mockNetLookupIP(mockAddrs)
+
+		resolver, err := NewEdgeAddrResolver(logrus.New())
+		if !assert.NoError(t, err) {
+			return false
+		}
+		assert.Equal(t, mockAddrs.numAddrs, resolver.AvailableAddrs(),
+			"every address should be initially available")
+
+		// Create several goroutines to simulate HA connections that acquire
+		// and replace IP addresses.
+		var wg sync.WaitGroup
+		wg.Add(mockAddrs.numAddrs)
+		for i := 0; i < mockAddrs.numAddrs; i++ {
+			go func() {
+				defer wg.Done()
+				const reconnectionCount = 50
+				for i := 0; i < reconnectionCount; i++ {
+					if resolver.AvailableAddrs() == 0 {
+						err = resolver.Refresh()
+						assert.NoError(t, err)
+					}
+					addr, err := resolver.Addr()
+					if !assert.NoError(t, err) {
+						return
+					}
+					time.Sleep(0) // allow some other goroutine to run
+					resolver.ReplaceAddr(addr)
+					time.Sleep(0) // allow some other goroutine to run
+				}
+			}()
+		}
+		wg.Wait()
+		assert.Equal(t, mockAddrs.numAddrs, resolver.AvailableAddrs(),
+			"every address should be available after replacement")
+		return !t.Failed()
+	}
+
+	badAddrWithRefresh := func(mockAddrs mockAddrs) bool {
+		netLookupSRV = mockNetLookupSRV(mockAddrs)
+		netLookupIP = mockNetLookupIP(mockAddrs)
+
+		resolver, err := NewEdgeAddrResolver(logrus.New())
+		if !assert.NoError(t, err) {
+			return false
+		}
+		assert.Equal(t, mockAddrs.numAddrs, resolver.AvailableAddrs(),
+			"every address should be initially available")
+
+		var addrs []*net.TCPAddr
+		for i := 0; i < mockAddrs.numAddrs; i++ {
+			assert.Equal(t, mockAddrs.numAddrs-i, resolver.AvailableAddrs())
+			addr, err := resolver.Addr()
+			assert.NoError(t, err)
+			addrs = append(addrs, addr)
+		}
+		assert.Equal(t, 0, resolver.AvailableAddrs(), "all addresses should have been taken")
+		_, err = resolver.Addr()
+		assert.Error(t, err)
+
+		anyAddr, err := resolver.AnyAddr()
+		assert.NoError(t, err, "should still be okay to call AnyAddr")
+
+		resolver.MarkAddrBad(anyAddr)
+
+		assert.Equal(t, 0, resolver.AvailableAddrs(), "all addresses should still be used")
+		_, err = resolver.Addr()
+		assert.Error(t, err, "all addresses should still be used")
+
+		err = resolver.Refresh()
+		assert.NoError(t, err, "Refresh() should have worked")
+
+		assert.Equal(t, 1, resolver.AvailableAddrs(),
+			"Refresh() should have reset the state of the 'bad' address")
+		addr, err := resolver.Addr()
+		assert.NoError(t, err)
+		assert.Equal(t, anyAddr, addr)
+
+		_, err = resolver.Addr()
+		assert.Error(t, err, "all addresses should be used again")
+
+		return !t.Failed()
+	}
+
+	assert.NoError(t, quick.Check(concurrentReplacement, nil))
+	assert.NoError(t, quick.Check(badAddrWithRefresh, nil))
+}
+
+// "White-box" test: runs Addr() and checks internal state
+func TestEdgeAddrResolver_Addr(t *testing.T) {
+	e := &EdgeAddrResolver{regions: nil}
+	addr, err := e.Addr()
+	assert.Error(t, err)
+
+	testRegions := func() []*region {
+		return []*region{
+			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}}, inUse: map[string]*net.TCPAddr{":2": &net.TCPAddr{Port: 2}, ":3": &net.TCPAddr{Port: 3}}},
+			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 4}, &net.TCPAddr{Port: 5}}, inUse: map[string]*net.TCPAddr{":6": &net.TCPAddr{Port: 6}}},
+			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 7}, &net.TCPAddr{Port: 8}}, inUse: map[string]*net.TCPAddr{":9": &net.TCPAddr{Port: 9}}},
+		}
+	}
+	e = &EdgeAddrResolver{regions: testRegions()}
+	addr, err = e.Addr()
+	assert.NoError(t, err)
+	assert.Equal(t, &net.TCPAddr{Port: 4}, addr)
+	var expected []*region
+	{
+		expected = testRegions()
+		expected[1].addrs = expected[1].addrs[1:]
+		expected[1].inUse[":4"] = &net.TCPAddr{Port: 4}
+	}
+	assert.Equal(t, expected, e.regions)
+}
+
+// "White-box" test: runs AnyAddr() and checks internal state
+func TestEdgeAddrResolver_AnyAddr(t *testing.T) {
+	e := &EdgeAddrResolver{regions: nil}
+	addr, err := e.AnyAddr()
+	assert.Error(t, err)
+
+	e = &EdgeAddrResolver{regions: []*region{&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}}, inUse: map[string]*net.TCPAddr{":2": &net.TCPAddr{Port: 2}}}}}
+	addr, err = e.AnyAddr()
+	assert.NoError(t, err)
+	assert.Equal(t, &net.TCPAddr{Port: 1}, addr, "should have chosen the inactive address")
+
+	e = &EdgeAddrResolver{regions: []*region{&region{inUse: map[string]*net.TCPAddr{":1": &net.TCPAddr{Port: 1}}}}}
+	addr, err = e.AnyAddr()
+	assert.NoError(t, err)
+	assert.Equal(t, &net.TCPAddr{Port: 1}, addr, "should have chosen an active address rather than nothing")
+}
+
+// "White-box" test: runs ReplaceAddr() and checks internal state
+func TestEdgeAddrResolver_ReplaceAddr(t *testing.T) {
+	e := &EdgeAddrResolver{regions: nil}
+	e.ReplaceAddr(&net.TCPAddr{Port: 1}) // this shouldn't panic, I guess
+
+	testRegions := func() []*region {
+		return []*region{
+			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}}, inUse: map[string]*net.TCPAddr{":2": &net.TCPAddr{Port: 2}, ":3": &net.TCPAddr{Port: 3}}},
+			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 4}, &net.TCPAddr{Port: 5}}, inUse: map[string]*net.TCPAddr{":6": &net.TCPAddr{Port: 6}}},
+			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 7}, &net.TCPAddr{Port: 8}}, inUse: map[string]*net.TCPAddr{":9": &net.TCPAddr{Port: 9}}},
+		}
+	}
+	e = &EdgeAddrResolver{regions: testRegions()}
+	e.ReplaceAddr(&net.TCPAddr{Port: 6})
+	var expected []*region
+	{
+		expected = testRegions()
+		delete(expected[1].inUse, ":6")
+		expected[1].addrs = append(expected[1].addrs, &net.TCPAddr{Port: 6})
+	}
+	assert.Equal(t, expected, e.regions)
+}
+
+// "White-box" test: runs MarkAddrBad() and checks internal state
+func TestEdgeAddrResolver_MarkAddrBad(t *testing.T) {
+	e := &EdgeAddrResolver{regions: nil}
+	e.ReplaceAddr(&net.TCPAddr{Port: 1}) // this shouldn't panic, I guess
+
+	testRegions := func() []*region {
+		return []*region{
+			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}}, inUse: map[string]*net.TCPAddr{":2": &net.TCPAddr{Port: 2}, ":3": &net.TCPAddr{Port: 3}}},
+			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 4}, &net.TCPAddr{Port: 5}}, inUse: map[string]*net.TCPAddr{":6": &net.TCPAddr{Port: 6}}},
+			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 7}, &net.TCPAddr{Port: 8}}, inUse: map[string]*net.TCPAddr{":9": &net.TCPAddr{Port: 9}}},
+		}
+	}
+	e = &EdgeAddrResolver{regions: testRegions()}
+	e.MarkAddrBad(&net.TCPAddr{Port: 6})
+	var expected []*region
+	{
+		expected = testRegions()
+		delete(expected[1].inUse, ":6")
+		expected[1].bad = append(expected[1].bad, &net.TCPAddr{Port: 6})
+	}
+	assert.Equal(t, expected, e.regions)
+}

connection/manager.go

@@ -0,0 +1,299 @@
+package connection
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/streamhandler"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+const (
+	quickStartLink       = "https://developers.cloudflare.com/argo-tunnel/quickstart/"
+	faqLink              = "https://developers.cloudflare.com/argo-tunnel/faq/"
+	defaultRetryAfter    = time.Second * 5
+	packageNamespace     = "connection"
+	edgeManagerSubsystem = "edgemanager"
+)
+
+// EdgeManager manages connections with the edge
+type EdgeManager struct {
+	// streamHandler handles stream opened by the edge
+	streamHandler *streamhandler.StreamHandler
+	// TLSConfig is the TLS configuration to connect with edge
+	tlsConfig *tls.Config
+	// cloudflaredConfig is the cloudflared configuration that is determined when the process first starts
+	cloudflaredConfig *CloudflaredConfig
+	// serviceDiscoverer returns the next edge addr to connect to
+	serviceDiscoverer EdgeServiceDiscoverer
+	// state is attributes of ConnectionManager that can change during runtime.
+	state *edgeManagerState
+
+	logger *logrus.Entry
+
+	metrics *metrics
+}
+
+type metrics struct {
+	// activeStreams is a gauge shared by all muxers of this process to expose the total number of active streams
+	activeStreams prometheus.Gauge
+}
+
+func newMetrics(namespace, subsystem string) *metrics {
+	return &metrics{
+		activeStreams: h2mux.NewActiveStreamsMetrics(namespace, subsystem),
+	}
+}
+
+// EdgeManagerConfigurable is the configurable attributes of a EdgeConnectionManager
+type EdgeManagerConfigurable struct {
+	TunnelHostnames []h2mux.TunnelHostname
+	*tunnelpogs.EdgeConnectionConfig
+}
+
+type CloudflaredConfig struct {
+	CloudflaredID uuid.UUID
+	Tags          []tunnelpogs.Tag
+	BuildInfo     *buildinfo.BuildInfo
+	IntentLabel   string
+}
+
+func NewEdgeManager(
+	streamHandler *streamhandler.StreamHandler,
+	edgeConnMgrConfigurable *EdgeManagerConfigurable,
+	userCredential []byte,
+	tlsConfig *tls.Config,
+	serviceDiscoverer EdgeServiceDiscoverer,
+	cloudflaredConfig *CloudflaredConfig,
+	logger *logrus.Logger,
+) *EdgeManager {
+	return &EdgeManager{
+		streamHandler:     streamHandler,
+		tlsConfig:         tlsConfig,
+		cloudflaredConfig: cloudflaredConfig,
+		serviceDiscoverer: serviceDiscoverer,
+		state:             newEdgeConnectionManagerState(edgeConnMgrConfigurable, userCredential),
+		logger:            logger.WithField("subsystem", "connectionManager"),
+		metrics:           newMetrics(packageNamespace, edgeManagerSubsystem),
+	}
+}
+
+func (em *EdgeManager) Run(ctx context.Context) error {
+	defer em.shutdown()
+
+	resolveEdgeIPTicker := time.Tick(resolveEdgeAddrTTL)
+	for {
+		select {
+		case <-ctx.Done():
+			return errors.Wrap(ctx.Err(), "EdgeConnectionManager terminated")
+		case <-resolveEdgeIPTicker:
+			if err := em.serviceDiscoverer.Refresh(); err != nil {
+				em.logger.WithError(err).Warn("Cannot refresh Cloudflare edge addresses")
+			}
+		default:
+			time.Sleep(1 * time.Second)
+		}
+		// Create/delete connection one at a time, so we don't need to adjust for connections that are being created/deleted
+		// in shouldCreateConnection or shouldReduceConnection calculation
+		if em.state.shouldCreateConnection(em.serviceDiscoverer.AvailableAddrs()) {
+			if connErr := em.newConnection(ctx); connErr != nil {
+				if !connErr.ShouldRetry {
+					em.logger.WithError(connErr).Error(em.noRetryMessage())
+					return connErr
+				}
+				em.logger.WithError(connErr).Error("cannot create new connection")
+			}
+		} else if em.state.shouldReduceConnection() {
+			if err := em.closeConnection(ctx); err != nil {
+				em.logger.WithError(err).Error("cannot close connection")
+			}
+		}
+	}
+}
+
+func (em *EdgeManager) UpdateConfigurable(newConfigurable *EdgeManagerConfigurable) {
+	em.logger.Infof("New edge connection manager configuration %+v", newConfigurable)
+	em.state.updateConfigurable(newConfigurable)
+}
+
+func (em *EdgeManager) newConnection(ctx context.Context) *tunnelpogs.ConnectError {
+	edgeTCPAddr, err := em.serviceDiscoverer.Addr()
+	if err != nil {
+		return retryConnection(fmt.Sprintf("edge address discovery error: %v", err))
+	}
+	configurable := em.state.getConfigurable()
+	edgeConn, err := DialEdge(ctx, configurable.Timeout, em.tlsConfig, edgeTCPAddr)
+	if err != nil {
+		return retryConnection(fmt.Sprintf("dial edge error: %v", err))
+	}
+	// Establish a muxed connection with the edge
+	// Client mux handshake with agent server
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, h2mux.MuxerConfig{
+		Timeout:           configurable.Timeout,
+		Handler:           em.streamHandler,
+		IsClient:          true,
+		HeartbeatInterval: configurable.HeartbeatInterval,
+		MaxHeartbeats:     configurable.MaxFailedHeartbeats,
+		Logger:            em.logger.WithField("subsystem", "muxer"),
+	}, em.metrics.activeStreams)
+	if err != nil {
+		retryConnection(fmt.Sprintf("couldn't perform handshake with edge: %v", err))
+	}
+
+	h2muxConn, err := newConnection(muxer, edgeTCPAddr)
+	if err != nil {
+		return retryConnection(fmt.Sprintf("couldn't create h2mux connection: %v", err))
+	}
+
+	go em.serveConn(ctx, h2muxConn)
+
+	connResult, err := h2muxConn.Connect(ctx, &tunnelpogs.ConnectParameters{
+		CloudflaredID:       em.cloudflaredConfig.CloudflaredID,
+		CloudflaredVersion:  em.cloudflaredConfig.BuildInfo.CloudflaredVersion,
+		NumPreviousAttempts: 0,
+		OriginCert:          em.state.getUserCredential(),
+		IntentLabel:         em.cloudflaredConfig.IntentLabel,
+		Tags:                em.cloudflaredConfig.Tags,
+	}, em.logger)
+	if err != nil {
+		h2muxConn.Shutdown()
+		return retryConnection(fmt.Sprintf("couldn't connect to edge: %v", err))
+	}
+
+	if connErr := connResult.ConnectError(); connErr != nil {
+		return connErr
+	}
+
+	em.state.newConnection(h2muxConn)
+	em.logger.Infof("connected to %s", connResult.ConnectedTo())
+
+	if connResult.ClientConfig() != nil {
+		em.streamHandler.UseConfiguration(ctx, connResult.ClientConfig())
+	}
+	return nil
+}
+
+func (em *EdgeManager) closeConnection(ctx context.Context) error {
+	conn := em.state.getFirstConnection()
+	if conn == nil {
+		return fmt.Errorf("no connection to close")
+	}
+	conn.Shutdown()
+	// teardown will be handled by EdgeManager.serveConn in another goroutine
+	return nil
+}
+
+func (em *EdgeManager) serveConn(ctx context.Context, conn *Connection) {
+	err := conn.Serve(ctx)
+	em.logger.WithError(err).Warn("Connection closed")
+	em.state.closeConnection(conn)
+	em.serviceDiscoverer.ReplaceAddr(conn.addr)
+}
+
+func (em *EdgeManager) noRetryMessage() string {
+	messageTemplate := "cloudflared could not register an Argo Tunnel on your account. Please confirm the following before trying again:" +
+		"1. You have Argo Smart Routing enabled in your account, See Enable Argo section of %s." +
+		"2. Your credential at %s is still valid. See %s."
+	return fmt.Sprintf(messageTemplate, quickStartLink, em.state.getConfigurable().UserCredentialPath, faqLink)
+}
+
+func (em *EdgeManager) shutdown() {
+	em.state.shutdown()
+}
+
+type edgeManagerState struct {
+	sync.RWMutex
+	configurable   *EdgeManagerConfigurable
+	userCredential []byte
+	conns          map[uuid.UUID]*Connection
+}
+
+func newEdgeConnectionManagerState(configurable *EdgeManagerConfigurable, userCredential []byte) *edgeManagerState {
+	return &edgeManagerState{
+		configurable:   configurable,
+		userCredential: userCredential,
+		conns:          make(map[uuid.UUID]*Connection),
+	}
+}
+
+func (ems *edgeManagerState) shouldCreateConnection(availableEdgeAddrs int) bool {
+	ems.RLock()
+	defer ems.RUnlock()
+	expectedHAConns := int(ems.configurable.NumHAConnections)
+	if availableEdgeAddrs < expectedHAConns {
+		expectedHAConns = availableEdgeAddrs
+	}
+	return len(ems.conns) < expectedHAConns
+}
+
+func (ems *edgeManagerState) shouldReduceConnection() bool {
+	ems.RLock()
+	defer ems.RUnlock()
+	return uint8(len(ems.conns)) > ems.configurable.NumHAConnections
+}
+
+func (ems *edgeManagerState) newConnection(conn *Connection) {
+	ems.Lock()
+	defer ems.Unlock()
+	ems.conns[conn.id] = conn
+}
+
+func (ems *edgeManagerState) closeConnection(conn *Connection) {
+	ems.Lock()
+	defer ems.Unlock()
+	delete(ems.conns, conn.id)
+}
+
+func (ems *edgeManagerState) getFirstConnection() *Connection {
+	ems.RLock()
+	defer ems.RUnlock()
+
+	for _, conn := range ems.conns {
+		return conn
+	}
+	return nil
+}
+
+func (ems *edgeManagerState) shutdown() {
+	ems.Lock()
+	defer ems.Unlock()
+	for _, conn := range ems.conns {
+		conn.Shutdown()
+	}
+}
+
+func (ems *edgeManagerState) getConfigurable() *EdgeManagerConfigurable {
+	ems.Lock()
+	defer ems.Unlock()
+	return ems.configurable
+}
+
+func (ems *edgeManagerState) updateConfigurable(newConfigurable *EdgeManagerConfigurable) {
+	ems.Lock()
+	defer ems.Unlock()
+	ems.configurable = newConfigurable
+}
+
+func (ems *edgeManagerState) getUserCredential() []byte {
+	ems.RLock()
+	defer ems.RUnlock()
+	return ems.userCredential
+}
+
+func retryConnection(cause string) *tunnelpogs.ConnectError {
+	return &tunnelpogs.ConnectError{
+		Cause:       cause,
+		RetryAfter:  defaultRetryAfter,
+		ShouldRetry: true,
+	}
+}

connection/manager_test.go

@@ -0,0 +1,75 @@
+package connection
+
+import (
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/streamhandler"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+var (
+	configurable = &EdgeManagerConfigurable{
+		[]h2mux.TunnelHostname{
+			"http.example.com",
+			"ws.example.com",
+			"hello.example.com",
+		},
+		&pogs.EdgeConnectionConfig{
+			NumHAConnections:    1,
+			HeartbeatInterval:   1 * time.Second,
+			Timeout:             5 * time.Second,
+			MaxFailedHeartbeats: 3,
+			UserCredentialPath:  "/etc/cloudflared/cert.pem",
+		},
+	}
+	cloudflaredConfig = &CloudflaredConfig{
+		CloudflaredID: uuid.New(),
+		Tags: []pogs.Tag{
+			{Name: "pool", Value: "east-6"},
+		},
+		BuildInfo: &buildinfo.BuildInfo{
+			GoOS:               "linux",
+			GoVersion:          "1.12",
+			GoArch:             "amd64",
+			CloudflaredVersion: "2019.6.0",
+		},
+	}
+)
+
+func mockEdgeManager() *EdgeManager {
+	newConfigChan := make(chan<- *pogs.ClientConfig)
+	useConfigResultChan := make(<-chan *pogs.UseConfigurationResult)
+	logger := logrus.New()
+	return NewEdgeManager(
+		streamhandler.NewStreamHandler(newConfigChan, useConfigResultChan, logger),
+		configurable,
+		[]byte{},
+		nil,
+		&mockEdgeServiceDiscoverer{},
+		cloudflaredConfig,
+		logrus.New(),
+	)
+}
+
+func TestUpdateConfigurable(t *testing.T) {
+	m := mockEdgeManager()
+	newConfigurable := &EdgeManagerConfigurable{
+		[]h2mux.TunnelHostname{
+			"second.example.com",
+		},
+		&pogs.EdgeConnectionConfig{
+			NumHAConnections: 2,
+		},
+	}
+	m.UpdateConfigurable(newConfigurable)
+
+	assert.Equal(t, newConfigurable, m.state.getConfigurable())
+}

connection/mocks_for_test.go

@@ -0,0 +1,118 @@
+package connection
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+	"net"
+	"reflect"
+	"testing/quick"
+)
+
+type mockAddrs struct {
+	// a set of synthetic SRV records
+	addrMap map[net.SRV][]*net.TCPAddr
+	// the total number of addresses, aggregated across addrMap.
+	// For the convenience of test code that would otherwise have to compute
+	// this by hand every time.
+	numAddrs int
+}
+
+func newMockAddrs(port uint16, numRegions uint8, numAddrsPerRegion uint8) mockAddrs {
+	addrMap := make(map[net.SRV][]*net.TCPAddr)
+	numAddrs := 0
+
+	for r := uint8(0); r < numRegions; r++ {
+		var (
+			srv   = net.SRV{Target: fmt.Sprintf("test-region-%v.example.com", r), Port: port}
+			addrs []*net.TCPAddr
+		)
+		for a := uint8(0); a < numAddrsPerRegion; a++ {
+			addrs = append(addrs, &net.TCPAddr{
+				IP:   net.ParseIP(fmt.Sprintf("10.0.%v.%v", r, a)),
+				Port: int(port),
+			})
+		}
+		addrMap[srv] = addrs
+		numAddrs += len(addrs)
+	}
+	return mockAddrs{addrMap: addrMap, numAddrs: numAddrs}
+}
+
+var _ quick.Generator = mockAddrs{}
+
+func (mockAddrs) Generate(rand *rand.Rand, size int) reflect.Value {
+	port := uint16(rand.Intn(math.MaxUint16))
+	numRegions := uint8(1 + rand.Intn(10))
+	numAddrsPerRegion := uint8(1 + rand.Intn(32))
+	result := newMockAddrs(port, numRegions, numAddrsPerRegion)
+	return reflect.ValueOf(result)
+}
+
+// Returns a function compatible with net.LookupSRV that will return the SRV
+// records from mockAddrs.
+func mockNetLookupSRV(
+	m mockAddrs,
+) func(service, proto, name string) (cname string, addrs []*net.SRV, err error) {
+	var addrs []*net.SRV
+	for k := range m.addrMap {
+		addr := k
+		addrs = append(addrs, &addr)
+		// We can't just do
+		//   addrs = append(addrs, &k)
+		// `k` will be reused by subsequent loop iterations,
+		// so all the copies of `&k` would point to the same location.
+	}
+	return func(_, _, _ string) (string, []*net.SRV, error) {
+		return "", addrs, nil
+	}
+}
+
+// Returns a function compatible with net.LookupIP that translates the SRV records
+// from mockAddrs into IP addresses, based on the TCP addresses in mockAddrs.
+func mockNetLookupIP(
+	m mockAddrs,
+) func(host string) ([]net.IP, error) {
+	return func(host string) ([]net.IP, error) {
+		for srv, tcpAddrs := range m.addrMap {
+			if srv.Target != host {
+				continue
+			}
+			result := make([]net.IP, len(tcpAddrs))
+			for i, tcpAddr := range tcpAddrs {
+				result[i] = tcpAddr.IP
+			}
+			return result, nil
+		}
+		return nil, fmt.Errorf("No IPs for %v", host)
+	}
+}
+
+type mockEdgeServiceDiscoverer struct {
+}
+
+func (mr *mockEdgeServiceDiscoverer) Addr() (*net.TCPAddr, error) {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("127.0.0.1"),
+		Port: 63102,
+	}, nil
+}
+
+func (mr *mockEdgeServiceDiscoverer) AnyAddr() (*net.TCPAddr, error) {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("127.0.0.1"),
+		Port: 63102,
+	}, nil
+}
+
+func (mr *mockEdgeServiceDiscoverer) ReplaceAddr(addr *net.TCPAddr) {}
+
+func (mr *mockEdgeServiceDiscoverer) MarkAddrBad(addr *net.TCPAddr) {}
+
+func (mr *mockEdgeServiceDiscoverer) AvailableAddrs() int {
+	return 1
+}
+
+func (mr *mockEdgeServiceDiscoverer) Refresh() error {
+	return nil
+}

connection/rpc.go

@@ -0,0 +1,49 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	rpc "zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// NewRPCClient creates and returns a new RPC client, which will communicate
+// using a stream on the given muxer
+func NewRPCClient(
+	ctx context.Context,
+	muxer *h2mux.Muxer,
+	logger *logrus.Entry,
+	openStreamTimeout time.Duration,
+) (client tunnelpogs.TunnelServer_PogsClient, err error) {
+	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer openStreamCancel()
+	stream, err := muxer.OpenRPCStream(openStreamCtx)
+	if err != nil {
+		return
+	}
+
+	if !isRPCStreamResponse(stream.Headers) {
+		stream.Close()
+		err = fmt.Errorf("rpc: bad response headers: %v", stream.Headers)
+		return
+	}
+
+	conn := rpc.NewConn(
+		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
+		tunnelrpc.ConnLog(logger),
+	)
+	client = tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
+	return client, nil
+}
+
+func isRPCStreamResponse(headers []h2mux.Header) bool {
+	return len(headers) == 1 &&
+		headers[0].Name == ":status" &&
+		headers[0].Value == "200"
+}

dbconnect/client.go

@@ -0,0 +1,145 @@
+package dbconnect
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+	"unicode"
+	"unicode/utf8"
+)
+
+// Client is an interface to talk to any database.
+//
+// Currently, the only implementation is SQLClient, but its structure
+// should be designed to handle a MongoClient or RedisClient in the future.
+type Client interface {
+	Ping(context.Context) error
+	Submit(context.Context, *Command) (interface{}, error)
+}
+
+// NewClient creates a database client based on its URL scheme.
+func NewClient(ctx context.Context, originURL *url.URL) (Client, error) {
+	return NewSQLClient(ctx, originURL)
+}
+
+// Command is a standard, non-vendor format for submitting database commands.
+//
+// When determining the scope of this struct, refer to the following litmus test:
+// Could this (roughly) conform to SQL, Document-based, and Key-value command formats?
+type Command struct {
+	Statement string        `json:"statement"`
+	Arguments Arguments     `json:"arguments,omitempty"`
+	Mode      string        `json:"mode,omitempty"`
+	Isolation string        `json:"isolation,omitempty"`
+	Timeout   time.Duration `json:"timeout,omitempty"`
+}
+
+// Validate enforces the contract of Command: non empty statement (both in length and logic),
+// lowercase mode and isolation, non-zero timeout, and valid Arguments.
+func (cmd *Command) Validate() error {
+	if cmd.Statement == "" {
+		return fmt.Errorf("cannot provide an empty statement")
+	}
+
+	if strings.Map(func(char rune) rune {
+		if char == ';' || unicode.IsSpace(char) {
+			return -1
+		}
+		return char
+	}, cmd.Statement) == "" {
+		return fmt.Errorf("cannot provide a statement with no logic: '%s'", cmd.Statement)
+	}
+
+	cmd.Mode = strings.ToLower(cmd.Mode)
+	cmd.Isolation = strings.ToLower(cmd.Isolation)
+
+	if cmd.Timeout.Nanoseconds() <= 0 {
+		cmd.Timeout = 24 * time.Hour
+	}
+
+	return cmd.Arguments.Validate()
+}
+
+// UnmarshalJSON converts a byte representation of JSON into a Command, which is also validated.
+func (cmd *Command) UnmarshalJSON(data []byte) error {
+	// Alias is required to avoid infinite recursion from the default UnmarshalJSON.
+	type Alias Command
+	alias := &struct {
+		*Alias
+	}{
+		Alias: (*Alias)(cmd),
+	}
+
+	err := json.Unmarshal(data, &alias)
+	if err == nil {
+		err = cmd.Validate()
+	}
+
+	return err
+}
+
+// Arguments is a wrapper for either map-based or array-based Command arguments.
+//
+// Each field is mutually-exclusive and some Client implementations may not
+// support both fields (eg. MySQL does not accept named arguments).
+type Arguments struct {
+	Named      map[string]interface{}
+	Positional []interface{}
+}
+
+// Validate enforces the contract of Arguments: non nil, mutually exclusive, and no empty or reserved keys.
+func (args *Arguments) Validate() error {
+	if args.Named == nil {
+		args.Named = map[string]interface{}{}
+	}
+	if args.Positional == nil {
+		args.Positional = []interface{}{}
+	}
+
+	if len(args.Named) > 0 && len(args.Positional) > 0 {
+		return fmt.Errorf("both named and positional arguments cannot be specified: %+v and %+v", args.Named, args.Positional)
+	}
+
+	for key := range args.Named {
+		if key == "" {
+			return fmt.Errorf("named arguments cannot contain an empty key: %+v", args.Named)
+		}
+		if !utf8.ValidString(key) {
+			return fmt.Errorf("named argument does not conform to UTF-8 encoding: %s", key)
+		}
+		if strings.HasPrefix(key, "_") {
+			return fmt.Errorf("named argument cannot start with a reserved keyword '_': %s", key)
+		}
+		if unicode.IsNumber([]rune(key)[0]) {
+			return fmt.Errorf("named argument cannot start with a number: %s", key)
+		}
+	}
+
+	return nil
+}
+
+// UnmarshalJSON converts a byte representation of JSON into Arguments, which is also validated.
+func (args *Arguments) UnmarshalJSON(data []byte) error {
+	var obj interface{}
+	err := json.Unmarshal(data, &obj)
+	if err != nil {
+		return err
+	}
+
+	named, ok := obj.(map[string]interface{})
+	if ok {
+		args.Named = named
+	} else {
+		positional, ok := obj.([]interface{})
+		if ok {
+			args.Positional = positional
+		} else {
+			return fmt.Errorf("arguments must either be an object {\"0\":\"val\"} or an array [\"val\"]: %s", string(data))
+		}
+	}
+
+	return args.Validate()
+}

dbconnect/client_test.go

@@ -0,0 +1,183 @@
+package dbconnect
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCommandValidateEmpty(t *testing.T) {
+	stmts := []string{
+		"",
+		";",
+		" \n\t",
+		";\n;\t;",
+	}
+
+	for _, stmt := range stmts {
+		cmd := Command{Statement: stmt}
+
+		assert.Error(t, cmd.Validate(), stmt)
+	}
+}
+
+func TestCommandValidateMode(t *testing.T) {
+	modes := []string{
+		"",
+		"query",
+		"ExEc",
+		"PREPARE",
+	}
+
+	for _, mode := range modes {
+		cmd := Command{Statement: "Ok", Mode: mode}
+
+		assert.NoError(t, cmd.Validate(), mode)
+		assert.Equal(t, strings.ToLower(mode), cmd.Mode)
+	}
+}
+
+func TestCommandValidateIsolation(t *testing.T) {
+	isos := []string{
+		"",
+		"default",
+		"read_committed",
+		"SNAPshot",
+	}
+
+	for _, iso := range isos {
+		cmd := Command{Statement: "Ok", Isolation: iso}
+
+		assert.NoError(t, cmd.Validate(), iso)
+		assert.Equal(t, strings.ToLower(iso), cmd.Isolation)
+	}
+}
+
+func TestCommandValidateTimeout(t *testing.T) {
+	cmd := Command{Statement: "Ok", Timeout: 0}
+
+	assert.NoError(t, cmd.Validate())
+	assert.NotZero(t, cmd.Timeout)
+
+	cmd = Command{Statement: "Ok", Timeout: 1 * time.Second}
+
+	assert.NoError(t, cmd.Validate())
+	assert.Equal(t, 1*time.Second, cmd.Timeout)
+}
+
+func TestCommandValidateArguments(t *testing.T) {
+	cmd := Command{Statement: "Ok", Arguments: Arguments{
+		Named:      map[string]interface{}{"key": "val"},
+		Positional: []interface{}{"val"},
+	}}
+
+	assert.Error(t, cmd.Validate())
+}
+
+func TestCommandUnmarshalJSON(t *testing.T) {
+	strs := []string{
+		"{\"statement\":\"Ok\"}",
+		"{\"statement\":\"Ok\",\"arguments\":[0, 3.14, \"apple\"],\"mode\":\"query\"}",
+		"{\"statement\":\"Ok\",\"isolation\":\"read_uncommitted\",\"timeout\":1000}",
+	}
+
+	for _, str := range strs {
+		var cmd Command
+		assert.NoError(t, json.Unmarshal([]byte(str), &cmd), str)
+	}
+
+	strs = []string{
+		"",
+		"\"",
+		"{}",
+		"{\"argument\":{\"key\":\"val\"}}",
+		"{\"statement\":[\"Ok\"]}",
+	}
+
+	for _, str := range strs {
+		var cmd Command
+		assert.Error(t, json.Unmarshal([]byte(str), &cmd), str)
+	}
+}
+
+func TestArgumentsValidateNotNil(t *testing.T) {
+	args := Arguments{}
+
+	assert.NoError(t, args.Validate())
+	assert.NotNil(t, args.Named)
+	assert.NotNil(t, args.Positional)
+}
+
+func TestArgumentsValidateMutuallyExclusive(t *testing.T) {
+	args := []Arguments{
+		Arguments{},
+		Arguments{Named: map[string]interface{}{"key": "val"}},
+		Arguments{Positional: []interface{}{"val"}},
+	}
+
+	for _, arg := range args {
+		assert.NoError(t, arg.Validate())
+		assert.False(t, len(arg.Named) > 0 && len(arg.Positional) > 0)
+	}
+
+	args = []Arguments{
+		Arguments{
+			Named:      map[string]interface{}{"key": "val"},
+			Positional: []interface{}{"val"},
+		},
+	}
+
+	for _, arg := range args {
+		assert.Error(t, arg.Validate())
+		assert.True(t, len(arg.Named) > 0 && len(arg.Positional) > 0)
+	}
+}
+
+func TestArgumentsValidateKeys(t *testing.T) {
+	keys := []string{
+		"",
+		"_",
+		"_key",
+		"1",
+		"1key",
+		"\xf0\x28\x8c\xbc", // non-utf8
+	}
+
+	for _, key := range keys {
+		args := Arguments{Named: map[string]interface{}{key: "val"}}
+
+		assert.Error(t, args.Validate(), key)
+	}
+}
+
+func TestArgumentsUnmarshalJSON(t *testing.T) {
+	strs := []string{
+		"{}",
+		"{\"key\":\"val\"}",
+		"{\"key\":[1, 3.14, {\"key\":\"val\"}]}",
+		"[]",
+		"[\"key\",\"val\"]",
+		"[{}]",
+	}
+
+	for _, str := range strs {
+		var args Arguments
+		assert.NoError(t, json.Unmarshal([]byte(str), &args), str)
+	}
+
+	strs = []string{
+		"",
+		"\"",
+		"1",
+		"\"key\"",
+		"{\"key\",\"val\"}",
+	}
+
+	for _, str := range strs {
+		var args Arguments
+		assert.Error(t, json.Unmarshal([]byte(str), &args), str)
+	}
+}

dbconnect/cmd.go

@@ -0,0 +1,157 @@
+package dbconnect
+
+import (
+	"context"
+	"log"
+	"net"
+	"strconv"
+
+	"gopkg.in/urfave/cli.v2"
+	"gopkg.in/urfave/cli.v2/altsrc"
+)
+
+// Cmd is the entrypoint command for dbconnect.
+//
+// The tunnel package is responsible for appending this to tunnel.Commands().
+func Cmd() *cli.Command {
+	return &cli.Command{
+		Category:  "Database Connect (ALPHA)",
+		Name:      "db-connect",
+		Usage:     "Access your SQL database from Cloudflare Workers or the browser",
+		ArgsUsage: " ",
+		Description: `
+		Creates a connection between your database and the Cloudflare edge.
+		Now you can execute SQL commands anywhere you can send HTTPS requests.
+
+		Connect your database with any of the following commands, you can also try the "playground" without a database:
+
+			cloudflared db-connect --hostname sql.mysite.com --url postgres://user:pass@localhost?sslmode=disable \
+			                       --auth-domain mysite.cloudflareaccess.com --application-aud my-access-policy-tag
+			cloudflared db-connect --hostname sql-dev.mysite.com --url mysql://localhost --insecure
+			cloudflared db-connect --playground
+		
+		Requests should be authenticated using Cloudflare Access, learn more about how to enable it here:
+
+			https://developers.cloudflare.com/access/service-auth/service-token/
+		`,
+		Flags: []cli.Flag{
+			altsrc.NewStringFlag(&cli.StringFlag{
+				Name:    "url",
+				Usage:   "URL to the database (eg. postgres://user:pass@localhost?sslmode=disable)",
+				EnvVars: []string{"TUNNEL_URL"},
+			}),
+			altsrc.NewStringFlag(&cli.StringFlag{
+				Name:    "hostname",
+				Usage:   "Hostname to accept commands over HTTPS (eg. sql.mysite.com)",
+				EnvVars: []string{"TUNNEL_HOSTNAME"},
+			}),
+			altsrc.NewStringFlag(&cli.StringFlag{
+				Name:    "auth-domain",
+				Usage:   "Cloudflare Access authentication domain for your account (eg. mysite.cloudflareaccess.com)",
+				EnvVars: []string{"TUNNEL_ACCESS_AUTH_DOMAIN"},
+			}),
+			altsrc.NewStringFlag(&cli.StringFlag{
+				Name:    "application-aud",
+				Usage:   "Cloudflare Access application \"AUD\" to verify JWTs from requests",
+				EnvVars: []string{"TUNNEL_ACCESS_APPLICATION_AUD"},
+			}),
+			altsrc.NewBoolFlag(&cli.BoolFlag{
+				Name:    "insecure",
+				Usage:   "Disable authentication, the database will be open to the Internet",
+				Value:   false,
+				EnvVars: []string{"TUNNEL_ACCESS_INSECURE"},
+			}),
+			altsrc.NewBoolFlag(&cli.BoolFlag{
+				Name:    "playground",
+				Usage:   "Run a temporary, in-memory SQLite3 database for testing",
+				Value:   false,
+				EnvVars: []string{"TUNNEL_HELLO_WORLD"},
+			}),
+			altsrc.NewStringFlag(&cli.StringFlag{
+				Name:    "loglevel",
+				Value:   "debug", // Make it more verbose than the tunnel default 'info'.
+				EnvVars: []string{"TUNNEL_LOGLEVEL"},
+				Hidden:  true,
+			}),
+		},
+		Before: CmdBefore,
+		Action: CmdAction,
+		Hidden: true,
+	}
+}
+
+// CmdBefore runs some validation checks before running the command.
+func CmdBefore(c *cli.Context) error {
+	// Show the help text is no flags are specified.
+	if c.NumFlags() == 0 {
+		return cli.ShowSubcommandHelp(c)
+	}
+
+	// Hello-world and playground are synonymous with each other,
+	// unset hello-world to prevent tunnel from initializing the hello package.
+	if c.IsSet("hello-world") {
+		c.Set("playground", "true")
+		c.Set("hello-world", "false")
+	}
+
+	// Unix-socket database urls are supported, but the logic is the same as url.
+	if c.IsSet("unix-socket") {
+		c.Set("url", c.String("unix-socket"))
+		c.Set("unix-socket", "")
+	}
+
+	// When playground mode is enabled, run with an in-memory database.
+	if c.IsSet("playground") {
+		c.Set("url", "sqlite3::memory:?cache=shared")
+		c.Set("insecure", strconv.FormatBool(!c.IsSet("auth-domain") && !c.IsSet("application-aud")))
+	}
+
+	// At this point, insecure configurations are valid.
+	if c.Bool("insecure") {
+		return nil
+	}
+
+	// Ensure that secure configurations specify a hostname, domain, and tag for JWT validation.
+	if !c.IsSet("hostname") || !c.IsSet("auth-domain") || !c.IsSet("application-aud") {
+		log.Fatal("must specify --hostname, --auth-domain, and --application-aud unless you want to run in --insecure mode")
+	}
+
+	return nil
+}
+
+// CmdAction starts the Proxy and sets the url in cli.Context to point to the Proxy address.
+func CmdAction(c *cli.Context) error {
+	// STOR-612: sync with context in tunnel daemon.
+	ctx := context.Background()
+
+	var proxy *Proxy
+	var err error
+	if c.Bool("insecure") {
+		proxy, err = NewInsecureProxy(ctx, c.String("url"))
+	} else {
+		proxy, err = NewSecureProxy(ctx, c.String("url"), c.String("auth-domain"), c.String("application-aud"))
+	}
+
+	if err != nil {
+		log.Fatal(err)
+		return err
+	}
+
+	listenerC := make(chan net.Listener)
+	defer close(listenerC)
+
+	// Since the Proxy should only talk to the tunnel daemon, find the next available
+	// localhost port and start to listen to requests.
+	go func() {
+		err := proxy.Start(ctx, "127.0.0.1:", listenerC)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}()
+
+	// Block until the the Proxy is online, retreive its address, and change the url to point to it.
+	// This is effectively "handing over" control to the tunnel package so it can run the tunnel daemon.
+	c.Set("url", "https://"+(<-listenerC).Addr().String())
+
+	return nil
+}

dbconnect/cmd_test.go

@@ -0,0 +1,27 @@
+package dbconnect
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"gopkg.in/urfave/cli.v2"
+)
+
+func TestCmd(t *testing.T) {
+	tests := [][]string{
+		{"cloudflared", "db-connect", "--playground"},
+		{"cloudflared", "db-connect", "--playground", "--hostname", "sql.mysite.com"},
+		{"cloudflared", "db-connect", "--url", "sqlite3::memory:?cache=shared", "--insecure"},
+		{"cloudflared", "db-connect", "--url", "sqlite3::memory:?cache=shared", "--hostname", "sql.mysite.com", "--auth-domain", "mysite.cloudflareaccess.com", "--application-aud", "aud"},
+	}
+
+	app := &cli.App{
+		Name:     "cloudflared",
+		Commands: []*cli.Command{Cmd()},
+	}
+
+	for _, test := range tests {
+		assert.NoError(t, app.Run(test))
+	}
+}

dbconnect/proxy.go

@@ -0,0 +1,271 @@
+package dbconnect
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/validation"
+	"github.com/gorilla/mux"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// Proxy is an HTTP server that proxies requests to a Client.
+type Proxy struct {
+	client          Client
+	accessValidator *validation.Access
+	logger          *logrus.Logger
+}
+
+// NewInsecureProxy creates a Proxy that talks to a Client at an origin.
+//
+// In insecure mode, the Proxy will allow all Command requests.
+func NewInsecureProxy(ctx context.Context, origin string) (*Proxy, error) {
+	originURL, err := url.Parse(origin)
+	if err != nil {
+		return nil, errors.Wrap(err, "must provide a valid database url")
+	}
+
+	client, err := NewClient(ctx, originURL)
+	if err != nil {
+		return nil, err
+	}
+
+	err = client.Ping(ctx)
+	if err != nil {
+		return nil, errors.Wrap(err, "could not connect to the database")
+	}
+
+	return &Proxy{client, nil, logrus.New()}, nil
+}
+
+// NewSecureProxy creates a Proxy that talks to a Client at an origin.
+//
+// In secure mode, the Proxy will reject any Command requests that are
+// not authenticated by Cloudflare Access with a valid JWT.
+func NewSecureProxy(ctx context.Context, origin, authDomain, applicationAUD string) (*Proxy, error) {
+	proxy, err := NewInsecureProxy(ctx, origin)
+	if err != nil {
+		return nil, err
+	}
+
+	validator, err := validation.NewAccessValidator(ctx, authDomain, authDomain, applicationAUD)
+	if err != nil {
+		return nil, err
+	}
+
+	proxy.accessValidator = validator
+
+	return proxy, err
+}
+
+// IsInsecure gets whether the Proxy will accept a Command from any source.
+func (proxy *Proxy) IsInsecure() bool {
+	return proxy.accessValidator == nil
+}
+
+// IsAllowed checks whether a http.Request is allowed to receive data.
+//
+// By default, requests must pass through Cloudflare Access for authentication.
+// If the proxy is explcitly set to insecure mode, all requests will be allowed.
+func (proxy *Proxy) IsAllowed(r *http.Request, verbose ...bool) bool {
+	if proxy.IsInsecure() {
+		return true
+	}
+
+	// Access and Tunnel should prevent bad JWTs from even reaching the origin,
+	// but validate tokens anyway as an abundance of caution.
+	err := proxy.accessValidator.ValidateRequest(r.Context(), r)
+	if err == nil {
+		return true
+	}
+
+	// Warn administrators that invalid JWTs are being rejected. This is indicative
+	// of either a misconfiguration of the CLI or a massive failure of upstream systems.
+	if len(verbose) > 0 {
+		proxy.httpLog(r, err).Error("Failed JWT authentication")
+	}
+
+	return false
+}
+
+// Start the Proxy at a given address and notify the listener channel when the server is online.
+func (proxy *Proxy) Start(ctx context.Context, addr string, listenerC chan<- net.Listener) error {
+	// STOR-611: use a seperate listener and consider web socket support.
+	httpListener, err := hello.CreateTLSListener(addr)
+	if err != nil {
+		return errors.Wrapf(err, "could not create listener at %s", addr)
+	}
+
+	errC := make(chan error)
+	defer close(errC)
+
+	// Starts the HTTP server and begins to serve requests.
+	go func() {
+		errC <- proxy.httpListen(ctx, httpListener)
+	}()
+
+	// Continually ping the server until it comes online or 10 attempts fail.
+	go func() {
+		var err error
+		for i := 0; i < 10; i++ {
+			_, err = http.Get("http://" + httpListener.Addr().String())
+
+			// Once no error was detected, notify the listener channel and return.
+			if err == nil {
+				listenerC <- httpListener
+				return
+			}
+
+			// Backoff between requests to ping the server.
+			<-time.After(1 * time.Second)
+		}
+		errC <- errors.Wrap(err, "took too long for the http server to start")
+	}()
+
+	return <-errC
+}
+
+// httpListen starts the httpServer and blocks until the context closes.
+func (proxy *Proxy) httpListen(ctx context.Context, listener net.Listener) error {
+	httpServer := &http.Server{
+		Addr:         listener.Addr().String(),
+		Handler:      proxy.httpRouter(),
+		ReadTimeout:  10 * time.Second,
+		WriteTimeout: 60 * time.Second,
+		IdleTimeout:  60 * time.Second,
+	}
+
+	go func() {
+		<-ctx.Done()
+		httpServer.Close()
+		listener.Close()
+	}()
+
+	return httpServer.Serve(listener)
+}
+
+// httpRouter creates a mux.Router for the Proxy.
+func (proxy *Proxy) httpRouter() *mux.Router {
+	router := mux.NewRouter()
+
+	router.HandleFunc("/ping", proxy.httpPing()).Methods("GET", "HEAD")
+	router.HandleFunc("/submit", proxy.httpSubmit()).Methods("POST")
+
+	return router
+}
+
+// httpPing tests the connection to the database.
+//
+// By default, this endpoint is unauthenticated to allow for health checks.
+// To enable authentication, Cloudflare Access must be enabled on this route.
+func (proxy *Proxy) httpPing() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		err := proxy.client.Ping(ctx)
+
+		if err == nil {
+			proxy.httpRespond(w, r, http.StatusOK, "")
+		} else {
+			proxy.httpRespondErr(w, r, http.StatusInternalServerError, err)
+		}
+	}
+}
+
+// httpSubmit sends a command to the database and returns its response.
+//
+// By default, this endpoint will reject requests that do not pass through Cloudflare Access.
+// To disable authentication, the --insecure flag must be specified in the command line.
+func (proxy *Proxy) httpSubmit() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if !proxy.IsAllowed(r, true) {
+			proxy.httpRespondErr(w, r, http.StatusForbidden, fmt.Errorf(""))
+			return
+		}
+
+		var cmd Command
+		err := json.NewDecoder(r.Body).Decode(&cmd)
+		if err != nil {
+			proxy.httpRespondErr(w, r, http.StatusBadRequest, err)
+			return
+		}
+
+		ctx := r.Context()
+		data, err := proxy.client.Submit(ctx, &cmd)
+
+		if err != nil {
+			proxy.httpRespondErr(w, r, http.StatusUnprocessableEntity, err)
+			return
+		}
+
+		w.Header().Set("Content-type", "application/json")
+		err = json.NewEncoder(w).Encode(data)
+		if err != nil {
+			proxy.httpRespondErr(w, r, http.StatusInternalServerError, err)
+		}
+	}
+}
+
+// httpRespond writes a status code and string response to the response writer.
+func (proxy *Proxy) httpRespond(w http.ResponseWriter, r *http.Request, status int, message string) {
+	w.WriteHeader(status)
+
+	// Only expose the message detail of the reponse if the request is not HEAD
+	// and the user is authenticated. For example, this prevents an unauthenticated
+	// failed health check from accidentally leaking sensitive information about the Client.
+	if r.Method != http.MethodHead && proxy.IsAllowed(r) {
+		if message == "" {
+			message = http.StatusText(status)
+		}
+		fmt.Fprint(w, message)
+	}
+}
+
+// httpRespondErr is similar to httpRespond, except it formats errors to be more friendly.
+func (proxy *Proxy) httpRespondErr(w http.ResponseWriter, r *http.Request, defaultStatus int, err error) {
+	status, err := httpError(defaultStatus, err)
+
+	proxy.httpRespond(w, r, status, err.Error())
+	if len(err.Error()) > 0 {
+		proxy.httpLog(r, err).Warn("Database proxy error")
+	}
+}
+
+// httpLog returns a logrus.Entry that is formatted to output a request Cf-ray.
+func (proxy *Proxy) httpLog(r *http.Request, err error) *logrus.Entry {
+	return proxy.logger.WithContext(r.Context()).WithField("CF-RAY", r.Header.Get("Cf-ray")).WithError(err)
+}
+
+// httpError extracts common errors and returns an status code and friendly error.
+func httpError(defaultStatus int, err error) (int, error) {
+	if err == nil {
+		return http.StatusNotImplemented, fmt.Errorf("error expected but found none")
+	}
+
+	if err == io.EOF {
+		return http.StatusBadRequest, fmt.Errorf("request body cannot be empty")
+	}
+
+	if err == context.DeadlineExceeded {
+		return http.StatusRequestTimeout, err
+	}
+
+	_, ok := err.(net.Error)
+	if ok {
+		return http.StatusRequestTimeout, err
+	}
+
+	if err == context.Canceled {
+		// Does not exist in Golang, but would be: http.StatusClientClosedWithoutResponse
+		return 444, err
+	}
+
+	return defaultStatus, err
+}

dbconnect/proxy_test.go

@@ -0,0 +1,238 @@
+package dbconnect
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/gorilla/mux"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewInsecureProxy(t *testing.T) {
+	origins := []string{
+		"",
+		":/",
+		"http://localhost",
+		"tcp://localhost:9000?debug=true",
+		"mongodb://127.0.0.1",
+	}
+
+	for _, origin := range origins {
+		proxy, err := NewInsecureProxy(context.Background(), origin)
+
+		assert.Error(t, err)
+		assert.Empty(t, proxy)
+	}
+}
+
+func TestProxyIsAllowed(t *testing.T) {
+	proxy := helperNewProxy(t)
+	req := httptest.NewRequest("GET", "https://1.1.1.1/ping", nil)
+	assert.True(t, proxy.IsAllowed(req))
+
+	proxy = helperNewProxy(t, true)
+	req.Header.Set("Cf-access-jwt-assertion", "xxx")
+	assert.False(t, proxy.IsAllowed(req))
+}
+
+func TestProxyStart(t *testing.T) {
+	proxy := helperNewProxy(t)
+	ctx := context.Background()
+	listenerC := make(chan net.Listener)
+
+	err := proxy.Start(ctx, "1.1.1.1:", listenerC)
+	assert.Error(t, err)
+
+	err = proxy.Start(ctx, "127.0.0.1:-1", listenerC)
+	assert.Error(t, err)
+
+	ctx, cancel := context.WithTimeout(ctx, 0)
+	defer cancel()
+
+	err = proxy.Start(ctx, "127.0.0.1:", listenerC)
+	assert.IsType(t, http.ErrServerClosed, err)
+}
+
+func TestProxyHTTPRouter(t *testing.T) {
+	proxy := helperNewProxy(t)
+	router := proxy.httpRouter()
+
+	tests := []struct {
+		path   string
+		method string
+		valid  bool
+	}{
+		{"", "GET", false},
+		{"/", "GET", false},
+		{"/ping", "GET", true},
+		{"/ping", "HEAD", true},
+		{"/ping", "POST", false},
+		{"/submit", "POST", true},
+		{"/submit", "GET", false},
+		{"/submit/extra", "POST", false},
+	}
+
+	for _, test := range tests {
+		match := &mux.RouteMatch{}
+		ok := router.Match(httptest.NewRequest(test.method, "https://1.1.1.1"+test.path, nil), match)
+
+		assert.True(t, ok == test.valid, test.path)
+	}
+}
+
+func TestProxyHTTPPing(t *testing.T) {
+	proxy := helperNewProxy(t)
+
+	server := httptest.NewServer(proxy.httpPing())
+	defer server.Close()
+	client := server.Client()
+
+	res, err := client.Get(server.URL)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, res.StatusCode)
+	assert.Equal(t, int64(2), res.ContentLength)
+
+	res, err = client.Head(server.URL)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusOK, res.StatusCode)
+	assert.Equal(t, int64(-1), res.ContentLength)
+}
+
+func TestProxyHTTPSubmit(t *testing.T) {
+	proxy := helperNewProxy(t)
+
+	server := httptest.NewServer(proxy.httpSubmit())
+	defer server.Close()
+	client := server.Client()
+
+	tests := []struct {
+		input  string
+		status int
+		output string
+	}{
+		{"", http.StatusBadRequest, "request body cannot be empty"},
+		{"{}", http.StatusBadRequest, "cannot provide an empty statement"},
+		{"{\"statement\":\"Ok\"}", http.StatusUnprocessableEntity, "cannot provide invalid sql mode: ''"},
+		{"{\"statement\":\"Ok\",\"mode\":\"query\"}", http.StatusUnprocessableEntity, "near \"Ok\": syntax error"},
+		{"{\"statement\":\"CREATE TABLE t (a INT);\",\"mode\":\"exec\"}", http.StatusOK, "{\"last_insert_id\":0,\"rows_affected\":0}\n"},
+	}
+
+	for _, test := range tests {
+		res, err := client.Post(server.URL, "application/json", strings.NewReader(test.input))
+
+		assert.NoError(t, err)
+		assert.Equal(t, test.status, res.StatusCode)
+		if res.StatusCode > http.StatusOK {
+			assert.Equal(t, "text/plain; charset=utf-8", res.Header.Get("Content-type"))
+		} else {
+			assert.Equal(t, "application/json", res.Header.Get("Content-type"))
+		}
+
+		data, err := ioutil.ReadAll(res.Body)
+		defer res.Body.Close()
+		str := string(data)
+
+		assert.NoError(t, err)
+		assert.Equal(t, test.output, str)
+	}
+}
+
+func TestProxyHTTPSubmitForbidden(t *testing.T) {
+	proxy := helperNewProxy(t, true)
+
+	server := httptest.NewServer(proxy.httpSubmit())
+	defer server.Close()
+	client := server.Client()
+
+	res, err := client.Get(server.URL)
+
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusForbidden, res.StatusCode)
+	assert.Zero(t, res.ContentLength)
+}
+
+func TestProxyHTTPRespond(t *testing.T) {
+	proxy := helperNewProxy(t)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		proxy.httpRespond(w, r, http.StatusAccepted, "Hello")
+	}))
+	defer server.Close()
+	client := server.Client()
+
+	res, err := client.Get(server.URL)
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusAccepted, res.StatusCode)
+	assert.Equal(t, int64(5), res.ContentLength)
+
+	data, err := ioutil.ReadAll(res.Body)
+	defer res.Body.Close()
+	assert.Equal(t, []byte("Hello"), data)
+}
+
+func TestProxyHTTPRespondForbidden(t *testing.T) {
+	proxy := helperNewProxy(t, true)
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		proxy.httpRespond(w, r, http.StatusAccepted, "Hello")
+	}))
+	defer server.Close()
+	client := server.Client()
+
+	res, err := client.Get(server.URL)
+
+	assert.NoError(t, err)
+	assert.Equal(t, http.StatusAccepted, res.StatusCode)
+	assert.Equal(t, int64(0), res.ContentLength)
+}
+
+func TestHTTPError(t *testing.T) {
+	_, errTimeout := net.DialTimeout("tcp", "127.0.0.1", 0)
+	assert.Error(t, errTimeout)
+
+	tests := []struct {
+		input  error
+		status int
+		output error
+	}{
+		{nil, http.StatusNotImplemented, fmt.Errorf("error expected but found none")},
+		{io.EOF, http.StatusBadRequest, fmt.Errorf("request body cannot be empty")},
+		{context.DeadlineExceeded, http.StatusRequestTimeout, nil},
+		{context.Canceled, 444, nil},
+		{errTimeout, http.StatusRequestTimeout, nil},
+		{fmt.Errorf(""), http.StatusInternalServerError, nil},
+	}
+
+	for _, test := range tests {
+		status, err := httpError(http.StatusInternalServerError, test.input)
+
+		assert.Error(t, err)
+		assert.Equal(t, test.status, status)
+		if test.output == nil {
+			test.output = test.input
+		}
+		assert.Equal(t, test.output, err)
+	}
+}
+
+func helperNewProxy(t *testing.T, secure ...bool) *Proxy {
+	t.Helper()
+
+	proxy, err := NewSecureProxy(context.Background(), "file::memory:?cache=shared", "test.cloudflareaccess.com", "")
+	assert.NoError(t, err)
+	assert.NotNil(t, proxy)
+
+	if len(secure) == 0 {
+		proxy.accessValidator = nil // Mark as insecure
+	}
+
+	return proxy
+}

dbconnect/sql.go

@@ -0,0 +1,318 @@
+package dbconnect
+
+import (
+	"context"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"net/url"
+	"reflect"
+	"strings"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/pkg/errors"
+	"github.com/xo/dburl"
+
+	// SQL drivers self-register with the database/sql package.
+	// https://github.com/golang/go/wiki/SQLDrivers
+	_ "github.com/denisenkom/go-mssqldb"
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/mattn/go-sqlite3"
+
+	"github.com/kshvakov/clickhouse"
+	"github.com/lib/pq"
+)
+
+// SQLClient is a Client that talks to a SQL database.
+type SQLClient struct {
+	Dialect string
+	driver  *sqlx.DB
+}
+
+// NewSQLClient creates a SQL client based on its URL scheme.
+func NewSQLClient(ctx context.Context, originURL *url.URL) (Client, error) {
+	res, err := dburl.Parse(originURL.String())
+	if err != nil {
+		helpText := fmt.Sprintf("supported drivers: %+q, see documentation for more details: %s", sql.Drivers(), "https://godoc.org/github.com/xo/dburl")
+		return nil, fmt.Errorf("could not parse sql database url '%s': %s\n%s", originURL, err.Error(), helpText)
+	}
+
+	// Establishes the driver, but does not test the connection.
+	driver, err := sqlx.Open(res.Driver, res.DSN)
+	if err != nil {
+		return nil, fmt.Errorf("could not open sql driver %s: %s\n%s", res.Driver, err.Error(), res.DSN)
+	}
+
+	// Closes the driver, will occur when the context finishes.
+	go func() {
+		<-ctx.Done()
+		driver.Close()
+	}()
+
+	return &SQLClient{driver.DriverName(), driver}, nil
+}
+
+// Ping verifies a connection to the database is still alive.
+func (client *SQLClient) Ping(ctx context.Context) error {
+	return client.driver.PingContext(ctx)
+}
+
+// Submit queries or executes a command to the SQL database.
+func (client *SQLClient) Submit(ctx context.Context, cmd *Command) (interface{}, error) {
+	txx, err := cmd.ValidateSQL(client.Dialect)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, cmd.Timeout)
+	defer cancel()
+
+	var res interface{}
+
+	// Get the next available sql.Conn and submit the Command.
+	err = sqlConn(ctx, client.driver, txx, func(conn *sql.Conn) error {
+		stmt := cmd.Statement
+		args := cmd.Arguments.Positional
+
+		if cmd.Mode == "query" {
+			res, err = sqlQuery(ctx, conn, stmt, args)
+		} else {
+			res, err = sqlExec(ctx, conn, stmt, args)
+		}
+
+		return err
+	})
+
+	return res, err
+}
+
+// ValidateSQL extends the contract of Command for SQL dialects:
+// mode is conformed, arguments are []sql.NamedArg, and isolation is a sql.IsolationLevel.
+//
+// When the command should not be wrapped in a transaction, *sql.TxOptions and error will both be nil.
+func (cmd *Command) ValidateSQL(dialect string) (*sql.TxOptions, error) {
+	err := cmd.Validate()
+	if err != nil {
+		return nil, err
+	}
+
+	mode, err := sqlMode(cmd.Mode)
+	if err != nil {
+		return nil, err
+	}
+
+	// Mutates Arguments to only use positional arguments with the type sql.NamedArg.
+	// This is a required by the sql.Driver before submitting arguments.
+	cmd.Arguments.sql(dialect)
+
+	iso, err := sqlIsolation(cmd.Isolation)
+	if err != nil {
+		return nil, err
+	}
+
+	// When isolation is out-of-range, this is indicative that no
+	// transaction should be executed and sql.TxOptions should be nil.
+	if iso < sql.LevelDefault {
+		return nil, nil
+	}
+
+	// In query mode, execute the transaction in read-only, unless it's Microsoft SQL
+	// which does not support that type of transaction.
+	readOnly := mode == "query" && dialect != "mssql"
+
+	return &sql.TxOptions{Isolation: iso, ReadOnly: readOnly}, nil
+}
+
+// sqlConn gets the next available sql.Conn in the connection pool and runs a function to use it.
+//
+// If the transaction options are nil, run the useIt function outside a transaction.
+// This is potentially an unsafe operation if the command does not clean up its state.
+func sqlConn(ctx context.Context, driver *sqlx.DB, txx *sql.TxOptions, useIt func(*sql.Conn) error) error {
+	conn, err := driver.Conn(ctx)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	// If transaction options are specified, begin and defer a rollback to catch errors.
+	var tx *sql.Tx
+	if txx != nil {
+		tx, err = conn.BeginTx(ctx, txx)
+		if err != nil {
+			return err
+		}
+		defer tx.Rollback()
+	}
+
+	err = useIt(conn)
+
+	// Check if useIt was successful and a transaction exists before committing.
+	if err == nil && tx != nil {
+		err = tx.Commit()
+	}
+
+	return err
+}
+
+// sqlQuery queries rows on a sql.Conn and returns an array of result objects.
+func sqlQuery(ctx context.Context, conn *sql.Conn, stmt string, args []interface{}) ([]map[string]interface{}, error) {
+	rows, err := conn.QueryContext(ctx, stmt, args...)
+	if err == nil {
+		return sqlRows(rows)
+	}
+	return nil, err
+}
+
+// sqlExec executes a command on a sql.Conn and returns the result of the operation.
+func sqlExec(ctx context.Context, conn *sql.Conn, stmt string, args []interface{}) (sqlResult, error) {
+	exec, err := conn.ExecContext(ctx, stmt, args...)
+	if err == nil {
+		return sqlResultFrom(exec), nil
+	}
+	return sqlResult{}, err
+}
+
+// sql mutates Arguments to contain a positional []sql.NamedArg.
+//
+// The actual return type is []interface{} due to the native Golang
+// function signatures for sql.Exec and sql.Query being generic.
+func (args *Arguments) sql(dialect string) {
+	result := args.Positional
+
+	for i, val := range result {
+		result[i] = sqlArg("", val, dialect)
+	}
+
+	for key, val := range args.Named {
+		result = append(result, sqlArg(key, val, dialect))
+	}
+
+	args.Positional = result
+	args.Named = map[string]interface{}{}
+}
+
+// sqlArg creates a sql.NamedArg from a key-value pair and an optional dialect.
+//
+// Certain dialects will need to wrap objects, such as arrays, to conform its driver requirements.
+func sqlArg(key, val interface{}, dialect string) sql.NamedArg {
+	switch reflect.ValueOf(val).Kind() {
+
+	// PostgreSQL and Clickhouse require arrays to be wrapped before
+	// being inserted into the driver interface.
+	case reflect.Slice, reflect.Array:
+		switch dialect {
+		case "postgres":
+			val = pq.Array(val)
+		case "clickhouse":
+			val = clickhouse.Array(val)
+		}
+	}
+
+	return sql.Named(fmt.Sprint(key), val)
+}
+
+// sqlIsolation tries to match a string to a sql.IsolationLevel.
+func sqlIsolation(str string) (sql.IsolationLevel, error) {
+	if str == "none" {
+		return sql.IsolationLevel(-1), nil
+	}
+
+	for iso := sql.LevelDefault; ; iso++ {
+		if iso > sql.LevelLinearizable {
+			return -1, fmt.Errorf("cannot provide an invalid sql isolation level: '%s'", str)
+		}
+
+		if str == "" || strings.EqualFold(iso.String(), strings.ReplaceAll(str, "_", " ")) {
+			return iso, nil
+		}
+	}
+}
+
+// sqlMode tries to match a string to a command mode: 'query' or 'exec' for now.
+func sqlMode(str string) (string, error) {
+	switch str {
+	case "query", "exec":
+		return str, nil
+	default:
+		return "", fmt.Errorf("cannot provide invalid sql mode: '%s'", str)
+	}
+}
+
+// sqlRows scans through a SQL result set and returns an array of objects.
+func sqlRows(rows *sql.Rows) ([]map[string]interface{}, error) {
+	columns, err := rows.Columns()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not extract columns from result")
+	}
+	defer rows.Close()
+
+	types, err := rows.ColumnTypes()
+	if err != nil {
+		// Some drivers do not support type extraction, so fail silently and continue.
+		types = make([]*sql.ColumnType, len(columns))
+	}
+
+	values := make([]interface{}, len(columns))
+	pointers := make([]interface{}, len(columns))
+
+	var results []map[string]interface{}
+	for rows.Next() {
+		for i := range columns {
+			pointers[i] = &values[i]
+		}
+		rows.Scan(pointers...)
+
+		// Convert a row, an array of values, into an object where
+		// each key is the name of its respective column.
+		entry := make(map[string]interface{})
+		for i, col := range columns {
+			entry[col] = sqlValue(values[i], types[i])
+		}
+		results = append(results, entry)
+	}
+
+	return results, nil
+}
+
+// sqlValue handles special cases where sql.Rows does not return a "human-readable" object.
+func sqlValue(val interface{}, col *sql.ColumnType) interface{} {
+	bytes, ok := val.([]byte)
+	if ok {
+		// Opportunistically check for embeded JSON and convert it to a first-class object.
+		var embeded interface{}
+		if json.Unmarshal(bytes, &embeded) == nil {
+			return embeded
+		}
+
+		// STOR-604: investigate a way to coerce PostgreSQL arrays '{a, b, ...}' into JSON.
+		// Although easy with strings, it becomes more difficult with special types like INET[].
+
+		return string(bytes)
+	}
+
+	return val
+}
+
+// sqlResult is a thin wrapper around sql.Result.
+type sqlResult struct {
+	LastInsertId int64 `json:"last_insert_id"`
+	RowsAffected int64 `json:"rows_affected"`
+}
+
+// sqlResultFrom converts sql.Result into a JSON-marshable sqlResult.
+func sqlResultFrom(res sql.Result) sqlResult {
+	insertID, errID := res.LastInsertId()
+	rowsAffected, errRows := res.RowsAffected()
+
+	// If an error occurs when extracting the result, it is because the
+	// driver does not support that specific field. Instead of passing this
+	// to the user, omit the field in the response.
+	if errID != nil {
+		insertID = -1
+	}
+	if errRows != nil {
+		rowsAffected = -1
+	}
+
+	return sqlResult{insertID, rowsAffected}
+}

dbconnect/sql_test.go

@@ -0,0 +1,336 @@
+package dbconnect
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/kshvakov/clickhouse"
+	"github.com/lib/pq"
+
+	"github.com/DATA-DOG/go-sqlmock"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNewSQLClient(t *testing.T) {
+	originURLs := []string{
+		"postgres://localhost",
+		"cockroachdb://localhost:1337",
+		"postgresql://user:pass@127.0.0.1",
+		"mysql://localhost",
+		"clickhouse://127.0.0.1:9000/?debug",
+		"sqlite3::memory:",
+		"file:test.db?cache=shared",
+	}
+
+	for _, originURL := range originURLs {
+		origin, _ := url.Parse(originURL)
+		_, err := NewSQLClient(context.Background(), origin)
+
+		assert.NoError(t, err, originURL)
+	}
+
+	originURLs = []string{
+		"",
+		"/",
+		"http://localhost",
+		"coolthing://user:pass@127.0.0.1",
+	}
+
+	for _, originURL := range originURLs {
+		origin, _ := url.Parse(originURL)
+		_, err := NewSQLClient(context.Background(), origin)
+
+		assert.Error(t, err, originURL)
+	}
+}
+
+func TestArgumentsSQL(t *testing.T) {
+	args := []Arguments{
+		Arguments{
+			Positional: []interface{}{
+				"val", 10, 3.14,
+			},
+		},
+		Arguments{
+			Named: map[string]interface{}{
+				"key": time.Unix(0, 0),
+			},
+		},
+	}
+
+	var nameType sql.NamedArg
+	for _, arg := range args {
+		arg.sql("")
+		for _, named := range arg.Positional {
+			assert.IsType(t, nameType, named)
+		}
+	}
+}
+
+func TestSQLArg(t *testing.T) {
+	tests := []struct {
+		key     interface{}
+		val     interface{}
+		dialect string
+		arg     sql.NamedArg
+	}{
+		{"key", "val", "mssql", sql.Named("key", "val")},
+		{0, 1, "sqlite3", sql.Named("0", 1)},
+		{1, []string{"a", "b", "c"}, "postgres", sql.Named("1", pq.Array([]string{"a", "b", "c"}))},
+		{"in", []uint{0, 1}, "clickhouse", sql.Named("in", clickhouse.Array([]uint{0, 1}))},
+		{"", time.Unix(0, 0), "", sql.Named("", time.Unix(0, 0))},
+	}
+
+	for _, test := range tests {
+		arg := sqlArg(test.key, test.val, test.dialect)
+		assert.Equal(t, test.arg, arg, test.key)
+	}
+}
+
+func TestSQLIsolation(t *testing.T) {
+	tests := []struct {
+		str string
+		iso sql.IsolationLevel
+	}{
+		{"", sql.LevelDefault},
+		{"DEFAULT", sql.LevelDefault},
+		{"read_UNcommitted", sql.LevelReadUncommitted},
+		{"serializable", sql.LevelSerializable},
+		{"none", sql.IsolationLevel(-1)},
+		{"SNAP shot", -2},
+		{"blah", -2},
+	}
+
+	for _, test := range tests {
+		iso, err := sqlIsolation(test.str)
+
+		if test.iso < -1 {
+			assert.Error(t, err, test.str)
+		} else {
+			assert.NoError(t, err)
+			assert.Equal(t, test.iso, iso, test.str)
+		}
+	}
+}
+
+func TestSQLMode(t *testing.T) {
+	modes := []string{
+		"query",
+		"exec",
+	}
+
+	for _, mode := range modes {
+		actual, err := sqlMode(mode)
+
+		assert.NoError(t, err)
+		assert.Equal(t, strings.ToLower(mode), actual, mode)
+	}
+
+	modes = []string{
+		"",
+		"blah",
+	}
+
+	for _, mode := range modes {
+		_, err := sqlMode(mode)
+
+		assert.Error(t, err)
+	}
+}
+
+func helperRows(mockRows *sqlmock.Rows) *sql.Rows {
+	db, mock, _ := sqlmock.New()
+	mock.ExpectQuery("SELECT").WillReturnRows(mockRows)
+	rows, _ := db.Query("SELECT")
+	return rows
+}
+
+func TestSQLRows(t *testing.T) {
+	actual, err := sqlRows(helperRows(sqlmock.NewRows(
+		[]string{"name", "age", "dept"}).
+		AddRow("alice", 19, "prod")))
+	expected := []map[string]interface{}{map[string]interface{}{
+		"name": "alice",
+		"age":  int64(19),
+		"dept": "prod"}}
+
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, expected, actual)
+}
+
+func TestSQLValue(t *testing.T) {
+	tests := []struct {
+		input  interface{}
+		output interface{}
+	}{
+		{"hello", "hello"},
+		{1, 1},
+		{false, false},
+		{[]byte("random"), "random"},
+		{[]byte("{\"json\":true}"), map[string]interface{}{"json": true}},
+		{[]byte("[]"), []interface{}{}},
+	}
+
+	for _, test := range tests {
+		assert.Equal(t, test.output, sqlValue(test.input, nil), test.input)
+	}
+}
+
+func TestSQLResultFrom(t *testing.T) {
+	res := sqlResultFrom(sqlmock.NewResult(1, 2))
+	assert.Equal(t, sqlResult{1, 2}, res)
+
+	res = sqlResultFrom(sqlmock.NewErrorResult(fmt.Errorf("")))
+	assert.Equal(t, sqlResult{-1, -1}, res)
+}
+
+func helperSQLite3(t *testing.T) (context.Context, Client) {
+	t.Helper()
+
+	ctx := context.Background()
+	url, _ := url.Parse("file::memory:?cache=shared")
+
+	sqlite3, err := NewSQLClient(ctx, url)
+	assert.NoError(t, err)
+
+	return ctx, sqlite3
+}
+
+func TestPing(t *testing.T) {
+	ctx, sqlite3 := helperSQLite3(t)
+	err := sqlite3.Ping(ctx)
+
+	assert.NoError(t, err)
+}
+
+func TestSubmit(t *testing.T) {
+	ctx, sqlite3 := helperSQLite3(t)
+
+	res, err := sqlite3.Submit(ctx, &Command{
+		Statement: "CREATE TABLE t (a INTEGER, b FLOAT, c TEXT, d BLOB);",
+		Mode:      "exec",
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, sqlResult{0, 0}, res)
+
+	res, err = sqlite3.Submit(ctx, &Command{
+		Statement: "SELECT * FROM t;",
+		Mode:      "query",
+	})
+	assert.NoError(t, err)
+	assert.Empty(t, res)
+
+	res, err = sqlite3.Submit(ctx, &Command{
+		Statement: "INSERT INTO t VALUES (?, ?, ?, ?);",
+		Mode:      "exec",
+		Arguments: Arguments{
+			Positional: []interface{}{
+				1,
+				3.14,
+				"text",
+				"blob",
+			},
+		},
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, sqlResult{1, 1}, res)
+
+	res, err = sqlite3.Submit(ctx, &Command{
+		Statement: "UPDATE t SET c = NULL;",
+		Mode:      "exec",
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, sqlResult{1, 1}, res)
+
+	res, err = sqlite3.Submit(ctx, &Command{
+		Statement: "SELECT * FROM t WHERE a = ?;",
+		Mode:      "query",
+		Arguments: Arguments{
+			Positional: []interface{}{1},
+		},
+	})
+	assert.NoError(t, err)
+	assert.Len(t, res, 1)
+
+	resf, ok := res.([]map[string]interface{})
+	assert.True(t, ok)
+	assert.EqualValues(t, map[string]interface{}{
+		"a": int64(1),
+		"b": 3.14,
+		"c": nil,
+		"d": "blob",
+	}, resf[0])
+
+	res, err = sqlite3.Submit(ctx, &Command{
+		Statement: "DROP TABLE t;",
+		Mode:      "exec",
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, sqlResult{1, 1}, res)
+}
+
+func TestSubmitTransaction(t *testing.T) {
+	ctx, sqlite3 := helperSQLite3(t)
+
+	res, err := sqlite3.Submit(ctx, &Command{
+		Statement: "BEGIN;",
+		Mode:      "exec",
+	})
+	assert.Error(t, err)
+	assert.Empty(t, res)
+
+	res, err = sqlite3.Submit(ctx, &Command{
+		Statement: "BEGIN; CREATE TABLE tt (a INT); COMMIT;",
+		Mode:      "exec",
+		Isolation: "none",
+	})
+	assert.NoError(t, err)
+	assert.Equal(t, sqlResult{0, 0}, res)
+
+	rows, err := sqlite3.Submit(ctx, &Command{
+		Statement: "SELECT * FROM tt;",
+		Mode:      "query",
+		Isolation: "repeatable_read",
+	})
+	assert.NoError(t, err)
+	assert.Empty(t, rows)
+}
+
+func TestSubmitTimeout(t *testing.T) {
+	ctx, sqlite3 := helperSQLite3(t)
+
+	res, err := sqlite3.Submit(ctx, &Command{
+		Statement: "SELECT * FROM t;",
+		Mode:      "query",
+		Timeout:   1 * time.Nanosecond,
+	})
+	assert.Error(t, err)
+	assert.Empty(t, res)
+}
+
+func TestSubmitMode(t *testing.T) {
+	ctx, sqlite3 := helperSQLite3(t)
+
+	res, err := sqlite3.Submit(ctx, &Command{
+		Statement: "SELECT * FROM t;",
+		Mode:      "notanoption",
+	})
+	assert.Error(t, err)
+	assert.Empty(t, res)
+}
+
+func TestSubmitEmpty(t *testing.T) {
+	ctx, sqlite3 := helperSQLite3(t)
+
+	res, err := sqlite3.Submit(ctx, &Command{
+		Statement: "; ; ; ;",
+		Mode:      "query",
+	})
+	assert.Error(t, err)
+	assert.Empty(t, res)
+}

dbconnect_tests/dbconnect.yaml

@@ -0,0 +1,78 @@
+# docker-compose -f ./dbconnect_tests/dbconnect.yaml up --build --force-recreate --renew-anon-volumes --exit-code-from cloudflared
+
+version: "2.3"
+networks:
+  test-dbconnect-network:
+    driver: bridge
+services:
+  cloudflared:
+    build:
+      context: ../
+      dockerfile: dev.Dockerfile
+    command: go test github.com/cloudflare/cloudflared/dbconnect_tests -v
+    depends_on:
+      postgres:
+        condition: service_healthy
+      mysql:
+        condition: service_healthy
+      mssql:
+        condition: service_healthy
+      clickhouse:
+        condition: service_healthy
+    environment:
+      DBCONNECT_INTEGRATION_TEST: "true"
+      POSTGRESQL_URL: postgres://postgres:secret@postgres/db?sslmode=disable
+      MYSQL_URL: mysql://root:secret@mysql/db?tls=false
+      MSSQL_URL: mssql://sa:secret12345!@mssql
+      CLICKHOUSE_URL: clickhouse://clickhouse:9000/db
+    networks:
+      - test-dbconnect-network
+  postgres:
+    image: postgres:11.4-alpine
+    environment:
+      POSTGRES_DB: db
+      POSTGRES_PASSWORD: secret
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "postgres"]
+      start_period: 3s
+      interval: 1s
+      timeout: 3s
+      retries: 10
+    networks:
+      - test-dbconnect-network
+  mysql:
+    image: mysql:8.0
+    environment:
+      MYSQL_DATABASE: db
+      MYSQL_ROOT_PASSWORD: secret
+    healthcheck:
+      test: ["CMD", "mysqladmin", "ping"]
+      start_period: 3s
+      interval: 1s
+      timeout: 3s
+      retries: 10
+    networks:
+      - test-dbconnect-network
+  mssql:
+    image: mcr.microsoft.com/mssql/server:2017-CU8-ubuntu
+    environment:
+      ACCEPT_EULA: "Y"
+      SA_PASSWORD: secret12345!
+    healthcheck:
+      test: ["CMD", "/opt/mssql-tools/bin/sqlcmd", "-S", "localhost", "-U", "sa", "-P", "secret12345!", "-Q", "SELECT 1"]
+      start_period: 3s
+      interval: 1s
+      timeout: 3s
+      retries: 10
+    networks:
+      - test-dbconnect-network
+  clickhouse:
+    image: yandex/clickhouse-server:19.11
+    healthcheck:
+      test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
+      start_period: 3s
+      interval: 1s
+      timeout: 3s
+      retries: 10
+    networks:
+      - test-dbconnect-network

dbconnect_tests/sql_test.go

@@ -0,0 +1,265 @@
+package dbconnect_tests
+
+import (
+	"context"
+	"log"
+	"net/url"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/dbconnect"
+)
+
+func TestIntegrationPostgreSQL(t *testing.T) {
+	ctx, pq := helperNewSQLClient(t, "POSTGRESQL_URL")
+
+	err := pq.Ping(ctx)
+	assert.NoError(t, err)
+
+	_, err = pq.Submit(ctx, &dbconnect.Command{
+		Statement: "CREATE TABLE t (a TEXT, b UUID, c JSON, d INET[], e SERIAL);",
+		Mode:      "exec",
+	})
+	assert.NoError(t, err)
+
+	_, err = pq.Submit(ctx, &dbconnect.Command{
+		Statement: "INSERT INTO t VALUES ($1, $2, $3, $4);",
+		Mode:      "exec",
+		Arguments: dbconnect.Arguments{
+			Positional: []interface{}{
+				"text",
+				"6b8d686d-bd8e-43bc-b09a-cfcbbe702c10",
+				"{\"bool\":true,\"array\":[\"a\", 1, 3.14],\"embed\":{\"num\":21}}",
+				[]string{"1.1.1.1", "1.0.0.1"},
+			},
+		},
+	})
+	assert.NoError(t, err)
+
+	_, err = pq.Submit(ctx, &dbconnect.Command{
+		Statement: "UPDATE t SET b = NULL;",
+		Mode:      "exec",
+	})
+	assert.NoError(t, err)
+
+	res, err := pq.Submit(ctx, &dbconnect.Command{
+		Statement: "SELECT * FROM t;",
+		Mode:      "query",
+	})
+	assert.NoError(t, err)
+	assert.IsType(t, make([]map[string]interface{}, 0), res)
+
+	actual := res.([]map[string]interface{})[0]
+	expected := map[string]interface{}{
+		"a": "text",
+		"b": nil,
+		"c": map[string]interface{}{
+			"bool":  true,
+			"array": []interface{}{"a", float64(1), 3.14},
+			"embed": map[string]interface{}{"num": float64(21)},
+		},
+		"d": "{1.1.1.1,1.0.0.1}",
+		"e": int64(1),
+	}
+	assert.EqualValues(t, expected, actual)
+
+	_, err = pq.Submit(ctx, &dbconnect.Command{
+		Statement: "DROP TABLE t;",
+		Mode:      "exec",
+	})
+	assert.NoError(t, err)
+}
+
+func TestIntegrationMySQL(t *testing.T) {
+	ctx, my := helperNewSQLClient(t, "MYSQL_URL")
+
+	err := my.Ping(ctx)
+	assert.NoError(t, err)
+
+	_, err = my.Submit(ctx, &dbconnect.Command{
+		Statement: "CREATE TABLE t (a CHAR, b TINYINT, c FLOAT, d JSON, e YEAR);",
+		Mode:      "exec",
+	})
+	assert.NoError(t, err)
+
+	_, err = my.Submit(ctx, &dbconnect.Command{
+		Statement: "INSERT INTO t VALUES (?, ?, ?, ?, ?);",
+		Mode:      "exec",
+		Arguments: dbconnect.Arguments{
+			Positional: []interface{}{
+				"a",
+				10,
+				3.14,
+				"{\"bool\":true}",
+				2000,
+			},
+		},
+	})
+	assert.NoError(t, err)
+
+	_, err = my.Submit(ctx, &dbconnect.Command{
+		Statement: "ALTER TABLE t ADD COLUMN f GEOMETRY;",
+		Mode:      "exec",
+	})
+	assert.NoError(t, err)
+
+	res, err := my.Submit(ctx, &dbconnect.Command{
+		Statement: "SELECT * FROM t;",
+		Mode:      "query",
+	})
+	assert.NoError(t, err)
+	assert.IsType(t, make([]map[string]interface{}, 0), res)
+
+	actual := res.([]map[string]interface{})[0]
+	expected := map[string]interface{}{
+		"a": "a",
+		"b": float64(10),
+		"c": 3.14,
+		"d": map[string]interface{}{"bool": true},
+		"e": float64(2000),
+		"f": nil,
+	}
+	assert.EqualValues(t, expected, actual)
+
+	_, err = my.Submit(ctx, &dbconnect.Command{
+		Statement: "DROP TABLE t;",
+		Mode:      "exec",
+	})
+	assert.NoError(t, err)
+}
+
+func TestIntegrationMSSQL(t *testing.T) {
+	ctx, ms := helperNewSQLClient(t, "MSSQL_URL")
+
+	err := ms.Ping(ctx)
+	assert.NoError(t, err)
+
+	_, err = ms.Submit(ctx, &dbconnect.Command{
+		Statement: "CREATE TABLE t (a BIT, b DECIMAL, c MONEY, d TEXT);",
+		Mode:      "exec"})
+	assert.NoError(t, err)
+
+	_, err = ms.Submit(ctx, &dbconnect.Command{
+		Statement: "INSERT INTO t VALUES (?, ?, ?, ?);",
+		Mode:      "exec",
+		Arguments: dbconnect.Arguments{
+			Positional: []interface{}{
+				0,
+				3,
+				"$0.99",
+				"text",
+			},
+		},
+	})
+	assert.NoError(t, err)
+
+	_, err = ms.Submit(ctx, &dbconnect.Command{
+		Statement: "UPDATE t SET d = NULL;",
+		Mode:      "exec",
+	})
+	assert.NoError(t, err)
+
+	res, err := ms.Submit(ctx, &dbconnect.Command{
+		Statement: "SELECT * FROM t;",
+		Mode:      "query",
+	})
+	assert.NoError(t, err)
+	assert.IsType(t, make([]map[string]interface{}, 0), res)
+
+	actual := res.([]map[string]interface{})[0]
+	expected := map[string]interface{}{
+		"a": false,
+		"b": float64(3),
+		"c": float64(0.99),
+		"d": nil,
+	}
+	assert.EqualValues(t, expected, actual)
+
+	_, err = ms.Submit(ctx, &dbconnect.Command{
+		Statement: "DROP TABLE t;",
+		Mode:      "exec",
+	})
+	assert.NoError(t, err)
+}
+
+func TestIntegrationClickhouse(t *testing.T) {
+	ctx, ch := helperNewSQLClient(t, "CLICKHOUSE_URL")
+
+	err := ch.Ping(ctx)
+	assert.NoError(t, err)
+
+	_, err = ch.Submit(ctx, &dbconnect.Command{
+		Statement: "CREATE TABLE t (a UUID, b String, c Float64, d UInt32, e Int16, f Array(Enum8('a'=1, 'b'=2, 'c'=3))) engine=Memory;",
+		Mode:      "exec",
+	})
+	assert.NoError(t, err)
+
+	_, err = ch.Submit(ctx, &dbconnect.Command{
+		Statement: "INSERT INTO t VALUES (?, ?, ?, ?, ?, ?);",
+		Mode:      "exec",
+		Arguments: dbconnect.Arguments{
+			Positional: []interface{}{
+				"ec65f626-6f50-4c86-9628-6314ef1edacd",
+				"",
+				3.14,
+				314,
+				-144,
+				[]string{"a", "b", "c"},
+			},
+		},
+	})
+	assert.NoError(t, err)
+
+	res, err := ch.Submit(ctx, &dbconnect.Command{
+		Statement: "SELECT * FROM t;",
+		Mode:      "query",
+	})
+	assert.NoError(t, err)
+	assert.IsType(t, make([]map[string]interface{}, 0), res)
+
+	actual := res.([]map[string]interface{})[0]
+	expected := map[string]interface{}{
+		"a": "ec65f626-6f50-4c86-9628-6314ef1edacd",
+		"b": "",
+		"c": float64(3.14),
+		"d": uint32(314),
+		"e": int16(-144),
+		"f": []string{"a", "b", "c"},
+	}
+	assert.EqualValues(t, expected, actual)
+
+	_, err = ch.Submit(ctx, &dbconnect.Command{
+		Statement: "DROP TABLE t;",
+		Mode:      "exec",
+	})
+	assert.NoError(t, err)
+}
+
+func helperNewSQLClient(t *testing.T, env string) (context.Context, dbconnect.Client) {
+	_, ok := os.LookupEnv("DBCONNECT_INTEGRATION_TEST")
+	if ok {
+		t.Helper()
+	} else {
+		t.SkipNow()
+	}
+
+	val, ok := os.LookupEnv(env)
+	if !ok {
+		log.Fatalf("must provide database url as environment variable: %s", env)
+	}
+
+	parsed, err := url.Parse(val)
+	if err != nil {
+		log.Fatalf("cannot provide invalid database url: %s=%s", env, val)
+	}
+
+	ctx := context.Background()
+	client, err := dbconnect.NewSQLClient(ctx, parsed)
+	if err != nil {
+		log.Fatalf("could not start test client: %s", err)
+	}
+
+	return ctx, client
+}

dev.Dockerfile

@@ -0,0 +1,4 @@
+FROM golang:1.12 as builder
+WORKDIR /go/src/github.com/cloudflare/cloudflared/
+RUN apt-get update
+COPY . .

go.mod

@@ -0,0 +1,72 @@
+module github.com/cloudflare/cloudflared
+
+go 1.12
+
+require (
+	github.com/DATA-DOG/go-sqlmock v1.3.3
+	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
+	github.com/aws/aws-sdk-go v1.25.8
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect
+	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
+	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
+	github.com/coredns/coredns v1.2.0
+	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
+	github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a
+	github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0
+	github.com/equinox-io/equinox v1.2.0
+	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
+	github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
+	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
+	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
+	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
+	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
+	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
+	github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0
+	github.com/go-sql-driver/mysql v1.4.1
+	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
+	github.com/google/certificate-transparency-go v1.1.0
+	github.com/google/uuid v1.1.1
+	github.com/gorilla/mux v1.7.3
+	github.com/gorilla/websocket v1.4.0
+	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
+	github.com/jmoiron/sqlx v1.2.0
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/kshvakov/clickhouse v1.3.11
+	github.com/lib/pq v1.2.0
+	github.com/mattn/go-colorable v0.1.4
+	github.com/mattn/go-isatty v0.0.10 // indirect
+	github.com/mattn/go-sqlite3 v1.11.0
+	github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b // indirect
+	github.com/miekg/dns v1.1.8
+	github.com/mitchellh/go-homedir v1.1.0
+	github.com/opentracing/opentracing-go v1.1.0 // indirect
+	github.com/philhofer/fwd v1.0.0 // indirect
+	github.com/pkg/errors v0.8.1
+	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
+	github.com/prometheus/client_golang v1.0.0
+	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
+	github.com/prometheus/common v0.7.0 // indirect
+	github.com/prometheus/procfs v0.0.5 // indirect
+	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
+	github.com/sirupsen/logrus v1.4.2
+	github.com/stretchr/testify v1.3.0
+	github.com/tinylib/msgp v1.1.0 // indirect
+	github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0
+	golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc
+	golang.org/x/net v0.0.0-20191007182048-72f939374954
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58
+	golang.org/x/sys v0.0.0-20191008105621-543471e840be
+	google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd // indirect
+	google.golang.org/grpc v1.24.0 // indirect
+	gopkg.in/coreos/go-oidc.v2 v2.1.0
+	gopkg.in/square/go-jose.v2 v2.4.0 // indirect
+	gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8
+	gopkg.in/yaml.v2 v2.2.4 // indirect
+	zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7
+)
+
+// ../../go/pkg/mod/github.com/coredns/coredns@v1.2.0/plugin/metrics/metrics.go:40:49: too many arguments in call to prometheus.NewProcessCollector
+// have (int, string)
+// want (prometheus.ProcessCollectorOpts)
+replace github.com/prometheus/client_golang => github.com/prometheus/client_golang v0.9.0-pre1

go.sum

@@ -0,0 +1,445 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw=
+contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
+github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
+github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/aws/aws-sdk-go v1.25.8 h1:n7I+HUUXjun2CsX7JK+1hpRIkZrlKhd3nayeb+Xmavs=
+github.com/aws/aws-sdk-go v1.25.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
+github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
+github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg=
+github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93 h1:QrGfkZDnMxcWHaYDdB7CmqS9i26OAnUj/xcus/abYkY=
+github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93/go.mod h1:QiTe66jFdP7cUKMCCf/WrvDyYdtdmdZfVcdoLbzaKVY=
+github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc h1:Dvk3ySBsOm5EviLx6VCyILnafPcQinXGP5jbTdHUJgE=
+github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc/go.mod h1:HlgKKR8V5a1wroIDDIz3/A+T+9Janfq+7n1P5sEFdi0=
+github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=
+github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
+github.com/coredns/coredns v1.2.0 h1:YEI38K2BJYzL/SxO2tZFD727T/C68DqVWkBQjT0sWPU=
+github.com/coredns/coredns v1.2.0/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73 h1:7CNPV0LWRCa1FNmqg700pbXhzvmoaXKyfxWRkjRym7Q=
+github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a h1:W8b4lQ4tFF21aspRGoBuCNV6V2fFJBF+pm1J6OY8Lys=
+github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0 h1:epsH3lb7KVbXHYk7LYGN5EiE0MxcevHU85CKITJ0wUY=
+github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/equinox-io/equinox v1.2.0 h1:bBS7Ou+Y7Jwgmy8TWSYxEh85WctuFn7FPlgbUzX4DBA=
+github.com/equinox-io/equinox v1.2.0/go.mod h1:6s3HJB0PYUNgs0mxmI8fHdfVl3TQ25ieA/PVfr+eyVo=
+github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
+github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
+github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 h1:wWke/RUCl7VRjQhwPlR/v0glZXNYzBHdNUzf/Am2Nmg=
+github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmAp6Sws4L7+Q/OokbWDAK1ibXYhB3PXFP1kol5hPg=
+github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434 h1:mOp33BLbcbJ8fvTAmZacbBiOASfxN+MLcLxymZCIrGE=
+github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434/go.mod h1:KigFdumBXUPSwzLDbeuzyt0elrL7+CP7TKuhrhT4bcU=
+github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
+github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
+github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y=
+github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
+github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/kkTFdWhctH96grJ7qiy7bMdiZcIvPKs=
+github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0 h1:gF8ngtda767ddth2SH0YSAhswhz6qUkvyI9EZFYCWJA=
+github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
+github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
+github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
+github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
+github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
+github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
+github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
+github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
+github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
+github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
+github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
+github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
+github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
+github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
+github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.17.2-0.20190910081718-bad04bb7378f/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg=
+github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
+github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
+github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
+github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/certificate-transparency-go v1.1.0 h1:10MlrYzh5wfkToxWI4yJzffsxLfxcEDlOATMx/V9Kzw=
+github.com/google/certificate-transparency-go v1.1.0/go.mod h1:i+Q7XY+ArBveOUT36jiHGfuSK1fHICIg6sUkRxPAbCs=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/monologue v0.0.0-20190606152607-4b11a32b5934/go.mod h1:6NTfaQoUpg5QmPsCUWLR3ig33FHrKXhTtWzF0DVdmuk=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/trillian v1.2.2-0.20190612132142-05461f4df60a/go.mod h1:YPmUVn5NGwgnDUgqlVyFGMTgaWlnSvH7W5p+NdOG8UA=
+github.com/google/trillian-examples v0.0.0-20190603134952-4e75ba15216c/go.mod h1:WgL3XZ3pA8/9cm7yxqWrZE6iZkESB2ItGxy5Fo6k2lk=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
+github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
+github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
+github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kshvakov/clickhouse v1.3.11 h1:dtzTJY0fCA+MWkLyuKZaNPkmSwdX4gh8+Klic9NB1Lw=
+github.com/kshvakov/clickhouse v1.3.11/go.mod h1:/SVBAcqF3u7rxQ9sTWCZwf8jzzvxiZGeQvtmSF2BBEc=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/letsencrypt/pkcs11key v2.0.1-0.20170608213348-396559074696+incompatible/go.mod h1:iGYXKqDXt0cpBthCHdr9ZdsQwyGlYFh/+8xa4WzIQ34=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
+github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
+github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b h1:/BbY4n99iMazlr2igipph+hj0MwlZIWpcsP8Iy+na+s=
+github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b/go.mod h1:Wb1PlT4DAYSqOEd03MsqkdkXnTxA8v9pKjdpxbqM1kY=
+github.com/miekg/dns v1.1.8 h1:1QYRAKU3lN5cRfLCkPU08hwvLJFhvjP6MqNMmQz6ZVI=
+github.com/miekg/dns v1.1.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
+github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
+github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
+github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU=
+github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
+github.com/prometheus/client_golang v0.9.0-pre1 h1:AWTOhsOI9qxeirTuA0A4By/1Es1+y9EcCGY6bBZ2fhM=
+github.com/prometheus/client_golang v0.9.0-pre1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
+github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
+github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
+github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
+github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
+github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
+github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
+github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0 h1:6DtWz8hNS4qbq0OCRPhdBMG9E2qKTSDKlwnP3dmZvuA=
+github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0/go.mod h1:A47W3pdWONaZmXuLZgfKLAVgUY0qvfTRM5vVDKS40S4=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/etcd v3.3.13+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
+golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191007182048-72f939374954 h1:JGZucVF/L/TotR719NbujzadOZ2AgnYlqphQGHDCKaU=
+golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190605220351-eb0b1bdb6ae6/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
+google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd h1:84VQPzup3IpKLxuIAZjHMhVjJ8fZ4/i3yUnj3k6fUdw=
+google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
+google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
+gopkg.in/coreos/go-oidc.v2 v2.1.0 h1:E8PjVFdj/SLDKB0hvb70KTbMbYVHjqztiQdSkIg8E+I=
+gopkg.in/coreos/go-oidc.v2 v2.1.0/go.mod h1:fYaTe2FS96wZZwR17YTDHwG+Mw6fmyqJNxN2eNCGPCI=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.4.0 h1:0kXPskUMGAXXWJlP05ktEMOV0vmzFQUWw6d+aZJQU8A=
+gopkg.in/square/go-jose.v2 v2.4.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8 h1:/pLAskKF+d5SawboKd8GB8ew4ClHDbt2c3K9EBFeRGU=
+gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
+mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
+sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
+zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7 h1:CZoOFlTPbKfAShKYrMuUfYbnXexFT1rYRUX1SPnrdE4=
+zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7/go.mod h1:TMGa8HWGJkXiq4nHe9Zu/JgRF5oUtg4XizFC+Vexbec=

h2mux/activestreammap.go

@@ -3,6 +3,7 @@ package h2mux
 import (
 	"sync"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/net/http2"
 )
 
@@ -12,23 +13,28 @@ type activeStreamMap struct {
 	sync.RWMutex
 	// streams tracks open streams.
 	streams map[uint32]*MuxedStream
-	// streamsEmpty is a chan that should be closed when no more streams are open.
-	streamsEmpty chan struct{}
 	// nextStreamID is the next ID to use on our side of the connection.
 	// This is odd for clients, even for servers.
 	nextStreamID uint32
 	// maxPeerStreamID is the ID of the most recent stream opened by the peer.
 	maxPeerStreamID uint32
+	// activeStreams is a gauge shared by all muxers of this process to expose the total number of active streams
+	activeStreams prometheus.Gauge
+
 	// ignoreNewStreams is true when the connection is being shut down. New streams
 	// cannot be registered.
 	ignoreNewStreams bool
+	// streamsEmpty is a chan that will be closed when no more streams are open.
+	streamsEmptyChan chan struct{}
+	closeOnce        sync.Once
 }
 
-func newActiveStreamMap(useClientStreamNumbers bool) *activeStreamMap {
+func newActiveStreamMap(useClientStreamNumbers bool, activeStreams prometheus.Gauge) *activeStreamMap {
 	m := &activeStreamMap{
-		streams:      make(map[uint32]*MuxedStream),
-		streamsEmpty: make(chan struct{}),
-		nextStreamID: 1,
+		streams:          make(map[uint32]*MuxedStream),
+		streamsEmptyChan: make(chan struct{}),
+		nextStreamID:     1,
+		activeStreams:    activeStreams,
 	}
 	// Client initiated stream uses odd stream ID, server initiated stream uses even stream ID
 	if !useClientStreamNumbers {
@@ -37,6 +43,13 @@ func newActiveStreamMap(useClientStreamNumbers bool) *activeStreamMap {
 	return m
 }
 
+// This function should be called while `m` is locked.
+func (m *activeStreamMap) notifyStreamsEmpty() {
+	m.closeOnce.Do(func() {
+		close(m.streamsEmptyChan)
+	})
+}
+
 // Len returns the number of active streams.
 func (m *activeStreamMap) Len() int {
 	m.RLock()
@@ -63,6 +76,7 @@ func (m *activeStreamMap) Set(newStream *MuxedStream) bool {
 		return false
 	}
 	m.streams[newStream.streamID] = newStream
+	m.activeStreams.Inc()
 	return true
 }
 
@@ -70,31 +84,33 @@ func (m *activeStreamMap) Set(newStream *MuxedStream) bool {
 func (m *activeStreamMap) Delete(streamID uint32) {
 	m.Lock()
 	defer m.Unlock()
-	delete(m.streams, streamID)
-	if len(m.streams) == 0 && m.streamsEmpty != nil {
-		close(m.streamsEmpty)
-		m.streamsEmpty = nil
+	if _, ok := m.streams[streamID]; ok {
+		delete(m.streams, streamID)
+		m.activeStreams.Dec()
+	}
+
+	// shutting down, and now the map is empty
+	if m.ignoreNewStreams && len(m.streams) == 0 {
+		m.notifyStreamsEmpty()
 	}
 }
 
-// Shutdown blocks new streams from being created. It returns a channel that receives an event
-// once the last stream has closed, or nil if a shutdown is in progress.
-func (m *activeStreamMap) Shutdown() <-chan struct{} {
+// Shutdown blocks new streams from being created.
+// It returns `done`, a channel that is closed once the last stream has closed
+// and `progress`, whether a shutdown was already in progress
+func (m *activeStreamMap) Shutdown() (done <-chan struct{}, alreadyInProgress bool) {
 	m.Lock()
 	defer m.Unlock()
 	if m.ignoreNewStreams {
 		// already shutting down
-		return nil
+		return m.streamsEmptyChan, true
 	}
 	m.ignoreNewStreams = true
-	done := make(chan struct{})
 	if len(m.streams) == 0 {
-		// nothing to shut down
-		close(done)
-		return done
+		// there are no streams to wait for
+		m.notifyStreamsEmpty()
 	}
-	m.streamsEmpty = done
-	return done
+	return m.streamsEmptyChan, false
 }
 
 // AcquireLocalID acquires a new stream ID for a stream you're opening.
@@ -162,4 +178,5 @@ func (m *activeStreamMap) Abort() {
 		stream.Close()
 	}
 	m.ignoreNewStreams = true
+	m.notifyStreamsEmpty()
 }

h2mux/activestreammap_test.go

@@ -0,0 +1,195 @@
+package h2mux
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestShutdown(t *testing.T) {
+	const numStreams = 1000
+	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+
+	// Add all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				stream := &MuxedStream{streamID: uint32(streamID)}
+				ok := m.Set(stream)
+				assert.True(t, ok)
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
+
+	shutdownChan, alreadyInProgress := m.Shutdown()
+	select {
+	case <-shutdownChan:
+		assert.Fail(t, "before Shutdown(), shutdownChan shouldn't be closed")
+	default:
+	}
+	assert.False(t, alreadyInProgress)
+
+	shutdownChan2, alreadyInProgress2 := m.Shutdown()
+	assert.Equal(t, shutdownChan, shutdownChan2, "repeated calls to Shutdown() should return the same channel")
+	assert.True(t, alreadyInProgress2, "repeated calls to Shutdown() should return true for 'in progress'")
+
+	// Delete all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				m.Delete(uint32(streamID))
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, 0, m.Len(), "All the streams should have been deleted")
+
+	select {
+	case <-shutdownChan:
+	default:
+		assert.Fail(t, "After all the streams are deleted, shutdownChan should have been closed")
+	}
+}
+
+func TestEmptyBeforeShutdown(t *testing.T) {
+	const numStreams = 1000
+	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+
+	// Add all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				stream := &MuxedStream{streamID: uint32(streamID)}
+				ok := m.Set(stream)
+				assert.True(t, ok)
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
+
+	// Delete all the streams, bringing m to size 0
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				m.Delete(uint32(streamID))
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, 0, m.Len(), "All the streams should have been deleted")
+
+	// Add one stream back
+	const soloStreamID = uint32(0)
+	ok := m.Set(&MuxedStream{streamID: soloStreamID})
+	assert.True(t, ok)
+
+	shutdownChan, alreadyInProgress := m.Shutdown()
+	select {
+	case <-shutdownChan:
+		assert.Fail(t, "before Shutdown(), shutdownChan shouldn't be closed")
+	default:
+	}
+	assert.False(t, alreadyInProgress)
+
+	shutdownChan2, alreadyInProgress2 := m.Shutdown()
+	assert.Equal(t, shutdownChan, shutdownChan2, "repeated calls to Shutdown() should return the same channel")
+	assert.True(t, alreadyInProgress2, "repeated calls to Shutdown() should return true for 'in progress'")
+
+	// Remove the remaining stream
+	m.Delete(soloStreamID)
+
+	select {
+	case <-shutdownChan:
+	default:
+		assert.Fail(t, "After all the streams are deleted, shutdownChan should have been closed")
+	}
+}
+
+type noopBuffer struct {
+	isClosed bool
+}
+
+func (t *noopBuffer) Read(p []byte) (n int, err error)  { return len(p), nil }
+func (t *noopBuffer) Write(p []byte) (n int, err error) { return len(p), nil }
+func (t *noopBuffer) Reset()                            {}
+func (t *noopBuffer) Len() int                          { return 0 }
+func (t *noopBuffer) Close() error                      { t.isClosed = true; return nil }
+func (t *noopBuffer) Closed() bool                      { return t.isClosed }
+
+type noopReadyList struct{}
+
+func (_ *noopReadyList) Signal(streamID uint32) {}
+
+func TestAbort(t *testing.T) {
+	const numStreams = 1000
+	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+
+	var openedStreams sync.Map
+
+	// Add all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				stream := &MuxedStream{
+					streamID:    uint32(streamID),
+					readBuffer:  &noopBuffer{},
+					writeBuffer: &noopBuffer{},
+					readyList:   &noopReadyList{},
+				}
+				ok := m.Set(stream)
+				assert.True(t, ok)
+
+				openedStreams.Store(stream.streamID, stream)
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
+
+	shutdownChan, alreadyInProgress := m.Shutdown()
+	select {
+	case <-shutdownChan:
+		assert.Fail(t, "before Abort(), shutdownChan shouldn't be closed")
+	default:
+	}
+	assert.False(t, alreadyInProgress)
+
+	m.Abort()
+	assert.Equal(t, numStreams, m.Len(), "Abort() shouldn't delete any streams")
+	openedStreams.Range(func(key interface{}, value interface{}) bool {
+		stream := value.(*MuxedStream)
+		readBuffer := stream.readBuffer.(*noopBuffer)
+		writeBuffer := stream.writeBuffer.(*noopBuffer)
+		return assert.True(t, readBuffer.isClosed && writeBuffer.isClosed, "Abort() should have closed all the streams")
+	})
+
+	select {
+	case <-shutdownChan:
+	default:
+		assert.Fail(t, "after Abort(), shutdownChan should have been closed")
+	}
+
+	// multiple aborts shouldn't cause any issues
+	m.Abort()
+	m.Abort()
+	m.Abort()
+}

h2mux/error.go

@@ -7,6 +7,7 @@ import (
 )
 
 var (
+	// HTTP2 error codes: https://http2.github.io/http2-spec/#ErrorCodes
 	ErrHandshakeTimeout               = MuxerHandshakeError{"1000 handshake timeout"}
 	ErrBadHandshakeNotSettings        = MuxerHandshakeError{"1001 unexpected response"}
 	ErrBadHandshakeUnexpectedAck      = MuxerHandshakeError{"1002 unexpected response"}
@@ -19,9 +20,12 @@ var (
 	ErrUnknownStream       = MuxerProtocolError{"2002 unknown stream", http2.ErrCodeProtocol}
 	ErrInvalidStream       = MuxerProtocolError{"2003 invalid stream", http2.ErrCodeProtocol}
 
-	ErrStreamHeadersSent = MuxerApplicationError{"3000 headers already sent"}
-	ErrConnectionClosed  = MuxerApplicationError{"3001 connection closed"}
-	ErrConnectionDropped = MuxerApplicationError{"3002 connection dropped"}
+	ErrStreamHeadersSent               = MuxerApplicationError{"3000 headers already sent"}
+	ErrStreamRequestConnectionClosed   = MuxerApplicationError{"3001 connection closed while opening stream"}
+	ErrConnectionDropped               = MuxerApplicationError{"3002 connection dropped"}
+	ErrStreamRequestTimeout            = MuxerApplicationError{"3003 open stream timeout"}
+	ErrResponseHeadersTimeout          = MuxerApplicationError{"3004 timeout waiting for initial response headers"}
+	ErrResponseHeadersConnectionClosed = MuxerApplicationError{"3005 connection closed while waiting for initial response headers"}
 
 	ErrClosedStream = MuxerStreamError{"4000 stream closed", http2.ErrCodeStreamClosed}
 )

h2mux/h2_dictionaries.go

@@ -542,7 +542,10 @@ func (w *h2DictWriter) Write(p []byte) (n int, err error) {
 }
 
 func (w *h2DictWriter) Close() error {
-	return w.comp.Close()
+	if w.comp != nil {
+		return w.comp.Close()
+	}
+	return nil
 }
 
 // From http2/hpack

h2mux/h2mux.go

@@ -1,13 +1,13 @@
 package h2mux
 
 import (
-	"bytes"
 	"context"
 	"io"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
@@ -15,11 +15,12 @@ import (
 )
 
 const (
-	defaultFrameSize  uint32        = 1 << 14 // Minimum frame size in http2 spec
-	defaultWindowSize uint32        = 65535
-	maxWindowSize     uint32        = (1 << 31) - 1 // 2^31-1 = 2147483647, max window size specified in http2 spec
-	defaultTimeout    time.Duration = 5 * time.Second
-	defaultRetries    uint64        = 5
+	defaultFrameSize         uint32        = 1 << 14       // Minimum frame size in http2 spec
+	defaultWindowSize        uint32        = (1 << 16) - 1 // Minimum window size in http2 spec
+	maxWindowSize            uint32        = (1 << 31) - 1 // 2^31-1 = 2147483647, max window size in http2 spec
+	defaultTimeout           time.Duration = 5 * time.Second
+	defaultRetries           uint64        = 5
+	defaultWriteBufferMaxLen int           = 1024 * 1024 // 1mb
 
 	SettingMuxerMagic http2.SettingID = 0x42db
 	MuxerMagicOrigin  uint32          = 0xa2e43c8b
@@ -49,6 +50,12 @@ type MuxerConfig struct {
 	// Logger to use
 	Logger             *log.Entry
 	CompressionQuality CompressionSetting
+	// Initial size for HTTP2 flow control windows
+	DefaultWindowSize uint32
+	// Largest allowable size for HTTP2 flow control windows
+	MaxWindowSize uint32
+	// Largest allowable capacity for the buffer of data to be sent
+	StreamWriteBufferMaxLen int
 }
 
 type Muxer struct {
@@ -64,7 +71,7 @@ type Muxer struct {
 	// muxWriter is the write process.
 	muxWriter *MuxWriter
 	// muxMetricsUpdater is the process to update metrics
-	muxMetricsUpdater *muxMetricsUpdater
+	muxMetricsUpdater muxMetricsUpdater
 	// newStreamChan is used to create new streams on the writer thread.
 	// The writer will assign the next available stream ID.
 	newStreamChan chan MuxedStreamRequest
@@ -87,17 +94,35 @@ type Header struct {
 	Name, Value string
 }
 
+func RPCHeaders() []Header {
+	return []Header{
+		{Name: ":method", Value: "RPC"},
+		{Name: ":scheme", Value: "capnp"},
+		{Name: ":path", Value: "*"},
+	}
+}
+
 // Handshake establishes a muxed connection with the peer.
 // After the handshake completes, it is possible to open and accept streams.
 func Handshake(
 	w io.WriteCloser,
 	r io.ReadCloser,
 	config MuxerConfig,
+	activeStreamsMetrics prometheus.Gauge,
 ) (*Muxer, error) {
 	// Set default config values
 	if config.Timeout == 0 {
 		config.Timeout = defaultTimeout
 	}
+	if config.DefaultWindowSize == 0 {
+		config.DefaultWindowSize = defaultWindowSize
+	}
+	if config.MaxWindowSize == 0 {
+		config.MaxWindowSize = maxWindowSize
+	}
+	if config.StreamWriteBufferMaxLen == 0 {
+		config.StreamWriteBufferMaxLen = defaultWriteBufferMaxLen
+	}
 	// Initialise connection state fields
 	m := &Muxer{
 		f:             http2.NewFramer(w, r), // A framer that writes to w and reads from r
@@ -107,7 +132,7 @@ func Handshake(
 		newStreamChan: make(chan MuxedStreamRequest),
 		abortChan:     make(chan struct{}),
 		readyList:     NewReadyList(),
-		streams:       newActiveStreamMap(config.IsClient),
+		streams:       newActiveStreamMap(config.IsClient, activeStreamsMetrics),
 	}
 
 	m.f.ReadMetaHeaders = hpack.NewDecoder(4096, func(hpack.HeaderField) {})
@@ -147,11 +172,6 @@ func Handshake(
 	// set up reader/writer pair ready for serve
 	streamErrors := NewStreamErrorMap()
 	goAwayChan := make(chan http2.ErrCode, 1)
-	updateRTTChan := make(chan *roundTripMeasurement, 1)
-	updateReceiveWindowChan := make(chan uint32, 1)
-	updateSendWindowChan := make(chan uint32, 1)
-	updateInBoundBytesChan := make(chan uint64)
-	updateOutBoundBytesChan := make(chan uint64)
 	inBoundCounter := NewAtomicCounter(0)
 	outBoundCounter := NewAtomicCounter(0)
 	pingTimestamp := NewPingTimestamp()
@@ -168,6 +188,14 @@ func Handshake(
 		config.Logger.Warn("Minimum number of unacked heartbeats to send before closing the connection has been adjusted to ", maxRetries)
 	}
 
+	compBytesBefore, compBytesAfter := NewAtomicCounter(0), NewAtomicCounter(0)
+
+	m.muxMetricsUpdater = newMuxMetricsUpdater(
+		m.abortChan,
+		compBytesBefore,
+		compBytesAfter,
+	)
+
 	m.explicitShutdown = NewBooleanFuse()
 	m.muxReader = &MuxReader{
 		f:                       m.f,
@@ -179,47 +207,30 @@ func Handshake(
 		abortChan:               m.abortChan,
 		pingTimestamp:           pingTimestamp,
 		connActive:              connActive,
-		initialStreamWindow:     defaultWindowSize,
-		streamWindowMax:         maxWindowSize,
+		initialStreamWindow:     m.config.DefaultWindowSize,
+		streamWindowMax:         m.config.MaxWindowSize,
+		streamWriteBufferMaxLen: m.config.StreamWriteBufferMaxLen,
 		r:                       m.r,
-		updateRTTChan:           updateRTTChan,
-		updateReceiveWindowChan: updateReceiveWindowChan,
-		updateSendWindowChan:    updateSendWindowChan,
+		metricsUpdater:          m.muxMetricsUpdater,
 		bytesRead:               inBoundCounter,
-		updateInBoundBytesChan:  updateInBoundBytesChan,
 	}
 	m.muxWriter = &MuxWriter{
-		f:                       m.f,
-		streams:                 m.streams,
-		streamErrors:            streamErrors,
-		readyStreamChan:         m.readyList.ReadyChannel(),
-		newStreamChan:           m.newStreamChan,
-		goAwayChan:              goAwayChan,
-		abortChan:               m.abortChan,
-		pingTimestamp:           pingTimestamp,
-		idleTimer:               NewIdleTimer(idleDuration, maxRetries),
-		connActiveChan:          connActive.WaitChannel(),
-		maxFrameSize:            defaultFrameSize,
-		updateReceiveWindowChan: updateReceiveWindowChan,
-		updateSendWindowChan:    updateSendWindowChan,
-		bytesWrote:              outBoundCounter,
-		updateOutBoundBytesChan: updateOutBoundBytesChan,
+		f:               m.f,
+		streams:         m.streams,
+		streamErrors:    streamErrors,
+		readyStreamChan: m.readyList.ReadyChannel(),
+		newStreamChan:   m.newStreamChan,
+		goAwayChan:      goAwayChan,
+		abortChan:       m.abortChan,
+		pingTimestamp:   pingTimestamp,
+		idleTimer:       NewIdleTimer(idleDuration, maxRetries),
+		connActiveChan:  connActive.WaitChannel(),
+		maxFrameSize:    defaultFrameSize,
+		metricsUpdater:  m.muxMetricsUpdater,
+		bytesWrote:      outBoundCounter,
 	}
 	m.muxWriter.headerEncoder = hpack.NewEncoder(&m.muxWriter.headerBuffer)
 
-	compBytesBefore, compBytesAfter := NewAtomicCounter(0), NewAtomicCounter(0)
-
-	m.muxMetricsUpdater = newMuxMetricsUpdater(
-		updateRTTChan,
-		updateReceiveWindowChan,
-		updateSendWindowChan,
-		updateInBoundBytesChan,
-		updateOutBoundBytesChan,
-		m.abortChan,
-		compBytesBefore,
-		compBytesAfter,
-	)
-
 	if m.compressionQuality.dictSize > 0 && m.compressionQuality.nDicts > 0 {
 		nd, sz := m.compressionQuality.nDicts, m.compressionQuality.dictSize
 		writeDicts, dictChan := newH2WriteDictionaries(
@@ -305,6 +316,12 @@ func joinErrorsWithTimeout(errChan <-chan error, receiveCount int, timeout time.
 	return nil
 }
 
+// Serve runs the event loops that comprise h2mux:
+// - MuxReader.run()
+// - MuxWriter.run()
+// - muxMetricsUpdater.run()
+// In the normal case, Shutdown() is called concurrently with Serve() to stop
+// these loops.
 func (m *Muxer) Serve(ctx context.Context) error {
 	errGroup, _ := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {
@@ -335,9 +352,12 @@ func (m *Muxer) Serve(ctx context.Context) error {
 	return nil
 }
 
-func (m *Muxer) Shutdown() {
+// Shutdown is called to initiate the "happy path" of muxer termination.
+// It blocks new streams from being created.
+// It returns a channel that is closed when the last stream has been closed.
+func (m *Muxer) Shutdown() <-chan struct{} {
 	m.explicitShutdown.Fuse(true)
-	m.muxReader.Shutdown()
+	return m.muxReader.Shutdown()
 }
 
 // IsUnexpectedTunnelError identifies errors that are expected when shutting down the h2mux tunnel.
@@ -370,41 +390,70 @@ func isConnectionClosedError(err error) bool {
 
 // OpenStream opens a new data stream with the given headers.
 // Called by proxy server and tunnel
-func (m *Muxer) OpenStream(headers []Header, body io.Reader) (*MuxedStream, error) {
-	stream := &MuxedStream{
-		responseHeadersReceived: make(chan struct{}),
-		readBuffer:              NewSharedBuffer(),
-		writeBuffer:             &bytes.Buffer{},
-		receiveWindow:           defaultWindowSize,
-		receiveWindowCurrentMax: defaultWindowSize, // Initial window size limit. exponentially increase it when receiveWindow is exhausted
-		receiveWindowMax:        maxWindowSize,
-		sendWindow:              defaultWindowSize,
-		readyList:               m.readyList,
-		writeHeaders:            headers,
-		dictionaries:            m.muxReader.dictionaries,
+func (m *Muxer) OpenStream(ctx context.Context, headers []Header, body io.Reader) (*MuxedStream, error) {
+	stream := m.NewStream(headers)
+	if err := m.MakeMuxedStreamRequest(ctx, NewMuxedStreamRequest(stream, body)); err != nil {
+		return nil, err
 	}
+	if err := m.AwaitResponseHeaders(ctx, stream); err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
 
-	select {
-	// Will be received by mux writer
-	case m.newStreamChan <- MuxedStreamRequest{stream: stream, body: body}:
-	case <-m.abortChan:
-		return nil, ErrConnectionClosed
+func (m *Muxer) OpenRPCStream(ctx context.Context) (*MuxedStream, error) {
+	stream := m.NewStream(RPCHeaders())
+	if err := m.MakeMuxedStreamRequest(ctx, NewMuxedStreamRequest(stream, nil)); err != nil {
+		return nil, err
 	}
+	if err := m.AwaitResponseHeaders(ctx, stream); err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
+
+func (m *Muxer) NewStream(headers []Header) *MuxedStream {
+	return NewStream(m.config, headers, m.readyList, m.muxReader.dictionaries)
+}
+
+func (m *Muxer) MakeMuxedStreamRequest(ctx context.Context, request MuxedStreamRequest) error {
 	select {
-	case <-stream.responseHeadersReceived:
-		return stream, nil
+	case <-ctx.Done():
+		return ErrStreamRequestTimeout
 	case <-m.abortChan:
-		return nil, ErrConnectionClosed
+		return ErrStreamRequestConnectionClosed
+	// Will be received by mux writer
+	case m.newStreamChan <- request:
+		return nil
+	}
+}
+
+func (m *Muxer) CloseStreamRead(stream *MuxedStream) {
+	stream.CloseRead()
+	if stream.WriteClosed() {
+		m.streams.Delete(stream.streamID)
+	}
+}
+
+func (m *Muxer) AwaitResponseHeaders(ctx context.Context, stream *MuxedStream) error {
+	select {
+	case <-ctx.Done():
+		return ErrResponseHeadersTimeout
+	case <-m.abortChan:
+		return ErrResponseHeadersConnectionClosed
+	case <-stream.responseHeadersReceived:
+		return nil
 	}
 }
 
 func (m *Muxer) Metrics() *MuxerMetrics {
-	return m.muxMetricsUpdater.Metrics()
+	return m.muxMetricsUpdater.metrics()
 }
 
 func (m *Muxer) abort() {
 	m.abortOnce.Do(func() {
 		close(m.abortChan)
+		m.readyList.Close()
 		m.streams.Abort()
 	})
 }

h2mux/h2mux_test.go

@@ -15,7 +15,15 @@ import (
 	"testing"
 	"time"
 
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	testOpenStreamTimeout = time.Millisecond * 5000
+	testHandshakeTimeout  = time.Millisecond * 1000
 )
 
 func TestMain(m *testing.M) {
@@ -35,86 +43,88 @@ type DefaultMuxerPair struct {
 	doneC           chan struct{}
 }
 
-func NewDefaultMuxerPair() *DefaultMuxerPair {
+func NewDefaultMuxerPair(t assert.TestingT, testName string, f MuxedStreamFunc) *DefaultMuxerPair {
 	origin, edge := net.Pipe()
-	return &DefaultMuxerPair{
+	p := &DefaultMuxerPair{
 		OriginMuxConfig: MuxerConfig{
-			Timeout: time.Second,
-			IsClient: true,
-			Name: "origin",
-			Logger: log.NewEntry(log.New()),
+			Timeout:                 testHandshakeTimeout,
+			Handler:                 f,
+			IsClient:                true,
+			Name:                    "origin",
+			Logger:                  log.NewEntry(log.New()),
+			DefaultWindowSize:       (1 << 8) - 1,
+			MaxWindowSize:           (1 << 15) - 1,
+			StreamWriteBufferMaxLen: 1024,
+			HeartbeatInterval:       defaultTimeout,
+			MaxHeartbeats:           defaultRetries,
 		},
-		OriginConn:      origin,
-		EdgeMuxConfig:   MuxerConfig{
-			Timeout: time.Second,
-			IsClient: false,
-			Name: "edge",
-			Logger: log.NewEntry(log.New()),
+		OriginConn: origin,
+		EdgeMuxConfig: MuxerConfig{
+			Timeout:                 testHandshakeTimeout,
+			IsClient:                false,
+			Name:                    "edge",
+			Logger:                  log.NewEntry(log.New()),
+			DefaultWindowSize:       (1 << 8) - 1,
+			MaxWindowSize:           (1 << 15) - 1,
+			StreamWriteBufferMaxLen: 1024,
+			HeartbeatInterval:       defaultTimeout,
+			MaxHeartbeats:           defaultRetries,
 		},
-		EdgeConn:        edge,
-		doneC:           make(chan struct{}),
+		EdgeConn: edge,
+		doneC:    make(chan struct{}),
 	}
+	assert.NoError(t, p.Handshake(testName))
+	return p
 }
 
-func NewCompressedMuxerPair(quality CompressionSetting) *DefaultMuxerPair {
+func NewCompressedMuxerPair(t assert.TestingT, testName string, quality CompressionSetting, f MuxedStreamFunc) *DefaultMuxerPair {
 	origin, edge := net.Pipe()
-	return &DefaultMuxerPair{
+	p := &DefaultMuxerPair{
 		OriginMuxConfig: MuxerConfig{
-			Timeout: time.Second,
-			IsClient: true,
-			Name: "origin",
+			Timeout:            time.Second,
+			Handler:            f,
+			IsClient:           true,
+			Name:               "origin",
 			CompressionQuality: quality,
-			Logger: log.NewEntry(log.New()),
+			Logger:             log.NewEntry(log.New()),
+			HeartbeatInterval:  defaultTimeout,
+			MaxHeartbeats:      defaultRetries,
 		},
-		OriginConn:      origin,
-		EdgeMuxConfig:   MuxerConfig{
-			Timeout: time.Second,
-			IsClient: false,
-			Name: "edge",
+		OriginConn: origin,
+		EdgeMuxConfig: MuxerConfig{
+			Timeout:            time.Second,
+			IsClient:           false,
+			Name:               "edge",
 			CompressionQuality: quality,
-			Logger: log.NewEntry(log.New()),
+			Logger:             log.NewEntry(log.New()),
+			HeartbeatInterval:  defaultTimeout,
+			MaxHeartbeats:      defaultRetries,
 		},
-		EdgeConn:        edge,
-		doneC:           make(chan struct{}),
+		EdgeConn: edge,
+		doneC:    make(chan struct{}),
 	}
+	assert.NoError(t, p.Handshake(testName))
+	return p
 }
 
-func (p *DefaultMuxerPair) Handshake(t *testing.T) {
-	edgeErrC := make(chan error)
-	originErrC := make(chan error)
-	go func() {
-		var err error
-		p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig)
-		edgeErrC <- err
-	}()
-	go func() {
-		var err error
-		p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig)
-		originErrC <- err
-	}()
+func (p *DefaultMuxerPair) Handshake(testName string) error {
+	ctx, cancel := context.WithTimeout(context.Background(), testHandshakeTimeout)
+	defer cancel()
+	errGroup, _ := errgroup.WithContext(ctx)
+	errGroup.Go(func() (err error) {
+		p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig, NewActiveStreamsMetrics(testName, "edge"))
+		return errors.Wrap(err, "edge handshake failure")
+	})
+	errGroup.Go(func() (err error) {
+		p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig, NewActiveStreamsMetrics(testName, "origin"))
+		return errors.Wrap(err, "origin handshake failure")
+	})
 
-	select {
-	case err := <-edgeErrC:
-		if err != nil {
-			t.Fatalf("edge handshake failure: %s", err)
-		}
-	case <-time.After(time.Second * 5):
-		t.Fatalf("edge handshake timeout")
-	}
-
-	select {
-	case err := <-originErrC:
-		if err != nil {
-			t.Fatalf("origin handshake failure: %s", err)
-		}
-	case <-time.After(time.Second * 5):
-		t.Fatalf("origin handshake timeout")
-	}
+	return errGroup.Wait()
 }
 
-func (p *DefaultMuxerPair) HandshakeAndServe(t *testing.T) {
+func (p *DefaultMuxerPair) Serve(t assert.TestingT) {
 	ctx := context.Background()
-	p.Handshake(t)
 	var wg sync.WaitGroup
 	wg.Add(2)
 	go func() {
@@ -149,18 +159,23 @@ func (p *DefaultMuxerPair) Wait(t *testing.T) {
 	}
 }
 
+func (p *DefaultMuxerPair) OpenEdgeMuxStream(headers []Header, body io.Reader) (*MuxedStream, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), testOpenStreamTimeout)
+	defer cancel()
+	return p.EdgeMux.OpenStream(ctx, headers, body)
+}
+
 func TestHandshake(t *testing.T) {
-	muxPair := NewDefaultMuxerPair()
-	muxPair.Handshake(t)
+	f := func(stream *MuxedStream) error {
+		return nil
+	}
+	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
 	AssertIfPipeReadable(t, muxPair.OriginConn)
 	AssertIfPipeReadable(t, muxPair.EdgeConn)
 }
 
 func TestSingleStream(t *testing.T) {
-	closeC := make(chan struct{})
-	muxPair := NewDefaultMuxerPair()
-	muxPair.OriginMuxConfig.Handler = MuxedStreamFunc(func(stream *MuxedStream) error {
-		defer close(closeC)
+	f := MuxedStreamFunc(func(stream *MuxedStream) error {
 		if len(stream.Headers) != 1 {
 			t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
 		}
@@ -175,8 +190,6 @@ func TestSingleStream(t *testing.T) {
 		})
 		buf := []byte("Hello world")
 		stream.Write(buf)
-		// after this receive, the edge closed the stream
-		<-closeC
 		n, err := io.ReadFull(stream, buf)
 		if n > 0 {
 			t.Fatalf("read %d bytes after EOF", n)
@@ -186,9 +199,10 @@ func TestSingleStream(t *testing.T) {
 		}
 		return nil
 	})
-	muxPair.HandshakeAndServe(t)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
+	muxPair.Serve(t)
 
-	stream, err := muxPair.EdgeMux.OpenStream(
+	stream, err := muxPair.OpenEdgeMuxStream(
 		[]Header{{Name: "test-header", Value: "headerValue"}},
 		nil,
 	)
@@ -216,7 +230,6 @@ func TestSingleStream(t *testing.T) {
 		t.Fatalf("expected response body %s, got %s", "Hello world", responseBody)
 	}
 	stream.Close()
-	closeC <- struct{}{}
 	n, err = stream.Write([]byte("aaaaa"))
 	if n > 0 {
 		t.Fatalf("wrote %d bytes after EOF", n)
@@ -224,14 +237,11 @@ func TestSingleStream(t *testing.T) {
 	if err != io.EOF {
 		t.Fatalf("expected EOF, got %s", err)
 	}
-	<-closeC
 }
 
 func TestSingleStreamLargeResponseBody(t *testing.T) {
-	muxPair := NewDefaultMuxerPair()
 	bodySize := 1 << 24
-	streamReady := make(chan struct{})
-	muxPair.OriginMuxConfig.Handler = MuxedStreamFunc(func(stream *MuxedStream) error {
+	f := MuxedStreamFunc(func(stream *MuxedStream) error {
 		if len(stream.Headers) != 1 {
 			t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
 		}
@@ -257,14 +267,13 @@ func TestSingleStreamLargeResponseBody(t *testing.T) {
 		if n != len(payload) {
 			t.Fatalf("origin short write: %d/%d bytes", n, len(payload))
 		}
-		t.Log("Payload written; signaling that the stream is ready")
-		streamReady <- struct{}{}
 
 		return nil
 	})
-	muxPair.HandshakeAndServe(t)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
+	muxPair.Serve(t)
 
-	stream, err := muxPair.EdgeMux.OpenStream(
+	stream, err := muxPair.OpenEdgeMuxStream(
 		[]Header{{Name: "test-header", Value: "headerValue"}},
 		nil,
 	)
@@ -282,9 +291,6 @@ func TestSingleStreamLargeResponseBody(t *testing.T) {
 	}
 	responseBody := make([]byte, bodySize)
 
-	<-streamReady
-	t.Log("Received stream ready signal; resuming the test")
-
 	n, err := io.ReadFull(stream, responseBody)
 	if err != nil {
 		t.Fatalf("error from (*MuxedStream).Read: %s", err)
@@ -295,10 +301,7 @@ func TestSingleStreamLargeResponseBody(t *testing.T) {
 }
 
 func TestMultipleStreams(t *testing.T) {
-	muxPair := NewDefaultMuxerPair()
-	maxStreams := 64
-	errorsC := make(chan error, maxStreams)
-	muxPair.OriginMuxConfig.Handler = MuxedStreamFunc(func(stream *MuxedStream) error {
+	f := MuxedStreamFunc(func(stream *MuxedStream) error {
 		if len(stream.Headers) != 1 {
 			t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
 		}
@@ -314,15 +317,18 @@ func TestMultipleStreams(t *testing.T) {
 		log.Debugf("Wrote body for stream %s", stream.Headers[0].Value)
 		return nil
 	})
-	muxPair.HandshakeAndServe(t)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
+	muxPair.Serve(t)
 
+	maxStreams := 64
+	errorsC := make(chan error, maxStreams)
 	var wg sync.WaitGroup
 	wg.Add(maxStreams)
 	for i := 0; i < maxStreams; i++ {
 		go func(tokenId int) {
 			defer wg.Done()
 			tokenString := fmt.Sprintf("%d", tokenId)
-			stream, err := muxPair.EdgeMux.OpenStream(
+			stream, err := muxPair.OpenEdgeMuxStream(
 				[]Header{{Name: "client-token", Value: tokenString}},
 				nil,
 			)
@@ -367,20 +373,18 @@ func TestMultipleStreams(t *testing.T) {
 		log.Error(err)
 	}
 	if testFail {
-		t.Fatalf("TestMultipleStreamsFlowControl failed")
+		t.Fatalf("TestMultipleStreams failed")
 	}
 }
 
 func TestMultipleStreamsFlowControl(t *testing.T) {
 	maxStreams := 32
-	errorsC := make(chan error, maxStreams)
-	streamReady := make(chan struct{})
 	responseSizes := make([]int32, maxStreams)
 	for i := 0; i < maxStreams; i++ {
 		responseSizes[i] = rand.Int31n(int32(defaultWindowSize << 4))
 	}
-	muxPair := NewDefaultMuxerPair()
-	muxPair.OriginMuxConfig.Handler = MuxedStreamFunc(func(stream *MuxedStream) error {
+
+	f := MuxedStreamFunc(func(stream *MuxedStream) error {
 		if len(stream.Headers) != 1 {
 			t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
 		}
@@ -398,7 +402,6 @@ func TestMultipleStreamsFlowControl(t *testing.T) {
 			payload[i] = byte(i % 256)
 		}
 		n, err := stream.Write(payload)
-		streamReady <- struct{}{}
 		if err != nil {
 			t.Fatalf("origin write error: %s", err)
 		}
@@ -407,64 +410,48 @@ func TestMultipleStreamsFlowControl(t *testing.T) {
 		}
 		return nil
 	})
-	muxPair.HandshakeAndServe(t)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
+	muxPair.Serve(t)
 
-	var wg sync.WaitGroup
-	wg.Add(maxStreams)
+	errGroup, _ := errgroup.WithContext(context.Background())
 	for i := 0; i < maxStreams; i++ {
-		go func(tokenId int) {
-			defer wg.Done()
-			stream, err := muxPair.EdgeMux.OpenStream(
+		errGroup.Go(func() error {
+			stream, err := muxPair.OpenEdgeMuxStream(
 				[]Header{{Name: "test-header", Value: "headerValue"}},
 				nil,
 			)
 			if err != nil {
-				errorsC <- fmt.Errorf("stream %d error in OpenStream: %s", stream.streamID, err)
-				return
+				return fmt.Errorf("error in OpenStream: %d %s", stream.streamID, err)
 			}
 			if len(stream.Headers) != 1 {
-				errorsC <- fmt.Errorf("stream %d expected %d headers, got %d", stream.streamID, 1, len(stream.Headers))
-				return
+				return fmt.Errorf("stream %d expected %d headers, got %d", stream.streamID, 1, len(stream.Headers))
 			}
 			if stream.Headers[0].Name != "response-header" {
-				errorsC <- fmt.Errorf("stream %d expected header name %s, got %s", stream.streamID, "response-header", stream.Headers[0].Name)
-				return
+				return fmt.Errorf("stream %d expected header name %s, got %s", stream.streamID, "response-header", stream.Headers[0].Name)
 			}
 			if stream.Headers[0].Value != "responseValue" {
-				errorsC <- fmt.Errorf("stream %d expected header value %s, got %s", stream.streamID, "responseValue", stream.Headers[0].Value)
-				return
+				return fmt.Errorf("stream %d expected header value %s, got %s", stream.streamID, "responseValue", stream.Headers[0].Value)
 			}
 
-			<-streamReady
 			responseBody := make([]byte, responseSizes[(stream.streamID-2)/2])
 			n, err := io.ReadFull(stream, responseBody)
 			if err != nil {
-				errorsC <- fmt.Errorf("stream %d error from (*MuxedStream).Read: %s", stream.streamID, err)
-				return
+				return fmt.Errorf("stream %d error from (*MuxedStream).Read: %s", stream.streamID, err)
 			}
 			if n != len(responseBody) {
-				errorsC <- fmt.Errorf("stream %d expected response body to have %d bytes, got %d", stream.streamID, len(responseBody), n)
-				return
+				return fmt.Errorf("stream %d expected response body to have %d bytes, got %d", stream.streamID, len(responseBody), n)
 			}
-		}(i)
-	}
-	wg.Wait()
-	close(errorsC)
-	testFail := false
-	for err := range errorsC {
-		testFail = true
-		log.Error(err)
-	}
-	if testFail {
-		t.Fatalf("TestMultipleStreamsFlowControl failed")
+			return nil
+		})
 	}
+	assert.NoError(t, errGroup.Wait())
 }
 
 func TestGracefulShutdown(t *testing.T) {
 	sendC := make(chan struct{})
 	responseBuf := bytes.Repeat([]byte("Hello world"), 65536)
-	muxPair := NewDefaultMuxerPair()
-	muxPair.OriginMuxConfig.Handler = MuxedStreamFunc(func(stream *MuxedStream) error {
+
+	f := MuxedStreamFunc(func(stream *MuxedStream) error {
 		stream.WriteHeaders([]Header{
 			{Name: "response-header", Value: "responseValue"},
 		})
@@ -482,18 +469,19 @@ func TestGracefulShutdown(t *testing.T) {
 		log.Debugf("Handler ends")
 		return nil
 	})
-	muxPair.HandshakeAndServe(t)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
+	muxPair.Serve(t)
 
-	stream, err := muxPair.EdgeMux.OpenStream(
+	stream, err := muxPair.OpenEdgeMuxStream(
 		[]Header{{Name: "test-header", Value: "headerValue"}},
 		nil,
 	)
-	// Start graceful shutdown of the edge mux - this should also close the origin mux when done
-	muxPair.EdgeMux.Shutdown()
-	close(sendC)
 	if err != nil {
 		t.Fatalf("error in OpenStream: %s", err)
 	}
+	// Start graceful shutdown of the edge mux - this should also close the origin mux when done
+	muxPair.EdgeMux.Shutdown()
+	close(sendC)
 	responseBody := make([]byte, len(responseBuf))
 	log.Debugf("Waiting for %d bytes", len(responseBuf))
 	n, err := io.ReadFull(stream, responseBody)
@@ -514,8 +502,8 @@ func TestUnexpectedShutdown(t *testing.T) {
 	sendC := make(chan struct{})
 	handlerFinishC := make(chan struct{})
 	responseBuf := bytes.Repeat([]byte("Hello world"), 65536)
-	muxPair := NewDefaultMuxerPair()
-	muxPair.OriginMuxConfig.Handler = MuxedStreamFunc(func(stream *MuxedStream) error {
+
+	f := MuxedStreamFunc(func(stream *MuxedStream) error {
 		defer close(handlerFinishC)
 		stream.WriteHeaders([]Header{
 			{Name: "response-header", Value: "responseValue"},
@@ -536,9 +524,10 @@ func TestUnexpectedShutdown(t *testing.T) {
 		}
 		return nil
 	})
-	muxPair.HandshakeAndServe(t)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
+	muxPair.Serve(t)
 
-	stream, err := muxPair.EdgeMux.OpenStream(
+	stream, err := muxPair.OpenEdgeMuxStream(
 		[]Header{{Name: "test-header", Value: "headerValue"}},
 		nil,
 	)
@@ -583,9 +572,8 @@ func EchoHandler(stream *MuxedStream) error {
 
 func TestOpenAfterDisconnect(t *testing.T) {
 	for i := 0; i < 3; i++ {
-		muxPair := NewDefaultMuxerPair()
-		muxPair.OriginMuxConfig.Handler = MuxedStreamFunc(EchoHandler)
-		muxPair.HandshakeAndServe(t)
+		muxPair := NewDefaultMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), i), EchoHandler)
+		muxPair.Serve(t)
 
 		switch i {
 		case 0:
@@ -600,22 +588,21 @@ func TestOpenAfterDisconnect(t *testing.T) {
 			muxPair.EdgeConn.Close()
 		}
 
-		_, err := muxPair.EdgeMux.OpenStream(
+		_, err := muxPair.OpenEdgeMuxStream(
 			[]Header{{Name: "test-header", Value: "headerValue"}},
 			nil,
 		)
-		if err != ErrConnectionClosed {
-			t.Fatalf("unexpected error in OpenStream: %s", err)
+		if err != ErrStreamRequestConnectionClosed && err != ErrResponseHeadersConnectionClosed {
+			t.Fatalf("case %v: unexpected error in OpenStream: %v", i, err)
 		}
 	}
 }
 
 func TestHPACK(t *testing.T) {
-	muxPair := NewDefaultMuxerPair()
-	muxPair.OriginMuxConfig.Handler = MuxedStreamFunc(EchoHandler)
-	muxPair.HandshakeAndServe(t)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), EchoHandler)
+	muxPair.Serve(t)
 
-	stream, err := muxPair.EdgeMux.OpenStream(
+	stream, err := muxPair.OpenEdgeMuxStream(
 		[]Header{
 			{Name: ":method", Value: "RPC"},
 			{Name: ":scheme", Value: "capnp"},
@@ -629,7 +616,7 @@ func TestHPACK(t *testing.T) {
 	stream.Close()
 
 	for i := 0; i < 3; i++ {
-		stream, err := muxPair.EdgeMux.OpenStream(
+		stream, err := muxPair.OpenEdgeMuxStream(
 			[]Header{
 				{Name: ":method", Value: "GET"},
 				{Name: ":scheme", Value: "https"},
@@ -691,8 +678,6 @@ func AssertIfPipeReadable(t *testing.T, pipe io.ReadCloser) {
 func TestMultipleStreamsWithDictionaries(t *testing.T) {
 
 	for q := CompressionNone; q <= CompressionMax; q++ {
-		muxPair := NewCompressedMuxerPair(q)
-
 		htmlBody := `<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"` +
 			`"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">` +
 			`<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">` +
@@ -715,7 +700,7 @@ func TestMultipleStreamsWithDictionaries(t *testing.T) {
 			`</body>` +
 			`</html>`
 
-		muxPair.OriginMuxConfig.Handler = MuxedStreamFunc(func(stream *MuxedStream) error {
+		f := MuxedStreamFunc(func(stream *MuxedStream) error {
 			var contentType string
 			var pathHeader Header
 
@@ -747,8 +732,8 @@ func TestMultipleStreamsWithDictionaries(t *testing.T) {
 
 			return nil
 		})
-
-		muxPair.HandshakeAndServe(t)
+		muxPair := NewCompressedMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), q), q, f)
+		muxPair.Serve(t)
 
 		var wg sync.WaitGroup
 
@@ -782,88 +767,105 @@ func TestMultipleStreamsWithDictionaries(t *testing.T) {
 		}
 
 		wg.Add(len(paths))
+		errorsC := make(chan error, len(paths))
 
 		for i, s := range paths {
-			go func(i int, path string) {
-				stream, err := muxPair.EdgeMux.OpenStream(
+			go func(index int, path string) {
+				defer wg.Done()
+				stream, err := muxPair.OpenEdgeMuxStream(
 					[]Header{
 						{Name: ":method", Value: "GET"},
 						{Name: ":scheme", Value: "https"},
 						{Name: ":authority", Value: "tunnel.otterlyadorable.co.uk"},
 						{Name: ":path", Value: path},
 						{Name: "cf-ray", Value: "378948953f044408-SFO-DOG"},
-						{Name: "idx", Value: strconv.Itoa(i)},
+						{Name: "idx", Value: strconv.Itoa(index)},
 						{Name: "accept-encoding", Value: "gzip, br"},
 					},
 					nil,
 				)
 				if err != nil {
-					t.Fatalf("error in OpenStream: %s", err)
+					errorsC <- fmt.Errorf("error in OpenStream: %v", err)
+					return
 				}
 
-				expectBody := strings.Replace(htmlBody, "paragraph", path, 1) + strconv.Itoa(i)
+				expectBody := strings.Replace(htmlBody, "paragraph", path, 1) + strconv.Itoa(index)
 				responseBody := make([]byte, len(expectBody)*2)
 				n, err := stream.Read(responseBody)
 				if err != nil {
-					log.Printf("error from (*MuxedStream).Read: %s", err)
-					t.Fatalf("error from (*MuxedStream).Read: %s", err)
+					errorsC <- fmt.Errorf("stream %d error from (*MuxedStream).Read: %s", stream.streamID, err)
+					return
 				}
 				if n != len(expectBody) {
-					log.Printf("expected response body to have %d bytes, got %d", len(expectBody), n)
-					t.Fatalf("expected response body to have %d bytes, got %d", len(expectBody), n)
+					errorsC <- fmt.Errorf("stream %d expected response body to have %d bytes, got %d", stream.streamID, len(expectBody), n)
+					return
 				}
 				if string(responseBody[:n]) != expectBody {
-					log.Printf("expected response body %s, got %s", expectBody, responseBody[:n])
-					t.Fatalf("expected response body %s, got %s", expectBody, responseBody[:n])
+					errorsC <- fmt.Errorf("stream %d expected response body %s, got %s", stream.streamID, expectBody, responseBody[:n])
+					return
 				}
-				wg.Done()
 			}(i, s)
-			time.Sleep(1 * time.Millisecond)
 		}
-		wg.Wait()
 
-		if q > CompressionNone && muxPair.OriginMux.muxMetricsUpdater.compBytesBefore.Value() <= 10*muxPair.OriginMux.muxMetricsUpdater.compBytesAfter.Value() {
+		wg.Wait()
+		close(errorsC)
+		testFail := false
+		for err := range errorsC {
+			testFail = true
+			log.Error(err)
+		}
+		if testFail {
+			t.Fatalf("TestMultipleStreams failed")
+		}
+
+		originMuxMetrics := muxPair.OriginMux.Metrics()
+		if q > CompressionNone && originMuxMetrics.CompBytesBefore.Value() <= 10*originMuxMetrics.CompBytesAfter.Value() {
 			t.Fatalf("Cross-stream compression is expected to give a better compression ratio")
 		}
 	}
 }
 
-func sampleSiteHandler(stream *MuxedStream) error {
-	var contentType string
-	var pathHeader Header
+func sampleSiteHandler(files map[string][]byte) MuxedStreamFunc {
+	return func(stream *MuxedStream) error {
+		var contentType string
+		var pathHeader Header
 
-	for _, h := range stream.Headers {
-		if h.Name == ":path" {
-			pathHeader = h
-			break
+		for _, h := range stream.Headers {
+			if h.Name == ":path" {
+				pathHeader = h
+				break
+			}
 		}
-	}
 
-	if pathHeader.Name != ":path" {
-		panic("Couldn't find :path header in test")
-	}
+		if pathHeader.Name != ":path" {
+			return fmt.Errorf("Couldn't find :path header in test")
+		}
 
-	if strings.Contains(pathHeader.Value, "html") {
-		contentType = "text/html; charset=utf-8"
-	} else if strings.Contains(pathHeader.Value, "js") {
-		contentType = "application/javascript"
-	} else if strings.Contains(pathHeader.Value, "css") {
-		contentType = "text/css"
-	} else {
-		contentType = "img/gif"
+		if strings.Contains(pathHeader.Value, "html") {
+			contentType = "text/html; charset=utf-8"
+		} else if strings.Contains(pathHeader.Value, "js") {
+			contentType = "application/javascript"
+		} else if strings.Contains(pathHeader.Value, "css") {
+			contentType = "text/css"
+		} else {
+			contentType = "img/gif"
+		}
+		stream.WriteHeaders([]Header{
+			Header{Name: "content-type", Value: contentType},
+		})
+		log.Debugf("Wrote headers for stream %s", pathHeader.Value)
+		file, ok := files[pathHeader.Value]
+		if !ok {
+			return fmt.Errorf("%s content is not preloaded", pathHeader.Value)
+		}
+		stream.Write(file)
+		log.Debugf("Wrote body for stream %s", pathHeader.Value)
+		return nil
 	}
-	stream.WriteHeaders([]Header{
-		Header{Name: "content-type", Value: contentType},
-	})
-	log.Debugf("Wrote headers for stream %s", pathHeader.Value)
-	b, _ := ioutil.ReadFile("./sample" + pathHeader.Value)
-	stream.Write(b)
-	log.Debugf("Wrote body for stream %s", pathHeader.Value)
-	return nil
 }
 
-func sampleSiteTest(t *testing.T, muxPair *DefaultMuxerPair, path string) {
-	stream, err := muxPair.EdgeMux.OpenStream(
+func sampleSiteTest(muxPair *DefaultMuxerPair, path string, files map[string][]byte) error {
+	stream, err := muxPair.OpenEdgeMuxStream(
 		[]Header{
 			{Name: ":method", Value: "GET"},
 			{Name: ":scheme", Value: "https"},
@@ -875,86 +877,152 @@ func sampleSiteTest(t *testing.T, muxPair *DefaultMuxerPair, path string) {
 		nil,
 	)
 	if err != nil {
-		t.Fatalf("error in OpenStream: %s", err)
+		return fmt.Errorf("error in OpenStream: %v", err)
 	}
-	expectBody, _ := ioutil.ReadFile("./sample" + path)
-	responseBody := make([]byte, len(expectBody))
+	file, ok := files[path]
+	if !ok {
+		return fmt.Errorf("%s content is not preloaded", path)
+	}
+	responseBody := make([]byte, len(file))
 	n, err := io.ReadFull(stream, responseBody)
-	log.Debugf("Got body for stream %s", path)
 	if err != nil {
-		t.Fatalf("error from (*MuxedStream).Read: %s", err)
+		return fmt.Errorf("error from (*MuxedStream).Read: %v", err)
 	}
-	if n != len(expectBody) {
-		t.Fatalf("expected response body to have %d bytes, got %d", len(expectBody), n)
+	if n != len(file) {
+		return fmt.Errorf("expected response body to have %d bytes, got %d", len(file), n)
 	}
-	if string(responseBody[:n]) != string(expectBody) {
-		t.Fatalf("expected response body %s, got %s", expectBody, responseBody[:n])
+	if string(responseBody[:n]) != string(file) {
+		return fmt.Errorf("expected response body %s, got %s", file, responseBody[:n])
 	}
+	return nil
+}
+
+func loadSampleFiles(paths []string) (map[string][]byte, error) {
+	files := make(map[string][]byte)
+	for _, path := range paths {
+		if _, ok := files[path]; !ok {
+			expectBody, err := ioutil.ReadFile(path)
+			if err != nil {
+				return nil, err
+			}
+			files[path] = expectBody
+		}
+	}
+	return files, nil
 }
 
 func TestSampleSiteWithDictionaries(t *testing.T) {
+	paths := []string{
+		"./sample/index.html",
+		"./sample/index2.html",
+		"./sample/index1.html",
+		"./sample/ghost-url.min.js",
+		"./sample/jquery.fitvids.js",
+		"./sample/index1.html",
+		"./sample/index2.html",
+		"./sample/index.html",
+	}
+	files, err := loadSampleFiles(paths)
+	assert.NoError(t, err)
+
 	for q := CompressionNone; q <= CompressionMax; q++ {
-		muxPair := NewCompressedMuxerPair(q)
-		muxPair.OriginMuxConfig.Handler = MuxedStreamFunc(sampleSiteHandler)
-		muxPair.HandshakeAndServe(t)
+		muxPair := NewCompressedMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), q), q, sampleSiteHandler(files))
+		muxPair.Serve(t)
 
 		var wg sync.WaitGroup
-
-		paths := []string{
-			"/index.html",
-			"/index2.html",
-			"/index1.html",
-			"/ghost-url.min.js",
-			"/jquery.fitvids.js",
-			"/index1.html",
-			"/index2.html",
-			"/index.html",
-		}
+		errC := make(chan error, len(paths))
 
 		wg.Add(len(paths))
 		for _, s := range paths {
 			go func(path string) {
-				sampleSiteTest(t, muxPair, path)
-				wg.Done()
+				defer wg.Done()
+				errC <- sampleSiteTest(muxPair, path, files)
 			}(s)
 		}
-		wg.Wait()
 
-		if q > CompressionNone && muxPair.OriginMux.muxMetricsUpdater.compBytesBefore.Value() <= 10*muxPair.OriginMux.muxMetricsUpdater.compBytesAfter.Value() {
+		wg.Wait()
+		close(errC)
+
+		for err := range errC {
+			assert.NoError(t, err)
+		}
+
+		originMuxMetrics := muxPair.OriginMux.Metrics()
+		if q > CompressionNone && originMuxMetrics.CompBytesBefore.Value() <= 10*originMuxMetrics.CompBytesAfter.Value() {
 			t.Fatalf("Cross-stream compression is expected to give a better compression ratio")
 		}
 	}
 }
 
 func TestLongSiteWithDictionaries(t *testing.T) {
+	paths := []string{
+		"./sample/index.html",
+		"./sample/index1.html",
+		"./sample/index2.html",
+		"./sample/ghost-url.min.js",
+		"./sample/jquery.fitvids.js",
+	}
+	files, err := loadSampleFiles(paths)
+	assert.NoError(t, err)
 	for q := CompressionNone; q <= CompressionMedium; q++ {
-		muxPair := NewCompressedMuxerPair(q)
-		muxPair.OriginMuxConfig.Handler = MuxedStreamFunc(sampleSiteHandler)
-		muxPair.HandshakeAndServe(t)
+		muxPair := NewCompressedMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), q), q, sampleSiteHandler(files))
+		muxPair.Serve(t)
 
-		var wg sync.WaitGroup
 		rand.Seed(time.Now().Unix())
 
-		paths := []string{
-			"/index.html",
-			"/index1.html",
-			"/index2.html",
-			"/ghost-url.min.js",
-			"/jquery.fitvids.js"}
-
-		tstLen := 1000
-		wg.Add(tstLen)
+		tstLen := 500
+		errGroup, _ := errgroup.WithContext(context.Background())
 		for i := 0; i < tstLen; i++ {
-			path := paths[rand.Int()%len(paths)]
-			go func(path string) {
-				sampleSiteTest(t, muxPair, path)
-				wg.Done()
-			}(path)
+			errGroup.Go(func() error {
+				path := paths[rand.Int()%len(paths)]
+				return sampleSiteTest(muxPair, path, files)
+			})
 		}
-		wg.Wait()
+		assert.NoError(t, errGroup.Wait())
 
-		if q > CompressionNone && muxPair.OriginMux.muxMetricsUpdater.compBytesBefore.Value() <= 100*muxPair.OriginMux.muxMetricsUpdater.compBytesAfter.Value() {
+		originMuxMetrics := muxPair.OriginMux.Metrics()
+		if q > CompressionNone && originMuxMetrics.CompBytesBefore.Value() <= 10*originMuxMetrics.CompBytesAfter.Value() {
 			t.Fatalf("Cross-stream compression is expected to give a better compression ratio")
 		}
 	}
 }
+
+func BenchmarkOpenStream(b *testing.B) {
+	const streams = 5000
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		f := MuxedStreamFunc(func(stream *MuxedStream) error {
+			if len(stream.Headers) != 1 {
+				b.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
+			}
+			if stream.Headers[0].Name != "test-header" {
+				b.Fatalf("expected header name %s, got %s", "test-header", stream.Headers[0].Name)
+			}
+			if stream.Headers[0].Value != "headerValue" {
+				b.Fatalf("expected header value %s, got %s", "headerValue", stream.Headers[0].Value)
+			}
+			stream.WriteHeaders([]Header{
+				{Name: "response-header", Value: "responseValue"},
+			})
+			return nil
+		})
+		muxPair := NewDefaultMuxerPair(b, fmt.Sprintf("%s_%d", b.Name(), i), f)
+		muxPair.Serve(b)
+		b.StartTimer()
+		openStreams(b, muxPair, streams)
+	}
+}
+
+func openStreams(b *testing.B, muxPair *DefaultMuxerPair, n int) {
+	errGroup, _ := errgroup.WithContext(context.Background())
+	for i := 0; i < n; i++ {
+		errGroup.Go(func() error {
+			_, err := muxPair.OpenEdgeMuxStream(
+				[]Header{{Name: "test-header", Value: "headerValue"}},
+				nil,
+			)
+			return err
+		})
+	}
+	assert.NoError(b, errGroup.Wait())
+}

h2mux/muxedstream.go

@@ -17,73 +17,150 @@ type ReadWriteClosedCloser interface {
 	Closed() bool
 }
 
-type MuxedStream struct {
-	Headers []Header
+// MuxedStreamDataSignaller is a write-only *ReadyList
+type MuxedStreamDataSignaller interface {
+	// Non-blocking: call this when data is ready to be sent for the given stream ID.
+	Signal(ID uint32)
+}
 
+// MuxedStream is logically an HTTP/2 stream, with an additional buffer for outgoing data.
+type MuxedStream struct {
 	streamID uint32
 
+	// The "Receive" end of the stream
+	readBufferLock sync.RWMutex
+	readBuffer     ReadWriteClosedCloser
+	// This is the amount of bytes that are in our receive window
+	// (how much data we can receive into this stream).
+	receiveWindow uint32
+	// current receive window size limit. Exponentially increase it when it's exhausted
+	receiveWindowCurrentMax uint32
+	// hard limit set in http2 spec. 2^31-1
+	receiveWindowMax uint32
+	// The desired size increment for receiveWindow.
+	// If this is nonzero, a WINDOW_UPDATE frame needs to be sent.
+	windowUpdate uint32
+	// The headers that were most recently received.
+	// Particularly:
+	//     * for an eyeball-initiated stream (as passed to TunnelHandler::ServeStream),
+	//       these are the request headers
+	//     * for a cloudflared-initiated stream (as created by Register/UnregisterTunnel),
+	//       these are the response headers.
+	// They are useful in both of these contexts; hence `Headers` is public.
+	Headers []Header
+	// For use in the context of a cloudflared-initiated stream.
 	responseHeadersReceived chan struct{}
 
-	readBuffer    ReadWriteClosedCloser
-	receiveWindow uint32
-	// current window size limit. Exponentially increase it when it's exhausted
-	receiveWindowCurrentMax uint32
-	// limit set in http2 spec. 2^31-1
-	receiveWindowMax uint32
-
-	// nonzero if a WINDOW_UPDATE frame for a stream needs to be sent
-	windowUpdate uint32
-
-	writeLock sync.Mutex
-	// The zero value for Buffer is an empty buffer ready to use.
+	// The "Send" end of the stream
+	writeLock   sync.Mutex
 	writeBuffer ReadWriteLengther
-
+	// The maximum capacity that the send buffer should grow to.
+	writeBufferMaxLen int
+	// A channel to be notified when the send buffer is not full.
+	writeBufferHasSpace chan struct{}
+	// This is the amount of bytes that are in the peer's receive window
+	// (how much data we can send from this stream).
 	sendWindow uint32
-
-	readyList    *ReadyList
+	// The muxer's readyList
+	readyList MuxedStreamDataSignaller
+	// The headers that should be sent, and a flag so we only send them once.
 	headersSent  bool
 	writeHeaders []Header
+
+	// EOF-related fields
 	// true if the write end of this stream has been closed
 	writeEOF bool
 	// true if we have sent EOF to the peer
 	sentEOF bool
 	// true if the peer sent us an EOF
 	receivedEOF bool
-
-	// dictionary that was used to compress the stream
+	// If valid, tunnelHostname is used to identify which origin service is the intended recipient of the request
+	tunnelHostname TunnelHostname
+	// Compression-related fields
 	receivedUseDict bool
 	method          string
 	contentType     string
 	path            string
 	dictionaries    h2Dictionaries
-	readBufferLock  sync.RWMutex
+}
+
+type TunnelHostname string
+
+func (th TunnelHostname) String() string {
+	return string(th)
+}
+
+func (th TunnelHostname) IsSet() bool {
+	return th != ""
+}
+
+func NewStream(config MuxerConfig, writeHeaders []Header, readyList MuxedStreamDataSignaller, dictionaries h2Dictionaries) *MuxedStream {
+	return &MuxedStream{
+		responseHeadersReceived: make(chan struct{}),
+		readBuffer:              NewSharedBuffer(),
+		writeBuffer:             &bytes.Buffer{},
+		writeBufferMaxLen:       config.StreamWriteBufferMaxLen,
+		writeBufferHasSpace:     make(chan struct{}, 1),
+		receiveWindow:           config.DefaultWindowSize,
+		receiveWindowCurrentMax: config.DefaultWindowSize,
+		receiveWindowMax:        config.MaxWindowSize,
+		sendWindow:              config.DefaultWindowSize,
+		readyList:               readyList,
+		writeHeaders:            writeHeaders,
+		dictionaries:            dictionaries,
+	}
 }
 
 func (s *MuxedStream) Read(p []byte) (n int, err error) {
+	var readBuffer ReadWriteClosedCloser
 	if s.dictionaries.read != nil {
 		s.readBufferLock.RLock()
-		b := s.readBuffer
+		readBuffer = s.readBuffer
 		s.readBufferLock.RUnlock()
-		return b.Read(p)
+	} else {
+		readBuffer = s.readBuffer
 	}
-	return s.readBuffer.Read(p)
+	n, err = readBuffer.Read(p)
+	s.replenishReceiveWindow(uint32(n))
+	return
 }
 
-func (s *MuxedStream) Write(p []byte) (n int, err error) {
+// Blocks until len(p) bytes have been written to the buffer
+func (s *MuxedStream) Write(p []byte) (int, error) {
+	// If assignDictToStream returns success, then it will have acquired the
+	// writeLock. Otherwise we must acquire it ourselves.
 	ok := assignDictToStream(s, p)
 	if !ok {
 		s.writeLock.Lock()
 	}
 	defer s.writeLock.Unlock()
+
 	if s.writeEOF {
 		return 0, io.EOF
 	}
-	n, err = s.writeBuffer.Write(p)
-	if n != len(p) || err != nil {
-		return n, err
+	totalWritten := 0
+	for totalWritten < len(p) {
+		// If the buffer is full, block till there is more room.
+		// Use a loop to recheck the buffer size after the lock is reacquired.
+		for s.writeBufferMaxLen <= s.writeBuffer.Len() {
+			s.awaitWriteBufferHasSpace()
+			if s.writeEOF {
+				return totalWritten, io.EOF
+			}
+		}
+		amountToWrite := len(p) - totalWritten
+		spaceAvailable := s.writeBufferMaxLen - s.writeBuffer.Len()
+		if spaceAvailable < amountToWrite {
+			amountToWrite = spaceAvailable
+		}
+		amountWritten, err := s.writeBuffer.Write(p[totalWritten : totalWritten+amountToWrite])
+		totalWritten += amountWritten
+		if err != nil {
+			return totalWritten, err
+		}
+		s.writeNotify()
 	}
-	s.writeNotify()
-	return n, nil
+	return totalWritten, nil
 }
 
 func (s *MuxedStream) Close() error {
@@ -118,10 +195,19 @@ func (s *MuxedStream) CloseWrite() error {
 	if c, ok := s.writeBuffer.(io.Closer); ok {
 		c.Close()
 	}
+	// Allow MuxedStream::Write() to terminate its loop with err=io.EOF, if needed
+	s.notifyWriteBufferHasSpace()
+	// We need to send something over the wire, even if it's an END_STREAM with no data
 	s.writeNotify()
 	return nil
 }
 
+func (s *MuxedStream) WriteClosed() bool {
+	s.writeLock.Lock()
+	defer s.writeLock.Unlock()
+	return s.writeEOF
+}
+
 func (s *MuxedStream) WriteHeaders(headers []Header) error {
 	s.writeLock.Lock()
 	defer s.writeLock.Unlock()
@@ -143,6 +229,42 @@ func (s *MuxedStream) WriteHeaders(headers []Header) error {
 	return nil
 }
 
+// IsRPCStream returns if the stream is used to transport RPC.
+func (s *MuxedStream) IsRPCStream() bool {
+	rpcHeaders := RPCHeaders()
+	if len(s.Headers) != len(rpcHeaders) {
+		return false
+	}
+	// The headers order matters, so RPC stream should be opened with OpenRPCStream method and let MuxWriter serializes the headers.
+	for i, rpcHeader := range rpcHeaders {
+		if s.Headers[i] != rpcHeader {
+			return false
+		}
+	}
+	return true
+}
+
+func (s *MuxedStream) TunnelHostname() TunnelHostname {
+	return s.tunnelHostname
+}
+
+// Block until a value is sent on writeBufferHasSpace.
+// Must be called while holding writeLock
+func (s *MuxedStream) awaitWriteBufferHasSpace() {
+	s.writeLock.Unlock()
+	<-s.writeBufferHasSpace
+	s.writeLock.Lock()
+}
+
+// Send a value on writeBufferHasSpace without blocking.
+// Must be called while holding writeLock
+func (s *MuxedStream) notifyWriteBufferHasSpace() {
+	select {
+	case s.writeBufferHasSpace <- struct{}{}:
+	default:
+	}
+}
+
 func (s *MuxedStream) getReceiveWindow() uint32 {
 	s.writeLock.Lock()
 	defer s.writeLock.Unlock()
@@ -164,9 +286,9 @@ func (s *MuxedStream) writeNotify() {
 // receive window (how much data we can send).
 func (s *MuxedStream) replenishSendWindow(bytes uint32) {
 	s.writeLock.Lock()
+	defer s.writeLock.Unlock()
 	s.sendWindow += bytes
 	s.writeNotify()
-	s.writeLock.Unlock()
 }
 
 // Call by muxreader when it receives a data frame
@@ -178,17 +300,30 @@ func (s *MuxedStream) consumeReceiveWindow(bytes uint32) bool {
 		return false
 	}
 	s.receiveWindow -= bytes
-	if s.receiveWindow < s.receiveWindowCurrentMax/2 {
+	if s.receiveWindow < s.receiveWindowCurrentMax/2 && s.receiveWindowCurrentMax < s.receiveWindowMax {
 		// exhausting client send window (how much data client can send)
-		if s.receiveWindowCurrentMax < s.receiveWindowMax {
-			s.receiveWindowCurrentMax <<= 1
+		// and there is room to grow the receive window
+		newMax := s.receiveWindowCurrentMax << 1
+		if newMax > s.receiveWindowMax {
+			newMax = s.receiveWindowMax
 		}
-		s.windowUpdate += s.receiveWindowCurrentMax - s.receiveWindow
+		s.windowUpdate += newMax - s.receiveWindowCurrentMax
+		s.receiveWindowCurrentMax = newMax
+		// notify MuxWriter to write WINDOW_UPDATE frame
 		s.writeNotify()
 	}
 	return true
 }
 
+// Arranges for the MuxWriter to send a WINDOW_UPDATE
+// Called by MuxedStream::Read when data has left the read buffer.
+func (s *MuxedStream) replenishReceiveWindow(bytes uint32) {
+	s.writeLock.Lock()
+	defer s.writeLock.Unlock()
+	s.windowUpdate += bytes
+	s.writeNotify()
+}
+
 // receiveEOF should be called when the peer indicates no more data will be sent.
 // Returns true if the socket is now closed (i.e. the write side is already closed).
 func (s *MuxedStream) receiveEOF() (closed bool) {
@@ -226,7 +361,8 @@ type streamChunk struct {
 	// true if a HEADERS frame should be sent
 	sendHeaders bool
 	headers     []Header
-	// nonzero if a WINDOW_UPDATE frame should be sent
+	// nonzero if a WINDOW_UPDATE frame should be sent;
+	// in that case, it is the increment value to use
 	windowUpdate uint32
 	// true if data frames should be sent
 	sendData bool
@@ -248,12 +384,20 @@ func (s *MuxedStream) getChunk() *streamChunk {
 		sendData:     !s.sentEOF,
 		eof:          s.writeEOF && uint32(s.writeBuffer.Len()) <= s.sendWindow,
 	}
-
-	// Copies at most s.sendWindow bytes
+	// Copy at most s.sendWindow bytes, adjust the sendWindow accordingly
 	writeLen, _ := io.CopyN(&chunk.buffer, s.writeBuffer, int64(s.sendWindow))
 	s.sendWindow -= uint32(writeLen)
+
+	// Allow MuxedStream::Write() to continue, if needed
+	if s.writeBuffer.Len() < s.writeBufferMaxLen {
+		s.notifyWriteBufferHasSpace()
+	}
+
+	// When we write the chunk, we'll write the WINDOW_UPDATE frame if needed
 	s.receiveWindow += s.windowUpdate
 	s.windowUpdate = 0
+
+	// When we write the chunk, we'll write the headers if needed
 	s.headersSent = true
 
 	// if this chunk contains the end of the stream, close the stream now

h2mux/muxedstream_test.go

@@ -23,47 +23,55 @@ func TestFlowControlSingleStream(t *testing.T) {
 		sendWindow:              testWindowSize,
 		readyList:               NewReadyList(),
 	}
+	var tempWindowUpdate uint32
+	var tempStreamChunk *streamChunk
+
 	assert.True(t, stream.consumeReceiveWindow(testWindowSize/2))
 	dataSent := testWindowSize / 2
 	assert.Equal(t, testWindowSize-dataSent, stream.receiveWindow)
 	assert.Equal(t, testWindowSize, stream.receiveWindowCurrentMax)
-	assert.Equal(t, uint32(0), stream.windowUpdate)
-	tempWindowUpdate := stream.windowUpdate
-
-	streamChunk := stream.getChunk()
-	assert.Equal(t, tempWindowUpdate, streamChunk.windowUpdate)
-	assert.Equal(t, testWindowSize-dataSent, stream.receiveWindow)
-	assert.Equal(t, uint32(0), stream.windowUpdate)
 	assert.Equal(t, testWindowSize, stream.sendWindow)
+	assert.Equal(t, uint32(0), stream.windowUpdate)
+
+	tempStreamChunk = stream.getChunk()
+	assert.Equal(t, uint32(0), tempStreamChunk.windowUpdate)
+	assert.Equal(t, testWindowSize-dataSent, stream.receiveWindow)
+	assert.Equal(t, testWindowSize, stream.receiveWindowCurrentMax)
+	assert.Equal(t, testWindowSize, stream.sendWindow)
+	assert.Equal(t, uint32(0), stream.windowUpdate)
 
 	assert.True(t, stream.consumeReceiveWindow(2))
 	dataSent += 2
 	assert.Equal(t, testWindowSize-dataSent, stream.receiveWindow)
 	assert.Equal(t, testWindowSize<<1, stream.receiveWindowCurrentMax)
-	assert.Equal(t, (testWindowSize<<1)-stream.receiveWindow, stream.windowUpdate)
+	assert.Equal(t, testWindowSize, stream.sendWindow)
+	assert.Equal(t, testWindowSize, stream.windowUpdate)
 	tempWindowUpdate = stream.windowUpdate
 
-	streamChunk = stream.getChunk()
-	assert.Equal(t, tempWindowUpdate, streamChunk.windowUpdate)
-	assert.Equal(t, testWindowSize<<1, stream.receiveWindow)
-	assert.Equal(t, uint32(0), stream.windowUpdate)
+	tempStreamChunk = stream.getChunk()
+	assert.Equal(t, tempWindowUpdate, tempStreamChunk.windowUpdate)
+	assert.Equal(t, (testWindowSize<<1)-dataSent, stream.receiveWindow)
+	assert.Equal(t, testWindowSize<<1, stream.receiveWindowCurrentMax)
 	assert.Equal(t, testWindowSize, stream.sendWindow)
+	assert.Equal(t, uint32(0), stream.windowUpdate)
 
 	assert.True(t, stream.consumeReceiveWindow(testWindowSize+10))
-	dataSent = testWindowSize + 10
+	dataSent += testWindowSize + 10
 	assert.Equal(t, (testWindowSize<<1)-dataSent, stream.receiveWindow)
 	assert.Equal(t, testWindowSize<<2, stream.receiveWindowCurrentMax)
-	assert.Equal(t, (testWindowSize<<2)-stream.receiveWindow, stream.windowUpdate)
+	assert.Equal(t, testWindowSize, stream.sendWindow)
+	assert.Equal(t, testWindowSize<<1, stream.windowUpdate)
 	tempWindowUpdate = stream.windowUpdate
 
-	streamChunk = stream.getChunk()
-	assert.Equal(t, tempWindowUpdate, streamChunk.windowUpdate)
-	assert.Equal(t, testWindowSize<<2, stream.receiveWindow)
-	assert.Equal(t, uint32(0), stream.windowUpdate)
+	tempStreamChunk = stream.getChunk()
+	assert.Equal(t, tempWindowUpdate, tempStreamChunk.windowUpdate)
+	assert.Equal(t, (testWindowSize<<2)-dataSent, stream.receiveWindow)
+	assert.Equal(t, testWindowSize<<2, stream.receiveWindowCurrentMax)
 	assert.Equal(t, testWindowSize, stream.sendWindow)
+	assert.Equal(t, uint32(0), stream.windowUpdate)
 
 	assert.False(t, stream.consumeReceiveWindow(testMaxWindowSize+1))
-	assert.Equal(t, testWindowSize<<2, stream.receiveWindow)
+	assert.Equal(t, (testWindowSize<<2)-dataSent, stream.receiveWindow)
 	assert.Equal(t, testMaxWindowSize, stream.receiveWindowCurrentMax)
 }
 
@@ -90,3 +98,30 @@ func TestMuxedStreamEOF(t *testing.T) {
 		assert.Equal(t, 0, n)
 	}
 }
+
+func TestIsRPCStream(t *testing.T) {
+	tests := []struct {
+		stream      *MuxedStream
+		isRPCStream bool
+	}{
+		{
+			stream:      &MuxedStream{},
+			isRPCStream: false,
+		},
+		{
+			stream:      &MuxedStream{Headers: RPCHeaders()},
+			isRPCStream: true,
+		},
+		{
+			stream: &MuxedStream{Headers: []Header{
+				{Name: ":method", Value: "rpc"},
+				{Name: ":scheme", Value: "Capnp"},
+				{Name: ":path", Value: "/"},
+			}},
+			isRPCStream: false,
+		},
+	}
+	for _, test := range tests {
+		assert.Equal(t, test.isRPCStream, test.stream.IsRPCStream())
+	}
+}

h2mux/muxmetrics.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/golang-collections/collections/queue"
+	"github.com/prometheus/client_golang/prometheus"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -17,7 +18,24 @@ const (
 	updateFreq = time.Second
 )
 
-type muxMetricsUpdater struct {
+type muxMetricsUpdater interface {
+	// metrics returns the latest metrics
+	metrics() *MuxerMetrics
+	// run is a blocking call to start the event loop
+	run(logger *log.Entry) error
+	// updateRTTChan is called by muxReader to report new RTT measurements
+	updateRTT(rtt *roundTripMeasurement)
+	//updateReceiveWindowChan is called by muxReader and muxWriter when receiveWindow size is updated
+	updateReceiveWindow(receiveWindow uint32)
+	//updateSendWindowChan is called by muxReader and muxWriter when sendWindow size is updated
+	updateSendWindow(sendWindow uint32)
+	// updateInBoundBytesChan is called  periodicallyby muxReader to report bytesRead
+	updateInBoundBytes(inBoundBytes uint64)
+	// updateOutBoundBytesChan is called periodically by muxWriter to report bytesWrote
+	updateOutBoundBytes(outBoundBytes uint64)
+}
+
+type muxMetricsUpdaterImpl struct {
 	// rttData keeps record of rtt, rttMin, rttMax and last measured time
 	rttData *rttData
 	// receiveWindowData keeps record of receive window measurement
@@ -28,16 +46,16 @@ type muxMetricsUpdater struct {
 	inBoundRate *rate
 	// outBoundRate is outgoing bytes/sec
 	outBoundRate *rate
-	// updateRTTChan is the channel to receive new RTT measurement from muxReader
-	updateRTTChan <-chan *roundTripMeasurement
-	//updateReceiveWindowChan is the channel to receive updated receiveWindow size from muxReader and muxWriter
-	updateReceiveWindowChan <-chan uint32
-	//updateSendWindowChan is the channel to receive updated sendWindow size from muxReader and muxWriter
-	updateSendWindowChan <-chan uint32
-	// updateInBoundBytesChan us the channel to receive bytesRead from muxReader
-	updateInBoundBytesChan <-chan uint64
-	// updateOutBoundBytesChan us the channel to receive bytesWrote from muxWriter
-	updateOutBoundBytesChan <-chan uint64
+	// updateRTTChan is the channel to receive new RTT measurement
+	updateRTTChan chan *roundTripMeasurement
+	//updateReceiveWindowChan is the channel to receive updated receiveWindow size
+	updateReceiveWindowChan chan uint32
+	//updateSendWindowChan is the channel to receive updated sendWindow size
+	updateSendWindowChan chan uint32
+	// updateInBoundBytesChan us the channel to receive bytesRead
+	updateInBoundBytesChan chan uint64
+	// updateOutBoundBytesChan us the channel to receive bytesWrote
+	updateOutBoundBytesChan chan uint64
 	// shutdownC is to signal the muxerMetricsUpdater to shutdown
 	abortChan <-chan struct{}
 
@@ -84,15 +102,16 @@ type rate struct {
 }
 
 func newMuxMetricsUpdater(
-	updateRTTChan <-chan *roundTripMeasurement,
-	updateReceiveWindowChan <-chan uint32,
-	updateSendWindowChan <-chan uint32,
-	updateInBoundBytesChan <-chan uint64,
-	updateOutBoundBytesChan <-chan uint64,
 	abortChan <-chan struct{},
 	compBytesBefore, compBytesAfter *AtomicCounter,
-) *muxMetricsUpdater {
-	return &muxMetricsUpdater{
+) muxMetricsUpdater {
+	updateRTTChan := make(chan *roundTripMeasurement, 1)
+	updateReceiveWindowChan := make(chan uint32, 1)
+	updateSendWindowChan := make(chan uint32, 1)
+	updateInBoundBytesChan := make(chan uint64)
+	updateOutBoundBytesChan := make(chan uint64)
+
+	return &muxMetricsUpdaterImpl{
 		rttData:                 newRTTData(),
 		receiveWindowData:       newFlowControlData(),
 		sendWindowData:          newFlowControlData(),
@@ -109,7 +128,7 @@ func newMuxMetricsUpdater(
 	}
 }
 
-func (updater *muxMetricsUpdater) Metrics() *MuxerMetrics {
+func (updater *muxMetricsUpdaterImpl) metrics() *MuxerMetrics {
 	m := &MuxerMetrics{}
 	m.RTT, m.RTTMin, m.RTTMax = updater.rttData.metrics()
 	m.ReceiveWindowAve, m.ReceiveWindowMin, m.ReceiveWindowMax = updater.receiveWindowData.metrics()
@@ -120,7 +139,7 @@ func (updater *muxMetricsUpdater) Metrics() *MuxerMetrics {
 	return m
 }
 
-func (updater *muxMetricsUpdater) run(parentLogger *log.Entry) error {
+func (updater *muxMetricsUpdaterImpl) run(parentLogger *log.Entry) error {
 	logger := parentLogger.WithFields(log.Fields{
 		"subsystem": "mux",
 		"dir":       "metrics",
@@ -152,6 +171,43 @@ func (updater *muxMetricsUpdater) run(parentLogger *log.Entry) error {
 	}
 }
 
+func (updater *muxMetricsUpdaterImpl) updateRTT(rtt *roundTripMeasurement) {
+	select {
+	case updater.updateRTTChan <- rtt:
+	case <-updater.abortChan:
+	}
+
+}
+
+func (updater *muxMetricsUpdaterImpl) updateReceiveWindow(receiveWindow uint32) {
+	select {
+	case updater.updateReceiveWindowChan <- receiveWindow:
+	case <-updater.abortChan:
+	}
+}
+
+func (updater *muxMetricsUpdaterImpl) updateSendWindow(sendWindow uint32) {
+	select {
+	case updater.updateSendWindowChan <- sendWindow:
+	case <-updater.abortChan:
+	}
+}
+
+func (updater *muxMetricsUpdaterImpl) updateInBoundBytes(inBoundBytes uint64) {
+	select {
+	case updater.updateInBoundBytesChan <- inBoundBytes:
+	case <-updater.abortChan:
+	}
+
+}
+
+func (updater *muxMetricsUpdaterImpl) updateOutBoundBytes(outBoundBytes uint64) {
+	select {
+	case updater.updateOutBoundBytesChan <- outBoundBytes:
+	case <-updater.abortChan:
+	}
+}
+
 func newRTTData() *rttData {
 	return &rttData{}
 }
@@ -244,3 +300,14 @@ func (r *rate) get() (curr, min, max uint64) {
 	defer r.lock.RUnlock()
 	return r.curr, r.min, r.max
 }
+
+func NewActiveStreamsMetrics(namespace, subsystem string) prometheus.Gauge {
+	activeStreams := prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      "active_streams",
+		Help:      "Number of active streams created by all muxers.",
+	})
+	prometheus.MustRegister(activeStreams)
+	return activeStreams
+}

h2mux/muxmetrics_test.go

@@ -86,24 +86,11 @@ func TestFlowControlDataUpdate(t *testing.T) {
 }
 
 func TestMuxMetricsUpdater(t *testing.T) {
-	t.Skip("Race condition")
-	updateRTTChan := make(chan *roundTripMeasurement)
-	updateReceiveWindowChan := make(chan uint32)
-	updateSendWindowChan := make(chan uint32)
-	updateInBoundBytesChan := make(chan uint64)
-	updateOutBoundBytesChan := make(chan uint64)
-	abortChan := make(chan struct{})
+	t.Skip("Inherently racy test due to muxMetricsUpdaterImpl.run()")
 	errChan := make(chan error)
+	abortChan := make(chan struct{})
 	compBefore, compAfter := NewAtomicCounter(0), NewAtomicCounter(0)
-	m := newMuxMetricsUpdater(updateRTTChan,
-		updateReceiveWindowChan,
-		updateSendWindowChan,
-		updateInBoundBytesChan,
-		updateOutBoundBytesChan,
-		abortChan,
-		compBefore,
-		compAfter,
-	)
+	m := newMuxMetricsUpdater(abortChan, compBefore, compAfter)
 	logger := log.NewEntry(log.New())
 
 	go func() {
@@ -116,42 +103,44 @@ func TestMuxMetricsUpdater(t *testing.T) {
 	// mock muxReader
 	readerStart := time.Now()
 	rm := &roundTripMeasurement{receiveTime: readerStart, sendTime: readerStart}
-	updateRTTChan <- rm
+	m.updateRTT(rm)
 	go func() {
 		defer wg.Done()
-		// Becareful if dataPoints is not divisibile by 4
+		assert.Equal(t, 0, dataPoints%4,
+			"dataPoints is not divisible by 4; this test should be adjusted accordingly")
 		readerSend := readerStart.Add(time.Millisecond)
 		for i := 1; i <= dataPoints/4; i++ {
 			readerReceive := readerSend.Add(time.Duration(i) * time.Millisecond)
 			rm := &roundTripMeasurement{receiveTime: readerReceive, sendTime: readerSend}
-			updateRTTChan <- rm
+			m.updateRTT(rm)
 			readerSend = readerReceive.Add(time.Millisecond)
+			m.updateReceiveWindow(uint32(i))
+			m.updateSendWindow(uint32(i))
 
-			updateReceiveWindowChan <- uint32(i)
-			updateSendWindowChan <- uint32(i)
-
-			updateInBoundBytesChan <- uint64(i)
+			m.updateInBoundBytes(uint64(i))
 		}
 	}()
 
 	// mock muxWriter
 	go func() {
 		defer wg.Done()
+		assert.Equal(t, 0, dataPoints%4,
+			"dataPoints is not divisible by 4; this test should be adjusted accordingly")
 		for j := dataPoints/4 + 1; j <= dataPoints/2; j++ {
-			updateReceiveWindowChan <- uint32(j)
-			updateSendWindowChan <- uint32(j)
+			m.updateReceiveWindow(uint32(j))
+			m.updateSendWindow(uint32(j))
 
-			// should always be disgard since the send time is before readerSend
+			// should always be disgarded since the send time is before readerSend
 			rm := &roundTripMeasurement{receiveTime: readerStart, sendTime: readerStart.Add(-time.Duration(j*dataPoints) * time.Millisecond)}
-			updateRTTChan <- rm
+			m.updateRTT(rm)
 
-			updateOutBoundBytesChan <- uint64(j)
+			m.updateOutBoundBytes(uint64(j))
 		}
 
 	}()
 	wg.Wait()
 
-	metrics := m.Metrics()
+	metrics := m.metrics()
 	points := dataPoints / 2
 	assert.Equal(t, time.Millisecond, metrics.RTTMin)
 	assert.Equal(t, time.Duration(dataPoints/4)*time.Millisecond, metrics.RTTMax)

h2mux/muxreader.go

@@ -11,6 +11,10 @@ import (
 	"golang.org/x/net/http2"
 )
 
+const (
+	CloudflaredProxyTunnelHostnameHeader = "cf-cloudflared-proxy-tunnel-hostname"
+)
+
 type MuxReader struct {
 	// f is used to read HTTP2 frames.
 	f *http2.Framer
@@ -35,26 +39,24 @@ type MuxReader struct {
 	initialStreamWindow uint32
 	// The max value for the send window of a stream.
 	streamWindowMax uint32
+	// The max size for the write buffer of a stream
+	streamWriteBufferMaxLen int
 	// r is a reference to the underlying connection used when shutting down.
 	r io.Closer
-	// updateRTTChan is the channel to send new RTT measurement to muxerMetricsUpdater
-	updateRTTChan chan<- *roundTripMeasurement
-	// updateReceiveWindowChan is the channel to update receiveWindow size to muxerMetricsUpdater
-	updateReceiveWindowChan chan<- uint32
-	// updateSendWindowChan is the channel to update sendWindow size to muxerMetricsUpdater
-	updateSendWindowChan chan<- uint32
-	// bytesRead is the amount of bytes read from data frame since the last time we send bytes read to metrics
+	// metricsUpdater is used to report metrics
+	metricsUpdater muxMetricsUpdater
+	// bytesRead is the amount of bytes read from data frames since the last time we called metricsUpdater.updateInBoundBytes()
 	bytesRead *AtomicCounter
-	// updateOutBoundBytesChan is the channel to send bytesWrote to muxerMetricsUpdater
-	updateInBoundBytesChan chan<- uint64
 	// dictionaries holds the h2 cross-stream compression dictionaries
 	dictionaries h2Dictionaries
 }
 
-func (r *MuxReader) Shutdown() {
-	done := r.streams.Shutdown()
-	if done == nil {
-		return
+// Shutdown blocks new streams from being created.
+// It returns a channel that is closed once the last stream has closed.
+func (r *MuxReader) Shutdown() <-chan struct{} {
+	done, alreadyInProgress := r.streams.Shutdown()
+	if alreadyInProgress {
+		return done
 	}
 	r.sendGoAway(http2.ErrCodeNo)
 	go func() {
@@ -62,6 +64,7 @@ func (r *MuxReader) Shutdown() {
 		<-done
 		r.r.Close()
 	}()
+	return done
 }
 
 func (r *MuxReader) run(parentLogger *log.Entry) error {
@@ -79,7 +82,7 @@ func (r *MuxReader) run(parentLogger *log.Entry) error {
 			case <-r.abortChan:
 				return
 			case <-tickC:
-				r.updateInBoundBytesChan <- r.bytesRead.Count()
+				r.metricsUpdater.updateInBoundBytes(r.bytesRead.Count())
 			}
 		}
 	}()
@@ -87,23 +90,33 @@ func (r *MuxReader) run(parentLogger *log.Entry) error {
 	for {
 		frame, err := r.f.ReadFrame()
 		if err != nil {
+			errLogger := logger.WithError(err)
+			if errorDetail := r.f.ErrorDetail(); errorDetail != nil {
+				errLogger = errLogger.WithField("errorDetail", errorDetail)
+			}
 			switch e := err.(type) {
 			case http2.StreamError:
-				logger.WithError(err).Warn("stream error")
-				r.streamError(e.StreamID, e.Code)
+				errLogger.Warn("stream error")
+				// Ideally we wouldn't return here, since that aborts the muxer.
+				// We should communicate the error to the relevant MuxedStream
+				// data structure, so that callers of MuxedStream.Read() and
+				// MuxedStream.Write() would see it. Then we could `continue`
+				// and keep the muxer going.
+				return r.streamError(e.StreamID, e.Code)
 			case http2.ConnectionError:
-				logger.WithError(err).Warn("connection error")
+				errLogger.Warn("connection error")
 				return r.connectionError(err)
 			default:
 				if isConnectionClosedError(err) {
 					if r.streams.Len() == 0 {
+						// don't log the error here -- that would just be extra noise
 						logger.Debug("shutting down")
 						return nil
 					}
-					logger.Warn("connection closed unexpectedly")
+					errLogger.Warn("connection closed unexpectedly")
 					return err
 				} else {
-					logger.WithError(err).Warn("frame read error")
+					errLogger.Warn("frame read error")
 					return r.connectionError(err)
 				}
 			}
@@ -120,6 +133,9 @@ func (r *MuxReader) run(parentLogger *log.Entry) error {
 			if streamID == 0 {
 				return ErrInvalidStream
 			}
+			if stream, ok := r.streams.Get(streamID); ok {
+				stream.Close()
+			}
 			r.streams.Delete(streamID)
 		case *http2.PingFrame:
 			r.receivePingData(f)
@@ -153,6 +169,8 @@ func (r *MuxReader) newMuxedStream(streamID uint32) *MuxedStream {
 		streamID:                streamID,
 		readBuffer:              NewSharedBuffer(),
 		writeBuffer:             &bytes.Buffer{},
+		writeBufferMaxLen:       r.streamWriteBufferMaxLen,
+		writeBufferHasSpace:     make(chan struct{}, 1),
 		receiveWindow:           r.initialStreamWindow,
 		receiveWindowCurrentMax: r.initialStreamWindow,
 		receiveWindowMax:        r.streamWindowMax,
@@ -237,6 +255,8 @@ func (r *MuxReader) receiveHeaderData(frame *http2.MetaHeadersFrame) error {
 			if r.dictionaries.write != nil {
 				continue
 			}
+		case CloudflaredProxyTunnelHostnameHeader:
+			stream.tunnelHostname = TunnelHostname(header.Value)
 		}
 		headers = append(headers, Header{Name: header.Name, Value: header.Value})
 	}
@@ -285,7 +305,7 @@ func (r *MuxReader) receiveFrameData(frame *http2.DataFrame, parentLogger *log.E
 	if !stream.consumeReceiveWindow(uint32(len(data))) {
 		return r.streamError(stream.streamID, http2.ErrCodeFlowControl)
 	}
-	r.updateReceiveWindowChan <- stream.getReceiveWindow()
+	r.metricsUpdater.updateReceiveWindow(stream.getReceiveWindow())
 	return nil
 }
 
@@ -297,13 +317,13 @@ func (r *MuxReader) receivePingData(frame *http2.PingFrame) {
 		return
 	}
 
-	// Update updates the computed values with a new measurement.
-	// outgoingTime is the time that the probe was sent.
-	// We assume that time.Now() is the time we received that probe.
-	r.updateRTTChan <- &roundTripMeasurement{
+	// Update the computed RTT aggregations with a new measurement.
+	// `ts` is the time that the probe was sent.
+	// We assume that `time.Now()` is the time we received that probe.
+	r.metricsUpdater.updateRTT(&roundTripMeasurement{
 		receiveTime: time.Now(),
 		sendTime:    time.Unix(0, ts),
-	}
+	})
 }
 
 // Receive a GOAWAY from the peer. Gracefully shut down our connection.
@@ -464,7 +484,7 @@ func (r *MuxReader) updateStreamWindow(frame *http2.WindowUpdateFrame) error {
 		return nil
 	}
 	stream.replenishSendWindow(frame.Increment)
-	r.updateSendWindowChan <- stream.getSendWindow()
+	r.metricsUpdater.updateSendWindow(stream.getSendWindow())
 	return nil
 }
 

h2mux/muxreader_test.go

@@ -0,0 +1,107 @@
+package h2mux
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	methodHeader = Header{
+		Name:  ":method",
+		Value: "GET",
+	}
+	schemeHeader = Header{
+		Name:  ":scheme",
+		Value: "https",
+	}
+	pathHeader = Header{
+		Name:  ":path",
+		Value: "/api/tunnels",
+	}
+	tunnelHostnameHeader = Header{
+		Name:  CloudflaredProxyTunnelHostnameHeader,
+		Value: "tunnel.example.com",
+	}
+	respStatusHeader = Header{
+		Name:  ":status",
+		Value: "200",
+	}
+)
+
+type mockOriginStreamHandler struct {
+	stream *MuxedStream
+}
+
+func (mosh *mockOriginStreamHandler) ServeStream(stream *MuxedStream) error {
+	mosh.stream = stream
+	// Echo tunnel hostname in header
+	stream.WriteHeaders([]Header{respStatusHeader})
+	return nil
+}
+
+func getCloudflaredProxyTunnelHostnameHeader(stream *MuxedStream) string {
+	for _, header := range stream.Headers {
+		if header.Name == CloudflaredProxyTunnelHostnameHeader {
+			return header.Value
+		}
+	}
+	return ""
+}
+
+func assertOpenStreamSucceed(t *testing.T, stream *MuxedStream, err error) {
+	assert.NoError(t, err)
+	assert.Len(t, stream.Headers, 1)
+	assert.Equal(t, respStatusHeader, stream.Headers[0])
+}
+
+func TestMissingHeaders(t *testing.T) {
+	originHandler := &mockOriginStreamHandler{}
+	muxPair := NewDefaultMuxerPair(t, t.Name(), originHandler.ServeStream)
+	muxPair.Serve(t)
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	reqHeaders := []Header{
+		{
+			Name:  "content-type",
+			Value: "application/json",
+		},
+	}
+
+	// Request doesn't contain CloudflaredProxyTunnelHostnameHeader
+	stream, err := muxPair.EdgeMux.OpenStream(ctx, reqHeaders, nil)
+	assertOpenStreamSucceed(t, stream, err)
+
+	assert.Empty(t, originHandler.stream.method)
+	assert.Empty(t, originHandler.stream.path)
+	assert.False(t, originHandler.stream.TunnelHostname().IsSet())
+}
+
+func TestReceiveHeaderData(t *testing.T) {
+	originHandler := &mockOriginStreamHandler{}
+	muxPair := NewDefaultMuxerPair(t, t.Name(), originHandler.ServeStream)
+	muxPair.Serve(t)
+
+	reqHeaders := []Header{
+		methodHeader,
+		schemeHeader,
+		pathHeader,
+		tunnelHostnameHeader,
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	reqHeaders = append(reqHeaders, tunnelHostnameHeader)
+	stream, err := muxPair.EdgeMux.OpenStream(ctx, reqHeaders, nil)
+	assertOpenStreamSucceed(t, stream, err)
+
+	assert.Equal(t, methodHeader.Value, originHandler.stream.method)
+	assert.Equal(t, pathHeader.Value, originHandler.stream.path)
+	assert.True(t, originHandler.stream.TunnelHostname().IsSet())
+	assert.Equal(t, tunnelHostnameHeader.Value, originHandler.stream.TunnelHostname().String())
+}

h2mux/muxwriter.go

@@ -40,14 +40,11 @@ type MuxWriter struct {
 	headerEncoder *hpack.Encoder
 	// headerBuffer is the temporary buffer used by headerEncoder.
 	headerBuffer bytes.Buffer
-	// updateReceiveWindowChan is the channel to update receiveWindow size to muxerMetricsUpdater
-	updateReceiveWindowChan chan<- uint32
-	// updateSendWindowChan is the channel to update sendWindow size to muxerMetricsUpdater
-	updateSendWindowChan chan<- uint32
-	// bytesWrote is the amount of bytes wrote to data frame since the last time we send bytes wrote to metrics
+
+	// metricsUpdater is used to report metrics
+	metricsUpdater muxMetricsUpdater
+	// bytesWrote is the amount of bytes written to data frames since the last time we called metricsUpdater.updateOutBoundBytes()
 	bytesWrote *AtomicCounter
-	// updateOutBoundBytesChan is the channel to send bytesWrote to muxerMetricsUpdater
-	updateOutBoundBytesChan chan<- uint64
 
 	useDictChan <-chan useDictRequest
 }
@@ -57,6 +54,13 @@ type MuxedStreamRequest struct {
 	body   io.Reader
 }
 
+func NewMuxedStreamRequest(stream *MuxedStream, body io.Reader) MuxedStreamRequest {
+	return MuxedStreamRequest{
+		stream: stream,
+		body:   body,
+	}
+}
+
 func (r *MuxedStreamRequest) flushBody() {
 	io.Copy(r.stream, r.body)
 	r.stream.CloseWrite()
@@ -83,7 +87,7 @@ func (w *MuxWriter) run(parentLogger *log.Entry) error {
 			case <-w.abortChan:
 				return
 			case <-tickC:
-				w.updateOutBoundBytesChan <- w.bytesWrote.Count()
+				w.metricsUpdater.updateOutBoundBytes(w.bytesWrote.Count())
 			}
 		}
 	}()
@@ -172,8 +176,8 @@ func (w *MuxWriter) run(parentLogger *log.Entry) error {
 func (w *MuxWriter) writeStreamData(stream *MuxedStream, logger *log.Entry) error {
 	logger.Debug("writable")
 	chunk := stream.getChunk()
-	w.updateReceiveWindowChan <- stream.getReceiveWindow()
-	w.updateSendWindowChan <- stream.getSendWindow()
+	w.metricsUpdater.updateReceiveWindow(stream.getReceiveWindow())
+	w.metricsUpdater.updateSendWindow(stream.getSendWindow())
 	if chunk.sendHeadersFrame() {
 		err := w.writeHeaders(chunk.streamID, chunk.headers)
 		if err != nil {

h2mux/readylist.go

@@ -1,15 +1,23 @@
 package h2mux
 
+import "sync"
+
 // ReadyList multiplexes several event signals onto a single channel.
 type ReadyList struct {
+	// signalC is used to signal that a stream can be enqueued
 	signalC chan uint32
-	waitC   chan uint32
+	// waitC is used to signal the ID of the first ready descriptor
+	waitC chan uint32
+	// doneC is used to signal that run should terminate
+	doneC     chan struct{}
+	closeOnce sync.Once
 }
 
 func NewReadyList() *ReadyList {
 	rl := &ReadyList{
 		signalC: make(chan uint32),
 		waitC:   make(chan uint32),
+		doneC:   make(chan struct{}),
 	}
 	go rl.run()
 	return rl
@@ -17,7 +25,11 @@ func NewReadyList() *ReadyList {
 
 // ID is the stream ID
 func (r *ReadyList) Signal(ID uint32) {
-	r.signalC <- ID
+	select {
+	case r.signalC <- ID:
+	// ReadyList already closed
+	case <-r.doneC:
+	}
 }
 
 func (r *ReadyList) ReadyChannel() <-chan uint32 {
@@ -25,7 +37,9 @@ func (r *ReadyList) ReadyChannel() <-chan uint32 {
 }
 
 func (r *ReadyList) Close() {
-	close(r.signalC)
+	r.closeOnce.Do(func() {
+		close(r.doneC)
+	})
 }
 
 func (r *ReadyList) run() {
@@ -35,28 +49,25 @@ func (r *ReadyList) run() {
 	activeDescriptors := newReadyDescriptorMap()
 	for {
 		if firstReady == nil {
-			// Wait for first ready descriptor
-			i, ok := <-r.signalC
-			if !ok {
-				// closed
+			select {
+			case i := <-r.signalC:
+				firstReady = activeDescriptors.SetIfMissing(i)
+			case <-r.doneC:
 				return
 			}
-			firstReady = activeDescriptors.SetIfMissing(i)
 		}
 		select {
 		case r.waitC <- firstReady.ID:
 			activeDescriptors.Delete(firstReady.ID)
 			firstReady = queue.Dequeue()
-		case i, ok := <-r.signalC:
-			if !ok {
-				// closed
-				return
-			}
+		case i := <-r.signalC:
 			newReady := activeDescriptors.SetIfMissing(i)
 			if newReady != nil {
 				// key doesn't exist
 				queue.Enqueue(newReady)
 			}
+		case <-r.doneC:
+			return
 		}
 	}
 }

h2mux/readylist_test.go

@@ -3,36 +3,59 @@ package h2mux
 import (
 	"testing"
 	"time"
+
+	"github.com/stretchr/testify/assert"
 )
 
-func TestReadyList(t *testing.T) {
+func assertEmpty(t *testing.T, rl *ReadyList) {
+	select {
+	case <-rl.ReadyChannel():
+		t.Fatal("Spurious wakeup")
+	default:
+	}
+}
+
+func assertClosed(t *testing.T, rl *ReadyList) {
+	select {
+	case _, ok := <-rl.ReadyChannel():
+		assert.False(t, ok, "ReadyChannel was not closed")
+	case <-time.After(100 * time.Millisecond):
+		t.Fatalf("Timeout")
+	}
+}
+
+func receiveWithTimeout(t *testing.T, rl *ReadyList) uint32 {
+	select {
+	case i := <-rl.ReadyChannel():
+		return i
+	case <-time.After(100 * time.Millisecond):
+		t.Fatalf("Timeout")
+		return 0
+	}
+}
+
+func TestReadyListEmpty(t *testing.T) {
 	rl := NewReadyList()
-	c := rl.ReadyChannel()
-	// helper functions
-	assertEmpty := func() {
-		select {
-		case <-c:
-			t.Fatalf("Spurious wakeup")
-		default:
-		}
-	}
-	receiveWithTimeout := func() uint32 {
-		select {
-		case i := <-c:
-			return i
-		case <-time.After(100 * time.Millisecond):
-			t.Fatalf("Timeout")
-			return 0
-		}
-	}
+
 	// no signals, receive should fail
-	assertEmpty()
+	assertEmpty(t, rl)
+}
+func TestReadyListSignal(t *testing.T) {
+	rl := NewReadyList()
+	assertEmpty(t, rl)
+
 	rl.Signal(0)
-	if receiveWithTimeout() != 0 {
+	if receiveWithTimeout(t, rl) != 0 {
 		t.Fatalf("Received wrong ID of signalled event")
 	}
-	// no new signals, receive should fail
-	assertEmpty()
+
+	assertEmpty(t, rl)
+}
+
+func TestReadyListMultipleSignals(t *testing.T) {
+	rl := NewReadyList()
+	assertEmpty(t, rl)
+
 	// Signals should not block;
 	// Duplicate unhandled signals should not cause multiple wakeups
 	signalled := [5]bool{}
@@ -42,12 +65,45 @@ func TestReadyList(t *testing.T) {
 	}
 	// All signals should be received once (in any order)
 	for range signalled {
-		i := receiveWithTimeout()
+		i := receiveWithTimeout(t, rl)
 		if signalled[i] {
 			t.Fatalf("Received signal %d more than once", i)
 		}
 		signalled[i] = true
 	}
+	for i := range signalled {
+		if !signalled[i] {
+			t.Fatalf("Never received signal %d", i)
+		}
+	}
+	assertEmpty(t, rl)
+}
+
+func TestReadyListClose(t *testing.T) {
+	rl := NewReadyList()
+	rl.Close()
+
+	// readyList.run() occurs in a separate goroutine,
+	// so there's no way to directly check that run() has terminated.
+	// Perform an indirect check: is the ready channel closed?
+	assertClosed(t, rl)
+
+	// a second rl.Close() shouldn't cause a panic
+	rl.Close()
+
+	// Signal shouldn't block after Close()
+	done := make(chan struct{})
+	go func() {
+		for i := 0; i < 5; i++ {
+			rl.Signal(uint32(i))
+		}
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(100 * time.Millisecond):
+		t.Fatal("Test timed out")
+	}
 }
 
 func TestReadyDescriptorQueue(t *testing.T) {

hello/hello.go

@@ -43,7 +43,7 @@ const indexTemplate = `
     <meta name="description" content="Argo Tunnel Connection">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <style>
-      html{line-height:1.15;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}body{margin:0}section{display:block}h1{font-size:2em;margin:.67em 0}a{background-color:transparent;-webkit-text-decoration-skip:objects}/* 1 */::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}/* 1 */a,body,dd,div,dl,dt,h1,h4,html,p,section{box-sizing:border-box}.bt{border-top-style:solid;border-top-width:1px}.bl{border-left-style:solid;border-left-width:1px}.b--orange{border-color:#f38020}.br1{border-radius:.125rem}.bw2{border-width:.25rem}.dib{display:inline-block}.sans-serif{font-family:open sans,-apple-system,BlinkMacSystemFont,avenir next,avenir,helvetica neue,helvetica,ubuntu,roboto,noto,segoe ui,arial,sans-serif}.code{font-family:Consolas,monaco,monospace}.b{font-weight:700}.fw3{font-weight:300}.fw4{font-weight:400}.fw5{font-weight:500}.fw6{font-weight:600}.lh-copy{line-height:1.5}.link{text-decoration:none}.link,.link:active,.link:focus,.link:hover,.link:link,.link:visited{transition:color .15s ease-in}.link:focus{outline:1px dotted currentColor}.mw-100{max-width:100%}.mw4{max-width:8rem}.mw7{max-width:48rem}.bg-light-gray{background-color:#f7f7f7}.link-hover:hover{background-color:#1f679e}.white{color:#fff}.bg-white{background-color:#fff}.bg-blue{background-color:#408bc9}.pb2{padding-bottom:.5rem}.pb6{padding-bottom:8rem}.pt3{padding-top:1rem}.pt5{padding-top:4rem}.pv2{padding-top:.5rem;padding-bottom:.5rem}.ph3{padding-left:1rem;padding-right:1rem}.ph4{padding-left:2rem;padding-right:2rem}.ml0{margin-left:0}.mb1{margin-bottom:.25rem}.mb2{margin-bottom:.5rem}.mb3{margin-bottom:1rem}.mt5{margin-top:4rem}.ttu{text-transform:uppercase}.f4{font-size:1.25rem}.f5{font-size:1rem}.f6{font-size:.875rem}.f7{font-size:.75rem}.measure{max-width:30em}.center{margin-left:auto}.center{margin-right:auto}@media screen and (min-width:30em){.f2-ns{font-size:2.25rem}}@media screen and (min-width:30em) and (max-width:60em){.f5-m{font-size:1rem}}@media screen and (min-width:60em){.f4-l{font-size:1.25rem}}
+      html{line-height:1.15;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}body{margin:0}section{display:block}h1{font-size:2em;margin:.67em 0}a{background-color:transparent;-webkit-text-decoration-skip:objects}/* 1 */::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}/* 1 */a,body,dd,div,dl,dt,h1,h4,html,p,section{box-sizing:border-box}.bt{border-top-style:solid;border-top-width:1px}.bl{border-left-style:solid;border-left-width:1px}.b--orange{border-color:#f38020}.br1{border-radius:.125rem}.bw2{border-width:.25rem}.dib{display:inline-block}.sans-serif{font-family:open sans,-apple-system,BlinkMacSystemFont,avenir next,avenir,helvetica neue,helvetica,ubuntu,roboto,noto,segoe ui,arial,sans-serif}.overflow-x-auto{overflow-x:auto}.code{font-family:Consolas,monaco,monospace}.b{font-weight:700}.fw3{font-weight:300}.fw4{font-weight:400}.fw5{font-weight:500}.fw6{font-weight:600}.lh-copy{line-height:1.5}.link{text-decoration:none}.link,.link:active,.link:focus,.link:hover,.link:link,.link:visited{transition:color .15s ease-in}.link:focus{outline:1px dotted currentColor}.mw-100{max-width:100%}.mw4{max-width:8rem}.mw7{max-width:48rem}.bg-light-gray{background-color:#f7f7f7}.link-hover:hover{background-color:#1f679e}.white{color:#fff}.bg-white{background-color:#fff}.bg-blue{background-color:#408bc9}.pb2{padding-bottom:.5rem}.pb6{padding-bottom:8rem}.pt3{padding-top:1rem}.pt5{padding-top:4rem}.pv2{padding-top:.5rem;padding-bottom:.5rem}.ph3{padding-left:1rem;padding-right:1rem}.ph4{padding-left:2rem;padding-right:2rem}.ml0{margin-left:0}.mb1{margin-bottom:.25rem}.mb2{margin-bottom:.5rem}.mb3{margin-bottom:1rem}.mt5{margin-top:4rem}.ttu{text-transform:uppercase}.f4{font-size:1.25rem}.f5{font-size:1rem}.f6{font-size:.875rem}.f7{font-size:.75rem}.measure{max-width:30em}.center{margin-left:auto}.center{margin-right:auto}@media screen and (min-width:30em){.f2-ns{font-size:2.25rem}}@media screen and (min-width:30em) and (max-width:60em){.f5-m{font-size:1rem}}@media screen and (min-width:60em){.f4-l{font-size:1.25rem}}
     .st0{fill:#FFF}.st1{fill:#f48120}.st2{fill:#faad3f}.st3{fill:#404041}
     </style>
   </head>
@@ -91,7 +91,6 @@ const indexTemplate = `
 </html>
 `
 
-
 func StartHelloWorldServer(logger *logrus.Logger, listener net.Listener, shutdownC <-chan struct{}) error {
 	logger.Infof("Starting Hello World server at %s", listener.Addr())
 	serverName := defaultServerName
@@ -104,15 +103,16 @@ func StartHelloWorldServer(logger *logrus.Logger, listener net.Listener, shutdow
 		WriteBufferSize: 1024,
 	}
 
-	httpServer := &http.Server{Addr: listener.Addr().String(), Handler: nil}
+	muxer := http.NewServeMux()
+	muxer.HandleFunc("/uptime", uptimeHandler(time.Now()))
+	muxer.HandleFunc("/ws", websocketHandler(logger, upgrader))
+	muxer.HandleFunc("/", rootHandler(serverName))
+	httpServer := &http.Server{Addr: listener.Addr().String(), Handler: muxer}
 	go func() {
 		<-shutdownC
 		httpServer.Close()
 	}()
 
-	http.HandleFunc("/uptime", uptimeHandler(time.Now()))
-	http.HandleFunc("/ws", websocketHandler(logger, upgrader))
-	http.HandleFunc("/", rootHandler(serverName))
 	err := httpServer.Serve(listener)
 	return err
 }
@@ -132,7 +132,6 @@ func CreateTLSListener(address string) (net.Listener, error) {
 	return listener, err
 }
 
-
 func uptimeHandler(startTime time.Time) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		// Note that if autoupdate is enabled, the uptime is reset when a new client

metrics/metrics.go

@@ -1,6 +1,8 @@
 package metrics
 
 import (
+	"context"
+	"fmt"
 	"net"
 	"net/http"
 	_ "net/http/pprof"
@@ -8,7 +10,6 @@ import (
 	"sync"
 	"time"
 
-	"golang.org/x/net/context"
 	"golang.org/x/net/trace"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -33,6 +34,9 @@ func ServeMetrics(l net.Listener, shutdownC <-chan struct{}, logger *logrus.Logg
 	}
 
 	http.Handle("/metrics", promhttp.Handler())
+	http.Handle("/healthcheck", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "OK\n")
+	}))
 
 	wg.Add(1)
 	go func() {

origin/backoffhandler.go

@@ -1,9 +1,8 @@
 package origin
 
 import (
+	"context"
 	"time"
-
-	"golang.org/x/net/context"
 )
 
 // Redeclare time functions so they can be overridden in tests.
@@ -93,3 +92,8 @@ func (b BackoffHandler) GetBaseTime() time.Duration {
 	}
 	return b.BaseTime
 }
+
+// Retries returns the number of retries consumed so far.
+func (b *BackoffHandler) Retries() int {
+	return int(b.retries)
+}

origin/backoffhandler_test.go

@@ -1,10 +1,9 @@
 package origin
 
 import (
+	"context"
 	"testing"
 	"time"
-
-	"golang.org/x/net/context"
 )
 
 func immediateTimeAfter(time.Duration) <-chan time.Time {

origin/build_info.go

@@ -1,19 +0,0 @@
-package origin
-
-import (
-	"runtime"
-)
-
-type BuildInfo struct {
-	GoOS               string `json:"go_os"`
-	GoVersion          string `json:"go_version"`
-	GoArch             string `json:"go_arch"`
-}
-
-func GetBuildInfo() *BuildInfo {
-	return &BuildInfo{
-		GoOS:      runtime.GOOS,
-		GoVersion: runtime.Version(),
-		GoArch:    runtime.GOARCH,
-	}
-}

origin/cloudflare_status_page.go

@@ -0,0 +1,117 @@
+package origin
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/cloudflare/golibs/lrucache"
+)
+
+// StatusPage.io API docs:
+// https://www.cloudflarestatus.com/api/v2/#incidents-unresolved
+const (
+	activeIncidentsURL    = "https://yh6f0r4529hb.statuspage.io/api/v2/incidents/unresolved.json"
+	argoTunnelKeyword     = "argo tunnel"
+	incidentDetailsPrefix = "https://www.cloudflarestatus.com/incidents/"
+)
+
+// IncidentLookup is an object that checks for active incidents in
+// the Cloudflare infrastructure.
+type IncidentLookup interface {
+	ActiveIncidents() []Incident
+}
+
+// NewIncidentLookup returns a new IncidentLookup instance that caches its
+// results with a 1-minute TTL.
+func NewIncidentLookup() IncidentLookup {
+	return newCachedIncidentLookup(fetchActiveIncidents)
+}
+
+type IncidentUpdate struct {
+	Body string
+}
+
+type Incident struct {
+	Name    string
+	ID      string           `json:"id"`
+	Updates []IncidentUpdate `json:"incident_updates"`
+}
+
+type StatusPage struct {
+	Incidents []Incident
+}
+
+func (i Incident) URL() string {
+	return incidentDetailsPrefix + i.ID
+}
+
+func parseStatusPage(data []byte) (*StatusPage, error) {
+	var result StatusPage
+	err := json.Unmarshal(data, &result)
+	return &result, err
+}
+
+func isArgoTunnelIncident(i Incident) bool {
+	if strings.Contains(strings.ToLower(i.Name), argoTunnelKeyword) {
+		return true
+	}
+	for _, u := range i.Updates {
+		if strings.Contains(strings.ToLower(u.Body), argoTunnelKeyword) {
+			return true
+		}
+	}
+	return false
+}
+
+func fetchActiveIncidents() (incidents []Incident) {
+	resp, err := http.Get(activeIncidentsURL)
+	if err != nil {
+		return
+	}
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+	statusPage, err := parseStatusPage(body)
+	if err != nil {
+		return
+	}
+	for _, i := range statusPage.Incidents {
+		if isArgoTunnelIncident(i) {
+			incidents = append(incidents, i)
+		}
+	}
+	return incidents
+}
+
+type cachedIncidentLookup struct {
+	cache          *lrucache.LRUCache
+	ttl            time.Duration
+	uncachedLookup func() []Incident
+}
+
+func newCachedIncidentLookup(uncachedLookup func() []Incident) *cachedIncidentLookup {
+	return &cachedIncidentLookup{
+		cache:          lrucache.NewLRUCache(1),
+		ttl:            time.Minute,
+		uncachedLookup: uncachedLookup,
+	}
+}
+
+// We only need one cache entry. Always use the empty string as its key.
+const cacheKey = ""
+
+func (c *cachedIncidentLookup) ActiveIncidents() []Incident {
+	if cached, ok := c.cache.GetNotStale(cacheKey); ok {
+		if incidents, ok := cached.([]Incident); ok {
+			return incidents
+		}
+	}
+	incidents := c.uncachedLookup()
+	c.cache.Set(cacheKey, incidents, time.Now().Add(c.ttl))
+	return incidents
+}

origin/cloudflare_status_page_test.go

@@ -0,0 +1,386 @@
+package origin
+
+import (
+	"testing"
+	"time"
+
+	"github.com/cloudflare/golibs/lrucache"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseStatusPage(t *testing.T) {
+	testCases := []struct {
+		input  []byte
+		output *StatusPage
+		fail   bool
+	}{
+		{
+			input: []byte(`<html>
+				<head><title>504 Gateway Time-out</title></head>
+				<body><center><h1>504 Gateway Time-out</h1></center></body>
+			</html>`),
+			output: nil,
+			fail:   true,
+		},
+		{
+			input: []byte(`{
+				"page": {
+					"id": "yh6f0r4529hb",
+					"name": "Cloudflare",
+					"url": "https://www.cloudflarestatus.com",
+					"time_zone": "Etc/UTC",
+					"updated_at": "2019-01-10T20:11:38.750Z"
+				},
+				"incidents": [
+					{
+						"name": "Cloudflare API service issues",
+						"status": "resolved",
+						"created_at": "2018-09-17T19:29:21.132Z",
+						"updated_at": "2018-09-18T07:45:41.313Z",
+						"monitoring_at": "2018-09-17T21:35:06.492Z",
+						"resolved_at": "2018-09-18T07:45:41.290Z",
+						"shortlink": "http://stspg.io/7f079791e",
+						"id": "q746ybtyb6q0",
+						"page_id": "yh6f0r4529hb",
+						"incident_updates": [
+							{
+								"status": "resolved",
+								"body": "Cloudflare has resolved the issue and the service have resumed normal operation.",
+								"created_at": "2018-09-18T07:45:41.290Z",
+								"updated_at": "2018-09-18T07:45:41.290Z",
+								"display_at": "2018-09-18T07:45:41.290Z",
+								"affected_components": [
+									{
+										"code": "g4tb35rs9yw7",
+										"name": "Cloudflare customer dashboard and APIs - Cloudflare APIs",
+										"old_status": "operational",
+										"new_status": "operational"
+									}
+								],
+								"deliver_notifications": true,
+								"tweet_id": null,
+								"id": "zl5g2pl5zhfs",
+								"incident_id": "q746ybtyb6q0",
+								"custom_tweet": null
+							},
+							{
+								"status": "monitoring",
+								"body": "Cloudflare has implemented a fix for this issue and is currently monitoring the results.\r\n\r\nWe will update the status once the issue is resolved.",
+								"created_at": "2018-09-17T21:35:06.492Z",
+								"updated_at": "2018-09-17T21:35:06.492Z",
+								"display_at": "2018-09-17T21:35:06.492Z",
+								"affected_components": [
+									{
+										"code": "g4tb35rs9yw7",
+										"name": "Cloudflare customer dashboard and APIs - Cloudflare APIs",
+										"old_status": "degraded_performance",
+										"new_status": "operational"
+									}
+								],
+								"deliver_notifications": false,
+								"tweet_id": null,
+								"id": "0001sv3chdnx",
+								"incident_id": "q746ybtyb6q0",
+								"custom_tweet": null
+							},
+							{
+								"status": "investigating",
+								"body": "We are continuing to investigate this issue.",
+								"created_at": "2018-09-17T19:30:08.049Z",
+								"updated_at": "2018-09-17T19:30:08.049Z",
+								"display_at": "2018-09-17T19:30:08.049Z",
+								"affected_components": [
+									{
+										"code": "g4tb35rs9yw7",
+										"name": "Cloudflare customer dashboard and APIs - Cloudflare APIs",
+										"old_status": "operational",
+										"new_status": "degraded_performance"
+									}
+								],
+								"deliver_notifications": false,
+								"tweet_id": null,
+								"id": "qdr164tfpq7m",
+								"incident_id": "q746ybtyb6q0",
+								"custom_tweet": null
+							},
+							{
+								"status": "investigating",
+								"body": "Cloudflare is investigating issues with APIs and Page Rule delays for Page Rule updates.  Cloudflare Page Rule service delivery is unaffected and is operating normally.  Also, these issues do not affect the Cloudflare CDN and therefore, do not impact customer websites.",
+								"created_at": "2018-09-17T19:29:21.201Z",
+								"updated_at": "2018-09-17T19:29:21.201Z",
+								"display_at": "2018-09-17T19:29:21.201Z",
+								"affected_components": [
+									{
+										"code": "g4tb35rs9yw7",
+										"name": "Cloudflare customer dashboard and APIs - Cloudflare APIs",
+										"old_status": "operational",
+										"new_status": "operational"
+									}
+								],
+								"deliver_notifications": false,
+								"tweet_id": null,
+								"id": "qzl2n0q8tskg",
+								"incident_id": "q746ybtyb6q0",
+								"custom_tweet": null
+							}
+						],
+						"components": [
+							{
+								"status": "operational",
+								"name": "Cloudflare APIs",
+								"created_at": "2014-10-09T03:32:07.158Z",
+								"updated_at": "2019-01-01T22:58:30.846Z",
+								"position": 2,
+								"description": null,
+								"showcase": false,
+								"id": "g4tb35rs9yw7",
+								"page_id": "yh6f0r4529hb",
+								"group_id": "1km35smx8p41",
+								"group": false,
+								"only_show_if_degraded": false,
+								"automation_email": "component+g4tb35rs9yw7@notifications.statuspage.io"
+							}
+						],
+						"impact": "minor"
+					},
+					{
+						"name": "Web Analytics Delays",
+						"status": "resolved",
+						"created_at": "2018-09-17T18:05:39.907Z",
+						"updated_at": "2018-09-17T22:53:05.078Z",
+						"monitoring_at": null,
+						"resolved_at": "2018-09-17T22:53:05.057Z",
+						"shortlink": "http://stspg.io/cb208928c",
+						"id": "wqfk9mzs5qt1",
+						"page_id": "yh6f0r4529hb",
+						"incident_updates": [
+							{
+								"status": "resolved",
+								"body": "Cloudflare has resolved the issue and Web Analytics have resumed normal operation.",
+								"created_at": "2018-09-17T22:53:05.057Z",
+								"updated_at": "2018-09-17T22:53:05.057Z",
+								"display_at": "2018-09-17T22:53:05.057Z",
+								"affected_components": [
+									{
+										"code": "4c231tkdlpcl",
+										"name": "Cloudflare customer dashboard and APIs - Analytics",
+										"old_status": "degraded_performance",
+										"new_status": "operational"
+									}
+								],
+								"deliver_notifications": false,
+								"tweet_id": null,
+								"id": "93y1w00yqzk4",
+								"incident_id": "wqfk9mzs5qt1",
+								"custom_tweet": null
+							},
+							{
+								"status": "investigating",
+								"body": "There is a delay in processing Cloudflare Web Analytics. This affects timely delivery of customer data.\n\nThese delays do not impact analytics for DNS and Rate Limiting.",
+								"created_at": "2018-09-17T18:05:40.033Z",
+								"updated_at": "2018-09-17T18:05:40.033Z",
+								"display_at": "2018-09-17T18:05:40.033Z",
+								"affected_components": [
+									{
+										"code": "4c231tkdlpcl",
+										"name": "Cloudflare customer dashboard and APIs - Analytics",
+										"old_status": "operational",
+										"new_status": "degraded_performance"
+									}
+								],
+								"deliver_notifications": false,
+								"tweet_id": null,
+								"id": "362t6lv0vrpk",
+								"incident_id": "wqfk9mzs5qt1",
+								"custom_tweet": null
+							}
+						],
+						"components": [
+							{
+								"status": "operational",
+								"name": "Analytics",
+								"created_at": "2014-11-13T11:54:10.191Z",
+								"updated_at": "2018-12-31T08:20:52.349Z",
+								"position": 3,
+								"description": "Customer data",
+								"showcase": false,
+								"id": "4c231tkdlpcl",
+								"page_id": "yh6f0r4529hb",
+								"group_id": "1km35smx8p41",
+								"group": false,
+								"only_show_if_degraded": false,
+								"automation_email": "component+4c231tkdlpcl@notifications.statuspage.io"
+							}
+						],
+						"impact": "minor"
+					}
+				]
+			}`),
+			output: &StatusPage{
+				Incidents: []Incident{
+					Incident{
+						Name: "Cloudflare API service issues",
+						ID:   "q746ybtyb6q0",
+						Updates: []IncidentUpdate{
+							IncidentUpdate{
+								Body: "Cloudflare has resolved the issue and the service have resumed normal operation.",
+							},
+							IncidentUpdate{
+								Body: "Cloudflare has implemented a fix for this issue and is currently monitoring the results.\r\n\r\nWe will update the status once the issue is resolved.",
+							},
+							IncidentUpdate{
+								Body: "We are continuing to investigate this issue.",
+							},
+							IncidentUpdate{
+								Body: "Cloudflare is investigating issues with APIs and Page Rule delays for Page Rule updates.  Cloudflare Page Rule service delivery is unaffected and is operating normally.  Also, these issues do not affect the Cloudflare CDN and therefore, do not impact customer websites.",
+							},
+						},
+					},
+					Incident{
+						Name: "Web Analytics Delays",
+						ID:   "wqfk9mzs5qt1",
+						Updates: []IncidentUpdate{
+							IncidentUpdate{
+								Body: "Cloudflare has resolved the issue and Web Analytics have resumed normal operation.",
+							},
+							IncidentUpdate{
+								Body: "There is a delay in processing Cloudflare Web Analytics. This affects timely delivery of customer data.\n\nThese delays do not impact analytics for DNS and Rate Limiting.",
+							},
+						},
+					},
+				},
+			},
+			fail: false,
+		},
+	}
+
+	for _, testCase := range testCases {
+		output, err := parseStatusPage(testCase.input)
+		if testCase.fail {
+			assert.Error(t, err)
+		} else {
+			assert.Nil(t, err)
+			assert.Equal(t, testCase.output, output)
+		}
+	}
+}
+
+func TestIsArgoTunnelIncident(t *testing.T) {
+	testCases := []struct {
+		input  Incident
+		output bool
+	}{
+		{
+			input:  Incident{},
+			output: false,
+		},
+		{
+			input:  Incident{Name: "An Argo Tunnel incident"},
+			output: true,
+		},
+		{
+			input:  Incident{Name: "an argo tunnel incident"},
+			output: true,
+		},
+		{
+			input:  Incident{Name: "an aRgO TuNnEl incident"},
+			output: true,
+		},
+		{
+			input:  Incident{Name: "an argotunnel incident"},
+			output: false,
+		},
+		{
+			input:  Incident{Name: "irrelevant"},
+			output: false,
+		},
+		{
+			input: Incident{
+				Name: "irrelevant",
+				Updates: []IncidentUpdate{
+					IncidentUpdate{Body: "irrelevant"},
+					IncidentUpdate{Body: "an Argo Tunnel incident"},
+					IncidentUpdate{Body: "irrelevant"},
+				},
+			},
+			output: true,
+		},
+		{
+			input: Incident{
+				Name: "an Argo Tunnel incident",
+				Updates: []IncidentUpdate{
+					IncidentUpdate{Body: "irrelevant"},
+					IncidentUpdate{Body: "irrelevant"},
+					IncidentUpdate{Body: "irrelevant"},
+				},
+			},
+			output: true,
+		},
+	}
+	for _, testCase := range testCases {
+		actual := isArgoTunnelIncident(testCase.input)
+		assert.Equal(t, testCase.output, actual, "Test case failed: %v", testCase.input)
+	}
+}
+
+func TestIncidentURL(t *testing.T) {
+	incident := Incident{
+		ID: "s6k0dnn5347b",
+	}
+	assert.Equal(t, "https://www.cloudflarestatus.com/incidents/s6k0dnn5347b", incident.URL())
+}
+
+func TestNewCachedIncidentLookup(t *testing.T) {
+	c := newCachedIncidentLookup(func() []Incident { return nil })
+	assert.Equal(t, time.Minute, c.ttl)
+	assert.Equal(t, 1, c.cache.Capacity())
+}
+
+func TestCachedIncidentLookup(t *testing.T) {
+	expected := []Incident{
+		Incident{
+			Name: "An incident",
+			ID:   "incidentID",
+		},
+	}
+
+	var shouldCallUncachedLookup bool
+	c := &cachedIncidentLookup{
+		cache: lrucache.NewLRUCache(1),
+		ttl:   50 * time.Millisecond,
+		uncachedLookup: func() []Incident {
+			if !shouldCallUncachedLookup {
+				t.Fatal("uncachedLookup shouldn't have been called")
+			}
+			return expected
+		},
+	}
+
+	shouldCallUncachedLookup = true
+	assert.Equal(t, expected, c.ActiveIncidents())
+
+	shouldCallUncachedLookup = false
+	assert.Equal(t, expected, c.ActiveIncidents())
+	assert.Equal(t, expected, c.ActiveIncidents())
+
+	time.Sleep(50 * time.Millisecond)
+	shouldCallUncachedLookup = true
+	assert.Equal(t, expected, c.ActiveIncidents())
+}
+
+func TestCachedIncidentLookupDoesntPanic(t *testing.T) {
+	expected := []Incident{
+		Incident{
+			Name: "An incident",
+			ID:   "incidentID",
+		},
+	}
+	c := &cachedIncidentLookup{
+		cache:          lrucache.NewLRUCache(1),
+		ttl:            50 * time.Millisecond,
+		uncachedLookup: func() []Incident { return expected },
+	}
+	c.cache.Set(cacheKey, 42, time.Now().Add(30*time.Minute))
+	actual := c.ActiveIncidents()
+	assert.Equal(t, expected, actual)
+}

origin/discovery.go

@@ -1,82 +0,0 @@
-package origin
-
-import (
-	"fmt"
-	"net"
-)
-
-const (
-	// Used to discover HA Warp servers
-	srvService = "warp"
-	srvProto   = "tcp"
-	srvName    = "cloudflarewarp.com"
-)
-
-func ResolveEdgeIPs(addresses []string) ([]*net.TCPAddr, error) {
-	if len(addresses) > 0 {
-		var tcpAddrs []*net.TCPAddr
-		for _, address := range addresses {
-			// Addresses specified (for testing, usually)
-			tcpAddr, err := net.ResolveTCPAddr("tcp", address)
-			if err != nil {
-				return nil, err
-			}
-			tcpAddrs = append(tcpAddrs, tcpAddr)
-		}
-		return tcpAddrs, nil
-	}
-	// HA service discovery lookup
-	_, addrs, err := net.LookupSRV(srvService, srvProto, srvName)
-	if err != nil {
-		return nil, err
-	}
-	var resolvedIPsPerCNAME [][]*net.TCPAddr
-	var lookupErr error
-	for _, addr := range addrs {
-		ips, err := ResolveSRVToTCP(addr)
-		if err != nil || len(ips) == 0 {
-			// don't return early, we might be able to resolve other addresses
-			lookupErr = err
-			continue
-		}
-		resolvedIPsPerCNAME = append(resolvedIPsPerCNAME, ips)
-	}
-	ips := FlattenServiceIPs(resolvedIPsPerCNAME)
-	if lookupErr == nil && len(ips) == 0 {
-		return nil, fmt.Errorf("Unknown service discovery error")
-	}
-	return ips, lookupErr
-}
-
-func ResolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
-	ips, err := net.LookupIP(srv.Target)
-	if err != nil {
-		return nil, err
-	}
-	addrs := make([]*net.TCPAddr, len(ips))
-	for i, ip := range ips {
-		addrs[i] = &net.TCPAddr{IP: ip, Port: int(srv.Port)}
-	}
-	return addrs, nil
-}
-
-// FlattenServiceIPs transposes and flattens the input slices such that the
-// first element of the n inner slices are the first n elements of the result.
-func FlattenServiceIPs(ipsByService [][]*net.TCPAddr) []*net.TCPAddr {
-	var result []*net.TCPAddr
-	for len(ipsByService) > 0 {
-		filtered := ipsByService[:0]
-		for _, ips := range ipsByService {
-			if len(ips) == 0 {
-				// sanity check
-				continue
-			}
-			result = append(result, ips[0])
-			if len(ips) > 1 {
-				filtered = append(filtered, ips[1:])
-			}
-		}
-		ipsByService = filtered
-	}
-	return result
-}

origin/discovery_test.go

@@ -1,45 +0,0 @@
-package origin
-
-import (
-	"net"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestFlattenServiceIPs(t *testing.T) {
-	result := FlattenServiceIPs([][]*net.TCPAddr{
-		[]*net.TCPAddr{
-			&net.TCPAddr{Port: 1},
-			&net.TCPAddr{Port: 2},
-			&net.TCPAddr{Port: 3},
-			&net.TCPAddr{Port: 4},
-		},
-		[]*net.TCPAddr{
-			&net.TCPAddr{Port: 10},
-			&net.TCPAddr{Port: 12},
-			&net.TCPAddr{Port: 13},
-		},
-		[]*net.TCPAddr{
-			&net.TCPAddr{Port: 21},
-			&net.TCPAddr{Port: 22},
-			&net.TCPAddr{Port: 23},
-			&net.TCPAddr{Port: 24},
-			&net.TCPAddr{Port: 25},
-		},
-	})
-	assert.EqualValues(t, []*net.TCPAddr{
-		&net.TCPAddr{Port: 1},
-		&net.TCPAddr{Port: 10},
-		&net.TCPAddr{Port: 21},
-		&net.TCPAddr{Port: 2},
-		&net.TCPAddr{Port: 12},
-		&net.TCPAddr{Port: 22},
-		&net.TCPAddr{Port: 3},
-		&net.TCPAddr{Port: 13},
-		&net.TCPAddr{Port: 23},
-		&net.TCPAddr{Port: 4},
-		&net.TCPAddr{Port: 24},
-		&net.TCPAddr{Port: 25},
-	}, result)
-}

origin/metrics.go

@@ -9,6 +9,12 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+const (
+	metricsNamespace = "cloudflared"
+	tunnelSubsystem  = "tunnel"
+	muxerSubsystem   = "muxer"
+)
+
 type muxerMetrics struct {
 	rtt              *prometheus.GaugeVec
 	rttMin           *prometheus.GaugeVec
@@ -32,6 +38,7 @@ type muxerMetrics struct {
 
 type TunnelMetrics struct {
 	haConnections     prometheus.Gauge
+	activeStreams     prometheus.Gauge
 	totalRequests     prometheus.Counter
 	requestsPerTunnel *prometheus.CounterVec
 	// concurrentRequestsLock is a mutex for concurrentRequests and maxConcurrentRequests
@@ -51,14 +58,24 @@ type TunnelMetrics struct {
 	// oldServerLocations stores the last server the tunnel was connected to
 	oldServerLocations map[string]string
 
-	muxerMetrics *muxerMetrics
+	regSuccess  *prometheus.CounterVec
+	regFail     *prometheus.CounterVec
+	authSuccess prometheus.Counter
+	authFail    *prometheus.CounterVec
+	rpcFail     *prometheus.CounterVec
+
+	muxerMetrics        *muxerMetrics
+	tunnelsHA           tunnelsForHA
+	userHostnamesCounts *prometheus.CounterVec
 }
 
 func newMuxerMetrics() *muxerMetrics {
 	rtt := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "rtt",
-			Help: "Round-trip time in millisecond",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt",
+			Help:      "Round-trip time in millisecond",
 		},
 		[]string{"connection_id"},
 	)
@@ -66,8 +83,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	rttMin := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "rtt_min",
-			Help: "Shortest round-trip time in millisecond",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_min",
+			Help:      "Shortest round-trip time in millisecond",
 		},
 		[]string{"connection_id"},
 	)
@@ -75,8 +94,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	rttMax := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "rtt_max",
-			Help: "Longest round-trip time in millisecond",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_max",
+			Help:      "Longest round-trip time in millisecond",
 		},
 		[]string{"connection_id"},
 	)
@@ -84,8 +105,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	receiveWindowAve := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "receive_window_ave",
-			Help: "Average receive window size in bytes",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_ave",
+			Help:      "Average receive window size in bytes",
 		},
 		[]string{"connection_id"},
 	)
@@ -93,8 +116,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	sendWindowAve := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "send_window_ave",
-			Help: "Average send window size in bytes",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_ave",
+			Help:      "Average send window size in bytes",
 		},
 		[]string{"connection_id"},
 	)
@@ -102,8 +127,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	receiveWindowMin := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "receive_window_min",
-			Help: "Smallest receive window size in bytes",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_min",
+			Help:      "Smallest receive window size in bytes",
 		},
 		[]string{"connection_id"},
 	)
@@ -111,8 +138,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	receiveWindowMax := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "receive_window_max",
-			Help: "Largest receive window size in bytes",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_max",
+			Help:      "Largest receive window size in bytes",
 		},
 		[]string{"connection_id"},
 	)
@@ -120,8 +149,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	sendWindowMin := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "send_window_min",
-			Help: "Smallest send window size in bytes",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_min",
+			Help:      "Smallest send window size in bytes",
 		},
 		[]string{"connection_id"},
 	)
@@ -129,8 +160,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	sendWindowMax := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "send_window_max",
-			Help: "Largest send window size in bytes",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_max",
+			Help:      "Largest send window size in bytes",
 		},
 		[]string{"connection_id"},
 	)
@@ -138,8 +171,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	inBoundRateCurr := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "inbound_bytes_per_sec_curr",
-			Help: "Current inbounding bytes per second, 0 if there is no incoming connection",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_curr",
+			Help:      "Current inbounding bytes per second, 0 if there is no incoming connection",
 		},
 		[]string{"connection_id"},
 	)
@@ -147,8 +182,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	inBoundRateMin := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "inbound_bytes_per_sec_min",
-			Help: "Minimum non-zero inbounding bytes per second",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero inbounding bytes per second",
 		},
 		[]string{"connection_id"},
 	)
@@ -156,8 +193,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	inBoundRateMax := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "inbound_bytes_per_sec_max",
-			Help: "Maximum inbounding bytes per second",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_max",
+			Help:      "Maximum inbounding bytes per second",
 		},
 		[]string{"connection_id"},
 	)
@@ -165,8 +204,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	outBoundRateCurr := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "outbound_bytes_per_sec_curr",
-			Help: "Current outbounding bytes per second, 0 if there is no outgoing traffic",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_curr",
+			Help:      "Current outbounding bytes per second, 0 if there is no outgoing traffic",
 		},
 		[]string{"connection_id"},
 	)
@@ -174,8 +215,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	outBoundRateMin := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "outbound_bytes_per_sec_min",
-			Help: "Minimum non-zero outbounding bytes per second",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero outbounding bytes per second",
 		},
 		[]string{"connection_id"},
 	)
@@ -183,8 +226,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	outBoundRateMax := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "outbound_bytes_per_sec_max",
-			Help: "Maximum outbounding bytes per second",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_max",
+			Help:      "Maximum outbounding bytes per second",
 		},
 		[]string{"connection_id"},
 	)
@@ -192,8 +237,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	compBytesBefore := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "comp_bytes_before",
-			Help: "Bytes sent via cross-stream compression, pre compression",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_before",
+			Help:      "Bytes sent via cross-stream compression, pre compression",
 		},
 		[]string{"connection_id"},
 	)
@@ -201,8 +248,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	compBytesAfter := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "comp_bytes_after",
-			Help: "Bytes sent via cross-stream compression, post compression",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_after",
+			Help:      "Bytes sent via cross-stream compression, post compression",
 		},
 		[]string{"connection_id"},
 	)
@@ -210,8 +259,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	compRateAve := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "comp_rate_ave",
-			Help: "Average outbound cross-stream compression ratio",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_rate_ave",
+			Help:      "Average outbound cross-stream compression ratio",
 		},
 		[]string{"connection_id"},
 	)
@@ -268,22 +319,30 @@ func convertRTTMilliSec(t time.Duration) float64 {
 func NewTunnelMetrics() *TunnelMetrics {
 	haConnections := prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Name: "ha_connections",
-			Help: "Number of active ha connections",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "ha_connections",
+			Help:      "Number of active ha connections",
 		})
 	prometheus.MustRegister(haConnections)
 
+	activeStreams := h2mux.NewActiveStreamsMetrics(metricsNamespace, tunnelSubsystem)
+
 	totalRequests := prometheus.NewCounter(
 		prometheus.CounterOpts{
-			Name: "total_requests",
-			Help: "Amount of requests proxied through all the tunnels",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "total_requests",
+			Help:      "Amount of requests proxied through all the tunnels",
 		})
 	prometheus.MustRegister(totalRequests)
 
 	requestsPerTunnel := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "requests_per_tunnel",
-			Help: "Amount of requests proxied through each tunnel",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "requests_per_tunnel",
+			Help:      "Amount of requests proxied through each tunnel",
 		},
 		[]string{"connection_id"},
 	)
@@ -291,8 +350,10 @@ func NewTunnelMetrics() *TunnelMetrics {
 
 	concurrentRequestsPerTunnel := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "concurrent_requests_per_tunnel",
-			Help: "Concurrent requests proxied through each tunnel",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "concurrent_requests_per_tunnel",
+			Help:      "Concurrent requests proxied through each tunnel",
 		},
 		[]string{"connection_id"},
 	)
@@ -300,8 +361,10 @@ func NewTunnelMetrics() *TunnelMetrics {
 
 	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "max_concurrent_requests_per_tunnel",
-			Help: "Largest number of concurrent requests proxied through each tunnel so far",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "max_concurrent_requests_per_tunnel",
+			Help:      "Largest number of concurrent requests proxied through each tunnel so far",
 		},
 		[]string{"connection_id"},
 	)
@@ -309,15 +372,19 @@ func NewTunnelMetrics() *TunnelMetrics {
 
 	timerRetries := prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Name: "timer_retries",
-			Help: "Unacknowledged heart beats count",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "timer_retries",
+			Help:      "Unacknowledged heart beats count",
 		})
 	prometheus.MustRegister(timerRetries)
 
 	responseByCode := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "response_by_code",
-			Help: "Count of responses by HTTP status code",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "response_by_code",
+			Help:      "Count of responses by HTTP status code",
 		},
 		[]string{"status_code"},
 	)
@@ -325,8 +392,10 @@ func NewTunnelMetrics() *TunnelMetrics {
 
 	responseCodePerTunnel := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "response_code_per_tunnel",
-			Help: "Count of responses by HTTP status code fore each tunnel",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "response_code_per_tunnel",
+			Help:      "Count of responses by HTTP status code fore each tunnel",
 		},
 		[]string{"connection_id", "status_code"},
 	)
@@ -334,15 +403,83 @@ func NewTunnelMetrics() *TunnelMetrics {
 
 	serverLocations := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "server_locations",
-			Help: "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "server_locations",
+			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
 		},
 		[]string{"connection_id", "location"},
 	)
 	prometheus.MustRegister(serverLocations)
 
+	rpcFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_rpc_fail",
+			Help:      "Count of RPC connection errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(rpcFail)
+
+	registerFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_register_fail",
+			Help:      "Count of tunnel registration errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(registerFail)
+
+	userHostnamesCounts := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "user_hostnames_counts",
+			Help:      "Which user hostnames cloudflared is serving",
+		},
+		[]string{"userHostname"},
+	)
+	prometheus.MustRegister(userHostnamesCounts)
+
+	registerSuccess := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_register_success",
+			Help:      "Count of successful tunnel registrations",
+		},
+		[]string{"rpcName"},
+	)
+	prometheus.MustRegister(registerSuccess)
+
+	authSuccess := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_success",
+			Help:      "Count of successful tunnel authenticate",
+		},
+	)
+	prometheus.MustRegister(authSuccess)
+
+	authFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_fail",
+			Help:      "Count of tunnel authenticate errors by type",
+		},
+		[]string{"error"},
+	)
+	prometheus.MustRegister(authFail)
+
 	return &TunnelMetrics{
 		haConnections:                  haConnections,
+		activeStreams:                  activeStreams,
 		totalRequests:                  totalRequests,
 		requestsPerTunnel:              requestsPerTunnel,
 		concurrentRequestsPerTunnel:    concurrentRequestsPerTunnel,
@@ -355,6 +492,13 @@ func NewTunnelMetrics() *TunnelMetrics {
 		serverLocations:                serverLocations,
 		oldServerLocations:             make(map[string]string),
 		muxerMetrics:                   newMuxerMetrics(),
+		tunnelsHA:                      NewTunnelsForHA(),
+		regSuccess:                     registerSuccess,
+		regFail:                        registerFail,
+		rpcFail:                        rpcFail,
+		userHostnamesCounts:            userHostnamesCounts,
+		authSuccess:                    authSuccess,
+		authFail:                       authFail,
 	}
 }
 
@@ -375,7 +519,7 @@ func (t *TunnelMetrics) incrementRequests(connectionID string) {
 	var concurrentRequests uint64
 	var ok bool
 	if concurrentRequests, ok = t.concurrentRequests[connectionID]; ok {
-		t.concurrentRequests[connectionID] += 1
+		t.concurrentRequests[connectionID]++
 		concurrentRequests++
 	} else {
 		t.concurrentRequests[connectionID] = 1
@@ -395,7 +539,7 @@ func (t *TunnelMetrics) incrementRequests(connectionID string) {
 func (t *TunnelMetrics) decrementConcurrentRequests(connectionID string) {
 	t.concurrentRequestsLock.Lock()
 	if _, ok := t.concurrentRequests[connectionID]; ok {
-		t.concurrentRequests[connectionID] -= 1
+		t.concurrentRequests[connectionID]--
 	}
 	t.concurrentRequestsLock.Unlock()
 

origin/supervisor.go

@@ -1,11 +1,20 @@
 package origin
 
 import (
+	"context"
+	"errors"
 	"fmt"
 	"net"
+	"sync"
 	"time"
 
-	"golang.org/x/net/context"
+	"github.com/google/uuid"
+	"github.com/sirupsen/logrus"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/signal"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
@@ -15,13 +24,25 @@ const (
 	resolveTTL = time.Hour
 	// Interval between registering new tunnels
 	registrationInterval = time.Second
+
+	subsystemRefreshAuth = "refresh_auth"
+	// Maximum exponent for 'Authenticate' exponential backoff
+	refreshAuthMaxBackoff = 10
+	// Waiting time before retrying a failed 'Authenticate' connection
+	refreshAuthRetryDuration = time.Second * 10
+	// Maximum time to make an Authenticate RPC
+	authTokenTimeout = time.Second * 30
+)
+
+var (
+	errJWTUnset         = errors.New("JWT unset")
+	errEventDigestUnset = errors.New("event digest unset")
 )
 
 type Supervisor struct {
-	config  *TunnelConfig
-	edgeIPs []*net.TCPAddr
-	// nextUnusedEdgeIP is the index of the next addr k edgeIPs to try
-	nextUnusedEdgeIP  int
+	cloudflaredUUID   uuid.UUID
+	config            *TunnelConfig
+	edgeIPs           connection.EdgeServiceDiscoverer
 	lastResolve       time.Time
 	resolverC         chan resolveResult
 	tunnelErrors      chan tunnelError
@@ -30,35 +51,73 @@ type Supervisor struct {
 	// currently-connecting tunnels to finish connecting so we can reset backoff timer
 	nextConnectedIndex  int
 	nextConnectedSignal chan struct{}
+
+	logger *logrus.Entry
+
+	jwtLock *sync.RWMutex
+	jwt     []byte
+
+	eventDigestLock *sync.RWMutex
+	eventDigest     []byte
 }
 
 type resolveResult struct {
-	edgeIPs []*net.TCPAddr
-	err     error
+	err error
 }
 
 type tunnelError struct {
 	index int
+	addr  *net.TCPAddr
 	err   error
 }
 
-func NewSupervisor(config *TunnelConfig) *Supervisor {
+func NewSupervisor(config *TunnelConfig, u uuid.UUID) (*Supervisor, error) {
+	var (
+		edgeIPs connection.EdgeServiceDiscoverer
+		err     error
+	)
+	if len(config.EdgeAddrs) > 0 {
+		edgeIPs, err = connection.NewEdgeHostnameResolver(config.EdgeAddrs)
+	} else {
+		edgeIPs, err = connection.NewEdgeAddrResolver(config.Logger)
+	}
+	if err != nil {
+		return nil, err
+	}
 	return &Supervisor{
+		cloudflaredUUID:   u,
 		config:            config,
+		edgeIPs:           edgeIPs,
 		tunnelErrors:      make(chan tunnelError),
 		tunnelsConnecting: map[int]chan struct{}{},
-	}
+		logger:            config.Logger.WithField("subsystem", "supervisor"),
+		jwtLock:           &sync.RWMutex{},
+		eventDigestLock:   &sync.RWMutex{},
+	}, nil
 }
 
-func (s *Supervisor) Run(ctx context.Context, connectedSignal chan struct{}) error {
+func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal) error {
 	logger := s.config.Logger
 	if err := s.initialize(ctx, connectedSignal); err != nil {
 		return err
 	}
 	var tunnelsWaiting []int
+	tunnelsActive := s.config.HAConnections
+
 	backoff := BackoffHandler{MaxRetries: s.config.Retries, BaseTime: tunnelRetryDuration, RetryForever: true}
 	var backoffTimer <-chan time.Time
-	tunnelsActive := s.config.HAConnections
+
+	refreshAuthBackoff := &BackoffHandler{MaxRetries: refreshAuthMaxBackoff, BaseTime: refreshAuthRetryDuration, RetryForever: true}
+	var refreshAuthBackoffTimer <-chan time.Time
+
+	if s.config.UseReconnectToken {
+		if timer, err := s.refreshAuth(ctx, refreshAuthBackoff, s.authenticate); err == nil {
+			refreshAuthBackoffTimer = timer
+		} else {
+			logger.WithError(err).Errorf("initial refreshAuth failed, retrying in %v", refreshAuthRetryDuration)
+			refreshAuthBackoffTimer = time.After(refreshAuthRetryDuration)
+		}
+	}
 
 	for {
 		select {
@@ -85,8 +144,8 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal chan struct{}) err
 				// If the error is a dial error, the problem is likely to be network related
 				// try another addr before refreshing since we are likely to get back the
 				// same IPs in the same order. Same problem with duplicate connection error.
-				if s.unusedIPs() {
-					s.replaceEdgeIP(tunnelError.index)
+				if s.unusedIPs() && tunnelError.addr != nil {
+					s.edgeIPs.MarkAddrBad(tunnelError.addr)
 				} else {
 					s.refreshEdgeIPs()
 				}
@@ -99,6 +158,16 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal chan struct{}) err
 			}
 			tunnelsActive += len(tunnelsWaiting)
 			tunnelsWaiting = nil
+		// Time to call Authenticate
+		case <-refreshAuthBackoffTimer:
+			newTimer, err := s.refreshAuth(ctx, refreshAuthBackoff, s.authenticate)
+			if err != nil {
+				logger.WithError(err).Error("Authentication failed")
+				// Permanent failure. Leave the `select` without setting the
+				// channel to be non-null, so we'll never hit this case of the `select` again.
+				continue
+			}
+			refreshAuthBackoffTimer = newTimer
 		// Tunnel successfully connected
 		case <-s.nextConnectedSignal:
 			if !s.waitForNextTunnel(s.nextConnectedIndex) && len(tunnelsWaiting) == 0 {
@@ -111,7 +180,6 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal chan struct{}) err
 			s.resolverC = nil
 			if result.err == nil {
 				logger.Debug("Service discovery refresh complete")
-				s.edgeIPs = result.edgeIPs
 			} else {
 				logger.WithError(result.err).Error("Service discovery error")
 			}
@@ -119,34 +187,35 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal chan struct{}) err
 	}
 }
 
-func (s *Supervisor) initialize(ctx context.Context, connectedSignal chan struct{}) error {
-	logger := s.config.Logger
-	edgeIPs, err := ResolveEdgeIPs(s.config.EdgeAddrs)
+// Returns nil if initialization succeeded, else the initialization error.
+func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal) error {
+	logger := s.logger
+
+	err := s.edgeIPs.Refresh()
 	if err != nil {
 		logger.Infof("ResolveEdgeIPs err")
 		return err
 	}
-	s.edgeIPs = edgeIPs
-	if s.config.HAConnections > len(edgeIPs) {
-		logger.Warnf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, len(edgeIPs))
-		s.config.HAConnections = len(edgeIPs)
-	}
 	s.lastResolve = time.Now()
-	// check entitlement and version too old error before attempting to register more tunnels
-	s.nextUnusedEdgeIP = s.config.HAConnections
+	availableAddrs := int(s.edgeIPs.AvailableAddrs())
+	if s.config.HAConnections > availableAddrs {
+		logger.Warnf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
+		s.config.HAConnections = availableAddrs
+	}
+
 	go s.startFirstTunnel(ctx, connectedSignal)
 	select {
 	case <-ctx.Done():
 		<-s.tunnelErrors
-		// Error can't be nil. A nil error signals that initialization succeed
-		return fmt.Errorf("context was canceled")
+		return ctx.Err()
 	case tunnelError := <-s.tunnelErrors:
 		return tunnelError.err
-	case <-connectedSignal:
+	case <-connectedSignal.Wait():
 	}
 	// At least one successful connection, so start the rest
 	for i := 1; i < s.config.HAConnections; i++ {
-		go s.startTunnel(ctx, i, make(chan struct{}))
+		ch := signal.New(make(chan struct{}))
+		go s.startTunnel(ctx, i, ch)
 		time.Sleep(registrationInterval)
 	}
 	return nil
@@ -154,45 +223,68 @@ func (s *Supervisor) initialize(ctx context.Context, connectedSignal chan struct
 
 // startTunnel starts the first tunnel connection. The resulting error will be sent on
 // s.tunnelErrors. It will send a signal via connectedSignal if registration succeed
-func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal chan struct{}) {
-	err := ServeTunnelLoop(ctx, s.config, s.getEdgeIP(0), 0, connectedSignal)
+func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *signal.Signal) {
+	var (
+		addr *net.TCPAddr
+		err  error
+	)
 	defer func() {
-		s.tunnelErrors <- tunnelError{index: 0, err: err}
+		s.tunnelErrors <- tunnelError{index: 0, addr: addr, err: err}
 	}()
 
+	addr, err = s.edgeIPs.Addr()
+	if err != nil {
+		return
+	}
+
+	err = ServeTunnelLoop(ctx, s, s.config, addr, 0, connectedSignal, s.cloudflaredUUID)
+
 	for s.unusedIPs() {
-		select {
-		case <-ctx.Done():
+		if ctx.Err() != nil {
 			return
-		default:
 		}
 		switch err.(type) {
 		case nil:
 			return
 		// try the next address if it was a dialError(network problem) or
 		// dupConnRegisterTunnelError
-		case dialError, dupConnRegisterTunnelError:
-			s.replaceEdgeIP(0)
+		case connection.DialError, dupConnRegisterTunnelError:
+			s.edgeIPs.MarkAddrBad(addr)
 		default:
 			return
 		}
-		err = ServeTunnelLoop(ctx, s.config, s.getEdgeIP(0), 0, connectedSignal)
+		addr, err = s.edgeIPs.Addr()
+		if err != nil {
+			return
+		}
+		err = ServeTunnelLoop(ctx, s, s.config, addr, 0, connectedSignal, s.cloudflaredUUID)
 	}
 }
 
 // startTunnel starts a new tunnel connection. The resulting error will be sent on
 // s.tunnelErrors.
-func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal chan struct{}) {
-	err := ServeTunnelLoop(ctx, s.config, s.getEdgeIP(index), uint8(index), connectedSignal)
-	s.tunnelErrors <- tunnelError{index: index, err: err}
+func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal *signal.Signal) {
+	var (
+		addr *net.TCPAddr
+		err  error
+	)
+	defer func() {
+		s.tunnelErrors <- tunnelError{index: index, addr: addr, err: err}
+	}()
+
+	addr, err = s.edgeIPs.Addr()
+	if err != nil {
+		return
+	}
+	err = ServeTunnelLoop(ctx, s, s.config, addr, uint8(index), connectedSignal, s.cloudflaredUUID)
 }
 
-func (s *Supervisor) newConnectedTunnelSignal(index int) chan struct{} {
-	signal := make(chan struct{})
-	s.tunnelsConnecting[index] = signal
-	s.nextConnectedSignal = signal
+func (s *Supervisor) newConnectedTunnelSignal(index int) *signal.Signal {
+	sig := make(chan struct{})
+	s.tunnelsConnecting[index] = sig
+	s.nextConnectedSignal = sig
 	s.nextConnectedIndex = index
-	return signal
+	return signal.New(sig)
 }
 
 func (s *Supervisor) waitForNextTunnel(index int) bool {
@@ -206,8 +298,8 @@ func (s *Supervisor) waitForNextTunnel(index int) bool {
 	return false
 }
 
-func (s *Supervisor) getEdgeIP(index int) *net.TCPAddr {
-	return s.edgeIPs[index%len(s.edgeIPs)]
+func (s *Supervisor) unusedIPs() bool {
+	return s.edgeIPs.AvailableAddrs() > s.config.HAConnections
 }
 
 func (s *Supervisor) refreshEdgeIPs() {
@@ -219,16 +311,125 @@ func (s *Supervisor) refreshEdgeIPs() {
 	}
 	s.resolverC = make(chan resolveResult)
 	go func() {
-		edgeIPs, err := ResolveEdgeIPs(s.config.EdgeAddrs)
-		s.resolverC <- resolveResult{edgeIPs: edgeIPs, err: err}
+		err := s.edgeIPs.Refresh()
+		s.resolverC <- resolveResult{err: err}
 	}()
 }
 
-func (s *Supervisor) unusedIPs() bool {
-	return s.nextUnusedEdgeIP < len(s.edgeIPs)
+func (s *Supervisor) ReconnectToken() ([]byte, error) {
+	s.jwtLock.RLock()
+	defer s.jwtLock.RUnlock()
+	if s.jwt == nil {
+		return nil, errJWTUnset
+	}
+	return s.jwt, nil
 }
 
-func (s *Supervisor) replaceEdgeIP(badIPIndex int) {
-	s.edgeIPs[badIPIndex] = s.edgeIPs[s.nextUnusedEdgeIP]
-	s.nextUnusedEdgeIP++
+func (s *Supervisor) SetReconnectToken(jwt []byte) {
+	s.jwtLock.Lock()
+	defer s.jwtLock.Unlock()
+	s.jwt = jwt
+}
+
+func (s *Supervisor) EventDigest() ([]byte, error) {
+	s.eventDigestLock.RLock()
+	defer s.eventDigestLock.RUnlock()
+	if s.eventDigest == nil {
+		return nil, errEventDigestUnset
+	}
+	return s.eventDigest, nil
+}
+
+func (s *Supervisor) SetEventDigest(eventDigest []byte) {
+	s.eventDigestLock.Lock()
+	defer s.eventDigestLock.Unlock()
+	s.eventDigest = eventDigest
+}
+
+func (s *Supervisor) refreshAuth(
+	ctx context.Context,
+	backoff *BackoffHandler,
+	authenticate func(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error),
+) (retryTimer <-chan time.Time, err error) {
+	logger := s.config.Logger.WithField("subsystem", subsystemRefreshAuth)
+	authOutcome, err := authenticate(ctx, backoff.Retries())
+	if err != nil {
+		s.config.Metrics.authFail.WithLabelValues(err.Error()).Inc()
+		if duration, ok := backoff.GetBackoffDuration(ctx); ok {
+			logger.WithError(err).Warnf("Retrying in %v", duration)
+			return backoff.BackoffTimer(), nil
+		}
+		return nil, err
+	}
+	// clear backoff timer
+	backoff.SetGracePeriod()
+
+	switch outcome := authOutcome.(type) {
+	case tunnelpogs.AuthSuccess:
+		s.SetReconnectToken(outcome.JWT())
+		s.config.Metrics.authSuccess.Inc()
+		return timeAfter(outcome.RefreshAfter()), nil
+	case tunnelpogs.AuthUnknown:
+		duration := outcome.RefreshAfter()
+		s.config.Metrics.authFail.WithLabelValues(outcome.Error()).Inc()
+		logger.WithError(outcome).Warnf("Retrying in %v", duration)
+		return timeAfter(duration), nil
+	case tunnelpogs.AuthFail:
+		s.config.Metrics.authFail.WithLabelValues(outcome.Error()).Inc()
+		return nil, outcome
+	default:
+		err := fmt.Errorf("Unexpected outcome type %T", authOutcome)
+		s.config.Metrics.authFail.WithLabelValues(err.Error()).Inc()
+		return nil, err
+	}
+}
+
+func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error) {
+	arbitraryEdgeIP, err := s.edgeIPs.AnyAddr()
+	if err != nil {
+		return nil, err
+	}
+
+	edgeConn, err := connection.DialEdge(ctx, dialTimeout, s.config.TlsConfig, arbitraryEdgeIP)
+	if err != nil {
+		return nil, err
+	}
+	defer edgeConn.Close()
+
+	handler := h2mux.MuxedStreamFunc(func(*h2mux.MuxedStream) error {
+		// This callback is invoked by h2mux when the edge initiates a stream.
+		return nil // noop
+	})
+	muxerConfig := s.config.muxerConfig(handler)
+	muxerConfig.Logger = muxerConfig.Logger.WithField("subsystem", subsystemRefreshAuth)
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, muxerConfig, s.config.Metrics.activeStreams)
+	if err != nil {
+		return nil, err
+	}
+	go muxer.Serve(ctx)
+	defer func() {
+		// If we don't wait for the muxer shutdown here, edgeConn.Close() runs before the muxer connections are done,
+		// and the user sees log noise: "error writing data", "connection closed unexpectedly"
+		<-muxer.Shutdown()
+	}()
+
+	tunnelServer, err := connection.NewRPCClient(ctx, muxer, s.logger.WithField("subsystem", subsystemRefreshAuth), openStreamTimeout)
+	if err != nil {
+		return nil, err
+	}
+	defer tunnelServer.Close()
+
+	const arbitraryConnectionID = uint8(0)
+	registrationOptions := s.config.RegistrationOptions(arbitraryConnectionID, edgeConn.LocalAddr().String(), s.cloudflaredUUID)
+	registrationOptions.NumPreviousAttempts = uint8(numPreviousAttempts)
+	authResponse, err := tunnelServer.Authenticate(
+		ctx,
+		s.config.OriginCert,
+		s.config.Hostname,
+		registrationOptions,
+	)
+	if err != nil {
+		return nil, err
+	}
+	return authResponse.Outcome(), nil
 }

origin/supervisor_test.go

@@ -0,0 +1,165 @@
+package origin
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+func testConfig(logger *logrus.Logger) *TunnelConfig {
+	metrics := TunnelMetrics{}
+
+	metrics.authSuccess = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_success",
+			Help:      "Count of successful tunnel authenticate",
+		},
+	)
+
+	metrics.authFail = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_fail",
+			Help:      "Count of tunnel authenticate errors by type",
+		},
+		[]string{"error"},
+	)
+	return &TunnelConfig{Logger: logger, Metrics: &metrics}
+}
+
+func TestRefreshAuthBackoff(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	var wait time.Duration
+	timeAfter = func(d time.Duration) <-chan time.Time {
+		wait = d
+		return time.After(d)
+	}
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return nil, fmt.Errorf("authentication failure")
+	}
+
+	// authentication failures should consume the backoff
+	for i := uint(0); i < backoff.MaxRetries; i++ {
+		retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+		assert.NoError(t, err)
+		assert.NotNil(t, retryChan)
+		assert.Equal(t, (1<<i)*time.Second, wait)
+	}
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.Error(t, err)
+	assert.Nil(t, retryChan)
+
+	// now we actually make contact with the remote server
+	_, _ = s.refreshAuth(context.Background(), backoff, func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
+	})
+
+	// The backoff timer should have been reset. To confirm this, make timeNow
+	// return a value after the backoff timer's grace period
+	timeNow = func() time.Time {
+		expectedGracePeriod := time.Duration(time.Second * 2 << backoff.MaxRetries)
+		return time.Now().Add(expectedGracePeriod * 2)
+	}
+	_, ok := backoff.GetBackoffDuration(context.Background())
+	assert.True(t, ok)
+}
+
+func TestRefreshAuthSuccess(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	var wait time.Duration
+	timeAfter = func(d time.Duration) <-chan time.Time {
+		wait = d
+		return time.After(d)
+	}
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthSuccess([]byte("jwt"), 19), nil
+	}
+
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.NoError(t, err)
+	assert.NotNil(t, retryChan)
+	assert.Equal(t, 19*time.Hour, wait)
+
+	token, err := s.ReconnectToken()
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("jwt"), token)
+}
+
+func TestRefreshAuthUnknown(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	var wait time.Duration
+	timeAfter = func(d time.Duration) <-chan time.Time {
+		wait = d
+		return time.After(d)
+	}
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
+	}
+
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.NoError(t, err)
+	assert.NotNil(t, retryChan)
+	assert.Equal(t, 19*time.Hour, wait)
+
+	token, err := s.ReconnectToken()
+	assert.Equal(t, errJWTUnset, err)
+	assert.Nil(t, token)
+}
+
+func TestRefreshAuthFail(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthFail(errors.New("auth fail")), nil
+	}
+
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.Error(t, err)
+	assert.Nil(t, retryChan)
+
+	token, err := s.ReconnectToken()
+	assert.Equal(t, errJWTUnset, err)
+	assert.Nil(t, token)
+}