Release 2020.5.1

Previously it was only being set after RegisterTunnel.

.gitignore

@@ -9,4 +9,8 @@ guide/public
 \#*\#
 cscope.*
 cloudflared
-!cmd/cloudflared/
+cloudflared.exe
+!cmd/cloudflared/
+.DS_Store
+*-session.log
+ssh_server_tests/.env

Dockerfile

@@ -1,12 +1,28 @@
-FROM golang:1.12 as builder
-WORKDIR /go/src/github.com/cloudflare/cloudflared/
-RUN apt-get update && apt-get install -y --no-install-recommends upx
-# Run after `apt-get update` to improve rebuild scenarios
-COPY . .
-RUN make cloudflared
-RUN upx --no-progress cloudflared
+# use a builder image for building cloudflare
+FROM golang:1.13.3 as builder
+ENV GO111MODULE=on
+ENV CGO_ENABLED=0
+ENV GOOS=linux
 
-FROM gcr.io/distroless/base
-COPY --from=builder /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+WORKDIR /go/src/github.com/cloudflare/cloudflared/
+
+# copy our sources into the builder image
+COPY . .
+
+# compile cloudflared
+RUN make cloudflared
+
+# ---
+
+# use a distroless base image with glibc
+FROM gcr.io/distroless/base-debian10:nonroot
+
+# copy our compiled binary
+COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+
+# run as non-privileged user
+USER nonroot
+
+# command / entrypoint of container
 ENTRYPOINT ["cloudflared", "--no-autoupdate"]
 CMD ["version"]

Gopkg.lock

@@ -1,788 +0,0 @@
-# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'.
-
-
-[[projects]]
-  digest = "1:3d3a509c5ba327e8573bb57f9da8430c63a46a06886eb1d2ffc8af4e76f31c72"
-  name = "cloud.google.com/go"
-  packages = ["civil"]
-  pruneopts = "UT"
-  revision = "cdaaf98f9226c39dc162b8e55083b2fbc67b4674"
-  version = "v0.43.0"
-
-[[projects]]
-  digest = "1:9f3b30d9f8e0d7040f729b82dcbc8f0dead820a133b3147ce355fc451f32d761"
-  name = "github.com/BurntSushi/toml"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005"
-  version = "v0.3.1"
-
-[[projects]]
-  digest = "1:c84a587136cb69cecc11f3dbe9f9001444044c0dba74997b07f7e4c150b07cda"
-  name = "github.com/DATA-DOG/go-sqlmock"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "3f9954f6f6697845b082ca57995849ddf614f450"
-  version = "v1.3.3"
-
-[[projects]]
-  digest = "1:d6afaeed1502aa28e80a4ed0981d570ad91b2579193404256ce672ed0a609e0d"
-  name = "github.com/beorn7/perks"
-  packages = ["quantile"]
-  pruneopts = "UT"
-  revision = "4b2b341e8d7715fae06375aa633dbb6e91b3fb46"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:fed1f537c2f1269fe475a8556c393fe466641682d73ef8fd0491cd3aa1e47bad"
-  name = "github.com/certifi/gocertifi"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "deb3ae2ef2610fde3330947281941c562861188b"
-  version = "2018.01.18"
-
-[[projects]]
-  digest = "1:e5003c19d396d8b3cf1324ea0bf49b00f13e9466d0297d1268b641f1c617c3a2"
-  name = "github.com/cloudflare/brotli-go"
-  packages = ["."]
-  pruneopts = "T"
-  revision = "18c9f6c67e3dfc12e0ddaca748d2887f97a7ac28"
-
-[[projects]]
-  digest = "1:6dbb2bbc7e6333e691c4d82fd86485f0695a35902fbb9b2df5f72e22ab0040f3"
-  name = "github.com/cloudflare/golibs"
-  packages = ["lrucache"]
-  pruneopts = "UT"
-  revision = "333127dbecfcc23a8db7d9a4f52785d23aff44a1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:05756b73bf7eae03d6f6c5f0ae83de42ce296e7d83336563dd9ce2072106d5e6"
-  name = "github.com/cloudflare/golz4"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "ef862a3cdc58a6f1fee4e3af3d44fbe279194cde"
-
-[[projects]]
-  digest = "1:3f9506ee991cdee1f05bf0cd3e34b5cd922dc00d6a950fb4beb4e07ab1c4d3d1"
-  name = "github.com/coredns/coredns"
-  packages = [
-    "core/dnsserver",
-    "coremain",
-    "pb",
-    "plugin",
-    "plugin/cache",
-    "plugin/cache/freq",
-    "plugin/etcd/msg",
-    "plugin/metrics",
-    "plugin/metrics/vars",
-    "plugin/pkg/cache",
-    "plugin/pkg/dnstest",
-    "plugin/pkg/dnsutil",
-    "plugin/pkg/doh",
-    "plugin/pkg/edns",
-    "plugin/pkg/fuzz",
-    "plugin/pkg/log",
-    "plugin/pkg/nonwriter",
-    "plugin/pkg/rcode",
-    "plugin/pkg/response",
-    "plugin/pkg/trace",
-    "plugin/pkg/uniq",
-    "plugin/pkg/watch",
-    "plugin/test",
-    "request",
-  ]
-  pruneopts = "UT"
-  revision = "2e322f6e8a54f18c6aef9c25a7c432c291a3d9f7"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "v2"
-  digest = "1:ca6ce07b0d28c6044411b9c966cd845233e27c2ff91d08a3a869631f329c9918"
-  name = "github.com/coreos/go-oidc"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "274971e2c94cd6fb17614979e2899edbc81af146"
-
-[[projects]]
-  digest = "1:1da3a221f0bc090792d3a2a080ff09008427c0e0f0533a4ed6abd8994421da73"
-  name = "github.com/coreos/go-systemd"
-  packages = ["daemon"]
-  pruneopts = "UT"
-  revision = "95778dfbb74eb7e4dbaf43bf7d71809650ef8076"
-  version = "v19"
-
-[[projects]]
-  digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec"
-  name = "github.com/davecgh/go-spew"
-  packages = ["spew"]
-  pruneopts = "UT"
-  revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"
-  version = "v1.1.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:27cbe99893a40975358d20736d80adc95ef395d3bf5923f802f3de259533c94a"
-  name = "github.com/denisenkom/go-mssqldb"
-  packages = [
-    ".",
-    "internal/cp",
-    "internal/querytext",
-  ]
-  pruneopts = "UT"
-  revision = "11b2859924c1e3c3da056fd4e0976b93883f3545"
-
-[[projects]]
-  digest = "1:d4268b2a09b1f736633577c4ac93f2a5356c73742fff5344e2451aeec60a7ad0"
-  name = "github.com/equinox-io/equinox"
-  packages = [
-    ".",
-    "internal/go-update",
-    "internal/go-update/internal/binarydist",
-    "internal/go-update/internal/osext",
-    "internal/osext",
-    "proto",
-  ]
-  pruneopts = "UT"
-  revision = "5205c98a6c11dc72747ce12fff6cd620a99fde05"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:433763f10d88dba9b533a7ea2fe9f5ee11e57e00306eb97a1f6090fd978e8fa1"
-  name = "github.com/facebookgo/grace"
-  packages = ["gracenet"]
-  pruneopts = "UT"
-  revision = "75cf19382434e82df4dd84953f566b8ad23d6e9e"
-
-[[projects]]
-  digest = "1:296cda2c4a6a7a54964d7b3b0815e2aed215da75217b72033060ffb1c4e4b6ae"
-  name = "github.com/felixge/httpsnoop"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "eadd4fad6aac69ae62379194fe0219f3dbc80fd3"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:50a46ab1d5edbbdd55125b4d37f1bf503d0807c26461f9ad7b358d6006641d09"
-  name = "github.com/flynn/go-shlex"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "3f9db97f856818214da2e1057f8ad84803971cff"
-
-[[projects]]
-  digest = "1:d4623fc7bf7e281d9107367cc4a9e76ed3e86b1eec1a4e30630c870bef1fedd0"
-  name = "github.com/getsentry/raven-go"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "ed7bcb39ff10f39ab08e317ce16df282845852fa"
-
-[[projects]]
-  digest = "1:ec6f9bf5e274c833c911923c9193867f3f18788c461f76f05f62bb1510e0ae65"
-  name = "github.com/go-sql-driver/mysql"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "72cd26f257d44c1114970e19afddcd812016007e"
-  version = "v1.4.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:3e6afc3ed8a72949aa735c00fddc23427dc9384ccfd51cf0d91a412e668da632"
-  name = "github.com/golang-collections/collections"
-  packages = ["queue"]
-  pruneopts = "UT"
-  revision = "604e922904d35e97f98a774db7881f049cd8d970"
-
-[[projects]]
-  branch = "master"
-  digest = "1:01745416ed734d0dbd8f2a25344536e399ee60319654873da12b107e9e3cb309"
-  name = "github.com/golang/gddo"
-  packages = ["httputil/header"]
-  pruneopts = "UT"
-  revision = "af0f2af80721261f4d211b2c9563f7b46b2aab06"
-
-[[projects]]
-  digest = "1:239c4c7fd2159585454003d9be7207167970194216193a8a210b8d29576f19c9"
-  name = "github.com/golang/protobuf"
-  packages = [
-    "proto",
-    "ptypes",
-    "ptypes/any",
-    "ptypes/duration",
-    "ptypes/timestamp",
-  ]
-  pruneopts = "UT"
-  revision = "b5d812f8a3706043e23a9cd5babf2e5423744d30"
-  version = "v1.3.1"
-
-[[projects]]
-  digest = "1:582b704bebaa06b48c29b0cec224a6058a09c86883aaddabde889cd1a5f73e1b"
-  name = "github.com/google/uuid"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "0cd6bf5da1e1c83f8b45653022c74f71af0538a4"
-  version = "v1.1.1"
-
-[[projects]]
-  digest = "1:d5f97fc268267ec1b61c3453058c738246fc3e746f14b1ae25161513b7367b0c"
-  name = "github.com/gorilla/mux"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "c5c6c98bc25355028a63748a498942a6398ccd22"
-  version = "v1.7.1"
-
-[[projects]]
-  digest = "1:43dd08a10854b2056e615d1b1d22ac94559d822e1f8b6fcc92c1a1057e85188e"
-  name = "github.com/gorilla/websocket"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "ea4d1f681babbce9545c9c5f3d5194a789c89f5b"
-  version = "v1.2.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:1a1206efd03a54d336dce7bb8719e74f2f8932f661cb9f57d5813a1d99c083d8"
-  name = "github.com/grpc-ecosystem/grpc-opentracing"
-  packages = ["go/otgrpc"]
-  pruneopts = "UT"
-  revision = "8e809c8a86450a29b90dcc9efbf062d0fe6d9746"
-
-[[projects]]
-  digest = "1:6c41d4f998a03b6604227ccad36edaed6126c397e5d78709ef4814a1145a6757"
-  name = "github.com/jmoiron/sqlx"
-  packages = [
-    ".",
-    "reflectx",
-  ]
-  pruneopts = "UT"
-  revision = "d161d7a76b5661016ad0b085869f77fd410f3e6a"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:31e761d97c76151dde79e9d28964a812c46efc5baee4085b86f68f0c654450de"
-  name = "github.com/konsorten/go-windows-terminal-sequences"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "f55edac94c9bbba5d6182a4be46d86a2c9b5b50e"
-  version = "v1.0.2"
-
-[[projects]]
-  digest = "1:177a18252d40bf39a867876456854717780b0b1212fdae231d31e93b6f885d80"
-  name = "github.com/kshvakov/clickhouse"
-  packages = [
-    ".",
-    "lib/binary",
-    "lib/cityhash102",
-    "lib/column",
-    "lib/data",
-    "lib/leakypool",
-    "lib/lz4",
-    "lib/protocol",
-    "lib/types",
-    "lib/writebuffer",
-  ]
-  pruneopts = "UT"
-  revision = "43e176a8d165376662f4d67203c71cc8a027c86b"
-  version = "v1.3.9"
-
-[[projects]]
-  digest = "1:12cb143f2148bf54bcd9fe622abac17325e85eeb1d84b8ec6caf1c80232108fd"
-  name = "github.com/lib/pq"
-  packages = [
-    ".",
-    "oid",
-    "scram",
-  ]
-  pruneopts = "UT"
-  revision = "3427c32cb71afc948325f299f040e53c1dd78979"
-  version = "v1.2.0"
-
-[[projects]]
-  digest = "1:2fa7b0155cd54479a755c629de26f888a918e13f8857a2c442205d825368e084"
-  name = "github.com/mattn/go-colorable"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "3a70a971f94a22f2fa562ffcc7a0eb45f5daf045"
-  version = "v0.1.1"
-
-[[projects]]
-  digest = "1:e150b5fafbd7607e2d638e4e5cf43aa4100124e5593385147b0a74e2733d8b0d"
-  name = "github.com/mattn/go-isatty"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "c2a7a6ca930a4cd0bc33a3f298eb71960732a3a7"
-  version = "v0.0.7"
-
-[[projects]]
-  digest = "1:79e87abf06b873987dee86598950f5b51732ac454d5a5cab6445a14330e6c9e3"
-  name = "github.com/mattn/go-sqlite3"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "b612a2feea6aa87c6d052d9086572551df06497e"
-  version = "v1.11.0"
-
-[[projects]]
-  digest = "1:ff5ebae34cfbf047d505ee150de27e60570e8c394b3b8fdbb720ff6ac71985fc"
-  name = "github.com/matttproud/golang_protobuf_extensions"
-  packages = ["pbutil"]
-  pruneopts = "UT"
-  revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c"
-  version = "v1.0.1"
-
-[[projects]]
-  digest = "1:75fa16a231ef40da3e462d651c20b9df20bde0777bdc1ac0982242c79057ee71"
-  name = "github.com/mholt/caddy"
-  packages = [
-    ".",
-    "caddyfile",
-    "telemetry",
-  ]
-  pruneopts = "UT"
-  revision = "d3b731e9255b72d4571a5aac125634cf1b6031dc"
-
-[[projects]]
-  digest = "1:2b4b4b2e5544c2a11a486c1b631357aa2ddf766e50c1b2483cf809da2c511234"
-  name = "github.com/miekg/dns"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "73601d4aed9d844322611759d7f3619110b7c88e"
-  version = "v1.1.8"
-
-[[projects]]
-  digest = "1:5d231480e1c64a726869bc4142d270184c419749d34f167646baa21008eb0a79"
-  name = "github.com/mitchellh/go-homedir"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "af06845cf3004701891bf4fdb884bfe4920b3727"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:dd3d50b46f8d3c88a8d3986f2e5344db582c457469a7cb3a0dda953c669cdf9a"
-  name = "github.com/mitchellh/go-server-timing"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "1ca01db910d57d64ec1c68f27105614b77558e28"
-  version = "v1.0.0"
-
-[[projects]]
-  digest = "1:53bc4cd4914cd7cd52139990d5170d6dc99067ae31c56530621b18b35fc30318"
-  name = "github.com/mitchellh/mapstructure"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "3536a929edddb9a5b34bd6861dc4a9647cb459fe"
-  version = "v1.1.2"
-
-[[projects]]
-  digest = "1:11e62d6050198055e6cd87ed57e5d8c669e84f839c16e16f192374d913d1a70d"
-  name = "github.com/opentracing/opentracing-go"
-  packages = [
-    ".",
-    "ext",
-    "log",
-  ]
-  pruneopts = "UT"
-  revision = "659c90643e714681897ec2521c60567dd21da733"
-  version = "v1.1.0"
-
-[[projects]]
-  digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747"
-  name = "github.com/pkg/errors"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
-  version = "v0.8.0"
-
-[[projects]]
-  digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe"
-  name = "github.com/pmezard/go-difflib"
-  packages = ["difflib"]
-  pruneopts = "UT"
-  revision = "792786c7400a136282c1664665ae0a8db921c6c2"
-  version = "v1.0.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:bd9efe4e0b0f768302a1e2f0c22458149278de533e521206e5ddc71848c269a0"
-  name = "github.com/pquerna/cachecontrol"
-  packages = [
-    ".",
-    "cacheobject",
-  ]
-  pruneopts = "UT"
-  revision = "1555304b9b35fdd2b425bccf1a5613677705e7d0"
-
-[[projects]]
-  digest = "1:c968b29db5d68ec97de404b6d058d5937fa015a141b3b4f7a0d87d5f8226f04c"
-  name = "github.com/prometheus/client_golang"
-  packages = [
-    "prometheus",
-    "prometheus/promhttp",
-  ]
-  pruneopts = "UT"
-  revision = "967789050ba94deca04a5e84cce8ad472ce313c1"
-  version = "v0.9.0-pre1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:2d5cd61daa5565187e1d96bae64dbbc6080dacf741448e9629c64fd93203b0d4"
-  name = "github.com/prometheus/client_model"
-  packages = ["go"]
-  pruneopts = "UT"
-  revision = "fd36f4220a901265f90734c3183c5f0c91daa0b8"
-
-[[projects]]
-  digest = "1:35cf6bdf68db765988baa9c4f10cc5d7dda1126a54bd62e252dbcd0b1fc8da90"
-  name = "github.com/prometheus/common"
-  packages = [
-    "expfmt",
-    "internal/bitbucket.org/ww/goautoneg",
-    "model",
-  ]
-  pruneopts = "UT"
-  revision = "a82f4c12f983cc2649298185f296632953e50d3e"
-  version = "v0.3.0"
-
-[[projects]]
-  branch = "master"
-  digest = "1:49b09905e781d7775c086604cc00083e1832d0783f1f421b79f42657c457d029"
-  name = "github.com/prometheus/procfs"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "8368d24ba045f26503eb745b624d930cbe214c79"
-
-[[projects]]
-  digest = "1:1a23fdd843129ef761ffe7651bc5fe7c5b09fbe933e92783ab06cc11c37b7b37"
-  name = "github.com/rifflock/lfshook"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "b9218ef580f59a2e72dad1aa33d660150445d05a"
-  version = "v2.4"
-
-[[projects]]
-  digest = "1:04457f9f6f3ffc5fea48e71d62f2ca256637dee0a04d710288e27e05c8b41976"
-  name = "github.com/sirupsen/logrus"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "839c75faf7f98a33d445d181f3018b5c3409a45e"
-  version = "v1.4.2"
-
-[[projects]]
-  digest = "1:7e8d267900c7fa7f35129a2a37596e38ed0f11ca746d6d9ba727980ee138f9f6"
-  name = "github.com/stretchr/testify"
-  packages = [
-    "assert",
-    "require",
-  ]
-  pruneopts = "UT"
-  revision = "12b6f73e6084dad08a7c6e575284b177ecafbc71"
-  version = "v1.2.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:cac263b8eb2e9ad2327b70e03be1de1227040bfb3be26b49c4b56cbed568afc7"
-  name = "github.com/xo/dburl"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "98997a05b24fc069c79fa13b05ccf376c22a326c"
-
-[[projects]]
-  branch = "master"
-  digest = "1:1a9391a80fe548ffcf90aee8ae35bfb0b188198b57a3fe2ef5bb099ba4b4f610"
-  name = "golang.org/x/crypto"
-  packages = [
-    "curve25519",
-    "ed25519",
-    "ed25519/internal/edwards25519",
-    "internal/chacha20",
-    "internal/subtle",
-    "md4",
-    "nacl/box",
-    "nacl/secretbox",
-    "pbkdf2",
-    "poly1305",
-    "salsa20/salsa",
-    "ssh",
-    "ssh/terminal",
-  ]
-  pruneopts = "UT"
-  revision = "f416ebab96af27ca70b6e5c23d6a0747530da626"
-
-[[projects]]
-  branch = "master"
-  digest = "1:6b3dbfffba73ea0d8131ecb5738fe87f6addc4c7b73154b9d58e2693ad6a6256"
-  name = "golang.org/x/net"
-  packages = [
-    "bpf",
-    "context",
-    "context/ctxhttp",
-    "http/httpguts",
-    "http2",
-    "http2/hpack",
-    "idna",
-    "internal/iana",
-    "internal/socket",
-    "internal/timeseries",
-    "ipv4",
-    "ipv6",
-    "trace",
-    "websocket",
-  ]
-  pruneopts = "UT"
-  revision = "1da14a5a36f220ea3f03470682b737b1dfd5de22"
-
-[[projects]]
-  branch = "master"
-  digest = "1:8d1c112fb1679fa097e9a9255a786ee47383fa2549a3da71bcb1334a693ebcfe"
-  name = "golang.org/x/oauth2"
-  packages = [
-    ".",
-    "internal",
-  ]
-  pruneopts = "UT"
-  revision = "0f29369cfe4552d0e4bcddc57cc75f4d7e672a33"
-
-[[projects]]
-  digest = "1:39ebcc2b11457b703ae9ee2e8cca0f68df21969c6102cb3b705f76cca0ea0239"
-  name = "golang.org/x/sync"
-  packages = ["errgroup"]
-  pruneopts = "UT"
-  revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
-
-[[projects]]
-  branch = "master"
-  digest = "1:77751d02e939d7078faedaeec10c09af575a09c528d84d18f2cb45a84bd1889a"
-  name = "golang.org/x/sys"
-  packages = [
-    "cpu",
-    "unix",
-    "windows",
-    "windows/registry",
-    "windows/svc",
-    "windows/svc/eventlog",
-    "windows/svc/mgr",
-  ]
-  pruneopts = "UT"
-  revision = "12500544f89f9420afe9529ba8940bf72d294972"
-
-[[projects]]
-  digest = "1:a2ab62866c75542dd18d2b069fec854577a20211d7c0ea6ae746072a1dccdd18"
-  name = "golang.org/x/text"
-  packages = [
-    "collate",
-    "collate/build",
-    "internal/colltab",
-    "internal/gen",
-    "internal/tag",
-    "internal/triegen",
-    "internal/ucd",
-    "language",
-    "secure/bidirule",
-    "transform",
-    "unicode/bidi",
-    "unicode/cldr",
-    "unicode/norm",
-    "unicode/rangetable",
-  ]
-  pruneopts = "UT"
-  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
-  version = "v0.3.0"
-
-[[projects]]
-  digest = "1:7e8b9c5ae49011b12ae8473834ac1a7bb8ac029ba201270c723e4c280c9e4855"
-  name = "google.golang.org/appengine"
-  packages = [
-    "cloudsql",
-    "internal",
-    "internal/base",
-    "internal/datastore",
-    "internal/log",
-    "internal/remote_api",
-    "internal/urlfetch",
-    "urlfetch",
-  ]
-  pruneopts = "UT"
-  revision = "b2f4a3cf3c67576a2ee09e1fe62656a5086ce880"
-  version = "v1.6.1"
-
-[[projects]]
-  branch = "master"
-  digest = "1:c3076e7defee87de1236f1814beb588f40a75544c60121e6eb38b3b3721783e2"
-  name = "google.golang.org/genproto"
-  packages = ["googleapis/rpc/status"]
-  pruneopts = "UT"
-  revision = "d1146b9035b912113a38af3b138eb2af567b2c67"
-
-[[projects]]
-  digest = "1:31d87f39886fb38a2b6c097ff3b9f985d6960772170d64a68246f7790e955746"
-  name = "google.golang.org/grpc"
-  packages = [
-    ".",
-    "balancer",
-    "balancer/base",
-    "balancer/roundrobin",
-    "binarylog/grpc_binarylog_v1",
-    "codes",
-    "connectivity",
-    "credentials",
-    "credentials/internal",
-    "encoding",
-    "encoding/proto",
-    "grpclog",
-    "internal",
-    "internal/backoff",
-    "internal/balancerload",
-    "internal/binarylog",
-    "internal/channelz",
-    "internal/envconfig",
-    "internal/grpcrand",
-    "internal/grpcsync",
-    "internal/syscall",
-    "internal/transport",
-    "keepalive",
-    "metadata",
-    "naming",
-    "peer",
-    "resolver",
-    "resolver/dns",
-    "resolver/passthrough",
-    "stats",
-    "status",
-    "tap",
-  ]
-  pruneopts = "UT"
-  revision = "236199dd5f8031d698fb64091194aecd1c3895b2"
-  version = "v1.20.0"
-
-[[projects]]
-  branch = "v1"
-  digest = "1:6f70106e7bc1c803e8a0a4519e09c12d154771acfa2559206e97b033bbd1dd38"
-  name = "gopkg.in/coreos/go-oidc.v1"
-  packages = ["jose"]
-  pruneopts = "UT"
-  revision = "e860bd55bfa7d7cb35d30d26a167982584f616b0"
-
-[[projects]]
-  digest = "1:8c05919580be8a5be668709d7e5a69d5cd19b8ee9f23d62ce5b10d3457bf6a13"
-  name = "gopkg.in/square/go-jose.v2"
-  packages = [
-    ".",
-    "cipher",
-    "json",
-  ]
-  pruneopts = "UT"
-  revision = "730df5f748271903322feb182be83b43ebbbe27d"
-  version = "v2.3.1"
-
-[[projects]]
-  branch = "altsrc-parse-durations"
-  digest = "1:0370b1bceda03dbfade3abbde639a43f1113bab711ec760452e5c0dcc0c14787"
-  name = "gopkg.in/urfave/cli.v2"
-  packages = [
-    ".",
-    "altsrc",
-  ]
-  pruneopts = "UT"
-  revision = "d604b6ffeee878fbf084fd2761466b6649989cee"
-  source = "https://github.com/cbranch/cli"
-
-[[projects]]
-  digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96"
-  name = "gopkg.in/yaml.v2"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "51d6538a90f86fe93ac480b35f37b2be17fef232"
-  version = "v2.2.2"
-
-[[projects]]
-  digest = "1:8ffc3ddc31414c0a71220957bb723b16510d7fcb5b3880dc0da4cf6d39c31642"
-  name = "zombiezen.com/go/capnproto2"
-  packages = [
-    ".",
-    "encoding/text",
-    "internal/fulfiller",
-    "internal/nodemap",
-    "internal/packed",
-    "internal/queue",
-    "internal/schema",
-    "internal/strquote",
-    "pogs",
-    "rpc",
-    "rpc/internal/refcount",
-    "schemas",
-    "server",
-    "std/capnp/rpc",
-  ]
-  pruneopts = "UT"
-  revision = "7cfd211c19c7f5783c695f3654efa46f0df259c3"
-  source = "https://github.com/zombiezen/go-capnproto2"
-  version = "v2.17.1"
-
-[solve-meta]
-  analyzer-name = "dep"
-  analyzer-version = 1
-  input-imports = [
-    "github.com/DATA-DOG/go-sqlmock",
-    "github.com/cloudflare/brotli-go",
-    "github.com/cloudflare/golibs/lrucache",
-    "github.com/coredns/coredns/core/dnsserver",
-    "github.com/coredns/coredns/plugin",
-    "github.com/coredns/coredns/plugin/cache",
-    "github.com/coredns/coredns/plugin/metrics/vars",
-    "github.com/coredns/coredns/plugin/pkg/dnstest",
-    "github.com/coredns/coredns/plugin/pkg/rcode",
-    "github.com/coredns/coredns/request",
-    "github.com/coreos/go-oidc",
-    "github.com/coreos/go-systemd/daemon",
-    "github.com/denisenkom/go-mssqldb",
-    "github.com/equinox-io/equinox",
-    "github.com/facebookgo/grace/gracenet",
-    "github.com/getsentry/raven-go",
-    "github.com/go-sql-driver/mysql",
-    "github.com/golang-collections/collections/queue",
-    "github.com/google/uuid",
-    "github.com/gorilla/mux",
-    "github.com/gorilla/websocket",
-    "github.com/jmoiron/sqlx",
-    "github.com/kshvakov/clickhouse",
-    "github.com/lib/pq",
-    "github.com/mattn/go-colorable",
-    "github.com/mattn/go-sqlite3",
-    "github.com/miekg/dns",
-    "github.com/mitchellh/go-homedir",
-    "github.com/mitchellh/go-server-timing",
-    "github.com/mitchellh/mapstructure",
-    "github.com/pkg/errors",
-    "github.com/prometheus/client_golang/prometheus",
-    "github.com/prometheus/client_golang/prometheus/promhttp",
-    "github.com/rifflock/lfshook",
-    "github.com/sirupsen/logrus",
-    "github.com/stretchr/testify/assert",
-    "github.com/stretchr/testify/require",
-    "github.com/xo/dburl",
-    "golang.org/x/crypto/nacl/box",
-    "golang.org/x/crypto/ssh",
-    "golang.org/x/crypto/ssh/terminal",
-    "golang.org/x/net/context",
-    "golang.org/x/net/http2",
-    "golang.org/x/net/http2/hpack",
-    "golang.org/x/net/idna",
-    "golang.org/x/net/trace",
-    "golang.org/x/net/websocket",
-    "golang.org/x/sync/errgroup",
-    "golang.org/x/sys/windows",
-    "golang.org/x/sys/windows/svc",
-    "golang.org/x/sys/windows/svc/eventlog",
-    "golang.org/x/sys/windows/svc/mgr",
-    "gopkg.in/coreos/go-oidc.v1/jose",
-    "gopkg.in/urfave/cli.v2",
-    "gopkg.in/urfave/cli.v2/altsrc",
-    "zombiezen.com/go/capnproto2",
-    "zombiezen.com/go/capnproto2/encoding/text",
-    "zombiezen.com/go/capnproto2/pogs",
-    "zombiezen.com/go/capnproto2/rpc",
-    "zombiezen.com/go/capnproto2/schemas",
-    "zombiezen.com/go/capnproto2/server",
-    "zombiezen.com/go/capnproto2/std/capnp/rpc",
-  ]
-  solver-name = "gps-cdcl"
-  solver-version = 1

Gopkg.toml

@@ -1,117 +0,0 @@
-[prune]
-  go-tests = true
-  unused-packages = true
-
-  [[prune.project]]
-    name = "github.com/cloudflare/brotli-go"
-    unused-packages = false
-
-[[constraint]]
-  name = "github.com/facebookgo/grace"
-  revision = "75cf19382434e82df4dd84953f566b8ad23d6e9e"
-
-[[constraint]]
-  name = "github.com/getsentry/raven-go"
-  revision = "ed7bcb39ff10f39ab08e317ce16df282845852fa"
-
-[[constraint]]
-  name = "github.com/pkg/errors"
-  version = "=0.8.0"
-
-[[constraint]]
-  name = "github.com/prometheus/client_golang"
-  version = "=0.9.0-pre1"
-
-[[constraint]]
-  name = "github.com/sirupsen/logrus"
-  version = "=1.4.2"
-
-[[constraint]]
-  name = "github.com/stretchr/testify"
-  version = "=1.2.1"
-
-[[constraint]]
-  name = "golang.org/x/net"
-  branch = "master"  # master required by github.com/miekg/dns
-
-[[constraint]]
-  name = "golang.org/x/sync"
-  revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca"
-
-[[constraint]]
-  name = "gopkg.in/urfave/cli.v2"
-  source = "https://github.com/cbranch/cli"
-  branch = "altsrc-parse-durations"
-
-[[constraint]]
-  name = "zombiezen.com/go/capnproto2"
-  source = "https://github.com/zombiezen/go-capnproto2"
-  version = "=2.17.1"
-
-[[constraint]]
-  name = "github.com/gorilla/websocket"
-  version = "=1.2.0"
-
-[[constraint]]
-  name = "github.com/coredns/coredns"
-  version = "=1.2.0"
-
-[[constraint]]
-  name = "github.com/miekg/dns"
-  version = "=1.1.8"
-
-[[constraint]]
-  name = "github.com/cloudflare/brotli-go"
-  revision = "18c9f6c67e3dfc12e0ddaca748d2887f97a7ac28"
-
-[[override]]
-  name = "github.com/mholt/caddy"
-  revision = "d3b731e9255b72d4571a5aac125634cf1b6031dc"
-
-[[constraint]]
-  name = "golang.org/x/crypto"
-  branch = "master"  # master required by github.com/miekg/dns
-
-[[constraint]]
-  name = "github.com/cloudflare/golibs"
-  revision = "333127dbecfcc23a8db7d9a4f52785d23aff44a1"
-
-[[constraint]]
-  name = "github.com/google/uuid"
-  version = "=1.1.1"
-
-[[constraint]]
-  name = "github.com/mitchellh/mapstructure"
-  version = "1.1.2"
-
-[[constraint]]
-  name = "github.com/jmoiron/sqlx"
-  version = "1.2.0"
-
-[[constraint]]
-  name = "github.com/go-sql-driver/mysql"
-  version = "1.4.1"
-
-[[constraint]]
-  name = "github.com/mattn/go-sqlite3"
-  version = "1.11.0"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/denisenkom/go-mssqldb"
-
-[[constraint]]
-  name = "github.com/kshvakov/clickhouse"
-  version = "1.3.9"
-
-[[constraint]]
-  name = "github.com/DATA-DOG/go-sqlmock"
-  version = "1.3.3"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/xo/dburl"
-
-[[constraint]]
-  branch = "v2"
-  name = "github.com/coreos/go-oidc"

Makefile

@@ -29,7 +29,7 @@ clean:
 
 .PHONY: cloudflared
 cloudflared: tunnel-deps
-	go build -v $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
+	go build -v -mod=vendor $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared
 
 .PHONY: container
 container:
@@ -37,7 +37,11 @@ container:
 
 .PHONY: test
 test: vet
-	go test -v -race $(VERSION_FLAGS) ./...
+	go test -v -mod=vendor -race $(VERSION_FLAGS) ./...
+
+.PHONY: test-ssh-server
+test-ssh-server:
+	docker-compose -f ssh_server_tests/docker-compose.yml up
 
 .PHONY: cloudflared-deb
 cloudflared-deb: cloudflared
@@ -78,6 +82,6 @@ tunnelrpc/tunnelrpc.capnp.go: tunnelrpc/tunnelrpc.capnp
 
 .PHONY: vet
 vet:
-	go vet ./...
+	go vet -mod=vendor ./...
 	which go-sumtype  # go get github.com/BurntSushi/go-sumtype
-	go-sumtype $$(go list ./...)
+	go-sumtype $$(go list -mod=vendor ./...)

RELEASE_NOTES

@@ -1,3 +1,186 @@
+2020.5.1
+- 2020-05-07 TUN-2860: Enable quick reconnect feature by default
+- 2020-05-07 AUTH-2564: error handling and minor fixes
+- 2020-05-01 AUTH-2588 add DoH to service mode
+
+2020.5.0
+- 2020-05-01 TUN-2943: Copy certutil from edge into cloudflared
+- 2020-05-05 TUN-2955: Fix connection and goroutine leaks when tunnel conection is terminated on error. Only unregister tunnels that had connected successfully. Close edge connection used to unregister the tunnel. Use buffered channels for error channels where receiver may quit early on context cancellation.
+- 2020-04-30 TUN-2940: Added delay parameter to stdin reconnect command.
+- 2020-04-27 TUN-2921: Rework address selection logic to avoid corner cases
+- 2020-04-28 TUN-2872: Exit with non-0 status code when the binary is updated so launchd will restart the service
+- 2020-04-13 AUTH-2587 add config watcher and reload logic for access client forwarder
+
+2020.4.0
+- 2020-04-10 TUN-2881: Parameterize response meta information header name in the generating function
+- 2020-04-11 TUN-2894: ResponseMetaHeader should be public
+- 2020-04-09 TUN-2880: Return metadata about source of the response from cloudflared
+- 2020-04-04 ARES-899: Fixes DoH client as system resolver. Fixes #91
+- 2020-03-31 AUTH-2394 added socks5 proxy
+- 2020-02-24 AUTH-2235 GetTokenIfExists now parses JWT payload for json expiry field to detect if the cached access token is expired
+
+2020.3.2
+- 2020-03-31 TUN-2854: Quick Reconnects should be an optional supported feature
+- 2020-03-30 TUN-2850: Tunnel stripping Cloudflare headers
+
+2020.3.1
+- 2020-03-27 TUN-2846: Trigger debug reconnects from stdin commands, not SIGUSR1
+
+2020.3.0
+- 2020-03-23 AUTH-2394 fixed header for websockets. Added TCP alias
+- 2020-03-10 TUN-2797: Fix panic in SetConnDigest by making mutexes values.
+- 2020-03-13 TUN-2807: cloudflared hello-world shouldn't assume it's my first tunnel
+- 2020-03-13 TUN-2756: Set connection digest after reconnect.
+- 2020-03-16 TUN-2812: Tunnel proxies and RPCs can share an edge address
+- 2020-03-18 TUN-2816: cloudflared metrics server should be more discoverable
+- 2020-03-19 TUN-2820: Serialized headers for Websockets
+- 2020-03-19 TUN-2819: cloudflared should close its connections when a signal is sent
+- 2020-03-19 TUN-2823: Bugfix. cloudflared would hang forever if error occurred.
+- 2020-03-10 TUN-2796: Implement HTTP2 CONTINUATION headers correctly
+- 2020-03-02 TUN-2779: update sample HTML pages
+- 2020-03-04 TUN-2785: Use reconnect token by default
+- 2020-03-05 TUN-2754: Add ConnDigest to cloudflared and its RPCs
+- 2020-03-06 TUN-2755: ReconnectTunnel RPC now transmits ConnectionDigest
+- 2020-03-06 TUN-2761: Use the new header management functions in cloudflared
+- 2020-03-06 TUN-2788: cloudflared should store one ConnDigest per HA connection
+- 2020-02-26 TUN-2767: Test for large headers
+- 2020-02-28 do not terminate tunnel if origin is not reachable on start-up (#177)
+- 2020-02-28 TUN-2776: Add header serialization feature in cloudflared
+- 2020-02-21 TUN-2748: Insecure randomness vulnerability in github.com/miekg/dns
+
+2020.2.1
+- 2020-02-20 TUN-2745: Rename existing header management functions
+- 2020-02-21 TUN-2746: Add the new header management functions
+- 2020-02-25 perf(cloudflared): reuse memory from buffer pool to get better throughput (#161)
+- 2020-02-25 Tweak HTTP host header. Fixes #107 (#168)
+- 2020-02-25 TUN-2765: Add list of features to tunnelrpc
+- 2020-02-19 TUN-2725: Specify in code that --edge is for internal testing only
+- 2020-02-19 TUN-2703: Muxer.Serve terminates when its context is Done
+- 2020-02-09 TUN-2717: Function to serialize/deserialize HTTP headers
+- 2020-02-05 TUN-2714: New edge discovery. Connections try to reconnect to the same edge IP.
+
+2020.2.0
+- 2020-01-30 TUN-2651: Fix panic in h2mux reader when a stream error is encountered
+- 2020-01-27 TUN-2645: Revert "TUN-2645: Turn on reconnect tokens"
+- 2020-01-28 TUN-2693: Metrics for ReconnectTunnel
+- 2020-01-28 TUN-2696: Add unknown registerRPCName
+- 2020-01-28 TUN-2699: Metrics for Authenticate RPCs
+- 2020-01-28 TUN-2690: cloudflared reconnect uses wrong context
+- 2020-01-29 TUN-2707: Inconsistent cardinality in tunnel error metrics
+- 2020-01-13 TUN-2645: Turn on reconnect tokens
+- 2019-12-23 TUN-2646: Make --edge flag work again for local development
+
+2019.12.0
+- 2019-12-11 TUN-2631: only notify that activeStreamMap is closed if ignoreNewStreams=true
+- 2019-12-17 bug(cloudflared): Set the MaxIdleConnsPerHost of http.Transport to proxy-keepalive-connections (#155)
+- 2019-12-17 refactor(docker): optimize Dockerfile (#126)
+- 2019-12-19 Fix timer scheduling for systemd update service (#159)
+- 2019-12-13 TUN-2637: Manage edge IPs in a region-aware manner
+- 2019-12-03 bug(cloudflared): nil pointer deference on h2DictWriter Close() (#154)
+- 2019-12-03 TUN-2608: h2mux.Muxer.Shutdown always returns a non-nil channel
+- 2019-12-04 TUN-2555: origin/supervisor.go calls Authenticate
+- 2019-12-06 TUN-2554: cloudflared calls ReconnectTunnel
+- 2019-11-20 TUN-2575: Constructors + simpler conversions for AuthOutcome
+- 2019-11-22 Fix Docker build failure (#149)
+- 2019-11-21 TUN-2573: Refactor TunnelRegistration into PermanentRegistrationError, RetryableRegistrationError and SuccessfulTunnelRegistration
+- 2019-11-22 TUN-2582: EventDigest field in tunnelrpc
+- 2019-11-22 Fix "happy eyeballs" not being disabled since Golang 1.12 upgrade * The Dialer.DualStack setting is now ignored and deprecated; RFC 6555 Fast Fallback ("Happy Eyeballs") is now enabled by default. To disable, set Dialer.FallbackDelay to a negative value.
+- 2019-11-25 TUN-2591: ReconnectTunnel now sends EventDigest
+- 2019-11-21 TUN-2606: add DialEdge helpers
+- 2019-11-21 TUN-2607: add RPC stream helpers
+
+2019.11.3
+- 2019-11-20 TUN-2562: Update Cloudflare Origin CA RSA root
+
+2019.11.2
+- 2019-11-18 TUN-2567: AuthOutcome can be turned back into AuthResponse
+- 2019-11-18 TUN-2563: Exposes config_version metrics
+
+2019.11.1
+- 2019-11-12 Add db-connect, a SQL over HTTPS server
+- 2019-11-12 TUN-2053: Add a /healthcheck endpoint to the metrics server
+- 2019-11-13 TUN-2178: public API to create new h2mux.MuxedStreamRequest
+- 2019-11-13 TUN-2490: respect original representation of HTTP request path
+- 2019-11-18 TUN-2547: TunnelRPC definitions for Authenticate flow
+- 2019-11-18 TUN-2551: TunnelRPC definitions for ReconnectTunnel flow
+- 2019-11-05 TUN-2506: Expose active streams metrics
+
+2019.11.0
+- 2019-11-04 TUN-2502: Switch to go modules
+- 2019-11-04 TUN-2500: Don't send client registration errors to Sentry
+- 2019-11-04 TUN-2489: Delete stream from activestreammap when read and write are both closed
+- 2019-11-05 TUN-2505: Terminate stream on receipt of RST_STREAM; MuxedStream.CloseWrite() should terminate the MuxedStream.Write() loop
+- 2019-10-30 TUN-2451: Log inavlid path
+- 2019-10-22 TUN-2425: Enable cloudflared to serve multiple Hello World servers by having each of them create its own ServeMux
+- 2019-10-22 AUTH-2173: Prepends access login url with scheme if one doesnt exist
+- 2019-10-23 TUN-2460: Configure according to the ClientConfig recevied from a successful Connect
+- 2019-10-23 AUTH-2177: Reads and writes error streams
+
+2019.10.4
+- 2019-10-21 TUN-2450: Remove Brew publishing formula
+
+2019.10.3
+- 2019-10-18 Fix #129: Excessive memory usage streaming large files (#142)
+
+2019.10.2
+- 2019-10-17 AUTH-2167: Adds CLI option for host key directory
+
+2019.10.1
+- 2019-10-17 Adds variable to fix windows build
+
+2019.10.0
+- 2019-10-11 AUTH-2105: Dont require --destination arg
+- 2019-10-14 TUN-2344: log more details: http2.Framer.ErrorDetail() if available, connectionID
+- 2019-10-16 AUTH-2159: Moves shutdownC close into error handling AUTH-2161: Lowers size of preamble length AUTH-2160: Fixes url parsing logic
+- 2019-10-16 AUTH-2135: Adds support for IPv6 and tests
+- 2019-10-02 AUTH-2105: Adds support for local forwarding. Refactor auditlogger creation. AUTH-2088: Adds dynamic destination routing
+- 2019-10-09 AUTH-2114: Uses short lived cert auth for outgoing client connection
+- 2019-09-30 AUTH-2089: Revise ssh server to function as a proxy
+
+2019.9.2
+- 2019-09-26 TUN-2355: Roll back TUN-2276
+
+2019.9.1
+- 2019-09-23 TUN-2334: remove tlsConfig.ServerName special case
+- 2019-09-23 AUTH-2077: Quotes open browser command in windows
+- 2019-09-11 AUTH-2050: Adds time.sleep to temporarily avoid hitting tunnel muxer dealock issue
+- 2019-09-10 AUTH-2056: Writes stderr to its own stream for non-pty connections
+- 2019-09-16 TUN-2307: Capnp is the only serialization format used in tunnelpogs
+- 2019-09-18 TUN-2315: Replace Scope with IntentLabel
+- 2019-09-17 TUN-2309: Split ConnectResult into ConnectError and ConnectSuccess, each implementing its own capnp serialization logic
+- 2019-09-18 AUTH-2052: Adds tests for SSH server
+- 2019-09-18 AUTH-2067: Log commands correctly
+- 2019-09-19 AUTH-2055: Verifies token at edge on access login
+- 2019-09-04 TUN-2201: change SRV records used by cloudflared
+- 2019-09-06 TUN-2280: Revert "TUN-2260: add name/group to CapnpConnectParameters, remove Scope"
+- 2019-09-03 AUTH-1943 hooked up uploader to logger, added timestamp to session logs, add tests
+- 2019-09-04 AUTH-2036: Refactor user retrieval, shutdown after ssh server stops, add custom version string
+- 2019-09-06 AUTH-1942 added event log to ssh server
+- 2019-09-04 AUTH-2037: Adds support for ssh port forwarding
+- 2019-09-05 TUN-2276: Path encoding broken
+
+2019.9.0
+- 2019-09-05 TUN-2279: Revert path encoding fix
+- 2019-08-30 AUTH-2021 - check error for failing tests
+- 2019-08-29 AUTH-2030: Support both authorized_key and short lived cert authentication simultaniously without specifiying at start time
+- 2019-08-29 AUTH-2026: Adds support for non-pty sessions and inline command exec
+- 2019-08-26 AUTH-1943: Adds session logging
+- 2019-08-26 TUN-2162: Decomplect OpenStream to allow finer-grained timeouts
+- 2019-08-29 TUN-2260: add name/group to CapnpConnectParameters, remove Scope
+
+2019.8.4
+- 2019-08-30 Fix #111: Add support for specifying a specific HTTP Host: header on the origin. (#114)
+- 2019-08-22 TUN-2165: Add ClientConfig to tunnelrpc.ConnectResult
+- 2019-08-20 AUTH-2014: Checks users login shell
+- 2019-08-26 TUN-2243: Revert "STOR-519: Add db-connect, a SQL over HTTPS server"
+- 2019-08-27 TUN-2244: Add NO_AUTOUPDATE env var
+- 2019-08-22 AUTH-2018: Adds support for authorized keys and short lived certs
+- 2019-08-28 AUTH-2022: Adds ssh timeout configuration
+- 2019-08-28 TUN-1968: Gracefully diff StreamHandler.UpdateConfig
+- 2019-08-26 AUTH-2021 - s3 bucket uploading for SSH logs
+- 2019-08-19 AUTH-2004: Adds static host key support
+- 2019-07-18 AUTH-1941: Adds initial SSH server implementation
+
 2019.8.3
 - 2019-08-20 STOR-519: Add db-connect, a SQL over HTTPS server
 - 2019-08-20 Release 2019.8.2

awsuploader/directory_upload_manager.go

@@ -0,0 +1,106 @@
+package awsuploader
+
+import (
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+// DirectoryUploadManager is used to manage file uploads on an interval from a directory
+type DirectoryUploadManager struct {
+	logger        *logrus.Logger
+	uploader      Uploader
+	rootDirectory string
+	sweepInterval time.Duration
+	ticker        *time.Ticker
+	shutdownC     chan struct{}
+	workQueue     chan string
+}
+
+// NewDirectoryUploadManager create a new DirectoryUploadManager
+// uploader is an Uploader to use as an actual uploading engine
+// directory is the directory to sweep for files to upload
+// sweepInterval is how often to iterate the directory and upload the files within
+func NewDirectoryUploadManager(logger *logrus.Logger, uploader Uploader, directory string, sweepInterval time.Duration, shutdownC chan struct{}) *DirectoryUploadManager {
+	workerCount := 10
+	manager := &DirectoryUploadManager{
+		logger:        logger,
+		uploader:      uploader,
+		rootDirectory: directory,
+		sweepInterval: sweepInterval,
+		shutdownC:     shutdownC,
+		workQueue:     make(chan string, workerCount),
+	}
+
+	//start workers
+	for i := 0; i < workerCount; i++ {
+		go manager.worker()
+	}
+
+	return manager
+}
+
+// Upload a file using the uploader
+// This is useful for "out of band" uploads that need to be triggered immediately instead of waiting for the sweep
+func (m *DirectoryUploadManager) Upload(filepath string) error {
+	return m.uploader.Upload(filepath)
+}
+
+// Start the upload ticker to walk the directories
+func (m *DirectoryUploadManager) Start() {
+	m.ticker = time.NewTicker(m.sweepInterval)
+	go m.run()
+}
+
+func (m *DirectoryUploadManager) run() {
+	for {
+		select {
+		case <-m.shutdownC:
+			m.ticker.Stop()
+			return
+		case <-m.ticker.C:
+			m.sweep()
+		}
+	}
+}
+
+// sweep the directory and kick off uploads
+func (m *DirectoryUploadManager) sweep() {
+	filepath.Walk(m.rootDirectory, func(path string, info os.FileInfo, err error) error {
+		if err != nil || info.IsDir() {
+			return nil
+		}
+		//30 days ago
+		retentionTime := 30 * (time.Hour * 24)
+		checkTime := time.Now().Add(-time.Duration(retentionTime))
+
+		//delete the file it is stale
+		if info.ModTime().Before(checkTime) {
+			os.Remove(path)
+			return nil
+		}
+		//add the upload to the work queue
+		go func() {
+			m.workQueue <- path
+		}()
+		return nil
+	})
+}
+
+// worker handles upload requests
+func (m *DirectoryUploadManager) worker() {
+	for {
+		select {
+		case <-m.shutdownC:
+			return
+		case filepath := <-m.workQueue:
+			if err := m.Upload(filepath); err != nil {
+				m.logger.WithError(err).Error("Cannot upload file to s3 bucket")
+			} else {
+				os.Remove(filepath)
+			}
+		}
+	}
+}

awsuploader/directory_upload_manager_test.go

@@ -0,0 +1,137 @@
+package awsuploader
+
+import (
+	"errors"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/sirupsen/logrus"
+)
+
+type MockUploader struct {
+	shouldFail bool
+}
+
+func (m *MockUploader) Upload(filepath string) error {
+	if m.shouldFail {
+		return errors.New("upload set to fail")
+	}
+	return nil
+}
+
+func NewMockUploader(shouldFail bool) Uploader {
+	return &MockUploader{shouldFail: shouldFail}
+}
+
+func getDirectoryPath(t *testing.T) string {
+	dir, err := os.Getwd()
+	if err != nil {
+		t.Fatal("couldn't create the test directory!", err)
+	}
+	return filepath.Join(dir, "uploads")
+}
+
+func setupTestDirectory(t *testing.T) string {
+	path := getDirectoryPath(t)
+	os.RemoveAll(path)
+	time.Sleep(100 * time.Millisecond) //short way to wait for the OS to delete the folder
+	err := os.MkdirAll(path, os.ModePerm)
+	if err != nil {
+		t.Fatal("couldn't create the test directory!", err)
+	}
+	return path
+}
+
+func createUploadManager(t *testing.T, shouldFailUpload bool) *DirectoryUploadManager {
+	rootDirectory := setupTestDirectory(t)
+	uploader := NewMockUploader(shouldFailUpload)
+	logger := logrus.New()
+	shutdownC := make(chan struct{})
+	return NewDirectoryUploadManager(logger, uploader, rootDirectory, 1*time.Second, shutdownC)
+}
+
+func createFile(t *testing.T, fileName string) (*os.File, string) {
+	path := filepath.Join(getDirectoryPath(t), fileName)
+	f, err := os.Create(path)
+	if err != nil {
+		t.Fatal("upload to create file for sweep test", err)
+	}
+	return f, path
+}
+
+func TestUploadSuccess(t *testing.T) {
+	manager := createUploadManager(t, false)
+	path := filepath.Join(getDirectoryPath(t), "test_file")
+	if err := manager.Upload(path); err != nil {
+		t.Fatal("the upload request method failed", err)
+	}
+}
+
+func TestUploadFailure(t *testing.T) {
+	manager := createUploadManager(t, true)
+	path := filepath.Join(getDirectoryPath(t), "test_file")
+	if err := manager.Upload(path); err == nil {
+		t.Fatal("the upload request method should have failed and didn't", err)
+	}
+}
+
+func TestSweepSuccess(t *testing.T) {
+	manager := createUploadManager(t, false)
+	f, path := createFile(t, "test_file")
+	defer f.Close()
+
+	manager.Start()
+	time.Sleep(2 * time.Second)
+	if _, err := os.Stat(path); os.IsExist(err) {
+		//the file should have been deleted
+		t.Fatal("the manager failed to delete the file", err)
+	}
+}
+
+func TestSweepFailure(t *testing.T) {
+	manager := createUploadManager(t, true)
+	f, path := createFile(t, "test_file")
+	defer f.Close()
+
+	manager.Start()
+	time.Sleep(2 * time.Second)
+	_, serr := f.Stat()
+	if serr != nil {
+		//the file should still exist
+		os.Remove(path)
+		t.Fatal("the manager failed to delete the file", serr)
+	}
+}
+
+func TestHighLoad(t *testing.T) {
+	manager := createUploadManager(t, false)
+	for i := 0; i < 30; i++ {
+		f, _ := createFile(t, randomString(6))
+		defer f.Close()
+	}
+	manager.Start()
+	time.Sleep(4 * time.Second)
+
+	directory := getDirectoryPath(t)
+	files, err := ioutil.ReadDir(directory)
+	if err != nil || len(files) > 0 {
+		t.Fatalf("the manager failed to upload all the files: %s files left: %d", err, len(files))
+	}
+}
+
+// LowerCase [a-z]
+const randSet = "abcdefghijklmnopqrstuvwxyz"
+
+// String returns a string of length 'n' from a set of letters 'lset'
+func randomString(n int) string {
+	b := make([]byte, n)
+	lsetLen := len(randSet)
+	for i := range b {
+		b[i] = randSet[rand.Intn(lsetLen)]
+	}
+	return string(b)
+}

awsuploader/file_uploader.go

@@ -0,0 +1,62 @@
+package awsuploader
+
+import (
+	"context"
+	"os"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+//FileUploader aws compliant bucket upload
+type FileUploader struct {
+	storage    *s3.S3
+	bucketName string
+	clientID   string
+	secretID   string
+}
+
+// NewFileUploader creates a new S3 compliant bucket uploader
+func NewFileUploader(bucketName, region, accessKeyID, secretID, token, s3Host string) (*FileUploader, error) {
+	sess, err := session.NewSession(&aws.Config{
+		Region:      aws.String(region),
+		Credentials: credentials.NewStaticCredentials(accessKeyID, secretID, token),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	var storage *s3.S3
+	if s3Host != "" {
+		storage = s3.New(sess, &aws.Config{Endpoint: aws.String(s3Host)})
+	} else {
+		storage = s3.New(sess)
+	}
+
+	return &FileUploader{
+		storage:    storage,
+		bucketName: bucketName,
+	}, nil
+}
+
+// Upload a file to the bucket
+func (u *FileUploader) Upload(filepath string) error {
+	info, err := os.Stat(filepath)
+	if err != nil {
+		return err
+	}
+	file, err := os.Open(filepath)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	_, serr := u.storage.PutObjectWithContext(context.Background(), &s3.PutObjectInput{
+		Bucket: aws.String(u.bucketName),
+		Key:    aws.String(info.Name()),
+		Body:   file,
+	})
+	return serr
+}

awsuploader/upload_manager.go

@@ -0,0 +1,7 @@
+package awsuploader
+
+// UploadManager is used to manage file uploads on an interval
+type UploadManager interface {
+	Upload(string) error
+	Start()
+}

awsuploader/uploader.go

@@ -0,0 +1,7 @@
+package awsuploader
+
+// Uploader the functions required to upload to a bucket
+type Uploader interface {
+	//Upload a file to the bucket
+	Upload(string) error
+}

buffer/pool.go

@@ -0,0 +1,29 @@
+package buffer
+
+import (
+	"sync"
+)
+
+type Pool struct {
+	// A Pool must not be copied after first use.
+	// https://golang.org/pkg/sync/#Pool
+	buffers sync.Pool
+}
+
+func NewPool(bufferSize int) *Pool {
+	return &Pool{
+		buffers: sync.Pool{
+			New: func() interface{} {
+				return make([]byte, bufferSize)
+			},
+		},
+	}
+}
+
+func (p *Pool) Get() []byte {
+	return p.buffers.Get().([]byte)
+}
+
+func (p *Pool) Put(buf []byte) {
+	p.buffers.Put(buf)
+}

carrier/carrier.go

@@ -11,9 +11,7 @@ import (
 	"strings"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
-	cloudflaredWebsocket "github.com/cloudflare/cloudflared/websocket"
-	"github.com/gorilla/websocket"
-	"github.com/sirupsen/logrus"
+	"github.com/pkg/errors"
 )
 
 type StartOptions struct {
@@ -21,6 +19,15 @@ type StartOptions struct {
 	Headers   http.Header
 }
 
+// Connection wraps up all the needed functions to forward over the tunnel
+type Connection interface {
+	// ServeStream is used to forward data from the client to the edge
+	ServeStream(*StartOptions, io.ReadWriter) error
+
+	// StartServer is used to listen for incoming connections from the edge to the origin
+	StartServer(net.Listener, string, <-chan struct{}) error
+}
+
 // StdinoutStream is empty struct for wrapping stdin/stdout
 // into a single ReadWriter
 type StdinoutStream struct {
@@ -44,92 +51,68 @@ func closeRespBody(resp *http.Response) {
 	}
 }
 
-// StartClient will copy the data from stdin/stdout over a WebSocket connection
-// to the edge (originURL)
-func StartClient(logger *logrus.Logger, stream io.ReadWriter, options *StartOptions) error {
-	return serveStream(logger, stream, options)
-}
-
-// StartServer will setup a listener on a specified address/port and then
+// StartForwarder will setup a listener on a specified address/port and then
 // forward connections to the origin by calling `Serve()`.
-func StartServer(logger *logrus.Logger, address string, shutdownC <-chan struct{}, options *StartOptions) error {
+func StartForwarder(conn Connection, address string, shutdownC <-chan struct{}, options *StartOptions) error {
 	listener, err := net.Listen("tcp", address)
 	if err != nil {
-		logger.WithError(err).Error("failed to start forwarding server")
-		return err
+		return errors.Wrap(err, "failed to start forwarding server")
 	}
-	logger.Info("Started listening on ", address)
-	return Serve(logger, listener, shutdownC, options)
+	return Serve(conn, listener, shutdownC, options)
+}
+
+// StartClient will copy the data from stdin/stdout over a WebSocket connection
+// to the edge (originURL)
+func StartClient(conn Connection, stream io.ReadWriter, options *StartOptions) error {
+	return serveStream(conn, stream, options)
 }
 
 // Serve accepts incoming connections on the specified net.Listener.
 // Each connection is handled in a new goroutine: its data is copied over a
 // WebSocket connection to the edge (originURL).
 // `Serve` always closes `listener`.
-func Serve(logger *logrus.Logger, listener net.Listener, shutdownC <-chan struct{}, options *StartOptions) error {
+func Serve(remoteConn Connection, listener net.Listener, shutdownC <-chan struct{}, options *StartOptions) error {
 	defer listener.Close()
-	for {
-		select {
-		case <-shutdownC:
-			return nil
-		default:
+	errChan := make(chan error)
+
+	go func() {
+		for {
 			conn, err := listener.Accept()
 			if err != nil {
-				return err
+				// don't block if parent goroutine quit early
+				select {
+				case errChan <- err:
+				default:
+				}
+				return
 			}
-			go serveConnection(logger, conn, options)
+			go serveConnection(remoteConn, conn, options)
 		}
+	}()
+
+	select {
+	case <-shutdownC:
+		return nil
+	case err := <-errChan:
+		return err
 	}
 }
 
 // serveConnection handles connections for the Serve() call
-func serveConnection(logger *logrus.Logger, c net.Conn, options *StartOptions) {
+func serveConnection(remoteConn Connection, c net.Conn, options *StartOptions) {
 	defer c.Close()
-	serveStream(logger, c, options)
+	serveStream(remoteConn, c, options)
 }
 
 // serveStream will serve the data over the WebSocket stream
-func serveStream(logger *logrus.Logger, conn io.ReadWriter, options *StartOptions) error {
-	wsConn, err := createWebsocketStream(options)
-	if err != nil {
-		logger.WithError(err).Errorf("failed to connect to %s\n", options.OriginURL)
-		return err
-	}
-	defer wsConn.Close()
-
-	cloudflaredWebsocket.Stream(wsConn, conn)
-
-	return nil
+func serveStream(remoteConn Connection, conn io.ReadWriter, options *StartOptions) error {
+	return remoteConn.ServeStream(options, conn)
 }
 
-// createWebsocketStream will create a WebSocket connection to stream data over
-// It also handles redirects from Access and will present that flow if
-// the token is not present on the request
-func createWebsocketStream(options *StartOptions) (*cloudflaredWebsocket.Conn, error) {
-	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header = options.Headers
-
-	wsConn, resp, err := cloudflaredWebsocket.ClientConnect(req, nil)
-	defer closeRespBody(resp)
-	if err != nil && isAccessResponse(resp) {
-		wsConn, err = createAccessAuthenticatedStream(options)
-		if err != nil {
-			return nil, err
-		}
-	} else if err != nil {
-		return nil, err
-	}
-
-	return &cloudflaredWebsocket.Conn{Conn: wsConn}, nil
-}
-
-// isAccessResponse checks the http Response to see if the url location
+// IsAccessResponse checks the http Response to see if the url location
 // contains the Access structure.
-func isAccessResponse(resp *http.Response) bool {
-	if resp == nil || resp.StatusCode <= 300 {
+func IsAccessResponse(resp *http.Response) bool {
+	if resp == nil || resp.StatusCode != http.StatusFound {
 		return false
 	}
 
@@ -144,50 +127,8 @@ func isAccessResponse(resp *http.Response) bool {
 	return false
 }
 
-// createAccessAuthenticatedStream will try load a token from storage and make
-// a connection with the token set on the request. If it still get redirect,
-// this probably means the token in storage is invalid (expired/revoked). If that
-// happens it deletes the token and runs the connection again, so the user can
-// login again and generate a new one.
-func createAccessAuthenticatedStream(options *StartOptions) (*websocket.Conn, error) {
-	wsConn, resp, err := createAccessWebSocketStream(options)
-	defer closeRespBody(resp)
-	if err == nil {
-		return wsConn, nil
-	}
-
-	if !isAccessResponse(resp) {
-		return nil, err
-	}
-
-	// Access Token is invalid for some reason. Go through regen flow
-	originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	if err := token.RemoveTokenIfExists(originReq.URL); err != nil {
-		return nil, err
-	}
-	wsConn, resp, err = createAccessWebSocketStream(options)
-	defer closeRespBody(resp)
-	if err != nil {
-		return nil, err
-	}
-
-	return wsConn, nil
-}
-
-// createAccessWebSocketStream builds an Access request and makes a connection
-func createAccessWebSocketStream(options *StartOptions) (*websocket.Conn, *http.Response, error) {
-	req, err := buildAccessRequest(options)
-	if err != nil {
-		return nil, nil, err
-	}
-	return cloudflaredWebsocket.ClientConnect(req, nil)
-}
-
-// buildAccessRequest builds an HTTP request with the Access token set
-func buildAccessRequest(options *StartOptions) (*http.Request, error) {
+// BuildAccessRequest builds an HTTP request with the Access token set
+func BuildAccessRequest(options *StartOptions) (*http.Request, error) {
 	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
 	if err != nil {
 		return nil, err

carrier/carrier_test.go

@@ -44,6 +44,7 @@ func (s *testStreamer) Write(p []byte) (int, error) {
 func TestStartClient(t *testing.T) {
 	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
 	logger := logrus.New()
+	wsConn := NewWSConnection(logger, false)
 	ts := newTestWebSocketServer()
 	defer ts.Close()
 
@@ -52,7 +53,7 @@ func TestStartClient(t *testing.T) {
 		OriginURL: "http://" + ts.Listener.Addr().String(),
 		Headers:   nil,
 	}
-	err := StartClient(logger, buf, options)
+	err := StartClient(wsConn, buf, options)
 	assert.NoError(t, err)
 	buf.Write([]byte(message))
 
@@ -69,6 +70,7 @@ func TestStartServer(t *testing.T) {
 	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
 	logger := logrus.New()
 	shutdownC := make(chan struct{})
+	wsConn := NewWSConnection(logger, false)
 	ts := newTestWebSocketServer()
 	defer ts.Close()
 	options := &StartOptions{
@@ -77,7 +79,7 @@ func TestStartServer(t *testing.T) {
 	}
 
 	go func() {
-		err := Serve(logger, listener, shutdownC, options)
+		err := Serve(wsConn, listener, shutdownC, options)
 		if err != nil {
 			t.Fatalf("Error running server: %v", err)
 		}
@@ -102,14 +104,14 @@ func TestIsAccessResponse(t *testing.T) {
 		ExpectedOut bool
 	}{
 		{"nil response", nil, false},
-		{"redirect with no location", &http.Response{StatusCode: http.StatusPermanentRedirect}, false},
+		{"redirect with no location", &http.Response{StatusCode: http.StatusFound}, false},
 		{"200 ok", &http.Response{StatusCode: http.StatusOK}, false},
-		{"redirect with location", &http.Response{StatusCode: http.StatusPermanentRedirect, Header: validLocationHeader}, true},
-		{"redirect with invalid location", &http.Response{StatusCode: http.StatusPermanentRedirect, Header: invalidLocationHeader}, false},
+		{"redirect with location", &http.Response{StatusCode: http.StatusFound, Header: validLocationHeader}, true},
+		{"redirect with invalid location", &http.Response{StatusCode: http.StatusFound, Header: invalidLocationHeader}, false},
 	}
 
 	for i, tc := range testCases {
-		if isAccessResponse(tc.In) != tc.ExpectedOut {
+		if IsAccessResponse(tc.In) != tc.ExpectedOut {
 			t.Fatalf("Failed case %d -- %s", i, tc.Description)
 		}
 	}

carrier/websocket.go

@@ -0,0 +1,137 @@
+package carrier
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
+	"github.com/cloudflare/cloudflared/socks"
+	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
+	"github.com/gorilla/websocket"
+	"github.com/sirupsen/logrus"
+)
+
+// Websocket is used to carry data via WS binary frames over the tunnel from client to the origin
+// This implements the functions for glider proxy (sock5) and the carrier interface
+type Websocket struct {
+	logger  *logrus.Logger
+	isSocks bool
+}
+
+type wsdialer struct {
+	conn *cfwebsocket.Conn
+}
+
+func (d *wsdialer) Dial(address string) (io.ReadWriteCloser, *socks.AddrSpec, error) {
+	local, ok := d.conn.LocalAddr().(*net.TCPAddr)
+	if !ok {
+		return nil, nil, fmt.Errorf("not a tcp connection")
+	}
+
+	addr := socks.AddrSpec{IP: local.IP, Port: local.Port}
+	return d.conn, &addr, nil
+}
+
+// NewWSConnection returns a new connection object
+func NewWSConnection(logger *logrus.Logger, isSocks bool) Connection {
+	return &Websocket{
+		logger:  logger,
+		isSocks: isSocks,
+	}
+}
+
+// ServeStream will create a Websocket client stream connection to the edge
+// it blocks and writes the raw data from conn over the tunnel
+func (ws *Websocket) ServeStream(options *StartOptions, conn io.ReadWriter) error {
+	wsConn, err := createWebsocketStream(options)
+	if err != nil {
+		ws.logger.WithError(err).Errorf("failed to connect to %s", options.OriginURL)
+		return err
+	}
+	defer wsConn.Close()
+
+	if ws.isSocks {
+		dialer := &wsdialer{conn: wsConn}
+		requestHandler := socks.NewRequestHandler(dialer)
+		socksServer := socks.NewConnectionHandler(requestHandler)
+
+		socksServer.Serve(conn)
+	} else {
+		cfwebsocket.Stream(wsConn, conn)
+	}
+	return nil
+}
+
+// StartServer creates a Websocket server to listen for connections.
+// This is used on the origin (tunnel) side to take data from the muxer and send it to the origin
+func (ws *Websocket) StartServer(listener net.Listener, remote string, shutdownC <-chan struct{}) error {
+	return cfwebsocket.StartProxyServer(ws.logger, listener, remote, shutdownC, cfwebsocket.DefaultStreamHandler)
+}
+
+// createWebsocketStream will create a WebSocket connection to stream data over
+// It also handles redirects from Access and will present that flow if
+// the token is not present on the request
+func createWebsocketStream(options *StartOptions) (*cfwebsocket.Conn, error) {
+	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header = options.Headers
+
+	wsConn, resp, err := cfwebsocket.ClientConnect(req, nil)
+	defer closeRespBody(resp)
+	if err != nil && IsAccessResponse(resp) {
+		wsConn, err = createAccessAuthenticatedStream(options)
+		if err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+
+	return &cfwebsocket.Conn{Conn: wsConn}, nil
+}
+
+// createAccessAuthenticatedStream will try load a token from storage and make
+// a connection with the token set on the request. If it still get redirect,
+// this probably means the token in storage is invalid (expired/revoked). If that
+// happens it deletes the token and runs the connection again, so the user can
+// login again and generate a new one.
+func createAccessAuthenticatedStream(options *StartOptions) (*websocket.Conn, error) {
+	wsConn, resp, err := createAccessWebSocketStream(options)
+	defer closeRespBody(resp)
+	if err == nil {
+		return wsConn, nil
+	}
+
+	if !IsAccessResponse(resp) {
+		return nil, err
+	}
+
+	// Access Token is invalid for some reason. Go through regen flow
+	originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	if err := token.RemoveTokenIfExists(originReq.URL); err != nil {
+		return nil, err
+	}
+	wsConn, resp, err = createAccessWebSocketStream(options)
+	defer closeRespBody(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	return wsConn, nil
+}
+
+// createAccessWebSocketStream builds an Access request and makes a connection
+func createAccessWebSocketStream(options *StartOptions) (*websocket.Conn, *http.Response, error) {
+	req, err := BuildAccessRequest(options)
+	if err != nil {
+		return nil, nil, err
+	}
+	return cfwebsocket.ClientConnect(req, nil)
+}

certutil/certutil.go

@@ -0,0 +1,94 @@
+package certutil
+
+import (
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"strings"
+)
+
+type namedTunnelToken struct {
+	ZoneID     string `json:"zoneID"`
+	AccountID  string `json:"accountID"`
+	ServiceKey string `json:"serviceKey"`
+}
+
+type OriginCert struct {
+	PrivateKey interface{}
+	Cert       *x509.Certificate
+	ZoneID     string
+	ServiceKey string
+	AccountID  string
+}
+
+func DecodeOriginCert(blocks []byte) (*OriginCert, error) {
+	if len(blocks) == 0 {
+		return nil, fmt.Errorf("Cannot decode empty certificate")
+	}
+	originCert := OriginCert{}
+	block, rest := pem.Decode(blocks)
+	for {
+		if block == nil {
+			break
+		}
+		switch block.Type {
+		case "PRIVATE KEY":
+			if originCert.PrivateKey != nil {
+				return nil, fmt.Errorf("Found multiple private key in the certificate")
+			}
+			// RSA private key
+			privateKey, err := x509.ParsePKCS8PrivateKey(block.Bytes)
+			if err != nil {
+				return nil, fmt.Errorf("Cannot parse private key")
+			}
+			originCert.PrivateKey = privateKey
+		case "CERTIFICATE":
+			if originCert.Cert != nil {
+				return nil, fmt.Errorf("Found multiple certificates in the certificate")
+			}
+			cert, err := x509.ParseCertificates(block.Bytes)
+			if err != nil {
+				return nil, fmt.Errorf("Cannot parse certificate")
+			} else if len(cert) > 1 {
+				return nil, fmt.Errorf("Found multiple certificates in the certificate")
+			}
+			originCert.Cert = cert[0]
+		case "WARP TOKEN", "ARGO TUNNEL TOKEN":
+			if originCert.ZoneID != "" || originCert.ServiceKey != "" {
+				return nil, fmt.Errorf("Found multiple tokens in the certificate")
+			}
+			// The token is a string,
+			// Try the newer JSON format
+			ntt := namedTunnelToken{}
+			if err := json.Unmarshal(block.Bytes, &ntt); err == nil {
+				originCert.ZoneID = ntt.ZoneID
+				originCert.ServiceKey = ntt.ServiceKey
+				originCert.AccountID = ntt.AccountID
+			} else {
+				// Try the older format, where the zoneID and service key are seperated by
+				// a new line character
+				token := string(block.Bytes)
+				s := strings.Split(token, "\n")
+				if len(s) != 2 {
+					return nil, fmt.Errorf("Cannot parse token")
+				}
+				originCert.ZoneID = s[0]
+				originCert.ServiceKey = s[1]
+			}
+		default:
+			return nil, fmt.Errorf("Unknown block %s in the certificate", block.Type)
+		}
+		block, rest = pem.Decode(rest)
+	}
+
+	if originCert.PrivateKey == nil {
+		return nil, fmt.Errorf("Missing private key in the certificate")
+	} else if originCert.Cert == nil {
+		return nil, fmt.Errorf("Missing certificate in the certificate")
+	} else if originCert.ZoneID == "" || originCert.ServiceKey == "" {
+		return nil, fmt.Errorf("Missing token in the certificate")
+	}
+
+	return &originCert, nil
+}

certutil/certutil_test.go

@@ -0,0 +1,67 @@
+package certutil
+
+import (
+	"fmt"
+	"io/ioutil"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLoadOriginCert(t *testing.T) {
+	cert, err := DecodeOriginCert([]byte{})
+	assert.Equal(t, fmt.Errorf("Cannot decode empty certificate"), err)
+	assert.Nil(t, cert)
+
+	blocks, err := ioutil.ReadFile("test-cert-no-key.pem")
+	assert.Nil(t, err)
+	cert, err = DecodeOriginCert(blocks)
+	assert.Equal(t, fmt.Errorf("Missing private key in the certificate"), err)
+	assert.Nil(t, cert)
+
+	blocks, err = ioutil.ReadFile("test-cert-two-certificates.pem")
+	assert.Nil(t, err)
+	cert, err = DecodeOriginCert(blocks)
+	assert.Equal(t, fmt.Errorf("Found multiple certificates in the certificate"), err)
+	assert.Nil(t, cert)
+
+	blocks, err = ioutil.ReadFile("test-cert-unknown-block.pem")
+	assert.Nil(t, err)
+	cert, err = DecodeOriginCert(blocks)
+	assert.Equal(t, fmt.Errorf("Unknown block RSA PRIVATE KEY in the certificate"), err)
+	assert.Nil(t, cert)
+
+	blocks, err = ioutil.ReadFile("test-cert.pem")
+	assert.Nil(t, err)
+	cert, err = DecodeOriginCert(blocks)
+	assert.Nil(t, err)
+	assert.NotNil(t, cert)
+	assert.Equal(t, "7b0a4d77dfb881c1a3b7d61ea9443e19", cert.ZoneID)
+	key := "v1.0-58bd4f9e28f7b3c28e05a35ff3e80ab4fd9644ef3fece537eb0d12e2e9258217-183442fbb0bbdb3e571558fec9b5589ebd77aafc87498ee3f09f64a4ad79ffe8791edbae08b36c1d8f1d70a8670de56922dff92b15d214a524f4ebfa1958859e-7ce80f79921312a6022c5d25e2d380f82ceaefe3fbdc43dd13b080e3ef1e26f7"
+	assert.Equal(t, key, cert.ServiceKey)
+}
+
+func TestNewlineArgoTunnelToken(t *testing.T) {
+	ArgoTunnelTokenTest(t, "test-argo-tunnel-cert.pem")
+}
+
+func TestJSONArgoTunnelToken(t *testing.T) {
+	// The given cert's Argo Tunnel Token was generated by base64 encoding this JSON:
+	// {
+	// "zoneID": "7b0a4d77dfb881c1a3b7d61ea9443e19",
+	// "serviceKey": "test-service-key",
+	// "accountID": "abcdabcdabcdabcd1234567890abcdef"
+	// }
+	ArgoTunnelTokenTest(t, "test-argo-tunnel-cert-json.pem")
+}
+
+func ArgoTunnelTokenTest(t *testing.T, path string) {
+	blocks, err := ioutil.ReadFile(path)
+	assert.Nil(t, err)
+	cert, err := DecodeOriginCert(blocks)
+	assert.Nil(t, err)
+	assert.NotNil(t, cert)
+	assert.Equal(t, "7b0a4d77dfb881c1a3b7d61ea9443e19", cert.ZoneID)
+	key := "test-service-key"
+	assert.Equal(t, key, cert.ServiceKey)
+}

certutil/test-argo-tunnel-cert-json.pem

@@ -0,0 +1,57 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
+sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
+1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
+z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
+6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
+x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
+1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
+IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
+xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
+sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
+2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
+sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
+Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
+AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
+m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
+QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
+P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
+pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
+G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
+6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
+LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
+OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
+W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
+M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
+y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
+uQicoQq3yzeQh20wtrtaXzTNmA==
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
+gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
+VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
+cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
+MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
+ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
+aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
+GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
+6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
+tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
+PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
+AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
+HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
+zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
+RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
+YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
+YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
+cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
+K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
+x+Yo/cL8fGfVpPt4UM8=
+-----END CERTIFICATE-----
+-----BEGIN ARGO TUNNEL TOKEN-----
+eyJ6b25lSUQiOiAiN2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkiLCAi
+c2VydmljZUtleSI6ICJ0ZXN0LXNlcnZpY2Uta2V5IiwgImFjY291bnRJRCI6ICJh
+YmNkYWJjZGFiY2RhYmNkMTIzNDU2Nzg5MGFiY2RlZiJ9
+-----END ARGO TUNNEL TOKEN-----

certutil/test-argo-tunnel-cert.pem

@@ -0,0 +1,56 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
+sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
+1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
+z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
+6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
+x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
+1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
+IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
+xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
+sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
+2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
+sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
+Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
+AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
+m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
+QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
+P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
+pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
+G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
+6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
+LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
+OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
+W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
+M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
+y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
+uQicoQq3yzeQh20wtrtaXzTNmA==
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
+gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
+VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
+cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
+MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
+ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
+aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
+GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
+6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
+tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
+PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
+AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
+HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
+zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
+RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
+YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
+YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
+cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
+K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
+x+Yo/cL8fGfVpPt4UM8=
+-----END CERTIFICATE-----
+-----BEGIN ARGO TUNNEL TOKEN-----
+N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdGVzdC1zZXJ2aWNlLWtl
+eQ==
+-----END ARGO TUNNEL TOKEN-----

certutil/test-cert-no-key.pem

@@ -0,0 +1,33 @@
+-----BEGIN CERTIFICATE-----
+MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
+gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
+VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
+cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
+MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
+ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
+aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
+GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
+6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
+tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
+PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
+AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
+HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
+zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
+RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
+YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
+YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
+cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
+K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
+x+Yo/cL8fGfVpPt4UM8=
+-----END CERTIFICATE-----
+-----BEGIN WARP TOKEN-----
+N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
+ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
+MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
+NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
+NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
+OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
+ZWYxZTI2Zjc=
+-----END WARP TOKEN-----

certutil/test-cert-two-certificates.pem

@@ -0,0 +1,85 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
+sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
+1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
+z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
+6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
+x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
+1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
+IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
+xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
+sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
+2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
+sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
+Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
+AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
+m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
+QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
+P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
+pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
+G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
+6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
+LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
+OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
+W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
+M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
+y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
+uQicoQq3yzeQh20wtrtaXzTNmA==
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
+gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
+VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
+cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
+MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
+ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
+aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
+GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
+6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
+tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
+PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
+AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
+HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
+zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
+RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
+YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
+YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
+cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
+K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
+x+Yo/cL8fGfVpPt4UM8=
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
+gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
+VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
+cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
+MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
+ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
+aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
+GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
+6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
+tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
+PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
+AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
+HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
+zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
+RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
+YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
+YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
+cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
+K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
+x+Yo/cL8fGfVpPt4UM8=
+-----END CERTIFICATE-----
+-----BEGIN WARP TOKEN-----
+N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
+ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
+MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
+NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
+NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
+OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
+ZWYxZTI2Zjc=
+-----END WARP TOKEN-----

certutil/test-cert-unknown-block.pem

@@ -0,0 +1,89 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
+sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
+1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
+z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
+6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
+x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
+1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
+IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
+xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
+sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
+2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
+sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
+Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
+AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
+m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
+QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
+P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
+pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
+G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
+6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
+LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
+OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
+W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
+M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
+y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
+uQicoQq3yzeQh20wtrtaXzTNmA==
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
+gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
+VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
+cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
+MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
+ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
+aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
+GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
+6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
+tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
+PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
+AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
+HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
+zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
+RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
+YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
+YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
+cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
+K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
+x+Yo/cL8fGfVpPt4UM8=
+-----END CERTIFICATE-----
+-----BEGIN WARP TOKEN-----
+N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
+ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
+MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
+NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
+NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
+OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
+ZWYxZTI2Zjc=
+-----END WARP TOKEN-----
+-----BEGIN RSA PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
+sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
+1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
+z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
+6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
+x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
+1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
+IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
+xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
+sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
+2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
+sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
+Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
+AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
+m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
+QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
+P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
+pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
+G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
+6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
+LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
+OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
+W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
+M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
+y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
+uQicoQq3yzeQh20wtrtaXzTNmA==
+-----END RSA PRIVATE KEY-----

certutil/test-cert.pem

@@ -0,0 +1,61 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCfGswL16Fz9Ei3
+sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng6yHR1H5oX1Lg
+1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bxtG0uyrXYh7Mt
+z0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyXPE6SuDvMHIeX
+6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZAzNOxVKrUsyS
+x7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOglHJ2n0sMcZ+Ja
+1Y649mPVAgMBAAECggEAEbPF0ah9fH0IzTU/CPbIeh3flyY8GDuMpR1HvwUurSWB
+IFI9bLyVAXKb8vYP1TMaTnXi5qmFof+/JShgyZc3+1tZtWTfoaiC8Y1bRfE2yk+D
+xmwddhDmijYGG7i8uEaeddSdFEh2GKAqkbV/QgBvN2Nl4EVmIOAJXXNe9l5LFyjy
+sR10aNVJRYV1FahrCTwZ3SovHP4d4AUvHh/3FFZDukHc37CFA0+CcR4uehp5yedi
+2UdqaszXqunFo/3h+Tn9dW2C7gTTZx4+mfyaws3p3YOmdYArXvpejxHIc0FGwLBm
+sb9K7wGVUiF0Bt0ch+C1mdYrCaFNHnPuDswjmm3FwQKBgQDYtxOwwSLA6ZyppozX
+Doyx9a7PhiMHCFKSdVB4l8rpK545a+AmpG6LRScTtBsMTHBhT3IQ3QPWlVm1AhjF
+AvXMa1rOeaGbCbDn1xqEoEVPtj4tys8eTfyWmtU73jWTFauOt4/xpf/urEpg91xj
+m+Gl/8qgBrpm5rQxV5Y4MysRlQKBgQC78jzzlhocXGNvw0wT/K2NsknyeoZXqpIE
+QYL60FMl4geZn6w9hwxaL1r+g/tUjTnpBPQtS1r2Ed2gXby5zspN1g/PW8U3t3to
+P7zHIJ/sLBXrCh5RJko3hUgGhDNOOCIQj4IaKUfvHYvEIbIxlyI0vdsXsgXgMuQ8
+pb9Yifn5QQKBgQCmGu0EtYQlyOlDP10EGSrN3Dm45l9CrKZdi326cN4eCkikSoLs
+G2x/YumouItiydP5QiNzuXOPrbmse4bwumwb2s0nJSMw6iSmDsFMlmuJxW2zO5e0
+6qGH7fUyhgcaTanJIfk6hrm7/mKkH/S4hGpYCc8NCRsmc/35M+D4AoAoYQKBgQC0
+LWpZaxDlF30MbAHHN3l6We2iU+vup0sMYXGb2ZOcwa/fir+ozIr++l8VmJmdWTan
+OWSM96zgMghx8Os4hhJTxF+rvqK242OfcVsc2x31X94zUaP2z+peh5uhA6Pb3Nxr
+W+iyA9k+Vujiwhr+h5D3VvtvH++aG6/KpGtoCf5nAQKBgQDXX2+d7bd5CLNLLFNd
+M2i4QoOFcSKIG+v4SuvgEJHgG8vGvxh2qlSxnMWuPV+7/1P5ATLqDj1PlKms+BNR
+y7sc5AT9PclkL3Y9MNzOu0LXyBkGYcl8M0EQfLv9VPbWT+NXiMg/O2CHiT02pAAz
+uQicoQq3yzeQh20wtrtaXzTNmA==
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIID+jCCA6CgAwIBAgIUJhFxUKEGvTRc3CjCok6dbPGH/P4wCgYIKoZIzj0EAwIw
+gagxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTgwNgYD
+VQQLEy9DbG91ZEZsYXJlIE9yaWdpbiBTU0wgRUNDIENlcnRpZmljYXRlIEF1dGhv
+cml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5p
+YTEXMBUGA1UEAxMOKGRldiB1c2Ugb25seSkwHhcNMTcxMDEzMTM1OTAwWhcNMzIx
+MDA5MTM1OTAwWjBiMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMR0wGwYDVQQL
+ExRDbG91ZEZsYXJlIE9yaWdpbiBDQTEmMCQGA1UEAxMdQ2xvdWRGbGFyZSBPcmln
+aW4gQ2VydGlmaWNhdGUwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCf
+GswL16Fz9Ei3sAg5AmBizoN2nZdyXHP8T57UxUMcrlJXEEXCVS5RR4m9l+EmK0ng
+6yHR1H5oX1Lg1WKyXgWwr0whwmdTD+qWFJW2M8HyefyBKLrsGPuxw4CVYT0h72bx
+tG0uyrXYh7Mtz0lHjGV90qrFpq5o0jx0sLbDlDvpFPbIO58uYzKG4Sn2VTC4rOyX
+PE6SuDvMHIeX6Ekw4wSVQ9eTbksLQqTyxSqM3zp2ygc56SjGjy1nGQT8ZBGFzSbZ
+AzNOxVKrUsySx7LzZVl+zCGCPlQwaYLKObKXadZJmrqSFmErC5jcbVgBz7oJQOgl
+HJ2n0sMcZ+Ja1Y649mPVAgMBAAGjggEgMIIBHDAOBgNVHQ8BAf8EBAMCBaAwEwYD
+VR0lBAwwCgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUzA6f2Ajq
+zhX67c6piY2a1uTiUkwwHwYDVR0jBBgwFoAU2qfBlqxKMZnf0QeTeYiMelfqJfgw
+RAYIKwYBBQUHAQEEODA2MDQGCCsGAQUFBzABhihodHRwOi8vb2NzcC5jbG91ZGZs
+YXJlLmNvbS9vcmlnaW5fZWNjX2NhMCMGA1UdEQQcMBqCDCouYXJub2xkLmNvbYIK
+YXJub2xkLmNvbTA8BgNVHR8ENTAzMDGgL6AthitodHRwOi8vY3JsLmNsb3VkZmxh
+cmUuY29tL29yaWdpbl9lY2NfY2EuY3JsMAoGCCqGSM49BAMCA0gAMEUCIDV7HoMj
+K5rShE/l+90YAOzHC89OH/wUz3I5KYOFuehoAiEA8e92aIf9XBkr0K6EvFCiSsD+
+x+Yo/cL8fGfVpPt4UM8=
+-----END CERTIFICATE-----
+-----BEGIN WARP TOKEN-----
+N2IwYTRkNzdkZmI4ODFjMWEzYjdkNjFlYTk0NDNlMTkKdjEuMC01OGJkNGY5ZTI4
+ZjdiM2MyOGUwNWEzNWZmM2U4MGFiNGZkOTY0NGVmM2ZlY2U1MzdlYjBkMTJlMmU5
+MjU4MjE3LTE4MzQ0MmZiYjBiYmRiM2U1NzE1NThmZWM5YjU1ODllYmQ3N2FhZmM4
+NzQ5OGVlM2YwOWY2NGE0YWQ3OWZmZTg3OTFlZGJhZTA4YjM2YzFkOGYxZDcwYTg2
+NzBkZTU2OTIyZGZmOTJiMTVkMjE0YTUyNGY0ZWJmYTE5NTg4NTllLTdjZTgwZjc5
+OTIxMzEyYTYwMjJjNWQyNWUyZDM4MGY4MmNlYWVmZTNmYmRjNDNkZDEzYjA4MGUz
+ZWYxZTI2Zjc=
+-----END WARP TOKEN-----

cfsetup.yaml

@@ -1,5 +1,5 @@
 pinned_go: &pinned_go go=1.12.7-1
-build_dir: &build_dir /cfsetup_build/src/github.com/cloudflare/cloudflared/
+build_dir: &build_dir /cfsetup_build
 default-flavor: stretch
 stretch: &stretch
   build:
@@ -8,7 +8,6 @@ stretch: &stretch
       - *pinned_go
       - build-essential
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared
@@ -20,7 +19,6 @@ stretch: &stretch
       - fakeroot
       - rubygem-fpm
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=amd64
       - make cloudflared-deb
@@ -30,7 +28,6 @@ stretch: &stretch
       - *pinned_go
       - build-essential
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=amd64
       - make release
@@ -41,7 +38,6 @@ stretch: &stretch
       - crossbuild-essential-armhf
       - gcc-arm-linux-gnueabihf
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=arm
       - export CC=arm-linux-gnueabihf-gcc
@@ -52,7 +48,6 @@ stretch: &stretch
       - *pinned_go
       - gcc-multilib
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=386
       - make release
@@ -62,7 +57,6 @@ stretch: &stretch
       - *pinned_go
       - gcc-mingw-w64
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=windows
       - export GOARCH=amd64
       - export CC=x86_64-w64-mingw32-gcc
@@ -73,7 +67,6 @@ stretch: &stretch
       - *pinned_go
       - gcc-mingw-w64
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=windows
       - export GOARCH=386
       - export CC=i686-w64-mingw32-gcc-win32
@@ -84,12 +77,11 @@ stretch: &stretch
       - *pinned_go
       - build-essential
     post-cache:
-      - export GOPATH=/cfsetup_build/
       - export GOOS=linux
       - export GOARCH=amd64
-      - sudo chown -R $(whoami) /cfsetup_build/
-      - go get github.com/BurntSushi/go-sumtype
-      - export PATH="$GOPATH/bin:$PATH"
+      # cd to a non-module directory: https://github.com/golang/go/issues/24250
+      - (cd / && go get github.com/BurntSushi/go-sumtype)
+      - export PATH="$HOME/go/bin:$PATH"
       - make test
 
 jessie: *stretch
@@ -103,4 +95,4 @@ test_dbconnect:
       - --abort-on-container-exit
       - --exit-code-from=cloudflared
     files:
-      - dbconnect/integration_test/dbconnect.yaml
+      - dbconnect_tests/dbconnect.yaml

cmd/cloudflared/access/carrier.go

@@ -12,6 +12,32 @@ import (
 	cli "gopkg.in/urfave/cli.v2"
 )
 
+// StartForwarder starts a client side websocket forward
+func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}) error {
+	validURLString, err := validation.ValidateUrl(forwarder.Listener)
+	if err != nil {
+		logger.WithError(err).Error("Error validating origin URL")
+		return errors.Wrap(err, "error validating origin URL")
+	}
+
+	validURL, err := url.Parse(validURLString)
+	if err != nil {
+		logger.WithError(err).Error("Error parsing origin URL")
+		return errors.Wrap(err, "error parsing origin URL")
+	}
+
+	options := &carrier.StartOptions{
+		OriginURL: forwarder.URL,
+		Headers:   make(http.Header), //TODO: TUN-2688 support custom headers from config file
+	}
+
+	// we could add a cmd line variable for this bool if we want the SOCK5 server to be on the client side
+	wsConn := carrier.NewWSConnection(logger, false)
+
+	logger.Infof("Start Websocket listener on: %s", validURL.Host)
+	return carrier.StartForwarder(wsConn, validURL.Host, shutdown, options)
+}
+
 // ssh will start a WS proxy server for server mode
 // or copy from stdin/stdout for client mode
 // useful for proxying other protocols (like ssh) over websockets
@@ -23,7 +49,7 @@ func ssh(c *cli.Context) error {
 	if err != nil || rawHostName == "" {
 		return cli.ShowCommandHelp(c, "ssh")
 	}
-	originURL := "https://" + hostname
+	originURL := ensureURLScheme(hostname)
 
 	// get the headers from the cmdline and add them
 	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
@@ -34,11 +60,19 @@ func ssh(c *cli.Context) error {
 		headers.Add("CF-Access-Client-Secret", c.String(sshTokenSecretFlag))
 	}
 
+	destination := c.String(sshDestinationFlag)
+	if destination != "" {
+		headers.Add("CF-Access-SSH-Destination", destination)
+	}
+
 	options := &carrier.StartOptions{
 		OriginURL: originURL,
 		Headers:   headers,
 	}
 
+	// we could add a cmd line variable for this bool if we want the SOCK5 server to be on the client side
+	wsConn := carrier.NewWSConnection(logger, false)
+
 	if c.NArg() > 0 || c.IsSet(sshURLFlag) {
 		localForwarder, err := config.ValidateUrl(c)
 		if err != nil {
@@ -50,10 +84,12 @@ func ssh(c *cli.Context) error {
 			logger.WithError(err).Error("Error validating origin URL")
 			return errors.Wrap(err, "error validating origin URL")
 		}
-		return carrier.StartServer(logger, forwarder.Host, shutdownC, options)
+
+		logger.Infof("Start Websocket listener on: %s", forwarder.Host)
+		return carrier.StartForwarder(wsConn, forwarder.Host, shutdownC, options)
 	}
 
-	return carrier.StartClient(logger, &carrier.StdinoutStream{}, options)
+	return carrier.StartClient(wsConn, &carrier.StdinoutStream{}, options)
 }
 
 func buildRequestHeaders(values []string) http.Header {

cmd/cloudflared/access/cmd.go

@@ -1,26 +1,30 @@
 package access
 
 import (
-	"errors"
 	"fmt"
+	"net/http"
 	"net/url"
 	"os"
 	"strings"
 	"text/template"
+	"time"
 
+	"github.com/cloudflare/cloudflared/carrier"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/shell"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
 	"github.com/cloudflare/cloudflared/sshgen"
 	"github.com/cloudflare/cloudflared/validation"
+	"github.com/pkg/errors"
 	"golang.org/x/net/idna"
 
 	"github.com/cloudflare/cloudflared/log"
-	raven "github.com/getsentry/raven-go"
-	cli "gopkg.in/urfave/cli.v2"
+	"github.com/getsentry/raven-go"
+	"gopkg.in/urfave/cli.v2"
 )
 
 const (
 	sshHostnameFlag    = "hostname"
+	sshDestinationFlag = "destination"
 	sshURLFlag         = "url"
 	sshHeaderFlag      = "header"
 	sshTokenIDFlag     = "service-token-id"
@@ -62,6 +66,20 @@ func Flags() []cli.Flag {
 	return []cli.Flag{} // no flags yet.
 }
 
+// Ensures exit with error code if actionFunc returns an error
+func errorHandler(actionFunc cli.ActionFunc) cli.ActionFunc {
+	return func(ctx *cli.Context) error {
+		err := actionFunc(ctx)
+
+		if err != nil {
+			// os.Exits with error code if err is cli.ExitCoder or cli.MultiError
+			cli.HandleExitCoder(err)
+			err = cli.Exit(err.Error(), 1)
+		}
+		return err
+	}
+}
+
 // Commands returns all the Access related subcommands
 func Commands() []*cli.Command {
 	return []*cli.Command{
@@ -77,7 +95,7 @@ func Commands() []*cli.Command {
 			Subcommands: []*cli.Command{
 				{
 					Name:   "login",
-					Action: login,
+					Action: errorHandler(login),
 					Usage:  "login <url of access application>",
 					Description: `The login subcommand initiates an authentication flow with your identity provider.
 					The subcommand will launch a browser. For headless systems, a url is provided.
@@ -93,7 +111,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:   "curl",
-					Action: curl,
+					Action: errorHandler(curl),
 					Usage:  "curl [--allow-request, -ar] <url> [<curl args>...]",
 					Description: `The curl subcommand wraps curl and automatically injects the JWT into a cf-access-token
 					header when using curl to reach an application behind Access.`,
@@ -102,7 +120,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "token",
-					Action:      generateToken,
+					Action:      errorHandler(generateToken),
 					Usage:       "token -app=<url of access application>",
 					ArgsUsage:   "url of Access application",
 					Description: `The token subcommand produces a JWT which can be used to authenticate requests.`,
@@ -114,8 +132,8 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "ssh",
-					Action:      ssh,
-					Aliases:     []string{"rdp"},
+					Action:      errorHandler(ssh),
+					Aliases:     []string{"rdp", "tcp"},
 					Usage:       "",
 					ArgsUsage:   "",
 					Description: `The ssh subcommand sends data over a proxy to the Cloudflare edge.`,
@@ -124,6 +142,10 @@ func Commands() []*cli.Command {
 							Name:  sshHostnameFlag,
 							Usage: "specify the hostname of your application.",
 						},
+						&cli.StringFlag{
+							Name:  sshDestinationFlag,
+							Usage: "specify the destination address of your SSH server.",
+						},
 						&cli.StringFlag{
 							Name:  sshURLFlag,
 							Usage: "specify the host:port to forward data to Cloudflare edge.",
@@ -147,7 +169,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "ssh-config",
-					Action:      sshConfig,
+					Action:      errorHandler(sshConfig),
 					Usage:       "",
 					Description: `Prints an example configuration ~/.ssh/config`,
 					Flags: []cli.Flag{
@@ -163,7 +185,7 @@ func Commands() []*cli.Command {
 				},
 				{
 					Name:        "ssh-gen",
-					Action:      sshGen,
+					Action:      errorHandler(sshGen),
 					Usage:       "",
 					Description: `Generates a short lived certificate for given hostname`,
 					Flags: []cli.Flag{
@@ -180,27 +202,50 @@ func Commands() []*cli.Command {
 
 // login pops up the browser window to do the actual login and JWT generation
 func login(c *cli.Context) error {
-	raven.SetDSN(sentryDSN)
+	if err := raven.SetDSN(sentryDSN); err != nil {
+		return err
+	}
 	logger := log.CreateLogger()
 	args := c.Args()
-	appURL, err := url.Parse(args.First())
+	rawURL := ensureURLScheme(args.First())
+	appURL, err := url.Parse(rawURL)
 	if args.Len() < 1 || err != nil {
 		logger.Errorf("Please provide the url of the Access application\n")
 		return err
 	}
-	token, err := token.FetchToken(appURL)
-	if err != nil {
-		logger.Errorf("Failed to fetch token: %s\n", err)
+	if err := verifyTokenAtEdge(appURL, c); err != nil {
+		logger.WithError(err).Error("Could not verify token")
 		return err
 	}
-	fmt.Fprintf(os.Stdout, "Successfully fetched your token:\n\n%s\n\n", string(token))
+
+	cfdToken, err := token.GetTokenIfExists(appURL)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Unable to find token for provided application.")
+		return err
+	} else if cfdToken == "" {
+		fmt.Fprintln(os.Stderr, "token for provided application was empty.")
+		return errors.New("empty application token")
+	}
+	fmt.Fprintf(os.Stdout, "Successfully fetched your token:\n\n%s\n\n", cfdToken)
 
 	return nil
 }
 
+// ensureURLScheme prepends a URL with https:// if it doesnt have a scheme. http:// URLs will not be converted.
+func ensureURLScheme(url string) string {
+	url = strings.Replace(strings.ToLower(url), "http://", "https://", 1)
+	if !strings.HasPrefix(url, "https://") {
+		url = fmt.Sprintf("https://%s", url)
+
+	}
+	return url
+}
+
 // curl provides a wrapper around curl, passing Access JWT along in request
 func curl(c *cli.Context) error {
-	raven.SetDSN(sentryDSN)
+	if err := raven.SetDSN(sentryDSN); err != nil {
+		return err
+	}
 	logger := log.CreateLogger()
 	args := c.Args()
 	if args.Len() < 1 {
@@ -234,7 +279,9 @@ func curl(c *cli.Context) error {
 
 // token dumps provided token to stdout
 func generateToken(c *cli.Context) error {
-	raven.SetDSN(sentryDSN)
+	if err := raven.SetDSN(sentryDSN); err != nil {
+		return err
+	}
 	appURL, err := url.Parse(c.String("app"))
 	if err != nil || c.NumFlags() < 1 {
 		fmt.Fprintln(os.Stderr, "Please provide a url.")
@@ -281,7 +328,7 @@ func sshGen(c *cli.Context) error {
 		return cli.ShowCommandHelp(c, "ssh-gen")
 	}
 
-	originURL, err := url.Parse("https://" + hostname)
+	originURL, err := url.Parse(ensureURLScheme(hostname))
 	if err != nil {
 		return err
 	}
@@ -289,12 +336,12 @@ func sshGen(c *cli.Context) error {
 	// this fetchToken function mutates the appURL param. We should refactor that
 	fetchTokenURL := &url.URL{}
 	*fetchTokenURL = *originURL
-	token, err := token.FetchToken(fetchTokenURL)
+	cfdToken, err := token.FetchToken(fetchTokenURL)
 	if err != nil {
 		return err
 	}
 
-	if err := sshgen.GenerateShortLivedCertificate(originURL, token); err != nil {
+	if err := sshgen.GenerateShortLivedCertificate(originURL, cfdToken); err != nil {
 		return err
 	}
 
@@ -372,3 +419,59 @@ func isFileThere(candidate string) bool {
 	}
 	return true
 }
+
+// verifyTokenAtEdge checks for a token on disk, or generates a new one.
+// Then makes a request to to the origin with the token to ensure it is valid.
+// Returns nil if token is valid.
+func verifyTokenAtEdge(appUrl *url.URL, c *cli.Context) error {
+	headers := buildRequestHeaders(c.StringSlice(sshHeaderFlag))
+	if c.IsSet(sshTokenIDFlag) {
+		headers.Add("CF-Access-Client-Id", c.String(sshTokenIDFlag))
+	}
+	if c.IsSet(sshTokenSecretFlag) {
+		headers.Add("CF-Access-Client-Secret", c.String(sshTokenSecretFlag))
+	}
+	options := &carrier.StartOptions{OriginURL: appUrl.String(), Headers: headers}
+
+	if valid, err := isTokenValid(options); err != nil {
+		return err
+	} else if valid {
+		return nil
+	}
+
+	if err := token.RemoveTokenIfExists(appUrl); err != nil {
+		return err
+	}
+
+	if valid, err := isTokenValid(options); err != nil {
+		return err
+	} else if !valid {
+		return errors.New("failed to verify token")
+	}
+
+	return nil
+}
+
+// isTokenValid makes a request to the origin and returns true if the response was not a 302.
+func isTokenValid(options *carrier.StartOptions) (bool, error) {
+	req, err := carrier.BuildAccessRequest(options)
+	if err != nil {
+		return false, errors.Wrap(err, "Could not create access request")
+	}
+
+	// Do not follow redirects
+	client := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+		Timeout: time.Second * 5,
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer resp.Body.Close()
+
+	// A redirect to login means the token was invalid.
+	return !carrier.IsAccessResponse(resp), nil
+}

cmd/cloudflared/access/cmd_test.go

@@ -0,0 +1,25 @@
+package access
+
+import "testing"
+
+func Test_ensureURLScheme(t *testing.T) {
+	type args struct {
+		url string
+	}
+	tests := []struct {
+		name string
+		args args
+		want string
+	}{
+		{"no scheme", args{"localhost:123"}, "https://localhost:123"},
+		{"http scheme", args{"http://test"}, "https://test"},
+		{"https scheme", args{"https://test"}, "https://test"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := ensureURLScheme(tt.args.url); got != tt.want {
+				t.Errorf("ensureURLScheme() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/cloudflared/app_forward_service.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+)
+
+// ForwardServiceType is used to identify what kind of overwatch service this is
+const ForwardServiceType = "forward"
+
+// ForwarderService is used to wrap the access package websocket forwarders
+// into a service model for the overwatch package.
+// it also holds a reference to the config object that represents its state
+type ForwarderService struct {
+	forwarder config.Forwarder
+	shutdown  chan struct{}
+}
+
+// NewForwardService creates a new forwarder service
+func NewForwardService(f config.Forwarder) *ForwarderService {
+	return &ForwarderService{forwarder: f, shutdown: make(chan struct{}, 1)}
+}
+
+// Name is used to figure out this service is related to the others (normally the addr it binds to)
+// e.g. localhost:78641 or 127.0.0.1:2222 since this is a websocket forwarder
+func (s *ForwarderService) Name() string {
+	return s.forwarder.Listener
+}
+
+// Type is used to identify what kind of overwatch service this is
+func (s *ForwarderService) Type() string {
+	return ForwardServiceType
+}
+
+// Hash is used to figure out if this forwarder is the unchanged or not from the config file updates
+func (s *ForwarderService) Hash() string {
+	return s.forwarder.Hash()
+}
+
+// Shutdown stops the websocket listener
+func (s *ForwarderService) Shutdown() {
+	s.shutdown <- struct{}{}
+}
+
+// Run is the run loop that is started by the overwatch service
+func (s *ForwarderService) Run() error {
+	return access.StartForwarder(s.forwarder, s.shutdown)
+}

cmd/cloudflared/app_resolver_service.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/tunneldns"
+	"github.com/sirupsen/logrus"
+)
+
+// ResolverServiceType is used to identify what kind of overwatch service this is
+const ResolverServiceType = "resolver"
+
+// ResolverService is used to wrap the tunneldns package's DNS over HTTP
+// into a service model for the overwatch package.
+// it also holds a reference to the config object that represents its state
+type ResolverService struct {
+	resolver config.DNSResolver
+	shutdown chan struct{}
+	logger   *logrus.Logger
+}
+
+// NewResolverService creates a new resolver service
+func NewResolverService(r config.DNSResolver, logger *logrus.Logger) *ResolverService {
+	return &ResolverService{resolver: r,
+		shutdown: make(chan struct{}),
+		logger:   logger,
+	}
+}
+
+// Name is used to figure out this service is related to the others (normally the addr it binds to)
+// this is just "resolver" since there can only be one DNS resolver running
+func (s *ResolverService) Name() string {
+	return ResolverServiceType
+}
+
+// Type is used to identify what kind of overwatch service this is
+func (s *ResolverService) Type() string {
+	return ResolverServiceType
+}
+
+// Hash is used to figure out if this forwarder is the unchanged or not from the config file updates
+func (s *ResolverService) Hash() string {
+	return s.resolver.Hash()
+}
+
+// Shutdown stops the tunneldns listener
+func (s *ResolverService) Shutdown() {
+	s.shutdown <- struct{}{}
+}
+
+// Run is the run loop that is started by the overwatch service
+func (s *ResolverService) Run() error {
+	// create a listener
+	l, err := tunneldns.CreateListener(s.resolver.AddressOrDefault(), s.resolver.PortOrDefault(),
+		s.resolver.UpstreamsOrDefault(), s.resolver.BootstrapsOrDefault())
+	if err != nil {
+		return err
+	}
+
+	// start the listener.
+	readySignal := make(chan struct{})
+	err = l.Start(readySignal)
+	if err != nil {
+		l.Stop()
+		return err
+	}
+	<-readySignal
+	s.logger.Infof("start resolver on: %s:%d", s.resolver.AddressOrDefault(), s.resolver.PortOrDefault())
+
+	// wait for shutdown signal
+	<-s.shutdown
+	s.logger.Infof("shutdown on: %s:%d", s.resolver.AddressOrDefault(), s.resolver.PortOrDefault())
+	return l.Stop()
+}

cmd/cloudflared/app_service.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/overwatch"
+	"github.com/sirupsen/logrus"
+)
+
+// AppService is the main service that runs when no command lines flags are passed to cloudflared
+// it manages all the running services such as tunnels, forwarders, DNS resolver, etc
+type AppService struct {
+	configManager    config.Manager
+	serviceManager   overwatch.Manager
+	shutdownC        chan struct{}
+	configUpdateChan chan config.Root
+	logger           *logrus.Logger
+}
+
+// NewAppService creates a new AppService with needed supporting services
+func NewAppService(configManager config.Manager, serviceManager overwatch.Manager, shutdownC chan struct{}, logger *logrus.Logger) *AppService {
+	return &AppService{
+		configManager:    configManager,
+		serviceManager:   serviceManager,
+		shutdownC:        shutdownC,
+		configUpdateChan: make(chan config.Root),
+		logger:           logger,
+	}
+}
+
+// Run starts the run loop to handle config updates and run forwarders, tunnels, etc
+func (s *AppService) Run() error {
+	go s.actionLoop()
+	return s.configManager.Start(s)
+}
+
+// Shutdown kills all the running services
+func (s *AppService) Shutdown() error {
+	s.configManager.Shutdown()
+	s.shutdownC <- struct{}{}
+	return nil
+}
+
+// ConfigDidUpdate is a delegate notification from the config manager
+// it is trigger when the config file has been updated and now the service needs
+// to update its services accordingly
+func (s *AppService) ConfigDidUpdate(c config.Root) {
+	s.configUpdateChan <- c
+}
+
+// actionLoop handles the actions from running processes
+func (s *AppService) actionLoop() {
+	for {
+		select {
+		case c := <-s.configUpdateChan:
+			s.handleConfigUpdate(c)
+		case <-s.shutdownC:
+			for _, service := range s.serviceManager.Services() {
+				service.Shutdown()
+			}
+			return
+		}
+	}
+}
+
+func (s *AppService) handleConfigUpdate(c config.Root) {
+	// handle the client forward listeners
+	activeServices := map[string]struct{}{}
+	for _, f := range c.Forwarders {
+		service := NewForwardService(f)
+		s.serviceManager.Add(service)
+		activeServices[service.Name()] = struct{}{}
+	}
+
+	// handle resolver changes
+	if c.Resolver.Enabled {
+		service := NewResolverService(c.Resolver, s.logger)
+		s.serviceManager.Add(service)
+		activeServices[service.Name()] = struct{}{}
+
+	}
+
+	// TODO: TUN-1451 - tunnels
+
+	// remove any services that are no longer active
+	for _, service := range s.serviceManager.Services() {
+		if _, ok := activeServices[service.Name()]; !ok {
+			s.serviceManager.Remove(service.Name())
+		}
+	}
+}

cmd/cloudflared/config/manager.go

@@ -0,0 +1,107 @@
+package config
+
+import (
+	"errors"
+	"os"
+
+	"github.com/cloudflare/cloudflared/watcher"
+
+	"github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+)
+
+// Notifier sends out config updates
+type Notifier interface {
+	ConfigDidUpdate(Root)
+}
+
+// Manager is the base functions of the config manager
+type Manager interface {
+	Start(Notifier) error
+	Shutdown()
+}
+
+// FileManager watches the yaml config for changes
+// sends updates to the service to reconfigure to match the updated config
+type FileManager struct {
+	watcher    watcher.Notifier
+	notifier   Notifier
+	configPath string
+	logger     *logrus.Logger
+	ReadConfig func(string) (Root, error)
+}
+
+// NewFileManager creates a config manager
+func NewFileManager(watcher watcher.Notifier, configPath string, logger *logrus.Logger) (*FileManager, error) {
+	m := &FileManager{
+		watcher:    watcher,
+		configPath: configPath,
+		logger:     logger,
+		ReadConfig: readConfigFromPath,
+	}
+	err := watcher.Add(configPath)
+	return m, err
+}
+
+// Start starts the runloop to watch for config changes
+func (m *FileManager) Start(notifier Notifier) error {
+	m.notifier = notifier
+
+	// update the notifier with a fresh config on start
+	config, err := m.GetConfig()
+	if err != nil {
+		return err
+	}
+	notifier.ConfigDidUpdate(config)
+
+	m.watcher.Start(m)
+	return nil
+}
+
+// GetConfig reads the yaml file from the disk
+func (m *FileManager) GetConfig() (Root, error) {
+	return m.ReadConfig(m.configPath)
+}
+
+// Shutdown stops the watcher
+func (m *FileManager) Shutdown() {
+	m.watcher.Shutdown()
+}
+
+func readConfigFromPath(configPath string) (Root, error) {
+	if configPath == "" {
+		return Root{}, errors.New("unable to find config file")
+	}
+
+	file, err := os.Open(configPath)
+	if err != nil {
+		return Root{}, err
+	}
+	defer file.Close()
+
+	var config Root
+	if err := yaml.NewDecoder(file).Decode(&config); err != nil {
+		return Root{}, err
+	}
+
+	return config, nil
+}
+
+// File change notifications from the watcher
+
+// WatcherItemDidChange triggers when the yaml config is updated
+// sends the updated config to the service to reload its state
+func (m *FileManager) WatcherItemDidChange(filepath string) {
+	config, err := m.GetConfig()
+	if err != nil {
+		m.logger.WithError(err).Error("Failed to read new config")
+		return
+	}
+	m.logger.Info("Config file has been updated")
+	m.notifier.ConfigDidUpdate(config)
+}
+
+// WatcherDidError notifies of errors with the file watcher
+func (m *FileManager) WatcherDidError(err error) {
+	m.logger.WithError(err).Error("Config watcher encountered an error")
+}

cmd/cloudflared/config/manager_test.go

@@ -0,0 +1,89 @@
+package config
+
+import (
+	"os"
+	"testing"
+
+	"github.com/cloudflare/cloudflared/log"
+	"github.com/cloudflare/cloudflared/watcher"
+	"github.com/stretchr/testify/assert"
+)
+
+type mockNotifier struct {
+	configs []Root
+}
+
+func (n *mockNotifier) ConfigDidUpdate(c Root) {
+	n.configs = append(n.configs, c)
+}
+
+type mockFileWatcher struct {
+	path     string
+	notifier watcher.Notification
+	ready    chan struct{}
+}
+
+func (w *mockFileWatcher) Start(n watcher.Notification) {
+	w.notifier = n
+	w.ready <- struct{}{}
+}
+
+func (w *mockFileWatcher) Add(string) error {
+	return nil
+}
+
+func (w *mockFileWatcher) Shutdown() {
+
+}
+
+func (w *mockFileWatcher) TriggerChange() {
+	w.notifier.WatcherItemDidChange(w.path)
+}
+
+func TestConfigChanged(t *testing.T) {
+	filePath := "config.yaml"
+	f, err := os.Create(filePath)
+	assert.NoError(t, err)
+	defer func() {
+		f.Close()
+		os.Remove(filePath)
+	}()
+	c := &Root{
+		OrgKey:          "abcd",
+		ConfigType:      "mytype",
+		CheckinInterval: 1,
+		Forwarders: []Forwarder{
+			{
+				URL:      "test.daltoniam.com",
+				Listener: "127.0.0.1:8080",
+			},
+		},
+	}
+	configRead := func(configPath string) (Root, error) {
+		return *c, nil
+	}
+	wait := make(chan struct{})
+	w := &mockFileWatcher{path: filePath, ready: wait}
+
+	logger := log.CreateLogger()
+	service, err := NewFileManager(w, filePath, logger)
+	service.ReadConfig = configRead
+	assert.NoError(t, err)
+
+	n := &mockNotifier{}
+	go service.Start(n)
+
+	<-wait
+	c.Forwarders = append(c.Forwarders, Forwarder{URL: "add.daltoniam.com", Listener: "127.0.0.1:8081"})
+	w.TriggerChange()
+
+	service.Shutdown()
+
+	assert.Len(t, n.configs, 2, "did not get 2 config updates as expected")
+	assert.Len(t, n.configs[0].Forwarders, 1, "not the amount of forwarders expected")
+	assert.Len(t, n.configs[1].Forwarders, 2, "not the amount of forwarders expected")
+
+	assert.Equal(t, n.configs[0].Forwarders[0].Hash(), c.Forwarders[0].Hash(), "forwarder hashes don't match")
+	assert.Equal(t, n.configs[1].Forwarders[0].Hash(), c.Forwarders[0].Hash(), "forwarder hashes don't match")
+	assert.Equal(t, n.configs[1].Forwarders[1].Hash(), c.Forwarders[1].Hash(), "forwarder hashes don't match")
+}

cmd/cloudflared/config/model.go

@@ -0,0 +1,96 @@
+package config
+
+import (
+	"crypto/md5"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// Forwarder represents a client side listener to forward traffic to the edge
+type Forwarder struct {
+	URL      string `json:"url"`
+	Listener string `json:"listener"`
+}
+
+// Tunnel represents a tunnel that should be started
+type Tunnel struct {
+	URL          string `json:"url"`
+	Origin       string `json:"origin"`
+	ProtocolType string `json:"type"`
+}
+
+// DNSResolver represents a client side DNS resolver
+type DNSResolver struct {
+	Enabled    bool     `json:"enabled"`
+	Address    string   `json:"address"`
+	Port       uint16   `json:"port"`
+	Upstreams  []string `json:"upstreams"`
+	Bootstraps []string `json:"bootstraps"`
+}
+
+// Root is the base options to configure the service
+type Root struct {
+	OrgKey          string      `json:"org_key"`
+	ConfigType      string      `json:"type"`
+	CheckinInterval int         `json:"checkin_interval"`
+	Forwarders      []Forwarder `json:"forwarders,omitempty"`
+	Tunnels         []Tunnel    `json:"tunnels,omitempty"`
+	Resolver        DNSResolver `json:"resolver"`
+}
+
+// Hash returns the computed values to see if the forwarder values change
+func (f *Forwarder) Hash() string {
+	h := md5.New()
+	io.WriteString(h, f.URL)
+	io.WriteString(h, f.Listener)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+// Hash returns the computed values to see if the forwarder values change
+func (r *DNSResolver) Hash() string {
+	h := md5.New()
+	io.WriteString(h, r.Address)
+	io.WriteString(h, strings.Join(r.Bootstraps, ","))
+	io.WriteString(h, strings.Join(r.Upstreams, ","))
+	io.WriteString(h, fmt.Sprintf("%d", r.Port))
+	io.WriteString(h, fmt.Sprintf("%v", r.Enabled))
+	return fmt.Sprintf("%x", h.Sum(nil))
+}
+
+// EnabledOrDefault returns the enabled property
+func (r *DNSResolver) EnabledOrDefault() bool {
+	return r.Enabled
+}
+
+// AddressOrDefault returns the address or returns the default if empty
+func (r *DNSResolver) AddressOrDefault() string {
+	if r.Address != "" {
+		return r.Address
+	}
+	return "localhost"
+}
+
+// PortOrDefault return the port or returns the default if 0
+func (r *DNSResolver) PortOrDefault() uint16 {
+	if r.Port > 0 {
+		return r.Port
+	}
+	return 53
+}
+
+// UpstreamsOrDefault returns the upstreams or returns the default if empty
+func (r *DNSResolver) UpstreamsOrDefault() []string {
+	if len(r.Upstreams) > 0 {
+		return r.Upstreams
+	}
+	return []string{"https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"}
+}
+
+// BootstrapsOrDefault returns the bootstraps or returns the default if empty
+func (r *DNSResolver) BootstrapsOrDefault() []string {
+	if len(r.Bootstraps) > 0 {
+		return r.Bootstraps
+	}
+	return []string{"https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"}
+}

cmd/cloudflared/linux_service.go

@@ -73,7 +73,7 @@ ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 64 ]; then s
 Description=Update Argo Tunnel
 
 [Timer]
-OnUnitActiveSec=1d
+OnCalendar=daily
 
 [Install]
 WantedBy=timers.target
@@ -118,10 +118,6 @@ case "$1" in
             echo "Starting $name"
             $cmd >> "$stdout_log" 2>> "$stderr_log" &
             echo $! > "$pid_file"
-            if ! is_running; then
-                echo "Unable to start, see $stdout_log and $stderr_log"
-                exit 1
-            fi
         fi
     ;;
     stop)

cmd/cloudflared/macos_service.go

@@ -60,7 +60,7 @@ func newLaunchdTemplate(installPath, stdoutPath, stderrPath string) *ServiceTemp
 			<false/>
 		</dict>
 		<key>ThrottleInterval</key>
-		<integer>20</integer>
+		<integer>5</integer>
 	</dict>
 </plist>`, launchdIdentifier, stdoutPath, stderrPath),
 	}

cmd/cloudflared/main.go

@@ -6,10 +6,13 @@ import (
 	"time"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/access"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/tunnel"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/log"
 	"github.com/cloudflare/cloudflared/metrics"
+	"github.com/cloudflare/cloudflared/overwatch"
+	"github.com/cloudflare/cloudflared/watcher"
 
 	raven "github.com/getsentry/raven-go"
 	homedir "github.com/mitchellh/go-homedir"
@@ -121,7 +124,7 @@ func isEmptyInvocation(c *cli.Context) bool {
 func action(version string, shutdownC, graceShutdownC chan struct{}) cli.ActionFunc {
 	return func(c *cli.Context) (err error) {
 		if isEmptyInvocation(c) {
-			cli.ShowAppHelpAndExit(c, 1)
+			return handleServiceMode(shutdownC)
 		}
 		tags := make(map[string]string)
 		tags["hostname"] = c.String("hostname")
@@ -161,3 +164,29 @@ func handleError(err error) {
 	}
 	raven.CaptureError(err, nil)
 }
+
+// cloudflared was started without any flags
+func handleServiceMode(shutdownC chan struct{}) error {
+	// start the main run loop that reads from the config file
+	f, err := watcher.NewFile()
+	if err != nil {
+		logger.WithError(err).Error("Cannot load config file")
+		return err
+	}
+
+	configPath := config.FindDefaultConfigPath()
+	configManager, err := config.NewFileManager(f, configPath, logger)
+	if err != nil {
+		logger.WithError(err).Error("Cannot setup config file for monitoring")
+		return err
+	}
+
+	serviceManager := overwatch.NewAppManager(nil)
+
+	appService := NewAppService(configManager, serviceManager, shutdownC, logger)
+	if err := appService.Run(); err != nil {
+		logger.WithError(err).Error("Failed to start app service")
+		return err
+	}
+	return nil
+}

cmd/cloudflared/shell/launch_browser_darwin.go

@@ -0,0 +1,11 @@
+//+build darwin
+
+package shell
+
+import (
+	"os/exec"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	return exec.Command("open", url)
+}

cmd/cloudflared/shell/launch_browser_other.go

@@ -0,0 +1,11 @@
+//+build !windows,!darwin,!linux,!netbsd,!freebsd,!openbsd
+
+package shell
+
+import (
+	"os/exec"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	return nil
+}

cmd/cloudflared/shell/launch_browser_unix.go

@@ -0,0 +1,11 @@
+//+build linux freebsd openbsd netbsd
+
+package shell
+
+import (
+	"os/exec"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	return exec.Command("xdg-open", url)
+}

cmd/cloudflared/shell/launch_browser_windows.go

@@ -0,0 +1,18 @@
+//+build windows
+
+package shell
+
+import (
+	"fmt"
+	"os/exec"
+	"syscall"
+)
+
+func getBrowserCmd(url string) *exec.Cmd {
+	cmd := exec.Command("cmd")
+	// CmdLine is only defined when compiling for windows.
+	// Empty string is the cmd proc "Title". Needs to be included because the start command will interpret the first
+	// quoted string as that field and we want to quote the URL.
+	cmd.SysProcAttr = &syscall.SysProcAttr{CmdLine: fmt.Sprintf(`/c start "" "%s"`, url)}
+	return cmd
+}

cmd/cloudflared/shell/shell.go

@@ -4,25 +4,11 @@ import (
 	"io"
 	"os"
 	"os/exec"
-	"runtime"
 )
 
 // OpenBrowser opens the specified URL in the default browser of the user
 func OpenBrowser(url string) error {
-	var cmd string
-	var args []string
-
-	switch runtime.GOOS {
-	case "windows":
-		cmd = "cmd"
-		args = []string{"/c", "start"}
-	case "darwin":
-		cmd = "open"
-	default: // "linux", "freebsd", "openbsd", "netbsd"
-		cmd = "xdg-open"
-	}
-	args = append(args, url)
-	return exec.Command(cmd, args...).Start()
+	return getBrowserCmd(url).Start()
 }
 
 // Run will kick off a shell task and pipe the results to the respective std pipes

cmd/cloudflared/token/token.go

@@ -2,19 +2,21 @@ package token
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/url"
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/path"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
 	"github.com/cloudflare/cloudflared/log"
 	"github.com/cloudflare/cloudflared/origin"
-	"gopkg.in/coreos/go-oidc.v1/jose"
+	"github.com/coreos/go-oidc/jose"
 )
 
 const (
@@ -34,6 +36,21 @@ type signalHandler struct {
 	signals    []os.Signal
 }
 
+type jwtPayload struct {
+	Aud   []string `json:"aud"`
+	Email string   `json:"email"`
+	Exp   int      `json:"exp"`
+	Iat   int      `json:"iat"`
+	Nbf   int      `json:"nbf"`
+	Iss   string   `json:"iss"`
+	Type  string   `json:"type"`
+	Subt  string   `json:"sub"`
+}
+
+func (p jwtPayload) isExpired() bool {
+	return int(time.Now().Unix()) > p.Exp
+}
+
 func (s *signalHandler) register(handler func()) {
 	s.sigChannel = make(chan os.Signal, 1)
 	signal.Notify(s.sigChannel, s.signals...)
@@ -147,7 +164,7 @@ func FetchToken(appURL *url.URL) (string, error) {
 	return string(token), nil
 }
 
-// GetTokenIfExists will return the token from local storage if it exists
+// GetTokenIfExists will return the token from local storage if it exists and not expired
 func GetTokenIfExists(url *url.URL) (string, error) {
 	path, err := path.GenerateFilePathFromURL(url, keyName)
 	if err != nil {
@@ -162,6 +179,17 @@ func GetTokenIfExists(url *url.URL) (string, error) {
 		return "", err
 	}
 
+	var payload jwtPayload
+	err = json.Unmarshal(token.Payload, &payload)
+	if err != nil {
+		return "", err
+	}
+
+	if payload.isExpired() {
+		err := os.Remove(path)
+		return "", err
+	}
+
 	return token.Encode(), nil
 }
 

cmd/cloudflared/transfer/transfer.go

@@ -18,7 +18,7 @@ import (
 )
 
 const (
-	baseStoreURL  = "https://login.cloudflarewarp.com/"
+	baseStoreURL  = "https://login.argotunnel.com/"
 	clientTimeout = time.Second * 60
 )
 
@@ -45,7 +45,7 @@ func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncr
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Please open the following URL and log in with your Cloudflare account:\n\n%s\n\nLeave cloudflared running to download the %s automatically.\n", requestURL, resourceName)
 	} else {
-		fmt.Fprintf(os.Stderr, "A browser window should have opened at the following URL:\n\n%s\n\nIf the browser failed to open, open it yourself and visit the URL above.\n", requestURL)
+		fmt.Fprintf(os.Stderr, "A browser window should have opened at the following URL:\n\n%s\n\nIf the browser failed to open, please visit the URL above directly in your browser.\n", requestURL)
 	}
 
 	var resourceData []byte

cmd/cloudflared/tunnel/cmd.go

@@ -1,6 +1,7 @@
 package tunnel
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"io/ioutil"
@@ -8,37 +9,82 @@ import (
 	"net/url"
 	"os"
 	"reflect"
+	"runtime"
 	"runtime/trace"
+	"strings"
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/dbconnect"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
-	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/supervisor"
-	"github.com/google/uuid"
-
-	"github.com/getsentry/raven-go"
-
+	"github.com/cloudflare/cloudflared/awsuploader"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/dbconnect"
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/signal"
+	"github.com/cloudflare/cloudflared/socks"
+	"github.com/cloudflare/cloudflared/sshlog"
+	"github.com/cloudflare/cloudflared/sshserver"
+	"github.com/cloudflare/cloudflared/supervisor"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	"github.com/cloudflare/cloudflared/tunneldns"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/websocket"
+
 	"github.com/coreos/go-systemd/daemon"
 	"github.com/facebookgo/grace/gracenet"
+	"github.com/getsentry/raven-go"
+	"github.com/gliderlabs/ssh"
+	"github.com/google/uuid"
+	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"gopkg.in/urfave/cli.v2"
 	"gopkg.in/urfave/cli.v2/altsrc"
 )
 
-const sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b:3e8827f6f9f740738eb11138f7bebb68@sentry.io/189878"
+const (
+	sentryDSN = "https://56a9c9fa5c364ab28f34b14f35ea0f1b:3e8827f6f9f740738eb11138f7bebb68@sentry.io/189878"
+
+	sshLogFileDirectory = "/usr/local/var/log/cloudflared/"
+
+	// sshPortFlag is the port on localhost the cloudflared ssh server will run on
+	sshPortFlag = "local-ssh-port"
+
+	// sshIdleTimeoutFlag defines the duration a SSH session can remain idle before being closed
+	sshIdleTimeoutFlag = "ssh-idle-timeout"
+
+	// sshMaxTimeoutFlag defines the max duration a SSH session can remain open for
+	sshMaxTimeoutFlag = "ssh-max-timeout"
+
+	// bucketNameFlag is the bucket name to use for the SSH log uploader
+	bucketNameFlag = "bucket-name"
+
+	// regionNameFlag is the AWS region name to use for the SSH log uploader
+	regionNameFlag = "region-name"
+
+	// secretIDFlag is the Secret id of SSH log uploader
+	secretIDFlag = "secret-id"
+
+	// accessKeyIDFlag is the Access key id of SSH log uploader
+	accessKeyIDFlag = "access-key-id"
+
+	// sessionTokenIDFlag is the Session token of SSH log uploader
+	sessionTokenIDFlag = "session-token"
+
+	// s3URLFlag is the S3 URL of SSH log uploader (e.g. don't use AWS s3 and use google storage bucket instead)
+	s3URLFlag = "s3-url-host"
+
+	// hostKeyPath is the path of the dir to save SSH host keys too
+	hostKeyPath = "host-key-path"
+
+	// socks5Flag is to enable the socks server to deframe
+	socks5Flag = "socks5"
+
+	noIntentMsg = "The --intent argument is required. Cloudflared looks up an Intent to determine what configuration to use (i.e. which tunnels to start). If you don't have any Intents yet, you can use a placeholder Intent Label for now. Then, when you make an Intent with that label, cloudflared will get notified and open the tunnels you specified in that Intent."
+)
 
 var (
 	shutdownC      chan struct{}
@@ -94,6 +140,12 @@ func Commands() []*cli.Command {
 					Value:   cli.NewStringSlice("https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"),
 					EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
 				},
+				&cli.StringSliceFlag{
+					Name:    "bootstrap",
+					Usage:   "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",
+					Value:   cli.NewStringSlice("https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"),
+					EnvVars: []string{"TUNNEL_DNS_BOOTSTRAP"},
+				},
 			},
 			ArgsUsage: " ", // can't be the empty string or we get the default output
 			Hidden:    false,
@@ -290,6 +342,57 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 		c.Set("url", "https://"+helloListener.Addr().String())
 	}
 
+	if c.IsSet("ssh-server") {
+		if runtime.GOOS != "darwin" && runtime.GOOS != "linux" {
+			msg := fmt.Sprintf("--ssh-server is not supported on %s", runtime.GOOS)
+			logger.Error(msg)
+			return errors.New(msg)
+
+		}
+
+		logger.Infof("ssh-server set")
+
+		logManager := sshlog.NewEmptyManager()
+		if c.IsSet(bucketNameFlag) && c.IsSet(regionNameFlag) && c.IsSet(accessKeyIDFlag) && c.IsSet(secretIDFlag) {
+			uploader, err := awsuploader.NewFileUploader(c.String(bucketNameFlag), c.String(regionNameFlag),
+				c.String(accessKeyIDFlag), c.String(secretIDFlag), c.String(sessionTokenIDFlag), c.String(s3URLFlag))
+			if err != nil {
+				msg := "Cannot create uploader for SSH Server"
+				logger.WithError(err).Error(msg)
+				return errors.Wrap(err, msg)
+			}
+
+			if err := os.MkdirAll(sshLogFileDirectory, 0700); err != nil {
+				msg := fmt.Sprintf("Cannot create SSH log file directory %s", sshLogFileDirectory)
+				logger.WithError(err).Errorf(msg)
+				return errors.Wrap(err, msg)
+			}
+
+			logManager = sshlog.New(sshLogFileDirectory)
+
+			uploadManager := awsuploader.NewDirectoryUploadManager(logger, uploader, sshLogFileDirectory, 30*time.Minute, shutdownC)
+			uploadManager.Start()
+		}
+
+		localServerAddress := "127.0.0.1:" + c.String(sshPortFlag)
+		server, err := sshserver.New(logManager, logger, version, localServerAddress, c.String("hostname"), c.Path(hostKeyPath), shutdownC, c.Duration(sshIdleTimeoutFlag), c.Duration(sshMaxTimeoutFlag))
+		if err != nil {
+			msg := "Cannot create new SSH Server"
+			logger.WithError(err).Error(msg)
+			return errors.Wrap(err, msg)
+		}
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if err = server.Start(); err != nil && err != ssh.ErrServerClosed {
+				logger.WithError(err).Error("SSH server error")
+				// TODO: remove when declarative tunnels are implemented.
+				close(shutdownC)
+			}
+		}()
+		c.Set("url", "ssh://"+localServerAddress)
+	}
+
 	if host := hostnameFromURI(c.String("url")); host != "" {
 		listener, err := net.Listen("tcp", "127.0.0.1:")
 		if err != nil {
@@ -299,7 +402,18 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			errC <- websocket.StartProxyServer(logger, listener, host, shutdownC)
+			streamHandler := websocket.DefaultStreamHandler
+			if c.IsSet(socks5Flag) {
+				logger.Info("SOCKS5 server started")
+				streamHandler = func(wsConn *websocket.Conn, remoteConn net.Conn) {
+					dialer := socks.NewConnDialer(remoteConn)
+					requestHandler := socks.NewRequestHandler(dialer)
+					socksServer := socks.NewConnectionHandler(requestHandler)
+
+					socksServer.Serve(wsConn)
+				}
+			}
+			errC <- websocket.StartProxyServer(logger, listener, host, shutdownC, streamHandler)
 		}()
 		c.Set("url", "http://"+listener.Addr().String())
 	}
@@ -309,12 +423,17 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 		return err
 	}
 
+	reconnectCh := make(chan origin.ReconnectSignal, 1)
+	if c.IsSet("stdin-control") {
+		logger.Warn("Enabling control through stdin")
+		go stdinControl(reconnectCh)
+	}
+
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		errC <- origin.StartTunnelDaemon(ctx, tunnelConfig, connectedSignal, cloudflaredID)
+		errC <- origin.StartTunnelDaemon(ctx, tunnelConfig, connectedSignal, cloudflaredID, reconnectCh)
 	}()
-
 	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"))
 }
 
@@ -395,21 +514,16 @@ func startDeclarativeTunnel(ctx context.Context,
 		return err
 	}
 
-	var scope pogs.Scope
-	if c.IsSet("group") == c.IsSet("system-name") {
-		err = fmt.Errorf("exactly one of --group or --system-name must be specified")
-		logger.WithError(err).Error("unable to determine scope")
-		return err
-	} else if c.IsSet("group") {
-		scope = pogs.NewGroup(c.String("group"))
-	} else {
-		scope = pogs.NewSystemName(c.String("system-name"))
+	intentLabel := c.String("intent")
+	if intentLabel == "" {
+		logger.Error("--intent was empty")
+		return fmt.Errorf(noIntentMsg)
 	}
 
 	cloudflaredConfig := &connection.CloudflaredConfig{
 		BuildInfo:     buildInfo,
 		CloudflaredID: cloudflaredID,
-		Scope:         scope,
+		IntentLabel:   intentLabel,
 		Tags:          tags,
 	}
 
@@ -493,9 +607,15 @@ func notifySystemd(waitForSignal *signal.Signal) {
 
 func writePidFile(waitForSignal *signal.Signal, pidFile string) {
 	<-waitForSignal.Wait()
-	file, err := os.Create(pidFile)
+	expandedPath, err := homedir.Expand(pidFile)
 	if err != nil {
-		logger.WithError(err).Errorf("Unable to write pid to %s", pidFile)
+		logger.WithError(err).Errorf("Unable to expand %s, try to use absolute path in --pidfile", pidFile)
+		return
+	}
+	file, err := os.Create(expandedPath)
+	if err != nil {
+		logger.WithError(err).Errorf("Unable to write pid to %s", expandedPath)
+		return
 	}
 	defer file.Close()
 	fmt.Fprintf(file, "%d", os.Getpid())
@@ -511,6 +631,8 @@ func hostnameFromURI(uri string) string {
 		return addPortIfMissing(u, 22)
 	case "rdp":
 		return addPortIfMissing(u, 3389)
+	case "tcp":
+		return addPortIfMissing(u, 7864) // just a random port since there isn't a default in this case
 	}
 	return ""
 }
@@ -591,10 +713,11 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Hidden: shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:   "no-autoupdate",
-			Usage:  "Disable periodic check for updates, restarting the server with the new version.",
-			Value:  false,
-			Hidden: shouldHide,
+			Name:    "no-autoupdate",
+			Usage:   "Disable periodic check for updates, restarting the server with the new version.",
+			EnvVars: []string{"NO_AUTOUPDATE"},
+			Value:   false,
+			Hidden:  shouldHide,
 		}),
 		altsrc.NewBoolFlag(&cli.BoolFlag{
 			Name:   "is-autoupdated",
@@ -604,7 +727,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
 			Name:    "edge",
-			Usage:   "Address of the Cloudflare tunnel server.",
+			Usage:   "Address of the Cloudflare tunnel server. Only works in Cloudflare's internal testing environment.",
 			EnvVars: []string{"TUNNEL_EDGE"},
 			Hidden:  true,
 		}),
@@ -652,6 +775,12 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_HOSTNAME"},
 			Hidden:  shouldHide,
 		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    "http-host-header",
+			Usage:   "Sets the HTTP Host header for the local webserver.",
+			EnvVars: []string{"TUNNEL_HTTP_HOST_HEADER"},
+			Hidden:  shouldHide,
+		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    "origin-server-name",
 			Usage:   "Hostname on the origin server certificate.",
@@ -749,6 +878,13 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_HELLO_WORLD"},
 			Hidden:  shouldHide,
 		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "ssh-server",
+			Value:   false,
+			Usage:   "Run an SSH Server",
+			EnvVars: []string{"TUNNEL_SSH_SERVER"},
+			Hidden:  true, // TODO: remove when feature is complete
+		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    "pidfile",
 			Usage:   "Write the application's PID to this file after first successful connection.",
@@ -840,6 +976,13 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
 			Hidden:  shouldHide,
 		}),
+		altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
+			Name:    "proxy-dns-bootstrap",
+			Usage:   "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",
+			Value:   cli.NewStringSlice("https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"),
+			EnvVars: []string{"TUNNEL_DNS_BOOTSTRAP"},
+			Hidden:  shouldHide,
+		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:    "grace-period",
 			Usage:   "Duration to accept new requests after cloudflared receives first SIGINT/SIGTERM. A second SIGINT/SIGTERM will force cloudflared to shutdown immediately.",
@@ -873,15 +1016,23 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Hidden:  true,
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "system-name",
-			Usage:   "Unique identifier for this cloudflared instance. It can be configured individually in the Declarative Tunnel UI. Mutually exclusive with `--group`.",
-			EnvVars: []string{"TUNNEL_SYSTEM_NAME"},
+			Name:    "intent",
+			Usage:   "The label of an Intent from which `cloudflared` should gets its tunnels from. Intents can be created in the Origin Registry UI.",
+			EnvVars: []string{"TUNNEL_INTENT"},
 			Hidden:  true,
 		}),
-		altsrc.NewStringFlag(&cli.StringFlag{
-			Name:    "group",
-			Usage:   "Name of a group of cloudflared instances, of which this instance should be an identical copy. They can be configured collectively in the Declarative Tunnel UI. Mutually exclusive with `--system-name`.",
-			EnvVars: []string{"TUNNEL_GROUP"},
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "use-reconnect-token",
+			Usage:   "Test reestablishing connections with the new 'reconnect token' flow.",
+			Value:   true,
+			EnvVars: []string{"TUNNEL_USE_RECONNECT_TOKEN"},
+			Hidden:  true,
+		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "use-quick-reconnects",
+			Usage:   "Test reestablishing connections with the new 'connection digest' flow.",
+			Value:   true,
+			EnvVars: []string{"TUNNEL_USE_QUICK_RECONNECTS"},
 			Hidden:  true,
 		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
@@ -891,5 +1042,113 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"DIAL_EDGE_TIMEOUT"},
 			Hidden:  true,
 		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    sshPortFlag,
+			Usage:   "Localhost port that cloudflared SSH server will run on",
+			Value:   "2222",
+			EnvVars: []string{"LOCAL_SSH_PORT"},
+			Hidden:  true,
+		}),
+		altsrc.NewDurationFlag(&cli.DurationFlag{
+			Name:    sshIdleTimeoutFlag,
+			Usage:   "Connection timeout after no activity",
+			EnvVars: []string{"SSH_IDLE_TIMEOUT"},
+			Hidden:  true,
+		}),
+		altsrc.NewDurationFlag(&cli.DurationFlag{
+			Name:    sshMaxTimeoutFlag,
+			Usage:   "Absolute connection timeout",
+			EnvVars: []string{"SSH_MAX_TIMEOUT"},
+			Hidden:  true,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    bucketNameFlag,
+			Usage:   "Bucket name of where to upload SSH logs",
+			EnvVars: []string{"BUCKET_ID"},
+			Hidden:  true,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    regionNameFlag,
+			Usage:   "Region name of where to upload SSH logs",
+			EnvVars: []string{"REGION_ID"},
+			Hidden:  true,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    accessKeyIDFlag,
+			Usage:   "Access Key ID of where to upload SSH logs",
+			EnvVars: []string{"ACCESS_CLIENT_ID"},
+			Hidden:  true,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    secretIDFlag,
+			Usage:   "Secret ID of where to upload SSH logs",
+			EnvVars: []string{"SECRET_ID"},
+			Hidden:  true,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    sessionTokenIDFlag,
+			Usage:   "Session Token to use in the configuration of SSH logs uploading",
+			EnvVars: []string{"SESSION_TOKEN_ID"},
+			Hidden:  true,
+		}),
+		altsrc.NewStringFlag(&cli.StringFlag{
+			Name:    s3URLFlag,
+			Usage:   "S3 url of where to upload SSH logs",
+			EnvVars: []string{"S3_URL"},
+			Hidden:  true,
+		}),
+		altsrc.NewPathFlag(&cli.PathFlag{
+			Name:    hostKeyPath,
+			Usage:   "Absolute path of directory to save SSH host keys in",
+			EnvVars: []string{"HOST_KEY_PATH"},
+			Hidden:  true,
+		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "stdin-control",
+			Usage:   "Control the process using commands sent through stdin",
+			EnvVars: []string{"STDIN-CONTROL"},
+			Hidden:  true,
+			Value:   false,
+		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    socks5Flag,
+			Usage:   "specify if this tunnel is running as a SOCK5 Server",
+			EnvVars: []string{"TUNNEL_SOCKS"},
+			Value:   false,
+			Hidden:  false,
+		}),
+	}
+}
+
+func stdinControl(reconnectCh chan origin.ReconnectSignal) {
+	for {
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			command := scanner.Text()
+			parts := strings.SplitN(command, " ", 2)
+
+			switch parts[0] {
+			case "":
+				break
+			case "reconnect":
+				var reconnect origin.ReconnectSignal
+				if len(parts) > 1 {
+					var err error
+					if reconnect.Delay, err = time.ParseDuration(parts[1]); err != nil {
+						logger.Error(err.Error())
+						continue
+					}
+				}
+				logger.Infof("Sending reconnect signal %+v", reconnect)
+				reconnectCh <- reconnect
+			default:
+				logger.Warn("Unknown command: ", command)
+				fallthrough
+			case "help":
+				logger.Info(`Supported command: 
+reconnect [delay] 
+- restarts one randomly chosen connection with optional delay before reconnect`)
+			}
+		}
 	}
 }

cmd/cloudflared/tunnel/configuration.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -197,17 +197,21 @@ func prepareTunnelConfig(
 	httpTransport := &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
 		MaxIdleConns:          c.Int("proxy-keepalive-connections"),
+		MaxIdleConnsPerHost:   c.Int("proxy-keepalive-connections"),
 		IdleConnTimeout:       c.Duration("proxy-keepalive-timeout"),
 		TLSHandshakeTimeout:   c.Duration("proxy-tls-timeout"),
 		ExpectContinueTimeout: 1 * time.Second,
 		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: c.IsSet("no-tls-verify")},
 	}
 
-	dialContext := (&net.Dialer{
+	dialer := &net.Dialer{
 		Timeout:   c.Duration("proxy-connect-timeout"),
 		KeepAlive: c.Duration("proxy-tcp-keepalive"),
-		DualStack: !c.Bool("proxy-no-happy-eyeballs"),
-	}).DialContext
+	}
+	if c.Bool("proxy-no-happy-eyeballs") {
+		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
+	}
+	dialContext := dialer.DialContext
 
 	if c.IsSet("unix-socket") {
 		unixSocket, err := config.ValidateUnixSocket(c)
@@ -233,7 +237,6 @@ func prepareTunnelConfig(
 	err = validation.ValidateHTTPService(originURL, hostname, httpTransport)
 	if err != nil {
 		logger.WithError(err).Error("unable to connect to the origin")
-		return nil, errors.Wrap(err, "unable to connect to the origin")
 	}
 
 	toEdgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c)
@@ -253,6 +256,7 @@ func prepareTunnelConfig(
 		HTTPTransport:        httpTransport,
 		HeartbeatInterval:    c.Duration("heartbeat-interval"),
 		Hostname:             hostname,
+		HTTPHostHeader:       c.String("http-host-header"),
 		IncidentLookup:       origin.NewIncidentLookup(),
 		IsAutoupdated:        c.Bool("is-autoupdated"),
 		IsFreeTunnel:         isFreeTunnel,
@@ -271,16 +275,18 @@ func prepareTunnelConfig(
 		TlsConfig:            toEdgeTLSConfig,
 		TransportLogger:      transportLogger,
 		UseDeclarativeTunnel: c.Bool("use-declarative-tunnels"),
+		UseReconnectToken:    c.Bool("use-reconnect-token"),
+		UseQuickReconnects:   c.Bool("use-quick-reconnects"),
 	}, nil
 }
 
-func serviceDiscoverer(c *cli.Context, logger *logrus.Logger) (connection.EdgeServiceDiscoverer, error) {
+func serviceDiscoverer(c *cli.Context, logger *logrus.Logger) (*edgediscovery.Edge, error) {
 	// If --edge is specfied, resolve edge server addresses
 	if len(c.StringSlice("edge")) > 0 {
-		return connection.NewEdgeHostnameResolver(c.StringSlice("edge"))
+		return edgediscovery.StaticEdge(logger, c.StringSlice("edge"))
 	}
 	// Otherwise lookup edge server addresses through service discovery
-	return connection.NewEdgeAddrResolver(logger)
+	return edgediscovery.ResolveEdge(logger)
 }
 
 func isRunningFromTerminal() bool {

cmd/cloudflared/tunnel/login.go

@@ -15,7 +15,7 @@ import (
 
 const (
 	baseLoginURL     = "https://dash.cloudflare.com/argotunnel"
-	callbackStoreURL = "https://login.cloudflarewarp.com/"
+	callbackStoreURL = "https://login.argotunnel.com/"
 )
 
 func login(c *cli.Context) error {

cmd/cloudflared/tunnel/server.go

@@ -14,7 +14,7 @@ func runDNSProxyServer(c *cli.Context, dnsReadySignal, shutdownC chan struct{})
 		logger.Errorf("The 'proxy-dns-port' must be a valid port number in <1, 65535> range.")
 		return errors.New("The 'proxy-dns-port' must be a valid port number in <1, 65535> range.")
 	}
-	listener, err := tunneldns.CreateListener(c.String("proxy-dns-address"), uint16(port), c.StringSlice("proxy-dns-upstream"))
+	listener, err := tunneldns.CreateListener(c.String("proxy-dns-address"), uint16(port), c.StringSlice("proxy-dns-upstream"), c.StringSlice("proxy-dns-bootstrap"))
 	if err != nil {
 		close(dnsReadySignal)
 		listener.Stop()

cmd/cloudflared/tunnel/signal.go

@@ -17,9 +17,11 @@ func waitForSignal(errC chan error, shutdownC chan struct{}) error {
 
 	select {
 	case err := <-errC:
+		logger.Infof("terminating due to error: %v", err)
 		close(shutdownC)
 		return err
-	case <-signals:
+	case s := <-signals:
+		logger.Infof("terminating due to signal %s", s)
 		close(shutdownC)
 	case <-shutdownC:
 	}
@@ -44,10 +46,12 @@ func waitForSignalWithGraceShutdown(errC chan error,
 
 	select {
 	case err := <-errC:
+		logger.Infof("Initiating graceful shutdown due to %v ...", err)
 		close(graceShutdownC)
 		close(shutdownC)
 		return err
-	case <-signals:
+	case s := <-signals:
+		logger.Infof("Initiating graceful shutdown due to signal %s ...", s)
 		close(graceShutdownC)
 		waitForGracePeriod(signals, errC, shutdownC, gracePeriod)
 	case <-graceShutdownC:
@@ -64,7 +68,6 @@ func waitForGracePeriod(signals chan os.Signal,
 	shutdownC chan struct{},
 	gracePeriod time.Duration,
 ) {
-	logger.Infof("Initiating graceful shutdown...")
 	// Unregister signal handler early, so the client can send a second SIGTERM/SIGINT
 	// to force shutdown cloudflared
 	signal.Stop(signals)

cmd/cloudflared/updater/update.go

@@ -2,6 +2,7 @@ package updater
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"runtime"
 	"time"
@@ -32,6 +33,33 @@ EKx0BZogHSor9Wy5VztdFaAaVbsJiCbO
 	logger = log.CreateLogger()
 )
 
+// BinaryUpdated implements ExitCoder interface, the app will exit with status code 11
+// https://pkg.go.dev/gopkg.in/urfave/cli.v2?tab=doc#ExitCoder
+type statusSuccess struct {
+	newVersion string
+}
+
+func (u *statusSuccess) Error() string {
+	return fmt.Sprintf("cloudflared has been updated to version %s", u.newVersion)
+}
+
+func (u *statusSuccess) ExitCode() int {
+	return 11
+}
+
+// UpdateErr implements ExitCoder interface, the app will exit with status code 10
+type statusErr struct {
+	err error
+}
+
+func (e *statusErr) Error() string {
+	return fmt.Sprintf("failed to update cloudflared: %v", e.err)
+}
+
+func (e *statusErr) ExitCode() int {
+	return 10
+}
+
 type UpdateOutcome struct {
 	Updated bool
 	Version string
@@ -67,14 +95,15 @@ func checkForUpdateAndApply() UpdateOutcome {
 func Update(_ *cli.Context) error {
 	updateOutcome := loggedUpdate()
 	if updateOutcome.Error != nil {
-		os.Exit(10)
+		return &statusErr{updateOutcome.Error}
 	}
 
 	if updateOutcome.noUpdate() {
 		logger.Infof("cloudflared is up to date (%s)", updateOutcome.Version)
+		return nil
 	}
 
-	return updateOutcome.Error
+	return &statusSuccess{newVersion: updateOutcome.Version}
 }
 
 // Checks for an update and applies it if one is available
@@ -126,15 +155,19 @@ func (a *AutoUpdater) Run(ctx context.Context) error {
 			updateOutcome := loggedUpdate()
 			if updateOutcome.Updated {
 				os.Args = append(os.Args, "--is-autoupdated=true")
-				pid, err := a.listeners.StartProcess()
-				if err != nil {
-					logger.WithError(err).Error("Unable to restart server automatically")
-					return err
+				if IsSysV() {
+					// SysV doesn't have a mechanism to keep service alive, we have to restart the process
+					logger.Infof("Restarting service managed by SysV...")
+					pid, err := a.listeners.StartProcess()
+					if err != nil {
+						logger.WithError(err).Error("Unable to restart server automatically")
+						return &statusErr{err: err}
+					}
+					// stop old process after autoupdate. Otherwise we create a new process
+					// after each update
+					logger.Infof("PID of the new process is %d", pid)
 				}
-				// stop old process after autoupdate. Otherwise we create a new process
-				// after each update
-				logger.Infof("PID of the new process is %d", pid)
-				return nil
+				return &statusSuccess{newVersion: updateOutcome.Version}
 			}
 		}
 		select {
@@ -187,3 +220,14 @@ func SupportAutoUpdate() bool {
 func isRunningFromTerminal() bool {
 	return terminal.IsTerminal(int(os.Stdout.Fd()))
 }
+
+func IsSysV() bool {
+	if runtime.GOOS != "linux" {
+		return false
+	}
+
+	if _, err := os.Stat("/run/systemd/system"); err == nil {
+		return false
+	}
+	return true
+}

connection/connection.go

@@ -5,35 +5,27 @@ import (
 	"net"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
-	rpc "zombiezen.com/go/capnproto2/rpc"
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
 	openStreamTimeout = 30 * time.Second
 )
 
-type dialError struct {
-	cause error
-}
-
-func (e dialError) Error() string {
-	return e.cause.Error()
-}
-
 type Connection struct {
-	id    uuid.UUID
-	muxer *h2mux.Muxer
+	id          uuid.UUID
+	muxer       *h2mux.Muxer
+	addr        *net.TCPAddr
+	isLongLived bool
+	longLivedID int
 }
 
-func newConnection(muxer *h2mux.Muxer, edgeIP *net.TCPAddr) (*Connection, error) {
+func newConnection(muxer *h2mux.Muxer, addr *net.TCPAddr) (*Connection, error) {
 	id, err := uuid.NewRandom()
 	if err != nil {
 		return nil, err
@@ -41,6 +33,7 @@ func newConnection(muxer *h2mux.Muxer, edgeIP *net.TCPAddr) (*Connection, error)
 	return &Connection{
 		id:    id,
 		muxer: muxer,
+		addr:  addr,
 	}, nil
 }
 
@@ -50,32 +43,15 @@ func (c *Connection) Serve(ctx context.Context) error {
 }
 
 // Connect is used to establish connections with cloudflare's edge network
-func (c *Connection) Connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters, logger *logrus.Entry) (*pogs.ConnectResult, error) {
-	openStreamCtx, cancel := context.WithTimeout(ctx, openStreamTimeout)
-	defer cancel()
-
-	rpcConn, err := c.newRPConn(openStreamCtx, logger)
+func (c *Connection) Connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters, logger *logrus.Entry) (tunnelpogs.ConnectResult, error) {
+	tsClient, err := NewRPCClient(ctx, c.muxer, logger.WithField("rpc", "connect"), openStreamTimeout)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot create new RPC connection")
 	}
-	defer rpcConn.Close()
-
-	tsClient := tunnelpogs.TunnelServer_PogsClient{Client: rpcConn.Bootstrap(ctx)}
-
+	defer tsClient.Close()
 	return tsClient.Connect(ctx, parameters)
 }
 
 func (c *Connection) Shutdown() {
 	c.muxer.Shutdown()
 }
-
-func (c *Connection) newRPConn(ctx context.Context, logger *logrus.Entry) (*rpc.Conn, error) {
-	stream, err := c.muxer.OpenRPCStream(ctx)
-	if err != nil {
-		return nil, err
-	}
-	return rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger.WithField("rpc", "connect"), rpc.StreamTransport(stream)),
-		tunnelrpc.ConnLog(logger.WithField("rpc", "connect")),
-	), nil
-}

connection/dial.go

@@ -0,0 +1,54 @@
+package connection
+
+import (
+	"context"
+	"crypto/tls"
+	"net"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// DialEdge makes a TLS connection to a Cloudflare edge node
+func DialEdge(
+	ctx context.Context,
+	timeout time.Duration,
+	tlsConfig *tls.Config,
+	edgeTCPAddr *net.TCPAddr,
+) (net.Conn, error) {
+	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
+	dialCtx, dialCancel := context.WithTimeout(ctx, timeout)
+	defer dialCancel()
+
+	dialer := net.Dialer{}
+	edgeConn, err := dialer.DialContext(dialCtx, "tcp", edgeTCPAddr.String())
+	if err != nil {
+		return nil, newDialError(err, "DialContext error")
+	}
+	tlsEdgeConn := tls.Client(edgeConn, tlsConfig)
+	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
+
+	if err = tlsEdgeConn.Handshake(); err != nil {
+		return nil, newDialError(err, "TLS handshake with edge error")
+	}
+	// clear the deadline on the conn; h2mux has its own timeouts
+	tlsEdgeConn.SetDeadline(time.Time{})
+	return tlsEdgeConn, nil
+}
+
+// DialError is an error returned from DialEdge
+type DialError struct {
+	cause error
+}
+
+func newDialError(err error, message string) error {
+	return DialError{cause: errors.Wrap(err, message)}
+}
+
+func (e DialError) Error() string {
+	return e.cause.Error()
+}
+
+func (e DialError) Cause() error {
+	return e.cause
+}

connection/discovery.go

@@ -1,254 +0,0 @@
-package connection
-
-import (
-	"context"
-	"crypto/tls"
-	"fmt"
-	"net"
-	"sync"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-)
-
-const (
-	// Used to discover HA Warp servers
-	srvService = "warp"
-	srvProto   = "tcp"
-	srvName    = "cloudflarewarp.com"
-
-	// Used to fallback to DoT when we can't use the default resolver to
-	// discover HA Warp servers (GitHub issue #75).
-	dotServerName = "cloudflare-dns.com"
-	dotServerAddr = "1.1.1.1:853"
-	dotTimeout    = time.Duration(15 * time.Second)
-
-	// SRV record resolution TTL
-	resolveEdgeAddrTTL = 1 * time.Hour
-)
-
-var friendlyDNSErrorLines = []string{
-	`Please try the following things to diagnose this issue:`,
-	`  1. ensure that cloudflarewarp.com is returning "warp" service records.`,
-	`     Run your system's equivalent of: dig srv _warp._tcp.cloudflarewarp.com`,
-	`  2. ensure that your DNS resolver is not returning compressed SRV records.`,
-	`     See GitHub issue https://github.com/golang/go/issues/27546`,
-	`     For example, you could use Cloudflare's 1.1.1.1 as your resolver:`,
-	`     https://developers.cloudflare.com/1.1.1.1/setting-up-1.1.1.1/`,
-}
-
-// EdgeServiceDiscoverer is an interface for looking up Cloudflare's edge network addresses
-type EdgeServiceDiscoverer interface {
-	// Addr returns an address to connect to cloudflare's edge network
-	Addr() *net.TCPAddr
-	// AvailableAddrs returns the number of unique addresses
-	AvailableAddrs() uint8
-	// Refresh rediscover Cloudflare's edge network addresses
-	Refresh() error
-}
-
-// EdgeAddrResolver discovers the addresses of Cloudflare's edge network through SRV record.
-// It implements EdgeServiceDiscoverer interface
-type EdgeAddrResolver struct {
-	sync.Mutex
-	// Addrs to connect to cloudflare's edge network
-	addrs []*net.TCPAddr
-	// index of the next element to use in addrs
-	nextAddrIndex int
-	logger        *logrus.Entry
-}
-
-func NewEdgeAddrResolver(logger *logrus.Logger) (EdgeServiceDiscoverer, error) {
-	r := &EdgeAddrResolver{
-		logger: logger.WithField("subsystem", " edgeAddrResolver"),
-	}
-	if err := r.Refresh(); err != nil {
-		return nil, err
-	}
-	return r, nil
-}
-
-func (r *EdgeAddrResolver) Addr() *net.TCPAddr {
-	r.Lock()
-	defer r.Unlock()
-	addr := r.addrs[r.nextAddrIndex]
-	r.nextAddrIndex = (r.nextAddrIndex + 1) % len(r.addrs)
-	return addr
-}
-
-func (r *EdgeAddrResolver) AvailableAddrs() uint8 {
-	r.Lock()
-	defer r.Unlock()
-	return uint8(len(r.addrs))
-}
-
-func (r *EdgeAddrResolver) Refresh() error {
-	newAddrs, err := EdgeDiscovery(r.logger)
-	if err != nil {
-		return err
-	}
-	r.Lock()
-	defer r.Unlock()
-	r.addrs = newAddrs
-	r.nextAddrIndex = 0
-	return nil
-}
-
-// HA service discovery lookup
-func EdgeDiscovery(logger *logrus.Entry) ([]*net.TCPAddr, error) {
-	_, addrs, err := net.LookupSRV(srvService, srvProto, srvName)
-	if err != nil {
-		// Try to fall back to DoT from Cloudflare directly.
-		//
-		// Note: Instead of DoT, we could also have used DoH. Either of these:
-		//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_warp._tcp.cloudflarewarp.com&type=srv)
-		//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
-		// But both of these cases miss out on a key feature from the stdlib:
-		//     "The returned records are sorted by priority and randomized by weight within a priority."
-		//     (https://golang.org/pkg/net/#Resolver.LookupSRV)
-		// Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it.
-		// See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552
-		r := fallbackResolver(dotServerName, dotServerAddr)
-		ctx, cancel := context.WithTimeout(context.Background(), dotTimeout)
-		defer cancel()
-		_, fallbackAddrs, fallbackErr := r.LookupSRV(ctx, srvService, srvProto, srvName)
-		if fallbackErr != nil || len(fallbackAddrs) == 0 {
-			// use the original DNS error `err` in messages, not `fallbackErr`
-			logger.Errorln("Error looking up Cloudflare edge IPs: the DNS query failed:", err)
-			for _, s := range friendlyDNSErrorLines {
-				logger.Errorln(s)
-			}
-			return nil, errors.Wrap(err, "Could not lookup srv records on _warp._tcp.cloudflarewarp.com")
-		}
-		// Accept the fallback results and keep going
-		addrs = fallbackAddrs
-	}
-	var resolvedIPsPerCNAME [][]*net.TCPAddr
-	var lookupErr error
-	for _, addr := range addrs {
-		ips, err := resolveSRVToTCP(addr)
-		if err != nil || len(ips) == 0 {
-			// don't return early, we might be able to resolve other addresses
-			lookupErr = err
-			continue
-		}
-		resolvedIPsPerCNAME = append(resolvedIPsPerCNAME, ips)
-	}
-	ips := flattenServiceIPs(resolvedIPsPerCNAME)
-	if lookupErr == nil && len(ips) == 0 {
-		return nil, fmt.Errorf("Unknown service discovery error")
-	}
-	return ips, lookupErr
-}
-
-func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
-	ips, err := net.LookupIP(srv.Target)
-	if err != nil {
-		return nil, err
-	}
-	addrs := make([]*net.TCPAddr, len(ips))
-	for i, ip := range ips {
-		addrs[i] = &net.TCPAddr{IP: ip, Port: int(srv.Port)}
-	}
-	return addrs, nil
-}
-
-// FlattenServiceIPs transposes and flattens the input slices such that the
-// first element of the n inner slices are the first n elements of the result.
-func flattenServiceIPs(ipsByService [][]*net.TCPAddr) []*net.TCPAddr {
-	var result []*net.TCPAddr
-	for len(ipsByService) > 0 {
-		filtered := ipsByService[:0]
-		for _, ips := range ipsByService {
-			if len(ips) == 0 {
-				// sanity check
-				continue
-			}
-			result = append(result, ips[0])
-			if len(ips) > 1 {
-				filtered = append(filtered, ips[1:])
-			}
-		}
-		ipsByService = filtered
-	}
-	return result
-}
-
-// Inspiration: https://github.com/artyom/dot/blob/master/dot.go
-func fallbackResolver(serverName, serverAddress string) *net.Resolver {
-	return &net.Resolver{
-		PreferGo: true,
-		Dial: func(ctx context.Context, _ string, _ string) (net.Conn, error) {
-			var dialer net.Dialer
-			conn, err := dialer.DialContext(ctx, "tcp", serverAddress)
-			if err != nil {
-				return nil, err
-			}
-			tlsConfig := &tls.Config{ServerName: serverName}
-			return tls.Client(conn, tlsConfig), nil
-		},
-	}
-}
-
-// EdgeHostnameResolver discovers the addresses of Cloudflare's edge network via a list of server hostnames.
-// It implements EdgeServiceDiscoverer interface, and is used mainly for testing connectivity.
-type EdgeHostnameResolver struct {
-	sync.Mutex
-	// hostnames of edge servers
-	hostnames []string
-	// Addrs to connect to cloudflare's edge network
-	addrs []*net.TCPAddr
-	// index of the next element to use in addrs
-	nextAddrIndex int
-}
-
-func NewEdgeHostnameResolver(edgeHostnames []string) (EdgeServiceDiscoverer, error) {
-	r := &EdgeHostnameResolver{
-		hostnames: edgeHostnames,
-	}
-	if err := r.Refresh(); err != nil {
-		return nil, err
-	}
-	return r, nil
-}
-
-func (r *EdgeHostnameResolver) Addr() *net.TCPAddr {
-	r.Lock()
-	defer r.Unlock()
-	addr := r.addrs[r.nextAddrIndex]
-	r.nextAddrIndex = (r.nextAddrIndex + 1) % len(r.addrs)
-	return addr
-}
-
-func (r *EdgeHostnameResolver) AvailableAddrs() uint8 {
-	r.Lock()
-	defer r.Unlock()
-	return uint8(len(r.addrs))
-}
-
-func (r *EdgeHostnameResolver) Refresh() error {
-	newAddrs, err := ResolveAddrs(r.hostnames)
-	if err != nil {
-		return err
-	}
-	r.Lock()
-	defer r.Unlock()
-	r.addrs = newAddrs
-	r.nextAddrIndex = 0
-	return nil
-}
-
-// Resolve TCP address given a list of addresses. Address can be a hostname, however, it will return at most one
-// of the hostname's IP addresses
-func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) {
-	var tcpAddrs []*net.TCPAddr
-	for _, addr := range addrs {
-		tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
-		if err != nil {
-			return nil, err
-		}
-		tcpAddrs = append(tcpAddrs, tcpAddr)
-	}
-	return tcpAddrs, nil
-}

connection/discovery_test.go

@@ -1,63 +0,0 @@
-package connection
-
-import (
-	"net"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-type mockEdgeServiceDiscoverer struct {
-}
-
-func (mr *mockEdgeServiceDiscoverer) Addr() *net.TCPAddr {
-	return &net.TCPAddr{
-		IP:   net.ParseIP("127.0.0.1"),
-		Port: 63102,
-	}
-}
-
-func (mr *mockEdgeServiceDiscoverer) AvailableAddrs() uint8 {
-	return 1
-}
-
-func (mr *mockEdgeServiceDiscoverer) Refresh() error {
-	return nil
-}
-
-func TestFlattenServiceIPs(t *testing.T) {
-	result := flattenServiceIPs([][]*net.TCPAddr{
-		[]*net.TCPAddr{
-			&net.TCPAddr{Port: 1},
-			&net.TCPAddr{Port: 2},
-			&net.TCPAddr{Port: 3},
-			&net.TCPAddr{Port: 4},
-		},
-		[]*net.TCPAddr{
-			&net.TCPAddr{Port: 10},
-			&net.TCPAddr{Port: 12},
-			&net.TCPAddr{Port: 13},
-		},
-		[]*net.TCPAddr{
-			&net.TCPAddr{Port: 21},
-			&net.TCPAddr{Port: 22},
-			&net.TCPAddr{Port: 23},
-			&net.TCPAddr{Port: 24},
-			&net.TCPAddr{Port: 25},
-		},
-	})
-	assert.EqualValues(t, []*net.TCPAddr{
-		&net.TCPAddr{Port: 1},
-		&net.TCPAddr{Port: 10},
-		&net.TCPAddr{Port: 21},
-		&net.TCPAddr{Port: 2},
-		&net.TCPAddr{Port: 12},
-		&net.TCPAddr{Port: 22},
-		&net.TCPAddr{Port: 3},
-		&net.TCPAddr{Port: 13},
-		&net.TCPAddr{Port: 23},
-		&net.TCPAddr{Port: 4},
-		&net.TCPAddr{Port: 24},
-		&net.TCPAddr{Port: 25},
-	}, result)
-}

connection/manager.go

@@ -4,58 +4,77 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"net"
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/streamhandler"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
-	quickStartLink = "https://developers.cloudflare.com/argo-tunnel/quickstart/"
-	faqLink        = "https://developers.cloudflare.com/argo-tunnel/faq/"
+	quickStartLink       = "https://developers.cloudflare.com/argo-tunnel/quickstart/"
+	faqLink              = "https://developers.cloudflare.com/argo-tunnel/faq/"
+	defaultRetryAfter    = time.Second * 5
+	packageNamespace     = "connection"
+	edgeManagerSubsystem = "edgemanager"
 )
 
 // EdgeManager manages connections with the edge
 type EdgeManager struct {
 	// streamHandler handles stream opened by the edge
-	streamHandler h2mux.MuxedStreamHandler
+	streamHandler *streamhandler.StreamHandler
 	// TLSConfig is the TLS configuration to connect with edge
 	tlsConfig *tls.Config
 	// cloudflaredConfig is the cloudflared configuration that is determined when the process first starts
 	cloudflaredConfig *CloudflaredConfig
 	// serviceDiscoverer returns the next edge addr to connect to
-	serviceDiscoverer EdgeServiceDiscoverer
+	serviceDiscoverer *edgediscovery.Edge
 	// state is attributes of ConnectionManager that can change during runtime.
 	state *edgeManagerState
 
 	logger *logrus.Entry
+
+	metrics *metrics
 }
 
-// EdgeConnectionManagerConfigurable is the configurable attributes of a EdgeConnectionManager
+type metrics struct {
+	// activeStreams is a gauge shared by all muxers of this process to expose the total number of active streams
+	activeStreams prometheus.Gauge
+}
+
+func newMetrics(namespace, subsystem string) *metrics {
+	return &metrics{
+		activeStreams: h2mux.NewActiveStreamsMetrics(namespace, subsystem),
+	}
+}
+
+// EdgeManagerConfigurable is the configurable attributes of a EdgeConnectionManager
 type EdgeManagerConfigurable struct {
 	TunnelHostnames []h2mux.TunnelHostname
-	*pogs.EdgeConnectionConfig
+	*tunnelpogs.EdgeConnectionConfig
 }
 
 type CloudflaredConfig struct {
 	CloudflaredID uuid.UUID
-	Tags          []pogs.Tag
+	Tags          []tunnelpogs.Tag
 	BuildInfo     *buildinfo.BuildInfo
-	Scope         pogs.Scope
+	IntentLabel   string
 }
 
 func NewEdgeManager(
-	streamHandler h2mux.MuxedStreamHandler,
+	streamHandler *streamhandler.StreamHandler,
 	edgeConnMgrConfigurable *EdgeManagerConfigurable,
 	userCredential []byte,
 	tlsConfig *tls.Config,
-	serviceDiscoverer EdgeServiceDiscoverer,
+	serviceDiscoverer *edgediscovery.Edge,
 	cloudflaredConfig *CloudflaredConfig,
 	logger *logrus.Logger,
 ) *EdgeManager {
@@ -66,29 +85,36 @@ func NewEdgeManager(
 		serviceDiscoverer: serviceDiscoverer,
 		state:             newEdgeConnectionManagerState(edgeConnMgrConfigurable, userCredential),
 		logger:            logger.WithField("subsystem", "connectionManager"),
+		metrics:           newMetrics(packageNamespace, edgeManagerSubsystem),
 	}
 }
 
 func (em *EdgeManager) Run(ctx context.Context) error {
 	defer em.shutdown()
 
-	resolveEdgeIPTicker := time.Tick(resolveEdgeAddrTTL)
+	// Currently, declarative tunnels don't have any concept of a stable connection
+	// Each edge connection is transient and when it dies, it is replaced by a different one,
+	// not restarted.
+	// So in the future we should really change this so that n connections are stored individually
+	connIndex := 0
 	for {
 		select {
 		case <-ctx.Done():
 			return errors.Wrap(ctx.Err(), "EdgeConnectionManager terminated")
-		case <-resolveEdgeIPTicker:
-			if err := em.serviceDiscoverer.Refresh(); err != nil {
-				em.logger.WithError(err).Warn("Cannot refresh Cloudflare edge addresses")
-			}
 		default:
 			time.Sleep(1 * time.Second)
 		}
 		// Create/delete connection one at a time, so we don't need to adjust for connections that are being created/deleted
 		// in shouldCreateConnection or shouldReduceConnection calculation
 		if em.state.shouldCreateConnection(em.serviceDiscoverer.AvailableAddrs()) {
-			if err := em.newConnection(ctx); err != nil {
-				em.logger.WithError(err).Error("cannot create new connection")
+			if connErr := em.newConnection(ctx, connIndex); connErr != nil {
+				if !connErr.ShouldRetry {
+					em.logger.WithError(connErr).Error(em.noRetryMessage())
+					return connErr
+				}
+				em.logger.WithError(connErr).Error("cannot create new connection")
+			} else {
+				connIndex++
 			}
 		} else if em.state.shouldReduceConnection() {
 			if err := em.closeConnection(ctx); err != nil {
@@ -103,13 +129,16 @@ func (em *EdgeManager) UpdateConfigurable(newConfigurable *EdgeManagerConfigurab
 	em.state.updateConfigurable(newConfigurable)
 }
 
-func (em *EdgeManager) newConnection(ctx context.Context) error {
-	edgeIP := em.serviceDiscoverer.Addr()
-	edgeConn, err := em.dialEdge(ctx, edgeIP)
+func (em *EdgeManager) newConnection(ctx context.Context, index int) *tunnelpogs.ConnectError {
+	edgeTCPAddr, err := em.serviceDiscoverer.GetAddr(index)
 	if err != nil {
-		return errors.Wrap(err, "dial edge error")
+		return retryConnection(fmt.Sprintf("edge address discovery error: %v", err))
 	}
 	configurable := em.state.getConfigurable()
+	edgeConn, err := DialEdge(ctx, configurable.Timeout, em.tlsConfig, edgeTCPAddr)
+	if err != nil {
+		return retryConnection(fmt.Sprintf("dial edge error: %v", err))
+	}
 	// Establish a muxed connection with the edge
 	// Client mux handshake with agent server
 	muxer, err := h2mux.Handshake(edgeConn, edgeConn, h2mux.MuxerConfig{
@@ -119,40 +148,41 @@ func (em *EdgeManager) newConnection(ctx context.Context) error {
 		HeartbeatInterval: configurable.HeartbeatInterval,
 		MaxHeartbeats:     configurable.MaxFailedHeartbeats,
 		Logger:            em.logger.WithField("subsystem", "muxer"),
-	})
+	}, em.metrics.activeStreams)
 	if err != nil {
-		return errors.Wrap(err, "couldn't perform handshake with edge")
+		retryConnection(fmt.Sprintf("couldn't perform handshake with edge: %v", err))
 	}
 
-	h2muxConn, err := newConnection(muxer, edgeIP)
+	h2muxConn, err := newConnection(muxer, edgeTCPAddr)
 	if err != nil {
-		return errors.Wrap(err, "couldn't create h2mux connection")
+		return retryConnection(fmt.Sprintf("couldn't create h2mux connection: %v", err))
 	}
 
 	go em.serveConn(ctx, h2muxConn)
 
-	connResult, err := h2muxConn.Connect(ctx, &pogs.ConnectParameters{
+	connResult, err := h2muxConn.Connect(ctx, &tunnelpogs.ConnectParameters{
 		CloudflaredID:       em.cloudflaredConfig.CloudflaredID,
 		CloudflaredVersion:  em.cloudflaredConfig.BuildInfo.CloudflaredVersion,
 		NumPreviousAttempts: 0,
 		OriginCert:          em.state.getUserCredential(),
-		Scope:               em.cloudflaredConfig.Scope,
+		IntentLabel:         em.cloudflaredConfig.IntentLabel,
 		Tags:                em.cloudflaredConfig.Tags,
 	}, em.logger)
 	if err != nil {
 		h2muxConn.Shutdown()
-		return errors.Wrap(err, "couldn't connect to edge")
+		return retryConnection(fmt.Sprintf("couldn't connect to edge: %v", err))
 	}
 
-	if connErr := connResult.Err; connErr != nil {
-		if !connErr.ShouldRetry {
-			return errors.Wrap(connErr, em.noRetryMessage())
-		}
-		return errors.Wrapf(connErr, "edge responded with RetryAfter=%v", connErr.RetryAfter)
+	if connErr := connResult.ConnectError(); connErr != nil {
+		return connErr
 	}
 
 	em.state.newConnection(h2muxConn)
-	em.logger.Infof("connected to %s", connResult.ServerInfo.LocationName)
+	em.logger.Infof("connected to %s", connResult.ConnectedTo())
+
+	if connResult.ClientConfig() != nil {
+		em.streamHandler.UseConfiguration(ctx, connResult.ClientConfig())
+	}
 	return nil
 }
 
@@ -162,6 +192,7 @@ func (em *EdgeManager) closeConnection(ctx context.Context) error {
 		return fmt.Errorf("no connection to close")
 	}
 	conn.Shutdown()
+	// teardown will be handled by EdgeManager.serveConn in another goroutine
 	return nil
 }
 
@@ -169,28 +200,7 @@ func (em *EdgeManager) serveConn(ctx context.Context, conn *Connection) {
 	err := conn.Serve(ctx)
 	em.logger.WithError(err).Warn("Connection closed")
 	em.state.closeConnection(conn)
-}
-
-func (em *EdgeManager) dialEdge(ctx context.Context, edgeIP *net.TCPAddr) (*tls.Conn, error) {
-	timeout := em.state.getConfigurable().Timeout
-	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
-	dialCtx, dialCancel := context.WithTimeout(ctx, timeout)
-	defer dialCancel()
-
-	dialer := net.Dialer{DualStack: true}
-	edgeConn, err := dialer.DialContext(dialCtx, "tcp", edgeIP.String())
-	if err != nil {
-		return nil, dialError{cause: errors.Wrap(err, "DialContext error")}
-	}
-	tlsEdgeConn := tls.Client(edgeConn, em.tlsConfig)
-	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
-
-	if err = tlsEdgeConn.Handshake(); err != nil {
-		return nil, dialError{cause: errors.Wrap(err, "Handshake with edge error")}
-	}
-	// clear the deadline on the conn; h2mux has its own timeouts
-	tlsEdgeConn.SetDeadline(time.Time{})
-	return tlsEdgeConn, nil
+	em.serviceDiscoverer.GiveBack(conn.addr)
 }
 
 func (em *EdgeManager) noRetryMessage() string {
@@ -219,14 +229,14 @@ func newEdgeConnectionManagerState(configurable *EdgeManagerConfigurable, userCr
 	}
 }
 
-func (ems *edgeManagerState) shouldCreateConnection(availableEdgeAddrs uint8) bool {
+func (ems *edgeManagerState) shouldCreateConnection(availableEdgeAddrs int) bool {
 	ems.RLock()
 	defer ems.RUnlock()
-	expectedHAConns := ems.configurable.NumHAConnections
+	expectedHAConns := int(ems.configurable.NumHAConnections)
 	if availableEdgeAddrs < expectedHAConns {
 		expectedHAConns = availableEdgeAddrs
 	}
-	return uint8(len(ems.conns)) < expectedHAConns
+	return len(ems.conns) < expectedHAConns
 }
 
 func (ems *edgeManagerState) shouldReduceConnection() bool {
@@ -282,3 +292,11 @@ func (ems *edgeManagerState) getUserCredential() []byte {
 	defer ems.RUnlock()
 	return ems.userCredential
 }
+
+func retryConnection(cause string) *tunnelpogs.ConnectError {
+	return &tunnelpogs.ConnectError{
+		Cause:       cause,
+		RetryAfter:  defaultRetryAfter,
+		ShouldRetry: true,
+	}
+}

connection/manager_test.go

@@ -1,16 +1,19 @@
 package connection
 
 import (
+	"net"
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
-	"github.com/stretchr/testify/assert"
-
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/streamhandler"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 var (
@@ -42,22 +45,19 @@ var (
 	}
 )
 
-type mockStreamHandler struct {
-}
-
-func (msh *mockStreamHandler) ServeStream(*h2mux.MuxedStream) error {
-	return nil
-}
-
 func mockEdgeManager() *EdgeManager {
+	newConfigChan := make(chan<- *pogs.ClientConfig)
+	useConfigResultChan := make(<-chan *pogs.UseConfigurationResult)
+	logger := logrus.New()
+	edge := edgediscovery.MockEdge(logger, []*net.TCPAddr{})
 	return NewEdgeManager(
-		&mockStreamHandler{},
+		streamhandler.NewStreamHandler(newConfigChan, useConfigResultChan, logger),
 		configurable,
 		[]byte{},
 		nil,
-		&mockEdgeServiceDiscoverer{},
+		edge,
 		cloudflaredConfig,
-		logrus.New(),
+		logger,
 	)
 }
 

connection/mocks_for_test.go

@@ -0,0 +1,118 @@
+package connection
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+	"net"
+	"reflect"
+	"testing/quick"
+)
+
+type mockAddrs struct {
+	// a set of synthetic SRV records
+	addrMap map[net.SRV][]*net.TCPAddr
+	// the total number of addresses, aggregated across addrMap.
+	// For the convenience of test code that would otherwise have to compute
+	// this by hand every time.
+	numAddrs int
+}
+
+func newMockAddrs(port uint16, numRegions uint8, numAddrsPerRegion uint8) mockAddrs {
+	addrMap := make(map[net.SRV][]*net.TCPAddr)
+	numAddrs := 0
+
+	for r := uint8(0); r < numRegions; r++ {
+		var (
+			srv   = net.SRV{Target: fmt.Sprintf("test-region-%v.example.com", r), Port: port}
+			addrs []*net.TCPAddr
+		)
+		for a := uint8(0); a < numAddrsPerRegion; a++ {
+			addrs = append(addrs, &net.TCPAddr{
+				IP:   net.ParseIP(fmt.Sprintf("10.0.%v.%v", r, a)),
+				Port: int(port),
+			})
+		}
+		addrMap[srv] = addrs
+		numAddrs += len(addrs)
+	}
+	return mockAddrs{addrMap: addrMap, numAddrs: numAddrs}
+}
+
+var _ quick.Generator = mockAddrs{}
+
+func (mockAddrs) Generate(rand *rand.Rand, size int) reflect.Value {
+	port := uint16(rand.Intn(math.MaxUint16))
+	numRegions := uint8(1 + rand.Intn(10))
+	numAddrsPerRegion := uint8(1 + rand.Intn(32))
+	result := newMockAddrs(port, numRegions, numAddrsPerRegion)
+	return reflect.ValueOf(result)
+}
+
+// Returns a function compatible with net.LookupSRV that will return the SRV
+// records from mockAddrs.
+func mockNetLookupSRV(
+	m mockAddrs,
+) func(service, proto, name string) (cname string, addrs []*net.SRV, err error) {
+	var addrs []*net.SRV
+	for k := range m.addrMap {
+		addr := k
+		addrs = append(addrs, &addr)
+		// We can't just do
+		//   addrs = append(addrs, &k)
+		// `k` will be reused by subsequent loop iterations,
+		// so all the copies of `&k` would point to the same location.
+	}
+	return func(_, _, _ string) (string, []*net.SRV, error) {
+		return "", addrs, nil
+	}
+}
+
+// Returns a function compatible with net.LookupIP that translates the SRV records
+// from mockAddrs into IP addresses, based on the TCP addresses in mockAddrs.
+func mockNetLookupIP(
+	m mockAddrs,
+) func(host string) ([]net.IP, error) {
+	return func(host string) ([]net.IP, error) {
+		for srv, tcpAddrs := range m.addrMap {
+			if srv.Target != host {
+				continue
+			}
+			result := make([]net.IP, len(tcpAddrs))
+			for i, tcpAddr := range tcpAddrs {
+				result[i] = tcpAddr.IP
+			}
+			return result, nil
+		}
+		return nil, fmt.Errorf("No IPs for %v", host)
+	}
+}
+
+type mockEdgeServiceDiscoverer struct {
+}
+
+func (mr *mockEdgeServiceDiscoverer) Addr() (*net.TCPAddr, error) {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("127.0.0.1"),
+		Port: 63102,
+	}, nil
+}
+
+func (mr *mockEdgeServiceDiscoverer) AnyAddr() (*net.TCPAddr, error) {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("127.0.0.1"),
+		Port: 63102,
+	}, nil
+}
+
+func (mr *mockEdgeServiceDiscoverer) ReplaceAddr(addr *net.TCPAddr) {}
+
+func (mr *mockEdgeServiceDiscoverer) MarkAddrBad(addr *net.TCPAddr) {}
+
+func (mr *mockEdgeServiceDiscoverer) AvailableAddrs() int {
+	return 1
+}
+
+func (mr *mockEdgeServiceDiscoverer) Refresh() error {
+	return nil
+}

connection/rpc.go

@@ -0,0 +1,49 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	rpc "zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// NewRPCClient creates and returns a new RPC client, which will communicate
+// using a stream on the given muxer
+func NewRPCClient(
+	ctx context.Context,
+	muxer *h2mux.Muxer,
+	logger *logrus.Entry,
+	openStreamTimeout time.Duration,
+) (client tunnelpogs.TunnelServer_PogsClient, err error) {
+	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer openStreamCancel()
+	stream, err := muxer.OpenRPCStream(openStreamCtx)
+	if err != nil {
+		return
+	}
+
+	if !isRPCStreamResponse(stream.Headers) {
+		stream.Close()
+		err = fmt.Errorf("rpc: bad response headers: %v", stream.Headers)
+		return
+	}
+
+	conn := rpc.NewConn(
+		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
+		tunnelrpc.ConnLog(logger),
+	)
+	client = tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
+	return client, nil
+}
+
+func isRPCStreamResponse(headers []h2mux.Header) bool {
+	return len(headers) == 1 &&
+		headers[0].Name == ":status" &&
+		headers[0].Value == "200"
+}

dbconnect/proxy.go

@@ -15,8 +15,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-
-	timing "github.com/mitchellh/go-server-timing"
 )
 
 // Proxy is an HTTP server that proxies requests to a Client.
@@ -64,6 +62,7 @@ func NewSecureProxy(ctx context.Context, origin, authDomain, applicationAUD stri
 	}
 
 	proxy.accessValidator = validator
+
 	return proxy, err
 }
 
@@ -138,7 +137,7 @@ func (proxy *Proxy) Start(ctx context.Context, addr string, listenerC chan<- net
 func (proxy *Proxy) httpListen(ctx context.Context, listener net.Listener) error {
 	httpServer := &http.Server{
 		Addr:         listener.Addr().String(),
-		Handler:      timing.Middleware(proxy.httpRouter(), nil),
+		Handler:      proxy.httpRouter(),
 		ReadTimeout:  10 * time.Second,
 		WriteTimeout: 60 * time.Second,
 		IdleTimeout:  60 * time.Second,
@@ -170,9 +169,7 @@ func (proxy *Proxy) httpRouter() *mux.Router {
 func (proxy *Proxy) httpPing() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		ctx := r.Context()
-		metric := timing.FromContext(ctx).NewMetric("db").Start()
 		err := proxy.client.Ping(ctx)
-		metric.Stop()
 
 		if err == nil {
 			proxy.httpRespond(w, r, http.StatusOK, "")
@@ -201,9 +198,7 @@ func (proxy *Proxy) httpSubmit() http.HandlerFunc {
 		}
 
 		ctx := r.Context()
-		metric := timing.FromContext(ctx).NewMetric("db").Start()
 		data, err := proxy.client.Submit(ctx, &cmd)
-		metric.Stop()
 
 		if err != nil {
 			proxy.httpRespondErr(w, r, http.StatusUnprocessableEntity, err)
@@ -238,7 +233,9 @@ func (proxy *Proxy) httpRespondErr(w http.ResponseWriter, r *http.Request, defau
 	status, err := httpError(defaultStatus, err)
 
 	proxy.httpRespond(w, r, status, err.Error())
-	proxy.httpLog(r, err).Warn("Database connect error")
+	if len(err.Error()) > 0 {
+		proxy.httpLog(r, err).Warn("Database proxy error")
+	}
 }
 
 // httpLog returns a logrus.Entry that is formatted to output a request Cf-ray.

dbconnect_tests/dbconnect.yaml

@@ -1,4 +1,4 @@
-# docker-compose -f ./dbconnect/integration_test/dbconnect.yaml up --build --force-recreate --renew-anon-volumes --exit-code-from cloudflared
+# docker-compose -f ./dbconnect_tests/dbconnect.yaml up --build --force-recreate --renew-anon-volumes --exit-code-from cloudflared
 
 version: "2.3"
 networks:
@@ -7,9 +7,9 @@ networks:
 services:
   cloudflared:
     build:
-      context: ../../
+      context: ../
       dockerfile: dev.Dockerfile
-    command: go test github.com/cloudflare/cloudflared/dbconnect/integration_test -v
+    command: go test github.com/cloudflare/cloudflared/dbconnect_tests -v
     depends_on:
       postgres:
         condition: service_healthy

dbconnect_tests/sql_test.go

@@ -1,4 +1,4 @@
-package dbconnect_test
+package dbconnect_tests
 
 import (
 	"context"

edgediscovery/allregions/discovery.go

@@ -0,0 +1,135 @@
+package allregions
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	// Used to discover HA origintunneld servers
+	srvService = "origintunneld"
+	srvProto   = "tcp"
+	srvName    = "argotunnel.com"
+
+	// Used to fallback to DoT when we can't use the default resolver to
+	// discover HA origintunneld servers (GitHub issue #75).
+	dotServerName = "cloudflare-dns.com"
+	dotServerAddr = "1.1.1.1:853"
+	dotTimeout    = time.Duration(15 * time.Second)
+
+	// SRV record resolution TTL
+	resolveEdgeAddrTTL = 1 * time.Hour
+
+	subsystemEdgeAddrResolver = "edgeAddrResolver"
+)
+
+// Redeclare network functions so they can be overridden in tests.
+var (
+	netLookupSRV = net.LookupSRV
+	netLookupIP  = net.LookupIP
+)
+
+// If the call to net.LookupSRV fails, try to fall back to DoT from Cloudflare directly.
+//
+// Note: Instead of DoT, we could also have used DoH. Either of these:
+//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
+//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
+// But both of these cases miss out on a key feature from the stdlib:
+//     "The returned records are sorted by priority and randomized by weight within a priority."
+//     (https://golang.org/pkg/net/#Resolver.LookupSRV)
+// Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it.
+// See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552
+var fallbackLookupSRV = lookupSRVWithDOT
+
+var friendlyDNSErrorLines = []string{
+	`Please try the following things to diagnose this issue:`,
+	`  1. ensure that argotunnel.com is returning "origintunneld" service records.`,
+	`     Run your system's equivalent of: dig srv _origintunneld._tcp.argotunnel.com`,
+	`  2. ensure that your DNS resolver is not returning compressed SRV records.`,
+	`     See GitHub issue https://github.com/golang/go/issues/27546`,
+	`     For example, you could use Cloudflare's 1.1.1.1 as your resolver:`,
+	`     https://developers.cloudflare.com/1.1.1.1/setting-up-1.1.1.1/`,
+}
+
+// EdgeDiscovery implements HA service discovery lookup.
+func edgeDiscovery(logger *logrus.Entry) ([][]*net.TCPAddr, error) {
+	_, addrs, err := netLookupSRV(srvService, srvProto, srvName)
+	if err != nil {
+		_, fallbackAddrs, fallbackErr := fallbackLookupSRV(srvService, srvProto, srvName)
+		if fallbackErr != nil || len(fallbackAddrs) == 0 {
+			// use the original DNS error `err` in messages, not `fallbackErr`
+			logger.Errorln("Error looking up Cloudflare edge IPs: the DNS query failed:", err)
+			for _, s := range friendlyDNSErrorLines {
+				logger.Errorln(s)
+			}
+			return nil, errors.Wrapf(err, "Could not lookup srv records on _%v._%v.%v", srvService, srvProto, srvName)
+		}
+		// Accept the fallback results and keep going
+		addrs = fallbackAddrs
+	}
+
+	var resolvedIPsPerCNAME [][]*net.TCPAddr
+	for _, addr := range addrs {
+		ips, err := resolveSRVToTCP(addr)
+		if err != nil {
+			return nil, err
+		}
+		resolvedIPsPerCNAME = append(resolvedIPsPerCNAME, ips)
+	}
+
+	return resolvedIPsPerCNAME, nil
+}
+
+func lookupSRVWithDOT(service, proto, name string) (cname string, addrs []*net.SRV, err error) {
+	// Inspiration: https://github.com/artyom/dot/blob/master/dot.go
+	r := &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, _ string, _ string) (net.Conn, error) {
+			var dialer net.Dialer
+			conn, err := dialer.DialContext(ctx, "tcp", dotServerAddr)
+			if err != nil {
+				return nil, err
+			}
+			tlsConfig := &tls.Config{ServerName: dotServerName}
+			return tls.Client(conn, tlsConfig), nil
+		},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), dotTimeout)
+	defer cancel()
+	return r.LookupSRV(ctx, srvService, srvProto, srvName)
+}
+
+func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
+	ips, err := netLookupIP(srv.Target)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Couldn't resolve SRV record %v", srv)
+	}
+	if len(ips) == 0 {
+		return nil, fmt.Errorf("SRV record %v had no IPs", srv)
+	}
+	addrs := make([]*net.TCPAddr, len(ips))
+	for i, ip := range ips {
+		addrs[i] = &net.TCPAddr{IP: ip, Port: int(srv.Port)}
+	}
+	return addrs, nil
+}
+
+// ResolveAddrs resolves TCP address given a list of addresses. Address can be a hostname, however, it will return at most one
+// of the hostname's IP addresses
+func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) {
+	var tcpAddrs []*net.TCPAddr
+	for _, addr := range addrs {
+		tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
+		if err != nil {
+			return nil, err
+		}
+		tcpAddrs = append(tcpAddrs, tcpAddr)
+	}
+	return tcpAddrs, nil
+}

edgediscovery/allregions/discovery_test.go

@@ -0,0 +1,32 @@
+package allregions
+
+import (
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEdgeDiscovery(t *testing.T) {
+	mockAddrs := newMockAddrs(19, 2, 5)
+	netLookupSRV = mockNetLookupSRV(mockAddrs)
+	netLookupIP = mockNetLookupIP(mockAddrs)
+
+	expectedAddrSet := map[string]bool{}
+	for _, addrs := range mockAddrs.addrMap {
+		for _, addr := range addrs {
+			expectedAddrSet[addr.String()] = true
+		}
+	}
+
+	addrLists, err := edgeDiscovery(logrus.New().WithFields(logrus.Fields{}))
+	assert.NoError(t, err)
+	actualAddrSet := map[string]bool{}
+	for _, addrs := range addrLists {
+		for _, addr := range addrs {
+			actualAddrSet[addr.String()] = true
+		}
+	}
+
+	assert.Equal(t, expectedAddrSet, actualAddrSet)
+}

edgediscovery/allregions/mocks_for_test.go

@@ -0,0 +1,89 @@
+package allregions
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+	"net"
+	"reflect"
+	"testing/quick"
+)
+
+type mockAddrs struct {
+	// a set of synthetic SRV records
+	addrMap map[net.SRV][]*net.TCPAddr
+	// the total number of addresses, aggregated across addrMap.
+	// For the convenience of test code that would otherwise have to compute
+	// this by hand every time.
+	numAddrs int
+}
+
+func newMockAddrs(port uint16, numRegions uint8, numAddrsPerRegion uint8) mockAddrs {
+	addrMap := make(map[net.SRV][]*net.TCPAddr)
+	numAddrs := 0
+
+	for r := uint8(0); r < numRegions; r++ {
+		var (
+			srv   = net.SRV{Target: fmt.Sprintf("test-region-%v.example.com", r), Port: port}
+			addrs []*net.TCPAddr
+		)
+		for a := uint8(0); a < numAddrsPerRegion; a++ {
+			addrs = append(addrs, &net.TCPAddr{
+				IP:   net.ParseIP(fmt.Sprintf("10.0.%v.%v", r, a)),
+				Port: int(port),
+			})
+		}
+		addrMap[srv] = addrs
+		numAddrs += len(addrs)
+	}
+	return mockAddrs{addrMap: addrMap, numAddrs: numAddrs}
+}
+
+var _ quick.Generator = mockAddrs{}
+
+func (mockAddrs) Generate(rand *rand.Rand, size int) reflect.Value {
+	port := uint16(rand.Intn(math.MaxUint16))
+	numRegions := uint8(1 + rand.Intn(10))
+	numAddrsPerRegion := uint8(1 + rand.Intn(32))
+	result := newMockAddrs(port, numRegions, numAddrsPerRegion)
+	return reflect.ValueOf(result)
+}
+
+// Returns a function compatible with net.LookupSRV that will return the SRV
+// records from mockAddrs.
+func mockNetLookupSRV(
+	m mockAddrs,
+) func(service, proto, name string) (cname string, addrs []*net.SRV, err error) {
+	var addrs []*net.SRV
+	for k := range m.addrMap {
+		addr := k
+		addrs = append(addrs, &addr)
+		// We can't just do
+		//   addrs = append(addrs, &k)
+		// `k` will be reused by subsequent loop iterations,
+		// so all the copies of `&k` would point to the same location.
+	}
+	return func(_, _, _ string) (string, []*net.SRV, error) {
+		return "", addrs, nil
+	}
+}
+
+// Returns a function compatible with net.LookupIP that translates the SRV records
+// from mockAddrs into IP addresses, based on the TCP addresses in mockAddrs.
+func mockNetLookupIP(
+	m mockAddrs,
+) func(host string) ([]net.IP, error) {
+	return func(host string) ([]net.IP, error) {
+		for srv, tcpAddrs := range m.addrMap {
+			if srv.Target != host {
+				continue
+			}
+			result := make([]net.IP, len(tcpAddrs))
+			for i, tcpAddr := range tcpAddrs {
+				result[i] = tcpAddr.IP
+			}
+			return result, nil
+		}
+		return nil, fmt.Errorf("No IPs for %v", host)
+	}
+}

edgediscovery/allregions/region.go

@@ -0,0 +1,84 @@
+package allregions
+
+import (
+	"net"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Region contains cloudflared edge addresses. The edge is partitioned into several regions for
+// redundancy purposes.
+type Region struct {
+	connFor map[*net.TCPAddr]UsedBy
+}
+
+// NewRegion creates a region with the given addresses, which are all unused.
+func NewRegion(addrs []*net.TCPAddr) Region {
+	// The zero value of UsedBy is Unused(), so we can just initialize the map's values with their
+	// zero values.
+	m := make(map[*net.TCPAddr]UsedBy)
+	for _, addr := range addrs {
+		m[addr] = Unused()
+	}
+	return Region{connFor: m}
+}
+
+// AddrUsedBy finds the address used by the given connection in this region.
+// Returns nil if the connection isn't using any IP.
+func (r *Region) AddrUsedBy(connID int) *net.TCPAddr {
+	for addr, used := range r.connFor {
+		if used.Used && used.ConnID == connID {
+			return addr
+		}
+	}
+	return nil
+}
+
+// AvailableAddrs counts how many unused addresses this region contains.
+func (r Region) AvailableAddrs() int {
+	n := 0
+	for _, usedby := range r.connFor {
+		if !usedby.Used {
+			n++
+		}
+	}
+	return n
+}
+
+// GetUnusedIP returns a random unused address in this region.
+// Returns nil if all addresses are in use.
+func (r Region) GetUnusedIP(excluding *net.TCPAddr) *net.TCPAddr {
+	for addr, usedby := range r.connFor {
+		if !usedby.Used && addr != excluding {
+			return addr
+		}
+	}
+	return nil
+}
+
+// Use the address, assigning it to a proxy connection.
+func (r Region) Use(addr *net.TCPAddr, connID int) {
+	if addr == nil {
+		logrus.Errorf("Attempted to use nil address for connection %d", connID)
+		return
+	}
+	r.connFor[addr] = InUse(connID)
+}
+
+// GetAnyAddress returns an arbitrary address from the region.
+func (r Region) GetAnyAddress() *net.TCPAddr {
+	for addr := range r.connFor {
+		return addr
+	}
+	return nil
+}
+
+// GiveBack the address, ensuring it is no longer assigned to an IP.
+// Returns true if the address is in this region.
+func (r Region) GiveBack(addr *net.TCPAddr) (ok bool) {
+	if _, ok := r.connFor[addr]; !ok {
+		return false
+	}
+	r.connFor[addr] = Unused()
+	return true
+}

edgediscovery/allregions/region_test.go

@@ -0,0 +1,287 @@
+package allregions
+
+import (
+	"fmt"
+	"net"
+	"reflect"
+	"testing"
+)
+
+func TestRegion_New(t *testing.T) {
+	r := NewRegion([]*net.TCPAddr{&addr0, &addr1, &addr2})
+	fmt.Println(r.connFor)
+	if r.AvailableAddrs() != 3 {
+		t.Errorf("r.AvailableAddrs() == %v but want 3", r.AvailableAddrs())
+	}
+}
+
+func TestRegion_AddrUsedBy(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	type args struct {
+		connID int
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   *net.TCPAddr
+	}{
+		{
+			name: "happy trivial test",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+			}},
+			args: args{connID: 0},
+			want: &addr0,
+		},
+		{
+			name: "sad trivial test",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+			}},
+			args: args{connID: 1},
+			want: nil,
+		},
+		{
+			name: "sad test",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			args: args{connID: 3},
+			want: nil,
+		},
+		{
+			name: "happy test",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			args: args{connID: 1},
+			want: &addr1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := &Region{
+				connFor: tt.fields.connFor,
+			}
+			if got := r.AddrUsedBy(tt.args.connID); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Region.AddrUsedBy() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestRegion_AvailableAddrs(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   int
+	}{
+		{
+			name: "contains addresses",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			}},
+			want: 1,
+		},
+		{
+			name: "all free",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: Unused(),
+				&addr1: Unused(),
+				&addr2: Unused(),
+			}},
+			want: 3,
+		},
+		{
+			name: "all used",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			want: 0,
+		},
+		{
+			name:   "empty",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{}},
+			want:   0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Region{
+				connFor: tt.fields.connFor,
+			}
+			if got := r.AvailableAddrs(); got != tt.want {
+				t.Errorf("Region.AvailableAddrs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestRegion_GetUnusedIP(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	type args struct {
+		excluding *net.TCPAddr
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   *net.TCPAddr
+	}{
+		{
+			name: "happy test with excluding set",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: Unused(),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			}},
+			args: args{excluding: &addr0},
+			want: &addr1,
+		},
+		{
+			name: "happy test with no excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			}},
+			args: args{excluding: nil},
+			want: &addr1,
+		},
+		{
+			name: "sad test with no excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			args: args{excluding: nil},
+			want: nil,
+		},
+		{
+			name: "sad test with excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: Unused(),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			args: args{excluding: &addr0},
+			want: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Region{
+				connFor: tt.fields.connFor,
+			}
+			if got := r.GetUnusedIP(tt.args.excluding); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Region.GetUnusedIP() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestRegion_GiveBack(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	type args struct {
+		addr *net.TCPAddr
+	}
+	tests := []struct {
+		name           string
+		fields         fields
+		args           args
+		wantOk         bool
+		availableAfter int
+	}{
+		{
+			name: "sad test with excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr1: InUse(1),
+			}},
+			args:           args{addr: &addr1},
+			wantOk:         true,
+			availableAfter: 1,
+		},
+		{
+			name: "sad test with excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr1: InUse(1),
+			}},
+			args:           args{addr: &addr2},
+			wantOk:         false,
+			availableAfter: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Region{
+				connFor: tt.fields.connFor,
+			}
+			if gotOk := r.GiveBack(tt.args.addr); gotOk != tt.wantOk {
+				t.Errorf("Region.GiveBack() = %v, want %v", gotOk, tt.wantOk)
+			}
+			if tt.availableAfter != r.AvailableAddrs() {
+				t.Errorf("Region.AvailableAddrs() = %v, want %v", r.AvailableAddrs(), tt.availableAfter)
+			}
+		})
+	}
+}
+
+func TestRegion_GetAnyAddress(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		wantNil bool
+	}{
+		{
+			name:    "Sad test -- GetAnyAddress should only fail if the region is empty",
+			fields:  fields{connFor: map[*net.TCPAddr]UsedBy{}},
+			wantNil: true,
+		},
+		{
+			name: "Happy test (all addresses unused)",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: Unused(),
+			}},
+			wantNil: false,
+		},
+		{
+			name: "Happy test (GetAnyAddress can still return addresses used by proxy conns)",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(2),
+			}},
+			wantNil: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Region{
+				connFor: tt.fields.connFor,
+			}
+			if got := r.GetAnyAddress(); tt.wantNil != (got == nil) {
+				t.Errorf("Region.GetAnyAddress() = %v, but should it return nil? %v", got, tt.wantNil)
+			}
+		})
+	}
+}

edgediscovery/allregions/regions.go

@@ -0,0 +1,125 @@
+package allregions
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Regions stores Cloudflare edge network IPs, partitioned into two regions.
+// This is NOT thread-safe. Users of this package should use it with a lock.
+type Regions struct {
+	region1 Region
+	region2 Region
+}
+
+// ------------------------------------
+// Constructors
+// ------------------------------------
+
+// ResolveEdge resolves the Cloudflare edge, returning all regions discovered.
+func ResolveEdge(logger *logrus.Entry) (*Regions, error) {
+	addrLists, err := edgeDiscovery(logger)
+	if err != nil {
+		return nil, err
+	}
+	if len(addrLists) < 2 {
+		return nil, fmt.Errorf("expected at least 2 Cloudflare Regions regions, but SRV only returned %v", len(addrLists))
+	}
+	return &Regions{
+		region1: NewRegion(addrLists[0]),
+		region2: NewRegion(addrLists[1]),
+	}, nil
+}
+
+// StaticEdge creates a list of edge addresses from the list of hostnames.
+// Mainly used for testing connectivity.
+func StaticEdge(hostnames []string) (*Regions, error) {
+	addrs, err := ResolveAddrs(hostnames)
+	if err != nil {
+		return nil, err
+	}
+	return NewNoResolve(addrs), nil
+}
+
+// NewNoResolve doesn't resolve the edge. Instead it just uses the given addresses.
+// You probably only need this for testing.
+func NewNoResolve(addrs []*net.TCPAddr) *Regions {
+	region1 := make([]*net.TCPAddr, 0)
+	region2 := make([]*net.TCPAddr, 0)
+	for i, v := range addrs {
+		if i%2 == 0 {
+			region1 = append(region1, v)
+		} else {
+			region2 = append(region2, v)
+		}
+	}
+
+	return &Regions{
+		region1: NewRegion(region1),
+		region2: NewRegion(region2),
+	}
+}
+
+// ------------------------------------
+// Methods
+// ------------------------------------
+
+// GetAnyAddress returns an arbitrary address from the larger region.
+func (rs *Regions) GetAnyAddress() *net.TCPAddr {
+	if addr := rs.region1.GetAnyAddress(); addr != nil {
+		return addr
+	}
+	return rs.region2.GetAnyAddress()
+}
+
+// AddrUsedBy finds the address used by the given connection.
+// Returns nil if the connection isn't using an address.
+func (rs *Regions) AddrUsedBy(connID int) *net.TCPAddr {
+	if addr := rs.region1.AddrUsedBy(connID); addr != nil {
+		return addr
+	}
+	return rs.region2.AddrUsedBy(connID)
+}
+
+// GetUnusedAddr gets an unused addr from the edge, excluding the given addr. Prefer to use addresses
+// evenly across both regions.
+func (rs *Regions) GetUnusedAddr(excluding *net.TCPAddr, connID int) *net.TCPAddr {
+	if rs.region1.AvailableAddrs() > rs.region2.AvailableAddrs() {
+		return getAddrs(excluding, connID, &rs.region1, &rs.region2)
+	}
+
+	return getAddrs(excluding, connID, &rs.region2, &rs.region1)
+}
+
+// getAddrs tries to grab address form `first` region, then `second` region
+// this is an unrolled loop over 2 element array
+func getAddrs(excluding *net.TCPAddr, connID int, first *Region, second *Region) *net.TCPAddr {
+	addr := first.GetUnusedIP(excluding)
+	if addr != nil {
+		first.Use(addr, connID)
+		return addr
+	}
+	addr = second.GetUnusedIP(excluding)
+	if addr != nil {
+		second.Use(addr, connID)
+		return addr
+	}
+
+	return nil
+}
+
+// AvailableAddrs returns how many edge addresses aren't used.
+func (rs *Regions) AvailableAddrs() int {
+	return rs.region1.AvailableAddrs() + rs.region2.AvailableAddrs()
+}
+
+// GiveBack the address so that other connections can use it.
+// Returns true if the address is in this edge.
+func (rs *Regions) GiveBack(addr *net.TCPAddr) bool {
+	if found := rs.region1.GiveBack(addr); found {
+		return found
+	}
+	return rs.region2.GiveBack(addr)
+}

edgediscovery/allregions/regions_test.go

@@ -0,0 +1,141 @@
+package allregions
+
+import (
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	addr0 = net.TCPAddr{
+		IP:   net.ParseIP("123.4.5.0"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr1 = net.TCPAddr{
+		IP:   net.ParseIP("123.4.5.1"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr2 = net.TCPAddr{
+		IP:   net.ParseIP("123.4.5.2"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr3 = net.TCPAddr{
+		IP:   net.ParseIP("123.4.5.3"),
+		Port: 8000,
+		Zone: "",
+	}
+)
+
+func makeRegions() Regions {
+	r1 := NewRegion([]*net.TCPAddr{&addr0, &addr1})
+	r2 := NewRegion([]*net.TCPAddr{&addr2, &addr3})
+	return Regions{region1: r1, region2: r2}
+}
+
+func TestRegions_AddrUsedBy(t *testing.T) {
+	rs := makeRegions()
+	addr1 := rs.GetUnusedAddr(nil, 1)
+	assert.Equal(t, addr1, rs.AddrUsedBy(1))
+	addr2 := rs.GetUnusedAddr(nil, 2)
+	assert.Equal(t, addr2, rs.AddrUsedBy(2))
+	addr3 := rs.GetUnusedAddr(nil, 3)
+	assert.Equal(t, addr3, rs.AddrUsedBy(3))
+}
+
+func TestRegions_Giveback_Region1(t *testing.T) {
+	rs := makeRegions()
+	rs.region1.Use(&addr0, 0)
+	rs.region1.Use(&addr1, 1)
+	rs.region2.Use(&addr2, 2)
+	rs.region2.Use(&addr3, 3)
+
+	assert.Equal(t, 0, rs.AvailableAddrs())
+
+	rs.GiveBack(&addr0)
+	assert.Equal(t, &addr0, rs.GetUnusedAddr(nil, 3))
+}
+
+func TestRegions_Giveback_Region2(t *testing.T) {
+	rs := makeRegions()
+	rs.region1.Use(&addr0, 0)
+	rs.region1.Use(&addr1, 1)
+	rs.region2.Use(&addr2, 2)
+	rs.region2.Use(&addr3, 3)
+
+	assert.Equal(t, 0, rs.AvailableAddrs())
+
+	rs.GiveBack(&addr2)
+	assert.Equal(t, &addr2, rs.GetUnusedAddr(nil, 2))
+}
+
+func TestRegions_GetUnusedAddr_OneAddrLeft(t *testing.T) {
+	rs := makeRegions()
+
+	rs.region1.Use(&addr0, 0)
+	rs.region1.Use(&addr1, 1)
+	rs.region2.Use(&addr2, 2)
+
+	assert.Equal(t, 1, rs.AvailableAddrs())
+	assert.Equal(t, &addr3, rs.GetUnusedAddr(nil, 3))
+}
+
+func TestRegions_GetUnusedAddr_Excluding_Region1(t *testing.T) {
+	rs := makeRegions()
+
+	rs.region1.Use(&addr0, 0)
+	rs.region1.Use(&addr1, 1)
+
+	assert.Equal(t, 2, rs.AvailableAddrs())
+	assert.Equal(t, &addr3, rs.GetUnusedAddr(&addr2, 3))
+}
+
+func TestRegions_GetUnusedAddr_Excluding_Region2(t *testing.T) {
+	rs := makeRegions()
+
+	rs.region2.Use(&addr2, 0)
+	rs.region2.Use(&addr3, 1)
+
+	assert.Equal(t, 2, rs.AvailableAddrs())
+	assert.Equal(t, &addr1, rs.GetUnusedAddr(&addr0, 1))
+}
+
+func TestNewNoResolveBalancesRegions(t *testing.T) {
+	type args struct {
+		addrs []*net.TCPAddr
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{
+			name: "one address",
+			args: args{addrs: []*net.TCPAddr{&addr0}},
+		},
+		{
+			name: "two addresses",
+			args: args{addrs: []*net.TCPAddr{&addr0, &addr1}},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			regions := NewNoResolve(tt.args.addrs)
+			RegionsIsBalanced(t, regions)
+		})
+	}
+}
+
+func RegionsIsBalanced(t *testing.T, rs *Regions) {
+	delta := rs.region1.AvailableAddrs() - rs.region2.AvailableAddrs()
+	assert.True(t, abs(delta) <= 1)
+}
+
+func abs(x int) int {
+	if x >= 0 {
+		return x
+	}
+	return -x
+}

edgediscovery/allregions/usedby.go

@@ -0,0 +1,14 @@
+package allregions
+
+type UsedBy struct {
+	ConnID int
+	Used   bool
+}
+
+func InUse(connID int) UsedBy {
+	return UsedBy{ConnID: connID, Used: true}
+}
+
+func Unused() UsedBy {
+	return UsedBy{}
+}

edgediscovery/edgediscovery.go

@@ -0,0 +1,144 @@
+package edgediscovery
+
+import (
+	"fmt"
+	"net"
+	"sync"
+
+	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
+
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	subsystem = "edgediscovery"
+)
+
+var errNoAddressesLeft = fmt.Errorf("There are no free edge addresses left")
+
+// Edge finds addresses on the Cloudflare edge and hands them out to connections.
+type Edge struct {
+	regions *allregions.Regions
+	sync.Mutex
+	logger *logrus.Entry
+}
+
+// ------------------------------------
+// Constructors
+// ------------------------------------
+
+// ResolveEdge runs the initial discovery of the Cloudflare edge, finding Addrs that can be allocated
+// to connections.
+func ResolveEdge(l *logrus.Logger) (*Edge, error) {
+	logger := l.WithField("subsystem", subsystem)
+	regions, err := allregions.ResolveEdge(logger)
+	if err != nil {
+		return new(Edge), err
+	}
+	return &Edge{
+		logger:  logger,
+		regions: regions,
+	}, nil
+}
+
+// StaticEdge creates a list of edge addresses from the list of hostnames. Mainly used for testing connectivity.
+func StaticEdge(l *logrus.Logger, hostnames []string) (*Edge, error) {
+	logger := l.WithField("subsystem", subsystem)
+	regions, err := allregions.StaticEdge(hostnames)
+	if err != nil {
+		return new(Edge), err
+	}
+	return &Edge{
+		logger:  logger,
+		regions: regions,
+	}, nil
+}
+
+// MockEdge creates a Cloudflare Edge from arbitrary TCP addresses. Used for testing.
+func MockEdge(l *logrus.Logger, addrs []*net.TCPAddr) *Edge {
+	logger := l.WithField("subsystem", subsystem)
+	regions := allregions.NewNoResolve(addrs)
+	return &Edge{
+		logger:  logger,
+		regions: regions,
+	}
+}
+
+// ------------------------------------
+// Methods
+// ------------------------------------
+
+// GetAddrForRPC gives this connection an edge Addr.
+func (ed *Edge) GetAddrForRPC() (*net.TCPAddr, error) {
+	ed.Lock()
+	defer ed.Unlock()
+	addr := ed.regions.GetAnyAddress()
+	if addr == nil {
+		return nil, errNoAddressesLeft
+	}
+	return addr, nil
+}
+
+// GetAddr gives this proxy connection an edge Addr. Prefer Addrs this connection has already used.
+func (ed *Edge) GetAddr(connID int) (*net.TCPAddr, error) {
+	ed.Lock()
+	defer ed.Unlock()
+	logger := ed.logger.WithFields(logrus.Fields{
+		"connID":   connID,
+		"function": "GetAddr",
+	})
+
+	// If this connection has already used an edge addr, return it.
+	if addr := ed.regions.AddrUsedBy(connID); addr != nil {
+		logger.Debug("Returning same address back to proxy connection")
+		return addr, nil
+	}
+
+	// Otherwise, give it an unused one
+	addr := ed.regions.GetUnusedAddr(nil, connID)
+	if addr == nil {
+		logger.Debug("No addresses left to give proxy connection")
+		return nil, errNoAddressesLeft
+	}
+	logger.Debugf("Giving connection its new address %s", addr)
+	return addr, nil
+}
+
+// GetDifferentAddr gives back the proxy connection's edge Addr and uses a new one.
+func (ed *Edge) GetDifferentAddr(connID int) (*net.TCPAddr, error) {
+	ed.Lock()
+	defer ed.Unlock()
+	logger := ed.logger.WithFields(logrus.Fields{
+		"connID":   connID,
+		"function": "GetDifferentAddr",
+	})
+
+	oldAddr := ed.regions.AddrUsedBy(connID)
+	if oldAddr != nil {
+		ed.regions.GiveBack(oldAddr)
+	}
+	addr := ed.regions.GetUnusedAddr(oldAddr, connID)
+	if addr == nil {
+		logger.Debug("No addresses left to give proxy connection")
+		// note: if oldAddr were not nil, it will become available on the next iteration
+		return nil, errNoAddressesLeft
+	}
+	logger.Debugf("Giving connection its new address %s", addr)
+	return addr, nil
+}
+
+// AvailableAddrs returns how many unused addresses there are left.
+func (ed *Edge) AvailableAddrs() int {
+	ed.Lock()
+	defer ed.Unlock()
+	return ed.regions.AvailableAddrs()
+}
+
+// GiveBack the address so that other connections can use it.
+// Returns true if the address is in this edge.
+func (ed *Edge) GiveBack(addr *net.TCPAddr) bool {
+	ed.Lock()
+	defer ed.Unlock()
+	ed.logger.WithField("function", "GiveBack").Debug("Address now unused")
+	return ed.regions.GiveBack(addr)
+}

edgediscovery/edgediscovery_test.go

@@ -0,0 +1,177 @@
+package edgediscovery
+
+import (
+	"net"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	addr0 = net.TCPAddr{
+		IP:   net.ParseIP("123.0.0.0"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr1 = net.TCPAddr{
+		IP:   net.ParseIP("123.0.0.1"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr2 = net.TCPAddr{
+		IP:   net.ParseIP("123.0.0.2"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr3 = net.TCPAddr{
+		IP:   net.ParseIP("123.0.0.3"),
+		Port: 8000,
+		Zone: "",
+	}
+)
+
+func TestGiveBack(t *testing.T) {
+	l := logrus.New()
+	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+
+	// Give this connection an address
+	assert.Equal(t, 4, edge.AvailableAddrs())
+	const connID = 0
+	addr, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+	assert.Equal(t, 3, edge.AvailableAddrs())
+
+	// Get it back
+	edge.GiveBack(addr)
+	assert.Equal(t, 4, edge.AvailableAddrs())
+}
+
+func TestRPCAndProxyShareSingleEdgeIP(t *testing.T) {
+	l := logrus.New()
+
+	// Make an edge with a single IP
+	edge := MockEdge(l, []*net.TCPAddr{&addr0})
+	tunnelConnID := 0
+
+	// Use the IP for a tunnel
+	addrTunnel, err := edge.GetAddr(tunnelConnID)
+	assert.NoError(t, err)
+
+	// Ensure the IP can be used for RPC too
+	addrRPC, err := edge.GetAddrForRPC()
+	assert.NoError(t, err)
+	assert.Equal(t, addrTunnel, addrRPC)
+}
+
+func TestGetAddrForRPC(t *testing.T) {
+	l := logrus.New()
+	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+
+	// Get a connection
+	assert.Equal(t, 4, edge.AvailableAddrs())
+	addr, err := edge.GetAddrForRPC()
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+
+	// Using an address for RPC shouldn't consume it
+	assert.Equal(t, 4, edge.AvailableAddrs())
+
+	// Get it back
+	edge.GiveBack(addr)
+	assert.Equal(t, 4, edge.AvailableAddrs())
+}
+
+func TestOnePerRegion(t *testing.T) {
+	l := logrus.New()
+
+	// Make an edge with only one address
+	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1})
+
+	// Use the only address
+	const connID = 0
+	a1, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, a1)
+
+	// if the first address is bad, get the second one
+	a2, err := edge.GetDifferentAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, a2)
+	assert.NotEqual(t, a1, a2)
+
+	// now that second one is bad, get the first one again
+	a3, err := edge.GetDifferentAddr(connID)
+	assert.NoError(t, err)
+	assert.Equal(t, a1, a3)
+}
+
+func TestOnlyOneAddrLeft(t *testing.T) {
+	l := logrus.New()
+
+	// Make an edge with only one address
+	edge := MockEdge(l, []*net.TCPAddr{&addr0})
+
+	// Use the only address
+	const connID = 0
+	addr, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+
+	// If that edge address is "bad", there's no alternative address.
+	_, err = edge.GetDifferentAddr(connID)
+	assert.Error(t, err)
+
+	// previously bad address should become available again on next iteration.
+	addr, err = edge.GetDifferentAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+}
+
+func TestNoAddrsLeft(t *testing.T) {
+	l := logrus.New()
+
+	// Make an edge with no addresses
+	edge := MockEdge(l, []*net.TCPAddr{})
+
+	_, err := edge.GetAddr(2)
+	assert.Error(t, err)
+	_, err = edge.GetAddrForRPC()
+	assert.Error(t, err)
+}
+
+func TestGetAddr(t *testing.T) {
+	l := logrus.New()
+	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+
+	// Give this connection an address
+	const connID = 0
+	addr, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+
+	// If the same connection requests another address, it should get the same one.
+	addr2, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.Equal(t, addr, addr2)
+}
+
+func TestGetDifferentAddr(t *testing.T) {
+	l := logrus.New()
+	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+
+	// Give this connection an address
+	assert.Equal(t, 4, edge.AvailableAddrs())
+	const connID = 0
+	addr, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+	assert.Equal(t, 3, edge.AvailableAddrs())
+
+	// If the same connection requests another address, it should get the same one.
+	addr2, err := edge.GetDifferentAddr(connID)
+	assert.NoError(t, err)
+	assert.NotEqual(t, addr, addr2)
+	assert.Equal(t, 3, edge.AvailableAddrs())
+}

go.mod

@@ -0,0 +1,77 @@
+module github.com/cloudflare/cloudflared
+
+go 1.12
+
+require (
+	github.com/BurntSushi/go-sumtype v0.0.0-20190304192233-fcb4a6205bdc // indirect
+	github.com/DATA-DOG/go-sqlmock v1.3.3
+	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
+	github.com/aws/aws-sdk-go v1.25.8
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
+	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
+	github.com/cloudflare/cfssl v0.0.0-20141119014638-2f7f44e802e2
+	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
+	github.com/coredns/coredns v1.2.0
+	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
+	github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a
+	github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0
+	github.com/equinox-io/equinox v1.2.0
+	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
+	github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
+	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
+	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
+	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
+	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
+	github.com/fsnotify/fsnotify v1.4.9
+	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
+	github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0
+	github.com/go-sql-driver/mysql v1.4.1
+	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
+	github.com/google/uuid v1.1.1
+	github.com/gorilla/mux v1.7.3
+	github.com/gorilla/websocket v1.4.0
+	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
+	github.com/jmoiron/sqlx v1.2.0
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/kshvakov/clickhouse v1.3.11
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/lib/pq v1.2.0
+	github.com/mattn/go-colorable v0.1.4
+	github.com/mattn/go-isatty v0.0.10 // indirect
+	github.com/mattn/go-sqlite3 v1.11.0
+	github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b // indirect
+	github.com/miekg/dns v1.1.27
+	github.com/mitchellh/go-homedir v1.1.0
+	github.com/opentracing/opentracing-go v1.1.0 // indirect
+	github.com/philhofer/fwd v1.0.0 // indirect
+	github.com/pkg/errors v0.8.1
+	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
+	github.com/prometheus/client_golang v1.0.0
+	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
+	github.com/prometheus/common v0.7.0 // indirect
+	github.com/prometheus/procfs v0.0.5 // indirect
+	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
+	github.com/sirupsen/logrus v1.4.2
+	github.com/stretchr/testify v1.3.0
+	github.com/tinylib/msgp v1.1.0 // indirect
+	github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0
+	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550
+	golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582
+	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect
+	golang.org/x/sync v0.0.0-20190423024810-112230192c58
+	golang.org/x/sys v0.0.0-20191020212454-3e7259c5e7c2
+	google.golang.org/appengine v1.5.0 // indirect
+	google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd // indirect
+	google.golang.org/grpc v1.24.0 // indirect
+	gopkg.in/coreos/go-oidc.v2 v2.1.0
+	gopkg.in/square/go-jose.v2 v2.4.0 // indirect
+	gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8
+	gopkg.in/yaml.v2 v2.2.4
+	zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7
+)
+
+// ../../go/pkg/mod/github.com/coredns/coredns@v1.2.0/plugin/metrics/metrics.go:40:49: too many arguments in call to prometheus.NewProcessCollector
+// have (int, string)
+// want (prometheus.ProcessCollectorOpts)
+replace github.com/prometheus/client_golang => github.com/prometheus/client_golang v0.9.0-pre1

go.sum

@@ -0,0 +1,284 @@
+bitbucket.org/liamstask/goose v0.0.0-20150115234039-8488cc47d90c/go.mod h1:hSVuE3qU7grINVSwrmzHfpg9k87ALBk+XaualNyUzI4=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/go-sumtype v0.0.0-20190304192233-fcb4a6205bdc h1:nvTP+jmloR0+J4YQur/rLRdLcGVEU4SquDgH+Bo7gBY=
+github.com/BurntSushi/go-sumtype v0.0.0-20190304192233-fcb4a6205bdc/go.mod h1:7yTWMMG2vOm4ABVciEt4EgNVP7fxwtcKIb/EuiLiKqY=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
+github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
+github.com/GeertJohan/go.incremental v1.0.0/go.mod h1:6fAjUhbVuX1KcMD3c8TEgVUqmo4seqhv0i0kdATSkM0=
+github.com/GeertJohan/go.rice v1.0.0/go.mod h1:eH6gbSOAUv07dQuZVnBmoDP8mgsM1rtixis4Tib9if0=
+github.com/akavel/rsrc v0.8.0/go.mod h1:uLoCtb9J+EyAqh+26kdrTgmzRBFPGOolLWKpdxkKq+c=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
+github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
+github.com/aws/aws-sdk-go v1.25.8 h1:n7I+HUUXjun2CsX7JK+1hpRIkZrlKhd3nayeb+Xmavs=
+github.com/aws/aws-sdk-go v1.25.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
+github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
+github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg=
+github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
+github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 h1:JLaf/iINcLyjwbtTsCJjc6rtlASgHeIJPrB6QmwURnA=
+github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudflare/backoff v0.0.0-20161212185259-647f3cdfc87a/go.mod h1:rzgs2ZOiguV6/NpiDgADjRLPNyZlApIWxKpkT+X8SdY=
+github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93 h1:QrGfkZDnMxcWHaYDdB7CmqS9i26OAnUj/xcus/abYkY=
+github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93/go.mod h1:QiTe66jFdP7cUKMCCf/WrvDyYdtdmdZfVcdoLbzaKVY=
+github.com/cloudflare/cfssl v0.0.0-20141119014638-2f7f44e802e2 h1:GOj9uxwfkVdMaHYGQqhab3hr1rjMDqfaQI+JjpKXgMM=
+github.com/cloudflare/cfssl v0.0.0-20141119014638-2f7f44e802e2/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA=
+github.com/cloudflare/go-metrics v0.0.0-20151117154305-6a9aea36fb41/go.mod h1:eaZPlJWD+G9wseg1BuRXlHnjntPMrywMsyxf+LTOdP4=
+github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc h1:Dvk3ySBsOm5EviLx6VCyILnafPcQinXGP5jbTdHUJgE=
+github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc/go.mod h1:HlgKKR8V5a1wroIDDIz3/A+T+9Janfq+7n1P5sEFdi0=
+github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg=
+github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
+github.com/cloudflare/redoctober v0.0.0-20171127175943-746a508df14c/go.mod h1:6Se34jNoqrd8bTxrmJB2Bg2aoZ2CdSXonils9NsiNgo=
+github.com/coredns/coredns v1.2.0 h1:YEI38K2BJYzL/SxO2tZFD727T/C68DqVWkBQjT0sWPU=
+github.com/coredns/coredns v1.2.0/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0=
+github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73 h1:7CNPV0LWRCa1FNmqg700pbXhzvmoaXKyfxWRkjRym7Q=
+github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
+github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a h1:W8b4lQ4tFF21aspRGoBuCNV6V2fFJBF+pm1J6OY8Lys=
+github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/daaku/go.zipexe v1.0.0/go.mod h1:z8IiR6TsVLEYKwXAoE/I+8ys/sDkgTzSL0CLnGVd57E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0 h1:epsH3lb7KVbXHYk7LYGN5EiE0MxcevHU85CKITJ0wUY=
+github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
+github.com/equinox-io/equinox v1.2.0 h1:bBS7Ou+Y7Jwgmy8TWSYxEh85WctuFn7FPlgbUzX4DBA=
+github.com/equinox-io/equinox v1.2.0/go.mod h1:6s3HJB0PYUNgs0mxmI8fHdfVl3TQ25ieA/PVfr+eyVo=
+github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
+github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
+github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 h1:wWke/RUCl7VRjQhwPlR/v0glZXNYzBHdNUzf/Am2Nmg=
+github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmAp6Sws4L7+Q/OokbWDAK1ibXYhB3PXFP1kol5hPg=
+github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434 h1:mOp33BLbcbJ8fvTAmZacbBiOASfxN+MLcLxymZCIrGE=
+github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434/go.mod h1:KigFdumBXUPSwzLDbeuzyt0elrL7+CP7TKuhrhT4bcU=
+github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
+github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
+github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y=
+github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
+github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/getsentry/raven-go v0.0.0-20180121060056-563b81fc02b7/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
+github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/kkTFdWhctH96grJ7qiy7bMdiZcIvPKs=
+github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
+github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0 h1:gF8ngtda767ddth2SH0YSAhswhz6qUkvyI9EZFYCWJA=
+github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
+github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/google/certificate-transparency-go v1.0.21 h1:Yf1aXowfZ2nuboBsg7iYGLmwsOARdV86pfH3g95wXmE=
+github.com/google/certificate-transparency-go v1.0.21/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
+github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
+github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
+github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548/go.mod h1:hGT6jSUVzF6no3QaDSMLGLEHtHSBSefs+MgcDWnmhmo=
+github.com/jmoiron/sqlx v0.0.0-20180124204410-05cef0741ade/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU=
+github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/sqlstruct v0.0.0-20150923205031-648daed35d49/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE=
+github.com/kisom/goutils v1.1.0/go.mod h1:+UBTfd78habUYWFbNWTJNG+jNG/i/lGURakr4A/yNRw=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kshvakov/clickhouse v1.3.11 h1:dtzTJY0fCA+MWkLyuKZaNPkmSwdX4gh8+Klic9NB1Lw=
+github.com/kshvakov/clickhouse v1.3.11/go.mod h1:/SVBAcqF3u7rxQ9sTWCZwf8jzzvxiZGeQvtmSF2BBEc=
+github.com/kylelemons/go-gypsy v0.0.0-20160905020020-08cad365cd28/go.mod h1:T/T7jsxVqf9k/zYOqbgNAsANsjxTd1Yq3htjDhQ1H0c=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lib/pq v0.0.0-20180201184707-88edab080323/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10=
+github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
+github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b h1:/BbY4n99iMazlr2igipph+hj0MwlZIWpcsP8Iy+na+s=
+github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b/go.mod h1:Wb1PlT4DAYSqOEd03MsqkdkXnTxA8v9pKjdpxbqM1kY=
+github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM=
+github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mreiferson/go-httpclient v0.0.0-20160630210159-31f0106b4474/go.mod h1:OQA4XLvDbMgS8P0CevmM4m9Q3Jq4phKUzcocxuGJ5m8=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nkovacs/streamquote v0.0.0-20170412213628-49af9bddb229/go.mod h1:0aYXnNPJ8l7uZxf45rWW1a/uME32OF0rhiYGNQ2oF2E=
+github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
+github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
+github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
+github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
+github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
+github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU=
+github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
+github.com/prometheus/client_golang v0.9.0-pre1 h1:AWTOhsOI9qxeirTuA0A4By/1Es1+y9EcCGY6bBZ2fhM=
+github.com/prometheus/client_golang v0.9.0-pre1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
+github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
+github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
+github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
+github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
+github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
+github.com/weppos/publicsuffix-go v0.4.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
+github.com/weppos/publicsuffix-go v0.5.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
+github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0 h1:6DtWz8hNS4qbq0OCRPhdBMG9E2qKTSDKlwnP3dmZvuA=
+github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0/go.mod h1:A47W3pdWONaZmXuLZgfKLAVgUY0qvfTRM5vVDKS40S4=
+github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
+github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE=
+github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is=
+github.com/zmap/zcrypto v0.0.0-20190729165852-9051775e6a2e/go.mod h1:w7kd3qXHh8FNaczNjslXqvFQiv5mMWRXlL9klTUAHc8=
+github.com/zmap/zlint v0.0.0-20190806154020-fd021b4cfbeb/go.mod h1:29UiAJNsiVdvTBFCJW8e3q6dcDbOoPkhMgttOSCIMMY=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582 h1:p9xBe/w/OzkeYVKm234g55gMdD1nSIooTir5kV11kfA=
+golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
+golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191020212454-3e7259c5e7c2 h1:nq114VpM8lsSlP+lyUbANecYHYiFcSNFtqcBlxRV+gA=
+golang.org/x/sys v0.0.0-20191020212454-3e7259c5e7c2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425 h1:VvQyQJN0tSuecqgcIxMWnnfG5kSmgy9KZR9sW3W5QeA=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd h1:84VQPzup3IpKLxuIAZjHMhVjJ8fZ4/i3yUnj3k6fUdw=
+google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
+google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/coreos/go-oidc.v2 v2.1.0 h1:E8PjVFdj/SLDKB0hvb70KTbMbYVHjqztiQdSkIg8E+I=
+gopkg.in/coreos/go-oidc.v2 v2.1.0/go.mod h1:fYaTe2FS96wZZwR17YTDHwG+Mw6fmyqJNxN2eNCGPCI=
+gopkg.in/square/go-jose.v2 v2.4.0 h1:0kXPskUMGAXXWJlP05ktEMOV0vmzFQUWw6d+aZJQU8A=
+gopkg.in/square/go-jose.v2 v2.4.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8 h1:/pLAskKF+d5SawboKd8GB8ew4ClHDbt2c3K9EBFeRGU=
+gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7 h1:CZoOFlTPbKfAShKYrMuUfYbnXexFT1rYRUX1SPnrdE4=
+zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7/go.mod h1:TMGa8HWGJkXiq4nHe9Zu/JgRF5oUtg4XizFC+Vexbec=

h2mux/activestreammap.go

@@ -3,6 +3,7 @@ package h2mux
 import (
 	"sync"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/net/http2"
 )
 
@@ -12,23 +13,28 @@ type activeStreamMap struct {
 	sync.RWMutex
 	// streams tracks open streams.
 	streams map[uint32]*MuxedStream
-	// streamsEmpty is a chan that should be closed when no more streams are open.
-	streamsEmpty chan struct{}
 	// nextStreamID is the next ID to use on our side of the connection.
 	// This is odd for clients, even for servers.
 	nextStreamID uint32
 	// maxPeerStreamID is the ID of the most recent stream opened by the peer.
 	maxPeerStreamID uint32
+	// activeStreams is a gauge shared by all muxers of this process to expose the total number of active streams
+	activeStreams prometheus.Gauge
+
 	// ignoreNewStreams is true when the connection is being shut down. New streams
 	// cannot be registered.
 	ignoreNewStreams bool
+	// streamsEmpty is a chan that will be closed when no more streams are open.
+	streamsEmptyChan chan struct{}
+	closeOnce        sync.Once
 }
 
-func newActiveStreamMap(useClientStreamNumbers bool) *activeStreamMap {
+func newActiveStreamMap(useClientStreamNumbers bool, activeStreams prometheus.Gauge) *activeStreamMap {
 	m := &activeStreamMap{
-		streams:      make(map[uint32]*MuxedStream),
-		streamsEmpty: make(chan struct{}),
-		nextStreamID: 1,
+		streams:          make(map[uint32]*MuxedStream),
+		streamsEmptyChan: make(chan struct{}),
+		nextStreamID:     1,
+		activeStreams:    activeStreams,
 	}
 	// Client initiated stream uses odd stream ID, server initiated stream uses even stream ID
 	if !useClientStreamNumbers {
@@ -37,6 +43,13 @@ func newActiveStreamMap(useClientStreamNumbers bool) *activeStreamMap {
 	return m
 }
 
+// This function should be called while `m` is locked.
+func (m *activeStreamMap) notifyStreamsEmpty() {
+	m.closeOnce.Do(func() {
+		close(m.streamsEmptyChan)
+	})
+}
+
 // Len returns the number of active streams.
 func (m *activeStreamMap) Len() int {
 	m.RLock()
@@ -63,6 +76,7 @@ func (m *activeStreamMap) Set(newStream *MuxedStream) bool {
 		return false
 	}
 	m.streams[newStream.streamID] = newStream
+	m.activeStreams.Inc()
 	return true
 }
 
@@ -70,31 +84,33 @@ func (m *activeStreamMap) Set(newStream *MuxedStream) bool {
 func (m *activeStreamMap) Delete(streamID uint32) {
 	m.Lock()
 	defer m.Unlock()
-	delete(m.streams, streamID)
-	if len(m.streams) == 0 && m.streamsEmpty != nil {
-		close(m.streamsEmpty)
-		m.streamsEmpty = nil
+	if _, ok := m.streams[streamID]; ok {
+		delete(m.streams, streamID)
+		m.activeStreams.Dec()
+	}
+
+	// shutting down, and now the map is empty
+	if m.ignoreNewStreams && len(m.streams) == 0 {
+		m.notifyStreamsEmpty()
 	}
 }
 
-// Shutdown blocks new streams from being created. It returns a channel that receives an event
-// once the last stream has closed, or nil if a shutdown is in progress.
-func (m *activeStreamMap) Shutdown() <-chan struct{} {
+// Shutdown blocks new streams from being created.
+// It returns `done`, a channel that is closed once the last stream has closed
+// and `progress`, whether a shutdown was already in progress
+func (m *activeStreamMap) Shutdown() (done <-chan struct{}, alreadyInProgress bool) {
 	m.Lock()
 	defer m.Unlock()
 	if m.ignoreNewStreams {
 		// already shutting down
-		return nil
+		return m.streamsEmptyChan, true
 	}
 	m.ignoreNewStreams = true
-	done := make(chan struct{})
 	if len(m.streams) == 0 {
-		// nothing to shut down
-		close(done)
-		return done
+		// there are no streams to wait for
+		m.notifyStreamsEmpty()
 	}
-	m.streamsEmpty = done
-	return done
+	return m.streamsEmptyChan, false
 }
 
 // AcquireLocalID acquires a new stream ID for a stream you're opening.
@@ -162,4 +178,5 @@ func (m *activeStreamMap) Abort() {
 		stream.Close()
 	}
 	m.ignoreNewStreams = true
+	m.notifyStreamsEmpty()
 }

h2mux/activestreammap_test.go

@@ -0,0 +1,195 @@
+package h2mux
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestShutdown(t *testing.T) {
+	const numStreams = 1000
+	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+
+	// Add all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				stream := &MuxedStream{streamID: uint32(streamID)}
+				ok := m.Set(stream)
+				assert.True(t, ok)
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
+
+	shutdownChan, alreadyInProgress := m.Shutdown()
+	select {
+	case <-shutdownChan:
+		assert.Fail(t, "before Shutdown(), shutdownChan shouldn't be closed")
+	default:
+	}
+	assert.False(t, alreadyInProgress)
+
+	shutdownChan2, alreadyInProgress2 := m.Shutdown()
+	assert.Equal(t, shutdownChan, shutdownChan2, "repeated calls to Shutdown() should return the same channel")
+	assert.True(t, alreadyInProgress2, "repeated calls to Shutdown() should return true for 'in progress'")
+
+	// Delete all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				m.Delete(uint32(streamID))
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, 0, m.Len(), "All the streams should have been deleted")
+
+	select {
+	case <-shutdownChan:
+	default:
+		assert.Fail(t, "After all the streams are deleted, shutdownChan should have been closed")
+	}
+}
+
+func TestEmptyBeforeShutdown(t *testing.T) {
+	const numStreams = 1000
+	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+
+	// Add all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				stream := &MuxedStream{streamID: uint32(streamID)}
+				ok := m.Set(stream)
+				assert.True(t, ok)
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
+
+	// Delete all the streams, bringing m to size 0
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				m.Delete(uint32(streamID))
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, 0, m.Len(), "All the streams should have been deleted")
+
+	// Add one stream back
+	const soloStreamID = uint32(0)
+	ok := m.Set(&MuxedStream{streamID: soloStreamID})
+	assert.True(t, ok)
+
+	shutdownChan, alreadyInProgress := m.Shutdown()
+	select {
+	case <-shutdownChan:
+		assert.Fail(t, "before Shutdown(), shutdownChan shouldn't be closed")
+	default:
+	}
+	assert.False(t, alreadyInProgress)
+
+	shutdownChan2, alreadyInProgress2 := m.Shutdown()
+	assert.Equal(t, shutdownChan, shutdownChan2, "repeated calls to Shutdown() should return the same channel")
+	assert.True(t, alreadyInProgress2, "repeated calls to Shutdown() should return true for 'in progress'")
+
+	// Remove the remaining stream
+	m.Delete(soloStreamID)
+
+	select {
+	case <-shutdownChan:
+	default:
+		assert.Fail(t, "After all the streams are deleted, shutdownChan should have been closed")
+	}
+}
+
+type noopBuffer struct {
+	isClosed bool
+}
+
+func (t *noopBuffer) Read(p []byte) (n int, err error)  { return len(p), nil }
+func (t *noopBuffer) Write(p []byte) (n int, err error) { return len(p), nil }
+func (t *noopBuffer) Reset()                            {}
+func (t *noopBuffer) Len() int                          { return 0 }
+func (t *noopBuffer) Close() error                      { t.isClosed = true; return nil }
+func (t *noopBuffer) Closed() bool                      { return t.isClosed }
+
+type noopReadyList struct{}
+
+func (_ *noopReadyList) Signal(streamID uint32) {}
+
+func TestAbort(t *testing.T) {
+	const numStreams = 1000
+	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+
+	var openedStreams sync.Map
+
+	// Add all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				stream := &MuxedStream{
+					streamID:    uint32(streamID),
+					readBuffer:  &noopBuffer{},
+					writeBuffer: &noopBuffer{},
+					readyList:   &noopReadyList{},
+				}
+				ok := m.Set(stream)
+				assert.True(t, ok)
+
+				openedStreams.Store(stream.streamID, stream)
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
+
+	shutdownChan, alreadyInProgress := m.Shutdown()
+	select {
+	case <-shutdownChan:
+		assert.Fail(t, "before Abort(), shutdownChan shouldn't be closed")
+	default:
+	}
+	assert.False(t, alreadyInProgress)
+
+	m.Abort()
+	assert.Equal(t, numStreams, m.Len(), "Abort() shouldn't delete any streams")
+	openedStreams.Range(func(key interface{}, value interface{}) bool {
+		stream := value.(*MuxedStream)
+		readBuffer := stream.readBuffer.(*noopBuffer)
+		writeBuffer := stream.writeBuffer.(*noopBuffer)
+		return assert.True(t, readBuffer.isClosed && writeBuffer.isClosed, "Abort() should have closed all the streams")
+	})
+
+	select {
+	case <-shutdownChan:
+	default:
+		assert.Fail(t, "after Abort(), shutdownChan should have been closed")
+	}
+
+	// multiple aborts shouldn't cause any issues
+	m.Abort()
+	m.Abort()
+	m.Abort()
+}

h2mux/error.go

@@ -20,11 +20,12 @@ var (
 	ErrUnknownStream       = MuxerProtocolError{"2002 unknown stream", http2.ErrCodeProtocol}
 	ErrInvalidStream       = MuxerProtocolError{"2003 invalid stream", http2.ErrCodeProtocol}
 
-	ErrStreamHeadersSent      = MuxerApplicationError{"3000 headers already sent"}
-	ErrConnectionClosed       = MuxerApplicationError{"3001 connection closed"}
-	ErrConnectionDropped      = MuxerApplicationError{"3002 connection dropped"}
-	ErrOpenStreamTimeout      = MuxerApplicationError{"3003 open stream timeout"}
-	ErrResponseHeadersTimeout = MuxerApplicationError{"3004 timeout waiting for initial response headers"}
+	ErrStreamHeadersSent               = MuxerApplicationError{"3000 headers already sent"}
+	ErrStreamRequestConnectionClosed   = MuxerApplicationError{"3001 connection closed while opening stream"}
+	ErrConnectionDropped               = MuxerApplicationError{"3002 connection dropped"}
+	ErrStreamRequestTimeout            = MuxerApplicationError{"3003 open stream timeout"}
+	ErrResponseHeadersTimeout          = MuxerApplicationError{"3004 timeout waiting for initial response headers"}
+	ErrResponseHeadersConnectionClosed = MuxerApplicationError{"3005 connection closed while waiting for initial response headers"}
 
 	ErrClosedStream = MuxerStreamError{"4000 stream closed", http2.ErrCodeStreamClosed}
 )

h2mux/h2_dictionaries.go

@@ -542,7 +542,10 @@ func (w *h2DictWriter) Write(p []byte) (n int, err error) {
 }
 
 func (w *h2DictWriter) Close() error {
-	return w.comp.Close()
+	if w.comp != nil {
+		return w.comp.Close()
+	}
+	return nil
 }
 
 // From http2/hpack

h2mux/h2mux.go

@@ -1,13 +1,13 @@
 package h2mux
 
 import (
-	"bytes"
 	"context"
 	"io"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/hpack"
@@ -20,7 +20,7 @@ const (
 	maxWindowSize            uint32        = (1 << 31) - 1 // 2^31-1 = 2147483647, max window size in http2 spec
 	defaultTimeout           time.Duration = 5 * time.Second
 	defaultRetries           uint64        = 5
-	defaultWriteBufferMaxLen int           = 1024 * 1024 * 512 // 500mb
+	defaultWriteBufferMaxLen int           = 1024 * 1024 // 1mb
 
 	SettingMuxerMagic http2.SettingID = 0x42db
 	MuxerMagicOrigin  uint32          = 0xa2e43c8b
@@ -90,10 +90,6 @@ type Muxer struct {
 	compressionQuality CompressionPreset
 }
 
-type Header struct {
-	Name, Value string
-}
-
 func RPCHeaders() []Header {
 	return []Header{
 		{Name: ":method", Value: "RPC"},
@@ -108,6 +104,7 @@ func Handshake(
 	w io.WriteCloser,
 	r io.ReadCloser,
 	config MuxerConfig,
+	activeStreamsMetrics prometheus.Gauge,
 ) (*Muxer, error) {
 	// Set default config values
 	if config.Timeout == 0 {
@@ -131,7 +128,7 @@ func Handshake(
 		newStreamChan: make(chan MuxedStreamRequest),
 		abortChan:     make(chan struct{}),
 		readyList:     NewReadyList(),
-		streams:       newActiveStreamMap(config.IsClient),
+		streams:       newActiveStreamMap(config.IsClient, activeStreamsMetrics),
 	}
 
 	m.f.ReadMetaHeaders = hpack.NewDecoder(4096, func(hpack.HeaderField) {})
@@ -209,9 +206,9 @@ func Handshake(
 		initialStreamWindow:     m.config.DefaultWindowSize,
 		streamWindowMax:         m.config.MaxWindowSize,
 		streamWriteBufferMaxLen: m.config.StreamWriteBufferMaxLen,
-		r:                       m.r,
-		metricsUpdater:          m.muxMetricsUpdater,
-		bytesRead:               inBoundCounter,
+		r:              m.r,
+		metricsUpdater: m.muxMetricsUpdater,
+		bytesRead:      inBoundCounter,
 	}
 	m.muxWriter = &MuxWriter{
 		f:               m.f,
@@ -324,24 +321,63 @@ func joinErrorsWithTimeout(errChan <-chan error, receiveCount int, timeout time.
 func (m *Muxer) Serve(ctx context.Context) error {
 	errGroup, _ := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {
-		err := m.muxReader.run(m.config.Logger)
-		m.explicitShutdown.Fuse(false)
-		m.r.Close()
-		m.abort()
-		return err
+		ch := make(chan error)
+		go func() {
+			err := m.muxReader.run(m.config.Logger)
+			m.explicitShutdown.Fuse(false)
+			m.r.Close()
+			m.abort()
+			// don't block if parent goroutine quit early
+			select {
+			case ch <- err:
+			default:
+			}
+		}()
+		select {
+		case err := <-ch:
+			return err
+		case <-ctx.Done():
+			return ctx.Err()
+		}
 	})
 
 	errGroup.Go(func() error {
-		err := m.muxWriter.run(m.config.Logger)
-		m.explicitShutdown.Fuse(false)
-		m.w.Close()
-		m.abort()
-		return err
+		ch := make(chan error)
+		go func() {
+			err := m.muxWriter.run(m.config.Logger)
+			m.explicitShutdown.Fuse(false)
+			m.w.Close()
+			m.abort()
+			// don't block if parent goroutine quit early
+			select {
+			case ch <- err:
+			default:
+			}
+		}()
+		select {
+		case err := <-ch:
+			return err
+		case <-ctx.Done():
+			return ctx.Err()
+		}
 	})
 
 	errGroup.Go(func() error {
-		err := m.muxMetricsUpdater.run(m.config.Logger)
-		return err
+		ch := make(chan error)
+		go func() {
+			err := m.muxMetricsUpdater.run(m.config.Logger)
+			// don't block if parent goroutine quit early
+			select {
+			case ch <- err:
+			default:
+			}
+		}()
+		select {
+		case err := <-ch:
+			return err
+		case <-ctx.Done():
+			return ctx.Err()
+		}
 	})
 
 	err := errGroup.Wait()
@@ -352,9 +388,11 @@ func (m *Muxer) Serve(ctx context.Context) error {
 }
 
 // Shutdown is called to initiate the "happy path" of muxer termination.
-func (m *Muxer) Shutdown() {
+// It blocks new streams from being created.
+// It returns a channel that is closed when the last stream has been closed.
+func (m *Muxer) Shutdown() <-chan struct{} {
 	m.explicitShutdown.Fuse(true)
-	m.muxReader.Shutdown()
+	return m.muxReader.Shutdown()
 }
 
 // IsUnexpectedTunnelError identifies errors that are expected when shutting down the h2mux tunnel.
@@ -388,72 +426,58 @@ func isConnectionClosedError(err error) bool {
 // OpenStream opens a new data stream with the given headers.
 // Called by proxy server and tunnel
 func (m *Muxer) OpenStream(ctx context.Context, headers []Header, body io.Reader) (*MuxedStream, error) {
-	stream := &MuxedStream{
-		responseHeadersReceived: make(chan struct{}),
-		readBuffer:              NewSharedBuffer(),
-		writeBuffer:             &bytes.Buffer{},
-		writeBufferMaxLen:       m.config.StreamWriteBufferMaxLen,
-		writeBufferHasSpace:     make(chan struct{}, 1),
-		receiveWindow:           m.config.DefaultWindowSize,
-		receiveWindowCurrentMax: m.config.DefaultWindowSize,
-		receiveWindowMax:        m.config.MaxWindowSize,
-		sendWindow:              m.config.DefaultWindowSize,
-		readyList:               m.readyList,
-		writeHeaders:            headers,
-		dictionaries:            m.muxReader.dictionaries,
+	stream := m.NewStream(headers)
+	if err := m.MakeMuxedStreamRequest(ctx, NewMuxedStreamRequest(stream, body)); err != nil {
+		return nil, err
 	}
-
-	select {
-	// Will be received by mux writer
-	case <-ctx.Done():
-		return nil, ErrOpenStreamTimeout
-	case <-m.abortChan:
-		return nil, ErrConnectionClosed
-	case m.newStreamChan <- MuxedStreamRequest{stream: stream, body: body}:
-	}
-
-	select {
-	case <-ctx.Done():
-		return nil, ErrResponseHeadersTimeout
-	case <-m.abortChan:
-		return nil, ErrConnectionClosed
-	case <-stream.responseHeadersReceived:
-		return stream, nil
+	if err := m.AwaitResponseHeaders(ctx, stream); err != nil {
+		return nil, err
 	}
+	return stream, nil
 }
 
 func (m *Muxer) OpenRPCStream(ctx context.Context) (*MuxedStream, error) {
-	stream := &MuxedStream{
-		responseHeadersReceived: make(chan struct{}),
-		readBuffer:              NewSharedBuffer(),
-		writeBuffer:             &bytes.Buffer{},
-		writeBufferMaxLen:       m.config.StreamWriteBufferMaxLen,
-		writeBufferHasSpace:     make(chan struct{}, 1),
-		receiveWindow:           m.config.DefaultWindowSize,
-		receiveWindowCurrentMax: m.config.DefaultWindowSize,
-		receiveWindowMax:        m.config.MaxWindowSize,
-		sendWindow:              m.config.DefaultWindowSize,
-		readyList:               m.readyList,
-		writeHeaders:            RPCHeaders(),
-		dictionaries:            m.muxReader.dictionaries,
+	stream := m.NewStream(RPCHeaders())
+	if err := m.MakeMuxedStreamRequest(ctx, NewMuxedStreamRequest(stream, nil)); err != nil {
+		return nil, err
 	}
+	if err := m.AwaitResponseHeaders(ctx, stream); err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
 
+func (m *Muxer) NewStream(headers []Header) *MuxedStream {
+	return NewStream(m.config, headers, m.readyList, m.muxReader.dictionaries)
+}
+
+func (m *Muxer) MakeMuxedStreamRequest(ctx context.Context, request MuxedStreamRequest) error {
 	select {
+	case <-ctx.Done():
+		return ErrStreamRequestTimeout
+	case <-m.abortChan:
+		return ErrStreamRequestConnectionClosed
 	// Will be received by mux writer
-	case <-ctx.Done():
-		return nil, ErrOpenStreamTimeout
-	case <-m.abortChan:
-		return nil, ErrConnectionClosed
-	case m.newStreamChan <- MuxedStreamRequest{stream: stream, body: nil}:
+	case m.newStreamChan <- request:
+		return nil
 	}
+}
 
+func (m *Muxer) CloseStreamRead(stream *MuxedStream) {
+	stream.CloseRead()
+	if stream.WriteClosed() {
+		m.streams.Delete(stream.streamID)
+	}
+}
+
+func (m *Muxer) AwaitResponseHeaders(ctx context.Context, stream *MuxedStream) error {
 	select {
 	case <-ctx.Done():
-		return nil, ErrResponseHeadersTimeout
+		return ErrResponseHeadersTimeout
 	case <-m.abortChan:
-		return nil, ErrConnectionClosed
+		return ErrResponseHeadersConnectionClosed
 	case <-stream.responseHeadersReceived:
-		return stream, nil
+		return nil
 	}
 }
 

h2mux/h2mux_test.go

@@ -43,7 +43,7 @@ type DefaultMuxerPair struct {
 	doneC           chan struct{}
 }
 
-func NewDefaultMuxerPair(t assert.TestingT, f MuxedStreamFunc) *DefaultMuxerPair {
+func NewDefaultMuxerPair(t assert.TestingT, testName string, f MuxedStreamFunc) *DefaultMuxerPair {
 	origin, edge := net.Pipe()
 	p := &DefaultMuxerPair{
 		OriginMuxConfig: MuxerConfig{
@@ -55,6 +55,8 @@ func NewDefaultMuxerPair(t assert.TestingT, f MuxedStreamFunc) *DefaultMuxerPair
 			DefaultWindowSize:       (1 << 8) - 1,
 			MaxWindowSize:           (1 << 15) - 1,
 			StreamWriteBufferMaxLen: 1024,
+			HeartbeatInterval:       defaultTimeout,
+			MaxHeartbeats:           defaultRetries,
 		},
 		OriginConn: origin,
 		EdgeMuxConfig: MuxerConfig{
@@ -65,15 +67,17 @@ func NewDefaultMuxerPair(t assert.TestingT, f MuxedStreamFunc) *DefaultMuxerPair
 			DefaultWindowSize:       (1 << 8) - 1,
 			MaxWindowSize:           (1 << 15) - 1,
 			StreamWriteBufferMaxLen: 1024,
+			HeartbeatInterval:       defaultTimeout,
+			MaxHeartbeats:           defaultRetries,
 		},
 		EdgeConn: edge,
 		doneC:    make(chan struct{}),
 	}
-	assert.NoError(t, p.Handshake())
+	assert.NoError(t, p.Handshake(testName))
 	return p
 }
 
-func NewCompressedMuxerPair(t assert.TestingT, quality CompressionSetting, f MuxedStreamFunc) *DefaultMuxerPair {
+func NewCompressedMuxerPair(t assert.TestingT, testName string, quality CompressionSetting, f MuxedStreamFunc) *DefaultMuxerPair {
 	origin, edge := net.Pipe()
 	p := &DefaultMuxerPair{
 		OriginMuxConfig: MuxerConfig{
@@ -83,6 +87,8 @@ func NewCompressedMuxerPair(t assert.TestingT, quality CompressionSetting, f Mux
 			Name:               "origin",
 			CompressionQuality: quality,
 			Logger:             log.NewEntry(log.New()),
+			HeartbeatInterval:  defaultTimeout,
+			MaxHeartbeats:      defaultRetries,
 		},
 		OriginConn: origin,
 		EdgeMuxConfig: MuxerConfig{
@@ -91,24 +97,26 @@ func NewCompressedMuxerPair(t assert.TestingT, quality CompressionSetting, f Mux
 			Name:               "edge",
 			CompressionQuality: quality,
 			Logger:             log.NewEntry(log.New()),
+			HeartbeatInterval:  defaultTimeout,
+			MaxHeartbeats:      defaultRetries,
 		},
 		EdgeConn: edge,
 		doneC:    make(chan struct{}),
 	}
-	assert.NoError(t, p.Handshake())
+	assert.NoError(t, p.Handshake(testName))
 	return p
 }
 
-func (p *DefaultMuxerPair) Handshake() error {
+func (p *DefaultMuxerPair) Handshake(testName string) error {
 	ctx, cancel := context.WithTimeout(context.Background(), testHandshakeTimeout)
 	defer cancel()
 	errGroup, _ := errgroup.WithContext(ctx)
 	errGroup.Go(func() (err error) {
-		p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig)
+		p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig, NewActiveStreamsMetrics(testName, "edge"))
 		return errors.Wrap(err, "edge handshake failure")
 	})
 	errGroup.Go(func() (err error) {
-		p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig)
+		p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig, NewActiveStreamsMetrics(testName, "origin"))
 		return errors.Wrap(err, "origin handshake failure")
 	})
 
@@ -161,7 +169,7 @@ func TestHandshake(t *testing.T) {
 	f := func(stream *MuxedStream) error {
 		return nil
 	}
-	muxPair := NewDefaultMuxerPair(t, f)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
 	AssertIfPipeReadable(t, muxPair.OriginConn)
 	AssertIfPipeReadable(t, muxPair.EdgeConn)
 }
@@ -191,7 +199,7 @@ func TestSingleStream(t *testing.T) {
 		}
 		return nil
 	})
-	muxPair := NewDefaultMuxerPair(t, f)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
 	muxPair.Serve(t)
 
 	stream, err := muxPair.OpenEdgeMuxStream(
@@ -262,7 +270,7 @@ func TestSingleStreamLargeResponseBody(t *testing.T) {
 
 		return nil
 	})
-	muxPair := NewDefaultMuxerPair(t, f)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
 	muxPair.Serve(t)
 
 	stream, err := muxPair.OpenEdgeMuxStream(
@@ -309,7 +317,7 @@ func TestMultipleStreams(t *testing.T) {
 		log.Debugf("Wrote body for stream %s", stream.Headers[0].Value)
 		return nil
 	})
-	muxPair := NewDefaultMuxerPair(t, f)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
 	muxPair.Serve(t)
 
 	maxStreams := 64
@@ -402,7 +410,7 @@ func TestMultipleStreamsFlowControl(t *testing.T) {
 		}
 		return nil
 	})
-	muxPair := NewDefaultMuxerPair(t, f)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
 	muxPair.Serve(t)
 
 	errGroup, _ := errgroup.WithContext(context.Background())
@@ -461,7 +469,7 @@ func TestGracefulShutdown(t *testing.T) {
 		log.Debugf("Handler ends")
 		return nil
 	})
-	muxPair := NewDefaultMuxerPair(t, f)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
 	muxPair.Serve(t)
 
 	stream, err := muxPair.OpenEdgeMuxStream(
@@ -516,7 +524,7 @@ func TestUnexpectedShutdown(t *testing.T) {
 		}
 		return nil
 	})
-	muxPair := NewDefaultMuxerPair(t, f)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), f)
 	muxPair.Serve(t)
 
 	stream, err := muxPair.OpenEdgeMuxStream(
@@ -564,7 +572,7 @@ func EchoHandler(stream *MuxedStream) error {
 
 func TestOpenAfterDisconnect(t *testing.T) {
 	for i := 0; i < 3; i++ {
-		muxPair := NewDefaultMuxerPair(t, EchoHandler)
+		muxPair := NewDefaultMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), i), EchoHandler)
 		muxPair.Serve(t)
 
 		switch i {
@@ -584,14 +592,14 @@ func TestOpenAfterDisconnect(t *testing.T) {
 			[]Header{{Name: "test-header", Value: "headerValue"}},
 			nil,
 		)
-		if err != ErrConnectionClosed {
-			t.Fatalf("unexpected error in OpenStream: %s", err)
+		if err != ErrStreamRequestConnectionClosed && err != ErrResponseHeadersConnectionClosed {
+			t.Fatalf("case %v: unexpected error in OpenStream: %v", i, err)
 		}
 	}
 }
 
 func TestHPACK(t *testing.T) {
-	muxPair := NewDefaultMuxerPair(t, EchoHandler)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), EchoHandler)
 	muxPair.Serve(t)
 
 	stream, err := muxPair.OpenEdgeMuxStream(
@@ -724,7 +732,7 @@ func TestMultipleStreamsWithDictionaries(t *testing.T) {
 
 			return nil
 		})
-		muxPair := NewCompressedMuxerPair(t, q, f)
+		muxPair := NewCompressedMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), q), q, f)
 		muxPair.Serve(t)
 
 		var wg sync.WaitGroup
@@ -918,7 +926,7 @@ func TestSampleSiteWithDictionaries(t *testing.T) {
 	assert.NoError(t, err)
 
 	for q := CompressionNone; q <= CompressionMax; q++ {
-		muxPair := NewCompressedMuxerPair(t, q, sampleSiteHandler(files))
+		muxPair := NewCompressedMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), q), q, sampleSiteHandler(files))
 		muxPair.Serve(t)
 
 		var wg sync.WaitGroup
@@ -957,7 +965,7 @@ func TestLongSiteWithDictionaries(t *testing.T) {
 	files, err := loadSampleFiles(paths)
 	assert.NoError(t, err)
 	for q := CompressionNone; q <= CompressionMedium; q++ {
-		muxPair := NewCompressedMuxerPair(t, q, sampleSiteHandler(files))
+		muxPair := NewCompressedMuxerPair(t, fmt.Sprintf("%s_%d", t.Name(), q), q, sampleSiteHandler(files))
 		muxPair.Serve(t)
 
 		rand.Seed(time.Now().Unix())
@@ -998,7 +1006,7 @@ func BenchmarkOpenStream(b *testing.B) {
 			})
 			return nil
 		})
-		muxPair := NewDefaultMuxerPair(b, f)
+		muxPair := NewDefaultMuxerPair(b, fmt.Sprintf("%s_%d", b.Name(), i), f)
 		muxPair.Serve(b)
 		b.StartTimer()
 		openStreams(b, muxPair, streams)

h2mux/header.go

@@ -0,0 +1,249 @@
+package h2mux
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+type Header struct {
+	Name, Value string
+}
+
+var headerEncoding = base64.RawStdEncoding
+
+const (
+	RequestUserHeadersField  = "cf-cloudflared-request-headers"
+	ResponseUserHeadersField = "cf-cloudflared-response-headers"
+
+	ResponseMetaHeaderField   = "cf-cloudflared-response-meta"
+	ResponseSourceCloudflared = "cloudflared"
+	ResponseSourceOrigin      = "origin"
+)
+
+// H2RequestHeadersToH1Request converts the HTTP/2 headers coming from origintunneld
+// to an HTTP/1 Request object destined for the local origin web service.
+// This operation includes conversion of the pseudo-headers into their closest
+// HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3
+func H2RequestHeadersToH1Request(h2 []Header, h1 *http.Request) error {
+	for _, header := range h2 {
+		if !IsControlHeader(header.Name) {
+			continue
+		}
+
+		switch strings.ToLower(header.Name) {
+		case ":method":
+			h1.Method = header.Value
+		case ":scheme":
+			// noop - use the preexisting scheme from h1.URL
+		case ":authority":
+			// Otherwise the host header will be based on the origin URL
+			h1.Host = header.Value
+		case ":path":
+			// We don't want to be an "opinionated" proxy, so ideally we would use :path as-is.
+			// However, this HTTP/1 Request object belongs to the Go standard library,
+			// whose URL package makes some opinionated decisions about the encoding of
+			// URL characters: see the docs of https://godoc.org/net/url#URL,
+			// in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath,
+			// which is always used when computing url.URL.String(), whether we'd like it or not.
+			//
+			// Well, not *always*. We could circumvent this by using url.URL.Opaque. But
+			// that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque
+			// is treated differently when HTTP_PROXY is set!
+			// See https://github.com/golang/go/issues/5684#issuecomment-66080888
+			//
+			// This means we are subject to the behavior of net/url's function `shouldEscape`
+			// (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101
+
+			if header.Value == "*" {
+				h1.URL.Path = "*"
+				continue
+			}
+			// Due to the behavior of validation.ValidateUrl, h1.URL may
+			// already have a partial value, with or without a trailing slash.
+			base := h1.URL.String()
+			base = strings.TrimRight(base, "/")
+			// But we know :path begins with '/', because we handled '*' above - see RFC7540
+			requestURL, err := url.Parse(base + header.Value)
+			if err != nil {
+				return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value))
+			}
+			h1.URL = requestURL
+		case "content-length":
+			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
+			if err != nil {
+				return fmt.Errorf("unparseable content length")
+			}
+			h1.ContentLength = contentLength
+		case RequestUserHeadersField:
+			// Do not forward the serialized headers to the origin -- deserialize them, and ditch the serialized version
+			// Find and parse user headers serialized into a single one
+			userHeaders, err := ParseUserHeaders(RequestUserHeadersField, h2)
+			if err != nil {
+				return errors.Wrap(err, "Unable to parse user headers")
+			}
+			for _, userHeader := range userHeaders {
+				h1.Header.Add(http.CanonicalHeaderKey(userHeader.Name), userHeader.Value)
+			}
+		default:
+			// All other control headers shall just be proxied transparently
+			h1.Header.Add(http.CanonicalHeaderKey(header.Name), header.Value)
+		}
+	}
+
+	return nil
+}
+
+func ParseUserHeaders(headerNameToParseFrom string, headers []Header) ([]Header, error) {
+	for _, header := range headers {
+		if header.Name == headerNameToParseFrom {
+			return DeserializeHeaders(header.Value)
+		}
+	}
+
+	return nil, fmt.Errorf("%v header not found", RequestUserHeadersField)
+}
+
+func IsControlHeader(headerName string) bool {
+	headerName = strings.ToLower(headerName)
+
+	return headerName == "content-length" ||
+		headerName == "connection" || headerName == "upgrade" || // Websocket headers
+		strings.HasPrefix(headerName, ":") ||
+		strings.HasPrefix(headerName, "cf-")
+}
+
+// IsWebsocketClientHeader returns true if the header name is required by the client to upgrade properly
+func IsWebsocketClientHeader(headerName string) bool {
+	headerName = strings.ToLower(headerName)
+
+	return headerName == "sec-websocket-accept" ||
+		headerName == "connection" ||
+		headerName == "upgrade"
+}
+
+func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
+	h2 = []Header{
+		{Name: ":status", Value: strconv.Itoa(h1.StatusCode)},
+	}
+	userHeaders := http.Header{}
+	for header, values := range h1.Header {
+		for _, value := range values {
+			if strings.ToLower(header) == "content-length" {
+				// This header has meaning in HTTP/2 and will be used by the edge,
+				// so it should be sent as an HTTP/2 response header.
+
+				// Since these are http2 headers, they're required to be lowercase
+				h2 = append(h2, Header{Name: strings.ToLower(header), Value: value})
+			} else if !IsControlHeader(header) || IsWebsocketClientHeader(header) {
+				// User headers, on the other hand, must all be serialized so that
+				// HTTP/2 header validation won't be applied to HTTP/1 header values
+				if _, ok := userHeaders[header]; ok {
+					userHeaders[header] = append(userHeaders[header], value)
+				} else {
+					userHeaders[header] = []string{value}
+				}
+			}
+		}
+	}
+
+	// Perform user header serialization and set them in the single header
+	h2 = append(h2, CreateSerializedHeaders(ResponseUserHeadersField, userHeaders)...)
+
+	return h2
+}
+
+// Serialize HTTP1.x headers by base64-encoding each header name and value,
+// and then joining them in the format of [key:value;]
+func SerializeHeaders(h1Headers http.Header) string {
+	var serializedHeaders []string
+	for headerName, headerValues := range h1Headers {
+		for _, headerValue := range headerValues {
+			encodedName := make([]byte, headerEncoding.EncodedLen(len(headerName)))
+			headerEncoding.Encode(encodedName, []byte(headerName))
+
+			encodedValue := make([]byte, headerEncoding.EncodedLen(len(headerValue)))
+			headerEncoding.Encode(encodedValue, []byte(headerValue))
+
+			serializedHeaders = append(
+				serializedHeaders,
+				strings.Join(
+					[]string{string(encodedName), string(encodedValue)},
+					":",
+				),
+			)
+		}
+	}
+
+	return strings.Join(serializedHeaders, ";")
+}
+
+// Deserialize headers serialized by `SerializeHeader`
+func DeserializeHeaders(serializedHeaders string) ([]Header, error) {
+	const unableToDeserializeErr = "Unable to deserialize headers"
+
+	var deserialized []Header
+	for _, serializedPair := range strings.Split(serializedHeaders, ";") {
+		if len(serializedPair) == 0 {
+			continue
+		}
+
+		serializedHeaderParts := strings.Split(serializedPair, ":")
+		if len(serializedHeaderParts) != 2 {
+			return nil, errors.New(unableToDeserializeErr)
+		}
+
+		serializedName := serializedHeaderParts[0]
+		serializedValue := serializedHeaderParts[1]
+		deserializedName := make([]byte, headerEncoding.DecodedLen(len(serializedName)))
+		deserializedValue := make([]byte, headerEncoding.DecodedLen(len(serializedValue)))
+
+		if _, err := headerEncoding.Decode(deserializedName, []byte(serializedName)); err != nil {
+			return nil, errors.Wrap(err, unableToDeserializeErr)
+		}
+		if _, err := headerEncoding.Decode(deserializedValue, []byte(serializedValue)); err != nil {
+			return nil, errors.Wrap(err, unableToDeserializeErr)
+		}
+
+		deserialized = append(deserialized, Header{
+			Name:  string(deserializedName),
+			Value: string(deserializedValue),
+		})
+	}
+
+	return deserialized, nil
+}
+
+func CreateSerializedHeaders(headersField string, headers ...http.Header) []Header {
+	var serializedHeaderChunks []string
+	for _, headerChunk := range headers {
+		serializedHeaderChunks = append(serializedHeaderChunks, SerializeHeaders(headerChunk))
+	}
+
+	return []Header{{
+		headersField,
+		strings.Join(serializedHeaderChunks, ";"),
+	}}
+}
+
+type ResponseMetaHeader struct {
+	Source string `json:"src"`
+}
+
+func CreateResponseMetaHeader(headerName, source string) Header {
+	jsonResponseMetaHeader, err := json.Marshal(ResponseMetaHeader{Source: source})
+	if err != nil {
+		panic(err)
+	}
+
+	return Header{
+		Name:  headerName,
+		Value: string(jsonResponseMetaHeader),
+	}
+}

h2mux/header_test.go

@@ -0,0 +1,652 @@
+package h2mux
+
+import (
+	"fmt"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"reflect"
+	"regexp"
+	"sort"
+	"strings"
+	"testing"
+	"testing/quick"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type ByName []Header
+
+func (a ByName) Len() int      { return len(a) }
+func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByName) Less(i, j int) bool {
+	if a[i].Name == a[j].Name {
+		return a[i].Value < a[j].Value
+	}
+
+	return a[i].Name < a[j].Name
+}
+
+func TestH2RequestHeadersToH1Request_RegularHeaders(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	mockHeaders := http.Header{
+		"Mock header 1": {"Mock value 1"},
+		"Mock header 2": {"Mock value 2"},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(CreateSerializedHeaders(RequestUserHeadersField, mockHeaders), request)
+
+	assert.True(t, reflect.DeepEqual(mockHeaders, request.Header))
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_NoHeaders(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	headersConversionErr := H2RequestHeadersToH1Request(
+		[]Header{{
+			RequestUserHeadersField,
+			SerializeHeaders(http.Header{}),
+		}},
+		request,
+	)
+
+	assert.True(t, reflect.DeepEqual(http.Header{}, request.Header))
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_InvalidHostPath(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	mockRequestHeaders := []Header{
+		{Name: ":path", Value: "//bad_path/"},
+		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
+
+	assert.Equal(t, http.Header{
+		"Mock header": []string{"Mock value"},
+	}, request.Header)
+
+	assert.Equal(t, "http://example.com//bad_path/", request.URL.String())
+
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_HostPathWithQuery(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
+	assert.NoError(t, err)
+
+	mockRequestHeaders := []Header{
+		{Name: ":path", Value: "/?query=mock%20value"},
+		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
+
+	assert.Equal(t, http.Header{
+		"Mock header": []string{"Mock value"},
+	}, request.Header)
+
+	assert.Equal(t, "http://example.com/?query=mock%20value", request.URL.String())
+
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_HostPathWithURLEncoding(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
+	assert.NoError(t, err)
+
+	mockRequestHeaders := []Header{
+		{Name: ":path", Value: "/mock%20path"},
+		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
+
+	assert.Equal(t, http.Header{
+		"Mock header": []string{"Mock value"},
+	}, request.Header)
+
+	assert.Equal(t, "http://example.com/mock%20path", request.URL.String())
+
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_WeirdURLs(t *testing.T) {
+	type testCase struct {
+		path string
+		want string
+	}
+	testCases := []testCase{
+		{
+			path: "",
+			want: "",
+		},
+		{
+			path: "/",
+			want: "/",
+		},
+		{
+			path: "//",
+			want: "//",
+		},
+		{
+			path: "/test",
+			want: "/test",
+		},
+		{
+			path: "//test",
+			want: "//test",
+		},
+		{
+			// https://github.com/cloudflare/cloudflared/issues/81
+			path: "//test/",
+			want: "//test/",
+		},
+		{
+			path: "/%2Ftest",
+			want: "/%2Ftest",
+		},
+		{
+			path: "//%20test",
+			want: "//%20test",
+		},
+		{
+			// https://github.com/cloudflare/cloudflared/issues/124
+			path: "/test?get=somthing%20a",
+			want: "/test?get=somthing%20a",
+		},
+		{
+			path: "/%20",
+			want: "/%20",
+		},
+		{
+			// stdlib's EscapedPath() will always percent-encode ' '
+			path: "/ ",
+			want: "/%20",
+		},
+		{
+			path: "/ a ",
+			want: "/%20a%20",
+		},
+		{
+			path: "/a%20b",
+			want: "/a%20b",
+		},
+		{
+			path: "/foo/bar;param?query#frag",
+			want: "/foo/bar;param?query#frag",
+		},
+		{
+			// stdlib's EscapedPath() will always percent-encode non-ASCII chars
+			path: "/a␠b",
+			want: "/a%E2%90%A0b",
+		},
+		{
+			path: "/a-umlaut-ä",
+			want: "/a-umlaut-%C3%A4",
+		},
+		{
+			path: "/a-umlaut-%C3%A4",
+			want: "/a-umlaut-%C3%A4",
+		},
+		{
+			path: "/a-umlaut-%c3%a4",
+			want: "/a-umlaut-%c3%a4",
+		},
+		{
+			// here the second '#' is treated as part of the fragment
+			path: "/a#b#c",
+			want: "/a#b%23c",
+		},
+		{
+			path: "/a#b␠c",
+			want: "/a#b%E2%90%A0c",
+		},
+		{
+			path: "/a#b%20c",
+			want: "/a#b%20c",
+		},
+		{
+			path: "/a#b c",
+			want: "/a#b%20c",
+		},
+		{
+			// stdlib's EscapedPath() will always percent-encode '\'
+			path: "/\\",
+			want: "/%5C",
+		},
+		{
+			path: "/a\\",
+			want: "/a%5C",
+		},
+		{
+			path: "/a,b.c.",
+			want: "/a,b.c.",
+		},
+		{
+			path: "/.",
+			want: "/.",
+		},
+		{
+			// stdlib's EscapedPath() will always percent-encode '`'
+			path: "/a`",
+			want: "/a%60",
+		},
+		{
+			path: "/a[0]",
+			want: "/a[0]",
+		},
+		{
+			path: "/?a[0]=5 &b[]=",
+			want: "/?a[0]=5 &b[]=",
+		},
+		{
+			path: "/?a=%22b%20%22",
+			want: "/?a=%22b%20%22",
+		},
+	}
+
+	for index, testCase := range testCases {
+		requestURL := "https://example.com"
+
+		request, err := http.NewRequest(http.MethodGet, requestURL, nil)
+		assert.NoError(t, err)
+
+		mockRequestHeaders := []Header{
+			{Name: ":path", Value: testCase.path},
+			{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		}
+
+		headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
+		assert.NoError(t, headersConversionErr)
+
+		assert.Equal(t,
+			http.Header{
+				"Mock header": []string{"Mock value"},
+			},
+			request.Header)
+
+		assert.Equal(t,
+			"https://example.com"+testCase.want,
+			request.URL.String(),
+			"Failed URL index: %v %#v", index, testCase)
+	}
+}
+
+func TestH2RequestHeadersToH1Request_QuickCheck(t *testing.T) {
+	config := &quick.Config{
+		Values: func(args []reflect.Value, rand *rand.Rand) {
+			args[0] = reflect.ValueOf(randomHTTP2Path(t, rand))
+		},
+	}
+
+	type testOrigin struct {
+		url string
+
+		expectedScheme   string
+		expectedBasePath string
+	}
+	testOrigins := []testOrigin{
+		{
+			url:              "http://origin.hostname.example.com:8080",
+			expectedScheme:   "http",
+			expectedBasePath: "http://origin.hostname.example.com:8080",
+		},
+		{
+			url:              "http://origin.hostname.example.com:8080/",
+			expectedScheme:   "http",
+			expectedBasePath: "http://origin.hostname.example.com:8080",
+		},
+		{
+			url:              "http://origin.hostname.example.com:8080/api",
+			expectedScheme:   "http",
+			expectedBasePath: "http://origin.hostname.example.com:8080/api",
+		},
+		{
+			url:              "http://origin.hostname.example.com:8080/api/",
+			expectedScheme:   "http",
+			expectedBasePath: "http://origin.hostname.example.com:8080/api",
+		},
+		{
+			url:              "https://origin.hostname.example.com:8080/api",
+			expectedScheme:   "https",
+			expectedBasePath: "https://origin.hostname.example.com:8080/api",
+		},
+	}
+
+	// use multiple schemes to demonstrate that the URL is based on the
+	// origin's scheme, not the :scheme header
+	for _, testScheme := range []string{"http", "https"} {
+		for _, testOrigin := range testOrigins {
+			assertion := func(testPath string) bool {
+				const expectedMethod = "POST"
+				const expectedHostname = "request.hostname.example.com"
+
+				h2 := []Header{
+					{Name: ":method", Value: expectedMethod},
+					{Name: ":scheme", Value: testScheme},
+					{Name: ":authority", Value: expectedHostname},
+					{Name: ":path", Value: testPath},
+					{Name: RequestUserHeadersField, Value: ""},
+				}
+				h1, err := http.NewRequest("GET", testOrigin.url, nil)
+				require.NoError(t, err)
+
+				err = H2RequestHeadersToH1Request(h2, h1)
+				return assert.NoError(t, err) &&
+					assert.Equal(t, expectedMethod, h1.Method) &&
+					assert.Equal(t, expectedHostname, h1.Host) &&
+					assert.Equal(t, testOrigin.expectedScheme, h1.URL.Scheme) &&
+					assert.Equal(t, testOrigin.expectedBasePath+testPath, h1.URL.String())
+			}
+			err := quick.Check(assertion, config)
+			assert.NoError(t, err)
+		}
+	}
+}
+
+func randomASCIIPrintableChar(rand *rand.Rand) int {
+	// smallest printable ASCII char is 32, largest is 126
+	const startPrintable = 32
+	const endPrintable = 127
+	return startPrintable + rand.Intn(endPrintable-startPrintable)
+}
+
+// randomASCIIText generates an ASCII string, some of whose characters may be
+// percent-encoded. Its "logical length" (ignoring percent-encoding) is
+// between 1 and `maxLength`.
+func randomASCIIText(rand *rand.Rand, minLength int, maxLength int) string {
+	length := minLength + rand.Intn(maxLength)
+	result := ""
+	for i := 0; i < length; i++ {
+		c := randomASCIIPrintableChar(rand)
+
+		// 1/4 chance of using percent encoding when not necessary
+		if c == '%' || rand.Intn(4) == 0 {
+			result += fmt.Sprintf("%%%02X", c)
+		} else {
+			result += string(c)
+		}
+	}
+	return result
+}
+
+// Calls `randomASCIIText` and ensures the result is a valid URL path,
+// i.e. one that can pass unchanged through url.URL.String()
+func randomHTTP1Path(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
+	text := randomASCIIText(rand, minLength, maxLength)
+	re, err := regexp.Compile("[^/;,]*")
+	require.NoError(t, err)
+	return "/" + re.ReplaceAllStringFunc(text, url.PathEscape)
+}
+
+// Calls `randomASCIIText` and ensures the result is a valid URL query,
+// i.e. one that can pass unchanged through url.URL.String()
+func randomHTTP1Query(rand *rand.Rand, minLength int, maxLength int) string {
+	text := randomASCIIText(rand, minLength, maxLength)
+	return "?" + strings.ReplaceAll(text, "#", "%23")
+}
+
+// Calls `randomASCIIText` and ensures the result is a valid URL fragment,
+// i.e. one that can pass unchanged through url.URL.String()
+func randomHTTP1Fragment(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
+	text := randomASCIIText(rand, minLength, maxLength)
+	u, err := url.Parse("#" + text)
+	require.NoError(t, err)
+	return u.String()
+}
+
+// Assemble a random :path pseudoheader that is legal by Go stdlib standards
+// (i.e. all characters will satisfy "net/url".shouldEscape for their respective locations)
+func randomHTTP2Path(t *testing.T, rand *rand.Rand) string {
+	result := randomHTTP1Path(t, rand, 1, 64)
+	if rand.Intn(2) == 1 {
+		result += randomHTTP1Query(rand, 1, 32)
+	}
+	if rand.Intn(2) == 1 {
+		result += randomHTTP1Fragment(t, rand, 1, 16)
+	}
+	return result
+}
+
+func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []Header) {
+	for name, values := range headers {
+		for _, value := range values {
+			h2muxHeaders = append(h2muxHeaders, Header{name, value})
+		}
+	}
+
+	return h2muxHeaders
+}
+
+func TestSerializeHeaders(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	mockHeaders := http.Header{
+		"Mock-Header-One":        {"Mock header one value", "three"},
+		"Mock-Header-Two-Long":   {"Mock header two value\nlong"},
+		":;":                     {":;", ";:"},
+		":":                      {":"},
+		";":                      {";"},
+		";;":                     {";;"},
+		"Empty values":           {"", ""},
+		"":                       {"Empty key"},
+		"control\tcharacter\b\n": {"value\n\b\t"},
+		";\v:":                   {":\v;"},
+	}
+
+	for header, values := range mockHeaders {
+		for _, value := range values {
+			// Note that Golang's http library is opinionated;
+			// at this point every header name will be title-cased in order to comply with the HTTP RFC
+			// This means our proxy is not completely transparent when it comes to proxying headers
+			request.Header.Add(header, value)
+		}
+	}
+
+	serializedHeaders := SerializeHeaders(request.Header)
+
+	// Sanity check: the headers serialized to something that's not an empty string
+	assert.NotEqual(t, "", serializedHeaders)
+
+	// Deserialize back, and ensure we get the same set of headers
+	deserializedHeaders, err := DeserializeHeaders(serializedHeaders)
+	assert.NoError(t, err)
+
+	assert.Equal(t, 13, len(deserializedHeaders))
+	h2muxExpectedHeaders := stdlibHeaderToH2muxHeader(mockHeaders)
+
+	sort.Sort(ByName(deserializedHeaders))
+	sort.Sort(ByName(h2muxExpectedHeaders))
+
+	assert.True(
+		t,
+		reflect.DeepEqual(h2muxExpectedHeaders, deserializedHeaders),
+		fmt.Sprintf("got = %#v, want = %#v\n", deserializedHeaders, h2muxExpectedHeaders),
+	)
+}
+
+func TestSerializeNoHeaders(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	serializedHeaders := SerializeHeaders(request.Header)
+	deserializedHeaders, err := DeserializeHeaders(serializedHeaders)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(deserializedHeaders))
+}
+
+func TestDeserializeMalformed(t *testing.T) {
+	var err error
+
+	malformedData := []string{
+		"malformed data",
+		"bW9jawo=",                   // "mock"
+		"bW9jawo=:ZGF0YQo=:bW9jawo=", // "mock:data:mock"
+		"::",
+	}
+
+	for _, malformedValue := range malformedData {
+		_, err = DeserializeHeaders(malformedValue)
+		assert.Error(t, err)
+	}
+}
+
+func TestParseHeaders(t *testing.T) {
+	mockUserHeadersToSerialize := http.Header{
+		"Mock-Header-One":   {"1", "1.5"},
+		"Mock-Header-Two":   {"2"},
+		"Mock-Header-Three": {"3"},
+	}
+
+	mockHeaders := []Header{
+		{Name: "One", Value: "1"},
+		{Name: "Cf-Two", Value: "cf-value-1"},
+		{Name: "Cf-Two", Value: "cf-value-2"},
+		{Name: RequestUserHeadersField, Value: SerializeHeaders(mockUserHeadersToSerialize)},
+	}
+
+	expectedHeaders := []Header{
+		{Name: "Mock-Header-One", Value: "1"},
+		{Name: "Mock-Header-One", Value: "1.5"},
+		{Name: "Mock-Header-Two", Value: "2"},
+		{Name: "Mock-Header-Three", Value: "3"},
+	}
+	parsedHeaders, err := ParseUserHeaders(RequestUserHeadersField, mockHeaders)
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, expectedHeaders, parsedHeaders)
+}
+
+func TestParseHeadersNoSerializedHeader(t *testing.T) {
+	mockHeaders := []Header{
+		{Name: "One", Value: "1"},
+		{Name: "Cf-Two", Value: "cf-value-1"},
+		{Name: "Cf-Two", Value: "cf-value-2"},
+	}
+
+	_, err := ParseUserHeaders(RequestUserHeadersField, mockHeaders)
+	assert.EqualError(t, err, fmt.Sprintf("%s header not found", RequestUserHeadersField))
+}
+
+func TestIsControlHeader(t *testing.T) {
+	controlHeaders := []string{
+		// Anything that begins with cf-
+		"cf-sample-header",
+		"CF-SAMPLE-HEADER",
+		"Cf-Sample-Header",
+
+		// Any http2 pseudoheader
+		":sample-pseudo-header",
+
+		// content-length is a special case, it has to be there
+		// for some requests to work (per the HTTP2 spec)
+		"content-length",
+	}
+
+	for _, header := range controlHeaders {
+		assert.True(t, IsControlHeader(header))
+	}
+}
+
+func TestIsNotControlHeader(t *testing.T) {
+	notControlHeaders := []string{
+		"Mock-header",
+		"Another-sample-header",
+	}
+
+	for _, header := range notControlHeaders {
+		assert.False(t, IsControlHeader(header))
+	}
+}
+
+func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
+	mockHeaders := http.Header{
+		"User-header-one": {""},
+		"User-header-two": {"1", "2"},
+		"cf-header":       {"cf-value"},
+		"Content-Length":  {"123"},
+	}
+	mockResponse := http.Response{
+		StatusCode: 200,
+		Header:     mockHeaders,
+	}
+
+	headers := H1ResponseToH2ResponseHeaders(&mockResponse)
+
+	serializedHeadersIndex := -1
+	for i, header := range headers {
+		if header.Name == ResponseUserHeadersField {
+			serializedHeadersIndex = i
+			break
+		}
+	}
+	assert.NotEqual(t, -1, serializedHeadersIndex)
+	actualControlHeaders := append(
+		headers[:serializedHeadersIndex],
+		headers[serializedHeadersIndex+1:]...,
+	)
+	expectedControlHeaders := []Header{
+		{Name: ":status", Value: "200"},
+		{Name: "content-length", Value: "123"},
+	}
+
+	assert.ElementsMatch(t, expectedControlHeaders, actualControlHeaders)
+
+	actualUserHeaders, err := DeserializeHeaders(headers[serializedHeadersIndex].Value)
+	expectedUserHeaders := []Header{
+		{Name: "User-header-one", Value: ""},
+		{Name: "User-header-two", Value: "1"},
+		{Name: "User-header-two", Value: "2"},
+	}
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, expectedUserHeaders, actualUserHeaders)
+}
+
+// The purpose of this test is to check that our code and the http.Header
+// implementation don't throw validation errors about header size
+func TestHeaderSize(t *testing.T) {
+	largeValue := randSeq(5 * 1024 * 1024) // 5Mb
+	largeHeaders := http.Header{
+		"User-header": {largeValue},
+	}
+	mockResponse := http.Response{
+		StatusCode: 200,
+		Header:     largeHeaders,
+	}
+
+	serializedHeaders := H1ResponseToH2ResponseHeaders(&mockResponse)
+	request, err := http.NewRequest(http.MethodGet, "https://example.com/", nil)
+	assert.NoError(t, err)
+	for _, header := range serializedHeaders {
+		request.Header.Set(header.Name, header.Value)
+	}
+
+	for _, header := range serializedHeaders {
+		if header.Name != ResponseUserHeadersField {
+			continue
+		}
+
+		deserializedHeaders, err := DeserializeHeaders(header.Value)
+		assert.NoError(t, err)
+		assert.Equal(t, largeValue, deserializedHeaders[0].Value)
+	}
+}
+
+func randSeq(n int) string {
+	randomizer := rand.New(rand.NewSource(17))
+	var letters = []rune(":;,+/=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letters[randomizer.Intn(len(letters))]
+	}
+	return string(b)
+}

h2mux/muxedstream.go

@@ -17,6 +17,12 @@ type ReadWriteClosedCloser interface {
 	Closed() bool
 }
 
+// MuxedStreamDataSignaller is a write-only *ReadyList
+type MuxedStreamDataSignaller interface {
+	// Non-blocking: call this when data is ready to be sent for the given stream ID.
+	Signal(ID uint32)
+}
+
 // MuxedStream is logically an HTTP/2 stream, with an additional buffer for outgoing data.
 type MuxedStream struct {
 	streamID uint32
@@ -55,8 +61,8 @@ type MuxedStream struct {
 	// This is the amount of bytes that are in the peer's receive window
 	// (how much data we can send from this stream).
 	sendWindow uint32
-	// Reference to the muxer's readyList; signal this for stream data to be sent.
-	readyList *ReadyList
+	// The muxer's readyList
+	readyList MuxedStreamDataSignaller
 	// The headers that should be sent, and a flag so we only send them once.
 	headersSent  bool
 	writeHeaders []Header
@@ -88,6 +94,23 @@ func (th TunnelHostname) IsSet() bool {
 	return th != ""
 }
 
+func NewStream(config MuxerConfig, writeHeaders []Header, readyList MuxedStreamDataSignaller, dictionaries h2Dictionaries) *MuxedStream {
+	return &MuxedStream{
+		responseHeadersReceived: make(chan struct{}),
+		readBuffer:              NewSharedBuffer(),
+		writeBuffer:             &bytes.Buffer{},
+		writeBufferMaxLen:       config.StreamWriteBufferMaxLen,
+		writeBufferHasSpace:     make(chan struct{}, 1),
+		receiveWindow:           config.DefaultWindowSize,
+		receiveWindowCurrentMax: config.DefaultWindowSize,
+		receiveWindowMax:        config.MaxWindowSize,
+		sendWindow:              config.DefaultWindowSize,
+		readyList:               readyList,
+		writeHeaders:            writeHeaders,
+		dictionaries:            dictionaries,
+	}
+}
+
 func (s *MuxedStream) Read(p []byte) (n int, err error) {
 	var readBuffer ReadWriteClosedCloser
 	if s.dictionaries.read != nil {
@@ -120,9 +143,10 @@ func (s *MuxedStream) Write(p []byte) (int, error) {
 		// If the buffer is full, block till there is more room.
 		// Use a loop to recheck the buffer size after the lock is reacquired.
 		for s.writeBufferMaxLen <= s.writeBuffer.Len() {
-			s.writeLock.Unlock()
-			<-s.writeBufferHasSpace
-			s.writeLock.Lock()
+			s.awaitWriteBufferHasSpace()
+			if s.writeEOF {
+				return totalWritten, io.EOF
+			}
 		}
 		amountToWrite := len(p) - totalWritten
 		spaceAvailable := s.writeBufferMaxLen - s.writeBuffer.Len()
@@ -171,10 +195,19 @@ func (s *MuxedStream) CloseWrite() error {
 	if c, ok := s.writeBuffer.(io.Closer); ok {
 		c.Close()
 	}
+	// Allow MuxedStream::Write() to terminate its loop with err=io.EOF, if needed
+	s.notifyWriteBufferHasSpace()
+	// We need to send something over the wire, even if it's an END_STREAM with no data
 	s.writeNotify()
 	return nil
 }
 
+func (s *MuxedStream) WriteClosed() bool {
+	s.writeLock.Lock()
+	defer s.writeLock.Unlock()
+	return s.writeEOF
+}
+
 func (s *MuxedStream) WriteHeaders(headers []Header) error {
 	s.writeLock.Lock()
 	defer s.writeLock.Unlock()
@@ -215,6 +248,23 @@ func (s *MuxedStream) TunnelHostname() TunnelHostname {
 	return s.tunnelHostname
 }
 
+// Block until a value is sent on writeBufferHasSpace.
+// Must be called while holding writeLock
+func (s *MuxedStream) awaitWriteBufferHasSpace() {
+	s.writeLock.Unlock()
+	<-s.writeBufferHasSpace
+	s.writeLock.Lock()
+}
+
+// Send a value on writeBufferHasSpace without blocking.
+// Must be called while holding writeLock
+func (s *MuxedStream) notifyWriteBufferHasSpace() {
+	select {
+	case s.writeBufferHasSpace <- struct{}{}:
+	default:
+	}
+}
+
 func (s *MuxedStream) getReceiveWindow() uint32 {
 	s.writeLock.Lock()
 	defer s.writeLock.Unlock()
@@ -334,17 +384,13 @@ func (s *MuxedStream) getChunk() *streamChunk {
 		sendData:     !s.sentEOF,
 		eof:          s.writeEOF && uint32(s.writeBuffer.Len()) <= s.sendWindow,
 	}
-
 	// Copy at most s.sendWindow bytes, adjust the sendWindow accordingly
 	writeLen, _ := io.CopyN(&chunk.buffer, s.writeBuffer, int64(s.sendWindow))
 	s.sendWindow -= uint32(writeLen)
 
-	// Non-blocking channel send. This will allow MuxedStream::Write() to continue, if needed
+	// Allow MuxedStream::Write() to continue, if needed
 	if s.writeBuffer.Len() < s.writeBufferMaxLen {
-		select {
-		case s.writeBufferHasSpace <- struct{}{}:
-		default:
-		}
+		s.notifyWriteBufferHasSpace()
 	}
 
 	// When we write the chunk, we'll write the WINDOW_UPDATE frame if needed

h2mux/muxmetrics.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/golang-collections/collections/queue"
+	"github.com/prometheus/client_golang/prometheus"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -299,3 +300,14 @@ func (r *rate) get() (curr, min, max uint64) {
 	defer r.lock.RUnlock()
 	return r.curr, r.min, r.max
 }
+
+func NewActiveStreamsMetrics(namespace, subsystem string) prometheus.Gauge {
+	activeStreams := prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      "active_streams",
+		Help:      "Number of active streams created by all muxers.",
+	})
+	prometheus.MustRegister(activeStreams)
+	return activeStreams
+}

h2mux/muxreader.go

@@ -51,10 +51,12 @@ type MuxReader struct {
 	dictionaries h2Dictionaries
 }
 
-func (r *MuxReader) Shutdown() {
-	done := r.streams.Shutdown()
-	if done == nil {
-		return
+// Shutdown blocks new streams from being created.
+// It returns a channel that is closed once the last stream has closed.
+func (r *MuxReader) Shutdown() <-chan struct{} {
+	done, alreadyInProgress := r.streams.Shutdown()
+	if alreadyInProgress {
+		return done
 	}
 	r.sendGoAway(http2.ErrCodeNo)
 	go func() {
@@ -62,6 +64,7 @@ func (r *MuxReader) Shutdown() {
 		<-done
 		r.r.Close()
 	}()
+	return done
 }
 
 func (r *MuxReader) run(parentLogger *log.Entry) error {
@@ -87,23 +90,33 @@ func (r *MuxReader) run(parentLogger *log.Entry) error {
 	for {
 		frame, err := r.f.ReadFrame()
 		if err != nil {
+			errLogger := logger.WithError(err)
+			if errorDetail := r.f.ErrorDetail(); errorDetail != nil {
+				errLogger = errLogger.WithField("errorDetail", errorDetail)
+			}
 			switch e := err.(type) {
 			case http2.StreamError:
-				logger.WithError(err).Warn("stream error")
-				r.streamError(e.StreamID, e.Code)
+				errLogger.Warn("stream error")
+				// Ideally we wouldn't return here, since that aborts the muxer.
+				// We should communicate the error to the relevant MuxedStream
+				// data structure, so that callers of MuxedStream.Read() and
+				// MuxedStream.Write() would see it. Then we could `continue`
+				// and keep the muxer going.
+				return r.streamError(e.StreamID, e.Code)
 			case http2.ConnectionError:
-				logger.WithError(err).Warn("connection error")
+				errLogger.Warn("connection error")
 				return r.connectionError(err)
 			default:
 				if isConnectionClosedError(err) {
 					if r.streams.Len() == 0 {
+						// don't log the error here -- that would just be extra noise
 						logger.Debug("shutting down")
 						return nil
 					}
-					logger.Warn("connection closed unexpectedly")
+					errLogger.Warn("connection closed unexpectedly")
 					return err
 				} else {
-					logger.WithError(err).Warn("frame read error")
+					errLogger.Warn("frame read error")
 					return r.connectionError(err)
 				}
 			}
@@ -120,6 +133,9 @@ func (r *MuxReader) run(parentLogger *log.Entry) error {
 			if streamID == 0 {
 				return ErrInvalidStream
 			}
+			if stream, ok := r.streams.Get(streamID); ok {
+				stream.Close()
+			}
 			r.streams.Delete(streamID)
 		case *http2.PingFrame:
 			r.receivePingData(f)

h2mux/muxreader_test.go

@@ -59,7 +59,7 @@ func assertOpenStreamSucceed(t *testing.T, stream *MuxedStream, err error) {
 
 func TestMissingHeaders(t *testing.T) {
 	originHandler := &mockOriginStreamHandler{}
-	muxPair := NewDefaultMuxerPair(t, originHandler.ServeStream)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), originHandler.ServeStream)
 	muxPair.Serve(t)
 
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
@@ -83,7 +83,7 @@ func TestMissingHeaders(t *testing.T) {
 
 func TestReceiveHeaderData(t *testing.T) {
 	originHandler := &mockOriginStreamHandler{}
-	muxPair := NewDefaultMuxerPair(t, originHandler.ServeStream)
+	muxPair := NewDefaultMuxerPair(t, t.Name(), originHandler.ServeStream)
 	muxPair.Serve(t)
 
 	reqHeaders := []Header{

h2mux/muxwriter.go

@@ -54,6 +54,13 @@ type MuxedStreamRequest struct {
 	body   io.Reader
 }
 
+func NewMuxedStreamRequest(stream *MuxedStream, body io.Reader) MuxedStreamRequest {
+	return MuxedStreamRequest{
+		stream: stream,
+		body:   body,
+	}
+}
+
 func (r *MuxedStreamRequest) flushBody() {
 	io.Copy(r.stream, r.body)
 	r.stream.CloseWrite()
@@ -243,28 +250,52 @@ func (w *MuxWriter) encodeHeaders(headers []Header) ([]byte, error) {
 // writeHeaders writes a block of encoded headers, splitting it into multiple frames if necessary.
 func (w *MuxWriter) writeHeaders(streamID uint32, headers []Header) error {
 	encodedHeaders, err := w.encodeHeaders(headers)
-	if err != nil {
+	if err != nil || len(encodedHeaders) == 0 {
 		return err
 	}
+
 	blockSize := int(w.maxFrameSize)
-	endHeaders := len(encodedHeaders) == 0
-	for !endHeaders && err == nil {
-		blockFragment := encodedHeaders
-		if len(encodedHeaders) > blockSize {
-			blockFragment = blockFragment[:blockSize]
-			encodedHeaders = encodedHeaders[blockSize:]
-			// Send CONTINUATION frame if the headers can't be fit into 1 frame
-			err = w.f.WriteContinuation(streamID, endHeaders, blockFragment)
-		} else {
-			endHeaders = true
-			err = w.f.WriteHeaders(http2.HeadersFrameParam{
-				StreamID:      streamID,
-				EndHeaders:    endHeaders,
-				BlockFragment: blockFragment,
-			})
+	// CONTINUATION is unnecessary; the headers fit within the blockSize
+	if len(encodedHeaders) < blockSize {
+		return w.f.WriteHeaders(http2.HeadersFrameParam{
+			StreamID:      streamID,
+			EndHeaders:    true,
+			BlockFragment: encodedHeaders,
+		})
+	}
+
+	choppedHeaders := chopEncodedHeaders(encodedHeaders, blockSize)
+	// len(choppedHeaders) is at least 2
+	if err := w.f.WriteHeaders(http2.HeadersFrameParam{StreamID: streamID, EndHeaders: false, BlockFragment: choppedHeaders[0]}); err != nil {
+		return err
+	}
+	for i := 1; i < len(choppedHeaders)-1; i++ {
+		if err := w.f.WriteContinuation(streamID, false, choppedHeaders[i]); err != nil {
+			return err
 		}
 	}
-	return err
+	if err := w.f.WriteContinuation(streamID, true, choppedHeaders[len(choppedHeaders)-1]); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Partition a slice of bytes into `len(slice) / blockSize` slices of length `blockSize`
+func chopEncodedHeaders(headers []byte, chunkSize int) [][]byte {
+	var divided [][]byte
+
+	for i := 0; i < len(headers); i += chunkSize {
+		end := i + chunkSize
+
+		if end > len(headers) {
+			end = len(headers)
+		}
+
+		divided = append(divided, headers[i:end])
+	}
+
+	return divided
 }
 
 func (w *MuxWriter) writeUseDictionary(dictRequest useDictRequest) error {

h2mux/muxwriter_test.go

@@ -0,0 +1,26 @@
+package h2mux
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestChopEncodedHeaders(t *testing.T) {
+	mockEncodedHeaders := make([]byte, 5)
+	for i := range mockEncodedHeaders {
+		mockEncodedHeaders[i] = byte(i)
+	}
+	chopped := chopEncodedHeaders(mockEncodedHeaders, 4)
+
+	assert.Equal(t, 2, len(chopped))
+	assert.Equal(t, []byte{0, 1, 2, 3}, chopped[0])
+	assert.Equal(t, []byte{4}, chopped[1])
+}
+
+func TestChopEncodedEmptyHeaders(t *testing.T) {
+	mockEncodedHeaders := make([]byte, 0)
+	chopped := chopEncodedHeaders(mockEncodedHeaders, 3)
+
+	assert.Equal(t, 0, len(chopped))
+}

h2mux/sample/index.html

@@ -80,7 +80,7 @@ ghost.init({
     <link rel="alternate" type="application/rss+xml" title="Cloudflare Blog" href="https://blog.cloudflare.com/rss/" />
     <meta name="msvalidate.01" content="CF295E1604697F9CAD18B5A232E871F6" />
 <meta class="swiftype" name="language" data-type="string" content="en" />
-<script src="https://s3-us-west-1.amazonaws.com/cf-ghost-assets-hotfix/js/index.js"></script>
+<script src="https://blog.cloudflare.com/assets/js/index.js"></script>
 <script type="text/javascript" src="//cdn.bizible.com/scripts/bizible.js" async=""></script>
 <script>
 var trackRecruitingLink = function(role, url) {
@@ -128,7 +128,7 @@ HTMLAttrToAdd.setAttribute("lang", "en");
         overflow-y: scroll;
     }
 </style>
-<link href="https://s3-us-west-1.amazonaws.com/cf-ghost-assets-hotfix/css/screen.css" rel="stylesheet">
+<link href="https://blog.cloudflare.com/assets/css/screen.css" rel="stylesheet">
 <link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.8.1/themes/prism.min.css" rel="stylesheet">
 
     <style>

h2mux/sample/index1.html

@@ -113,7 +113,7 @@ ghost.init({
     <link rel="alternate" type="application/rss+xml" title="Cloudflare Blog" href="https://blog.cloudflare.com/rss/" />
     <meta name="msvalidate.01" content="CF295E1604697F9CAD18B5A232E871F6" />
 <meta class="swiftype" name="language" data-type="string" content="en" />
-<script src="https://s3-us-west-1.amazonaws.com/cf-ghost-assets-hotfix/js/index.js"></script>
+<script src="https://blog.cloudflare.com/assets/js/index.js"></script>
 <script type="text/javascript" src="//cdn.bizible.com/scripts/bizible.js" async=""></script>
 <script>
 var trackRecruitingLink = function(role, url) {
@@ -161,7 +161,7 @@ HTMLAttrToAdd.setAttribute("lang", "en");
         overflow-y: scroll;
     }
 </style>
-<link href="https://s3-us-west-1.amazonaws.com/cf-ghost-assets-hotfix/css/screen.css" rel="stylesheet">
+<link href="https://blog.cloudflare.com/assets/css/screen.css" rel="stylesheet">
 <link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.8.1/themes/prism.min.css" rel="stylesheet">
 
     <style>

h2mux/sample/index2.html

@@ -109,7 +109,7 @@ ghost.init({
     <link rel="alternate" type="application/rss+xml" title="Cloudflare Blog" href="https://blog.cloudflare.com/rss/" />
     <meta name="msvalidate.01" content="CF295E1604697F9CAD18B5A232E871F6" />
 <meta class="swiftype" name="language" data-type="string" content="en" />
-<script src="https://s3-us-west-1.amazonaws.com/cf-ghost-assets-hotfix/js/index.js"></script>
+<script src="https://blog.cloudflare.com/assets/js/index.js"></script>
 <script type="text/javascript" src="//cdn.bizible.com/scripts/bizible.js" async=""></script>
 <script>
 var trackRecruitingLink = function(role, url) {
@@ -157,7 +157,7 @@ HTMLAttrToAdd.setAttribute("lang", "en");
         overflow-y: scroll;
     }
 </style>
-<link href="https://s3-us-west-1.amazonaws.com/cf-ghost-assets-hotfix/css/screen.css" rel="stylesheet">
+<link href="https://blog.cloudflare.com/assets/css/screen.css" rel="stylesheet">
 <link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.8.1/themes/prism.min.css" rel="stylesheet">
 
     <style>

hello/hello.go

@@ -56,7 +56,7 @@ const indexTemplate = `
           <path class="st2" d="M93.6 12.8h-.5c-.1 0-.2.1-.3.2l-.7 2.4c-.3 1-.2 2 .3 2.6.5.6 1.2 1 2.1 1.1l3.7.2c.1 0 .2.1.3.1.1.1.1.2 0 .3-.1.2-.2.3-.4.3l-3.8.2c-2.1.1-4.3 1.8-5.1 3.8l-.2.9c-.1.1 0 .3.2.3h13.2c.2 0 .3-.1.3-.3.2-.8.4-1.7.4-2.6 0-5.2-4.3-9.5-9.5-9.5"/>
           <path class="st3" d="M104.4 30.8c-.5 0-.9-.4-.9-.9s.4-.9.9-.9.9.4.9.9-.4.9-.9.9m0-1.6c-.4 0-.7.3-.7.7 0 .4.3.7.7.7.4 0 .7-.3.7-.7 0-.4-.3-.7-.7-.7m.4 1.2h-.2l-.2-.3h-.2v.3h-.2v-.9h.5c.2 0 .3.1.3.3 0 .1-.1.2-.2.3l.2.3zm-.3-.5c.1 0 .1 0 .1-.1s-.1-.1-.1-.1h-.3v.3h.3zM14.8 29H17v6h3.8v1.9h-6zM23.1 32.9c0-2.3 1.8-4.1 4.3-4.1s4.2 1.8 4.2 4.1-1.8 4.1-4.3 4.1c-2.4 0-4.2-1.8-4.2-4.1m6.3 0c0-1.2-.8-2.2-2-2.2s-2 1-2 2.1.8 2.1 2 2.1c1.2.2 2-.8 2-2M34.3 33.4V29h2.2v4.4c0 1.1.6 1.7 1.5 1.7s1.5-.5 1.5-1.6V29h2.2v4.4c0 2.6-1.5 3.7-3.7 3.7-2.3-.1-3.7-1.2-3.7-3.7M45 29h3.1c2.8 0 4.5 1.6 4.5 3.9s-1.7 4-4.5 4h-3V29zm3.1 5.9c1.3 0 2.2-.7 2.2-2s-.9-2-2.2-2h-.9v4h.9zM55.7 29H62v1.9h-4.1v1.3h3.7V34h-3.7v2.9h-2.2zM65.1 29h2.2v6h3.8v1.9h-6zM76.8 28.9H79l3.4 8H80l-.6-1.4h-3.1l-.6 1.4h-2.3l3.4-8zm2 4.9l-.9-2.2-.9 2.2h1.8zM85.2 29h3.7c1.2 0 2 .3 2.6.9.5.5.7 1.1.7 1.8 0 1.2-.6 2-1.6 2.4l1.9 2.8H90l-1.6-2.4h-1v2.4h-2.2V29zm3.6 3.8c.7 0 1.2-.4 1.2-.9 0-.6-.5-.9-1.2-.9h-1.4v1.9h1.4zM95.3 29h6.4v1.8h-4.2V32h3.8v1.8h-3.8V35h4.3v1.9h-6.5zM10 33.9c-.3.7-1 1.2-1.8 1.2-1.2 0-2-1-2-2.1s.8-2.1 2-2.1c.9 0 1.6.6 1.9 1.3h2.3c-.4-1.9-2-3.3-4.2-3.3-2.4 0-4.3 1.8-4.3 4.1s1.8 4.1 4.2 4.1c2.1 0 3.7-1.4 4.2-3.2H10z"/>
         </svg>
-        <h1 class="f4 f2-ns mt5 fw5">Congrats! You created your first tunnel!</h1>
+        <h1 class="f4 f2-ns mt5 fw5">Congrats! You created a tunnel!</h1>
         <p class="f6 f5-m f4-l measure lh-copy fw3">
           Argo Tunnel exposes locally running applications to the internet by
           running an encrypted, virtual tunnel from your laptop or server to
@@ -103,15 +103,16 @@ func StartHelloWorldServer(logger *logrus.Logger, listener net.Listener, shutdow
 		WriteBufferSize: 1024,
 	}
 
-	httpServer := &http.Server{Addr: listener.Addr().String(), Handler: nil}
+	muxer := http.NewServeMux()
+	muxer.HandleFunc("/uptime", uptimeHandler(time.Now()))
+	muxer.HandleFunc("/ws", websocketHandler(logger, upgrader))
+	muxer.HandleFunc("/", rootHandler(serverName))
+	httpServer := &http.Server{Addr: listener.Addr().String(), Handler: muxer}
 	go func() {
 		<-shutdownC
 		httpServer.Close()
 	}()
 
-	http.HandleFunc("/uptime", uptimeHandler(time.Now()))
-	http.HandleFunc("/ws", websocketHandler(logger, upgrader))
-	http.HandleFunc("/", rootHandler(serverName))
 	err := httpServer.Serve(listener)
 	return err
 }

metrics/metrics.go

@@ -2,6 +2,7 @@ package metrics
 
 import (
 	"context"
+	"fmt"
 	"net"
 	"net/http"
 	_ "net/http/pprof"
@@ -33,13 +34,16 @@ func ServeMetrics(l net.Listener, shutdownC <-chan struct{}, logger *logrus.Logg
 	}
 
 	http.Handle("/metrics", promhttp.Handler())
+	http.Handle("/healthcheck", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintf(w, "OK\n")
+	}))
 
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
 		err = server.Serve(l)
 	}()
-	logger.WithField("addr", l.Addr()).Info("Starting metrics server")
+	logger.WithField("addr", fmt.Sprintf("%v/metrics", l.Addr())).Info("Starting metrics server")
 	// server.Serve will hang if server.Shutdown is called before the server is
 	// fully started up. So add artificial delay.
 	time.Sleep(startupTime)

origin/backoffhandler.go

@@ -92,3 +92,8 @@ func (b BackoffHandler) GetBaseTime() time.Duration {
 	}
 	return b.BaseTime
 }
+
+// Retries returns the number of retries consumed so far.
+func (b *BackoffHandler) Retries() int {
+	return int(b.retries)
+}

origin/external_control.go

@@ -0,0 +1,21 @@
+package origin
+
+import (
+	"time"
+)
+
+type ReconnectSignal struct {
+	// wait this many seconds before re-establish the connection
+	Delay time.Duration
+}
+
+// Error allows us to use ReconnectSignal as a special error to force connection abort
+func (r *ReconnectSignal) Error() string {
+	return "reconnect signal"
+}
+
+func (r *ReconnectSignal) DelayBeforeReconnect() {
+	if r.Delay > 0 {
+		time.Sleep(r.Delay)
+	}
+}

origin/metrics.go

@@ -9,6 +9,12 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
+const (
+	metricsNamespace = "cloudflared"
+	tunnelSubsystem  = "tunnel"
+	muxerSubsystem   = "muxer"
+)
+
 type muxerMetrics struct {
 	rtt              *prometheus.GaugeVec
 	rttMin           *prometheus.GaugeVec
@@ -32,6 +38,7 @@ type muxerMetrics struct {
 
 type TunnelMetrics struct {
 	haConnections     prometheus.Gauge
+	activeStreams     prometheus.Gauge
 	totalRequests     prometheus.Counter
 	requestsPerTunnel *prometheus.CounterVec
 	// concurrentRequestsLock is a mutex for concurrentRequests and maxConcurrentRequests
@@ -51,9 +58,11 @@ type TunnelMetrics struct {
 	// oldServerLocations stores the last server the tunnel was connected to
 	oldServerLocations map[string]string
 
-	regSuccess prometheus.Counter
-	regFail    *prometheus.CounterVec
-	rpcFail    *prometheus.CounterVec
+	regSuccess  *prometheus.CounterVec
+	regFail     *prometheus.CounterVec
+	authSuccess prometheus.Counter
+	authFail    *prometheus.CounterVec
+	rpcFail     *prometheus.CounterVec
 
 	muxerMetrics        *muxerMetrics
 	tunnelsHA           tunnelsForHA
@@ -63,8 +72,10 @@ type TunnelMetrics struct {
 func newMuxerMetrics() *muxerMetrics {
 	rtt := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "rtt",
-			Help: "Round-trip time in millisecond",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt",
+			Help:      "Round-trip time in millisecond",
 		},
 		[]string{"connection_id"},
 	)
@@ -72,8 +83,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	rttMin := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "rtt_min",
-			Help: "Shortest round-trip time in millisecond",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_min",
+			Help:      "Shortest round-trip time in millisecond",
 		},
 		[]string{"connection_id"},
 	)
@@ -81,8 +94,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	rttMax := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "rtt_max",
-			Help: "Longest round-trip time in millisecond",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_max",
+			Help:      "Longest round-trip time in millisecond",
 		},
 		[]string{"connection_id"},
 	)
@@ -90,8 +105,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	receiveWindowAve := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "receive_window_ave",
-			Help: "Average receive window size in bytes",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_ave",
+			Help:      "Average receive window size in bytes",
 		},
 		[]string{"connection_id"},
 	)
@@ -99,8 +116,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	sendWindowAve := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "send_window_ave",
-			Help: "Average send window size in bytes",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_ave",
+			Help:      "Average send window size in bytes",
 		},
 		[]string{"connection_id"},
 	)
@@ -108,8 +127,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	receiveWindowMin := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "receive_window_min",
-			Help: "Smallest receive window size in bytes",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_min",
+			Help:      "Smallest receive window size in bytes",
 		},
 		[]string{"connection_id"},
 	)
@@ -117,8 +138,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	receiveWindowMax := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "receive_window_max",
-			Help: "Largest receive window size in bytes",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_max",
+			Help:      "Largest receive window size in bytes",
 		},
 		[]string{"connection_id"},
 	)
@@ -126,8 +149,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	sendWindowMin := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "send_window_min",
-			Help: "Smallest send window size in bytes",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_min",
+			Help:      "Smallest send window size in bytes",
 		},
 		[]string{"connection_id"},
 	)
@@ -135,8 +160,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	sendWindowMax := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "send_window_max",
-			Help: "Largest send window size in bytes",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_max",
+			Help:      "Largest send window size in bytes",
 		},
 		[]string{"connection_id"},
 	)
@@ -144,8 +171,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	inBoundRateCurr := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "inbound_bytes_per_sec_curr",
-			Help: "Current inbounding bytes per second, 0 if there is no incoming connection",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_curr",
+			Help:      "Current inbounding bytes per second, 0 if there is no incoming connection",
 		},
 		[]string{"connection_id"},
 	)
@@ -153,8 +182,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	inBoundRateMin := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "inbound_bytes_per_sec_min",
-			Help: "Minimum non-zero inbounding bytes per second",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero inbounding bytes per second",
 		},
 		[]string{"connection_id"},
 	)
@@ -162,8 +193,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	inBoundRateMax := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "inbound_bytes_per_sec_max",
-			Help: "Maximum inbounding bytes per second",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_max",
+			Help:      "Maximum inbounding bytes per second",
 		},
 		[]string{"connection_id"},
 	)
@@ -171,8 +204,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	outBoundRateCurr := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "outbound_bytes_per_sec_curr",
-			Help: "Current outbounding bytes per second, 0 if there is no outgoing traffic",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_curr",
+			Help:      "Current outbounding bytes per second, 0 if there is no outgoing traffic",
 		},
 		[]string{"connection_id"},
 	)
@@ -180,8 +215,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	outBoundRateMin := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "outbound_bytes_per_sec_min",
-			Help: "Minimum non-zero outbounding bytes per second",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero outbounding bytes per second",
 		},
 		[]string{"connection_id"},
 	)
@@ -189,8 +226,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	outBoundRateMax := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "outbound_bytes_per_sec_max",
-			Help: "Maximum outbounding bytes per second",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_max",
+			Help:      "Maximum outbounding bytes per second",
 		},
 		[]string{"connection_id"},
 	)
@@ -198,8 +237,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	compBytesBefore := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "comp_bytes_before",
-			Help: "Bytes sent via cross-stream compression, pre compression",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_before",
+			Help:      "Bytes sent via cross-stream compression, pre compression",
 		},
 		[]string{"connection_id"},
 	)
@@ -207,8 +248,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	compBytesAfter := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "comp_bytes_after",
-			Help: "Bytes sent via cross-stream compression, post compression",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_after",
+			Help:      "Bytes sent via cross-stream compression, post compression",
 		},
 		[]string{"connection_id"},
 	)
@@ -216,8 +259,10 @@ func newMuxerMetrics() *muxerMetrics {
 
 	compRateAve := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "comp_rate_ave",
-			Help: "Average outbound cross-stream compression ratio",
+			Namespace: metricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_rate_ave",
+			Help:      "Average outbound cross-stream compression ratio",
 		},
 		[]string{"connection_id"},
 	)
@@ -274,22 +319,30 @@ func convertRTTMilliSec(t time.Duration) float64 {
 func NewTunnelMetrics() *TunnelMetrics {
 	haConnections := prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Name: "ha_connections",
-			Help: "Number of active ha connections",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "ha_connections",
+			Help:      "Number of active ha connections",
 		})
 	prometheus.MustRegister(haConnections)
 
+	activeStreams := h2mux.NewActiveStreamsMetrics(metricsNamespace, tunnelSubsystem)
+
 	totalRequests := prometheus.NewCounter(
 		prometheus.CounterOpts{
-			Name: "total_requests",
-			Help: "Amount of requests proxied through all the tunnels",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "total_requests",
+			Help:      "Amount of requests proxied through all the tunnels",
 		})
 	prometheus.MustRegister(totalRequests)
 
 	requestsPerTunnel := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "requests_per_tunnel",
-			Help: "Amount of requests proxied through each tunnel",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "requests_per_tunnel",
+			Help:      "Amount of requests proxied through each tunnel",
 		},
 		[]string{"connection_id"},
 	)
@@ -297,8 +350,10 @@ func NewTunnelMetrics() *TunnelMetrics {
 
 	concurrentRequestsPerTunnel := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "concurrent_requests_per_tunnel",
-			Help: "Concurrent requests proxied through each tunnel",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "concurrent_requests_per_tunnel",
+			Help:      "Concurrent requests proxied through each tunnel",
 		},
 		[]string{"connection_id"},
 	)
@@ -306,8 +361,10 @@ func NewTunnelMetrics() *TunnelMetrics {
 
 	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "max_concurrent_requests_per_tunnel",
-			Help: "Largest number of concurrent requests proxied through each tunnel so far",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "max_concurrent_requests_per_tunnel",
+			Help:      "Largest number of concurrent requests proxied through each tunnel so far",
 		},
 		[]string{"connection_id"},
 	)
@@ -315,15 +372,19 @@ func NewTunnelMetrics() *TunnelMetrics {
 
 	timerRetries := prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Name: "timer_retries",
-			Help: "Unacknowledged heart beats count",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "timer_retries",
+			Help:      "Unacknowledged heart beats count",
 		})
 	prometheus.MustRegister(timerRetries)
 
 	responseByCode := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "response_by_code",
-			Help: "Count of responses by HTTP status code",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "response_by_code",
+			Help:      "Count of responses by HTTP status code",
 		},
 		[]string{"status_code"},
 	)
@@ -331,8 +392,10 @@ func NewTunnelMetrics() *TunnelMetrics {
 
 	responseCodePerTunnel := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "response_code_per_tunnel",
-			Help: "Count of responses by HTTP status code fore each tunnel",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "response_code_per_tunnel",
+			Help:      "Count of responses by HTTP status code fore each tunnel",
 		},
 		[]string{"connection_id", "status_code"},
 	)
@@ -340,8 +403,10 @@ func NewTunnelMetrics() *TunnelMetrics {
 
 	serverLocations := prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Name: "server_locations",
-			Help: "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "server_locations",
+			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
 		},
 		[]string{"connection_id", "location"},
 	)
@@ -349,40 +414,72 @@ func NewTunnelMetrics() *TunnelMetrics {
 
 	rpcFail := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "tunnel_rpc_fail",
-			Help: "Count of RPC connection errors by type",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_rpc_fail",
+			Help:      "Count of RPC connection errors by type",
 		},
-		[]string{"error"},
+		[]string{"error", "rpcName"},
 	)
 	prometheus.MustRegister(rpcFail)
 
 	registerFail := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "tunnel_register_fail",
-			Help: "Count of tunnel registration errors by type",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_register_fail",
+			Help:      "Count of tunnel registration errors by type",
 		},
-		[]string{"error"},
+		[]string{"error", "rpcName"},
 	)
 	prometheus.MustRegister(registerFail)
 
 	userHostnamesCounts := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "user_hostnames_counts",
-			Help: "Which user hostnames cloudflared is serving",
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "user_hostnames_counts",
+			Help:      "Which user hostnames cloudflared is serving",
 		},
 		[]string{"userHostname"},
 	)
 	prometheus.MustRegister(userHostnamesCounts)
 
-	registerSuccess := prometheus.NewCounter(
+	registerSuccess := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Name: "tunnel_register_success",
-			Help: "Count of successful tunnel registrations",
-		})
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_register_success",
+			Help:      "Count of successful tunnel registrations",
+		},
+		[]string{"rpcName"},
+	)
 	prometheus.MustRegister(registerSuccess)
 
+	authSuccess := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_success",
+			Help:      "Count of successful tunnel authenticate",
+		},
+	)
+	prometheus.MustRegister(authSuccess)
+
+	authFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_fail",
+			Help:      "Count of tunnel authenticate errors by type",
+		},
+		[]string{"error"},
+	)
+	prometheus.MustRegister(authFail)
+
 	return &TunnelMetrics{
 		haConnections:                  haConnections,
+		activeStreams:                  activeStreams,
 		totalRequests:                  totalRequests,
 		requestsPerTunnel:              requestsPerTunnel,
 		concurrentRequestsPerTunnel:    concurrentRequestsPerTunnel,
@@ -400,6 +497,8 @@ func NewTunnelMetrics() *TunnelMetrics {
 		regFail:                        registerFail,
 		rpcFail:                        rpcFail,
 		userHostnamesCounts:            userHostnamesCounts,
+		authSuccess:                    authSuccess,
+		authFail:                       authFail,
 	}
 }
 

origin/supervisor.go

@@ -2,16 +2,21 @@ package origin
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
+	"sync"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
 
+	"github.com/cloudflare/cloudflared/buffer"
 	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/signal"
-
-	"github.com/google/uuid"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
@@ -21,13 +26,27 @@ const (
 	resolveTTL = time.Hour
 	// Interval between registering new tunnels
 	registrationInterval = time.Second
+
+	subsystemRefreshAuth = "refresh_auth"
+	// Maximum exponent for 'Authenticate' exponential backoff
+	refreshAuthMaxBackoff = 10
+	// Waiting time before retrying a failed 'Authenticate' connection
+	refreshAuthRetryDuration = time.Second * 10
+	// Maximum time to make an Authenticate RPC
+	authTokenTimeout = time.Second * 30
 )
 
+var (
+	errJWTUnset         = errors.New("JWT unset")
+	errEventDigestUnset = errors.New("event digest unset")
+)
+
+// Supervisor manages non-declarative tunnels. Establishes TCP connections with the edge, and
+// reconnects them if they disconnect.
 type Supervisor struct {
-	config  *TunnelConfig
-	edgeIPs []*net.TCPAddr
-	// nextUnusedEdgeIP is the index of the next addr k edgeIPs to try
-	nextUnusedEdgeIP  int
+	cloudflaredUUID   uuid.UUID
+	config            *TunnelConfig
+	edgeIPs           *edgediscovery.Edge
 	lastResolve       time.Time
 	resolverC         chan resolveResult
 	tunnelErrors      chan tunnelError
@@ -38,36 +57,76 @@ type Supervisor struct {
 	nextConnectedSignal chan struct{}
 
 	logger *logrus.Entry
+
+	jwtLock sync.RWMutex
+	jwt     []byte
+
+	eventDigestLock sync.RWMutex
+	eventDigest     []byte
+
+	connDigestLock sync.RWMutex
+	connDigest     map[uint8][]byte
+
+	bufferPool *buffer.Pool
 }
 
 type resolveResult struct {
-	edgeIPs []*net.TCPAddr
-	err     error
+	err error
 }
 
 type tunnelError struct {
 	index int
+	addr  *net.TCPAddr
 	err   error
 }
 
-func NewSupervisor(config *TunnelConfig) *Supervisor {
+func NewSupervisor(config *TunnelConfig, u uuid.UUID) (*Supervisor, error) {
+	var (
+		edgeIPs *edgediscovery.Edge
+		err     error
+	)
+	if len(config.EdgeAddrs) > 0 {
+		edgeIPs, err = edgediscovery.StaticEdge(config.Logger, config.EdgeAddrs)
+	} else {
+		edgeIPs, err = edgediscovery.ResolveEdge(config.Logger)
+	}
+	if err != nil {
+		return nil, err
+	}
 	return &Supervisor{
+		cloudflaredUUID:   u,
 		config:            config,
+		edgeIPs:           edgeIPs,
 		tunnelErrors:      make(chan tunnelError),
 		tunnelsConnecting: map[int]chan struct{}{},
 		logger:            config.Logger.WithField("subsystem", "supervisor"),
-	}
+		connDigest:        make(map[uint8][]byte),
+		bufferPool:        buffer.NewPool(512 * 1024),
+	}, nil
 }
 
-func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, u uuid.UUID) error {
+func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) error {
 	logger := s.config.Logger
-	if err := s.initialize(ctx, connectedSignal, u); err != nil {
+	if err := s.initialize(ctx, connectedSignal, reconnectCh); err != nil {
 		return err
 	}
 	var tunnelsWaiting []int
+	tunnelsActive := s.config.HAConnections
+
 	backoff := BackoffHandler{MaxRetries: s.config.Retries, BaseTime: tunnelRetryDuration, RetryForever: true}
 	var backoffTimer <-chan time.Time
-	tunnelsActive := s.config.HAConnections
+
+	refreshAuthBackoff := &BackoffHandler{MaxRetries: refreshAuthMaxBackoff, BaseTime: refreshAuthRetryDuration, RetryForever: true}
+	var refreshAuthBackoffTimer <-chan time.Time
+
+	if s.config.UseReconnectToken {
+		if timer, err := s.refreshAuth(ctx, refreshAuthBackoff, s.authenticate); err == nil {
+			refreshAuthBackoffTimer = timer
+		} else {
+			logger.WithError(err).Errorf("initial refreshAuth failed, retrying in %v", refreshAuthRetryDuration)
+			refreshAuthBackoffTimer = time.After(refreshAuthRetryDuration)
+		}
+	}
 
 	for {
 		select {
@@ -91,23 +150,27 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, u
 					backoffTimer = backoff.BackoffTimer()
 				}
 
-				// If the error is a dial error, the problem is likely to be network related
-				// try another addr before refreshing since we are likely to get back the
-				// same IPs in the same order. Same problem with duplicate connection error.
-				if s.unusedIPs() {
-					s.replaceEdgeIP(tunnelError.index)
-				} else {
-					s.refreshEdgeIPs()
-				}
+				// Previously we'd mark the edge address as bad here, but now we'll just silently use
+				// another.
 			}
 		// Backoff was set and its timer expired
 		case <-backoffTimer:
 			backoffTimer = nil
 			for _, index := range tunnelsWaiting {
-				go s.startTunnel(ctx, index, s.newConnectedTunnelSignal(index), u)
+				go s.startTunnel(ctx, index, s.newConnectedTunnelSignal(index), reconnectCh)
 			}
 			tunnelsActive += len(tunnelsWaiting)
 			tunnelsWaiting = nil
+		// Time to call Authenticate
+		case <-refreshAuthBackoffTimer:
+			newTimer, err := s.refreshAuth(ctx, refreshAuthBackoff, s.authenticate)
+			if err != nil {
+				logger.WithError(err).Error("Authentication failed")
+				// Permanent failure. Leave the `select` without setting the
+				// channel to be non-null, so we'll never hit this case of the `select` again.
+				continue
+			}
+			refreshAuthBackoffTimer = newTimer
 		// Tunnel successfully connected
 		case <-s.nextConnectedSignal:
 			if !s.waitForNextTunnel(s.nextConnectedIndex) && len(tunnelsWaiting) == 0 {
@@ -120,7 +183,6 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, u
 			s.resolverC = nil
 			if result.err == nil {
 				logger.Debug("Service discovery refresh complete")
-				s.edgeIPs = result.edgeIPs
 			} else {
 				logger.WithError(result.err).Error("Service discovery error")
 			}
@@ -128,29 +190,22 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, u
 	}
 }
 
-func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal, u uuid.UUID) error {
+// Returns nil if initialization succeeded, else the initialization error.
+func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) error {
 	logger := s.logger
 
-	edgeIPs, err := s.resolveEdgeIPs()
-
-	if err != nil {
-		logger.Infof("ResolveEdgeIPs err")
-		return err
-	}
-	s.edgeIPs = edgeIPs
-	if s.config.HAConnections > len(edgeIPs) {
-		logger.Warnf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, len(edgeIPs))
-		s.config.HAConnections = len(edgeIPs)
-	}
 	s.lastResolve = time.Now()
-	// check entitlement and version too old error before attempting to register more tunnels
-	s.nextUnusedEdgeIP = s.config.HAConnections
-	go s.startFirstTunnel(ctx, connectedSignal, u)
+	availableAddrs := int(s.edgeIPs.AvailableAddrs())
+	if s.config.HAConnections > availableAddrs {
+		logger.Warnf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
+		s.config.HAConnections = availableAddrs
+	}
+
+	go s.startFirstTunnel(ctx, connectedSignal, reconnectCh)
 	select {
 	case <-ctx.Done():
 		<-s.tunnelErrors
-		// Error can't be nil. A nil error signals that initialization succeed
-		return fmt.Errorf("context was canceled")
+		return ctx.Err()
 	case tunnelError := <-s.tunnelErrors:
 		return tunnelError.err
 	case <-connectedSignal.Wait():
@@ -158,7 +213,7 @@ func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Sig
 	// At least one successful connection, so start the rest
 	for i := 1; i < s.config.HAConnections; i++ {
 		ch := signal.New(make(chan struct{}))
-		go s.startTunnel(ctx, i, ch, u)
+		go s.startTunnel(ctx, i, ch, reconnectCh)
 		time.Sleep(registrationInterval)
 	}
 	return nil
@@ -166,37 +221,64 @@ func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Sig
 
 // startTunnel starts the first tunnel connection. The resulting error will be sent on
 // s.tunnelErrors. It will send a signal via connectedSignal if registration succeed
-func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *signal.Signal, u uuid.UUID) {
-	err := ServeTunnelLoop(ctx, s.config, s.getEdgeIP(0), 0, connectedSignal, u)
+func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) {
+	var (
+		addr *net.TCPAddr
+		err  error
+	)
+	const thisConnID = 0
 	defer func() {
-		s.tunnelErrors <- tunnelError{index: 0, err: err}
+		s.tunnelErrors <- tunnelError{index: thisConnID, addr: addr, err: err}
 	}()
 
+	addr, err = s.edgeIPs.GetAddr(thisConnID)
+	if err != nil {
+		return
+	}
+
+	err = ServeTunnelLoop(ctx, s, s.config, addr, thisConnID, connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
+	// If the first tunnel disconnects, keep restarting it.
+	edgeErrors := 0
 	for s.unusedIPs() {
-		select {
-		case <-ctx.Done():
+		if ctx.Err() != nil {
 			return
-		default:
 		}
 		switch err.(type) {
 		case nil:
 			return
 		// try the next address if it was a dialError(network problem) or
 		// dupConnRegisterTunnelError
-		case dialError, dupConnRegisterTunnelError:
-			s.replaceEdgeIP(0)
+		case connection.DialError, dupConnRegisterTunnelError:
+			edgeErrors++
 		default:
 			return
 		}
-		err = ServeTunnelLoop(ctx, s.config, s.getEdgeIP(0), 0, connectedSignal, u)
+		if edgeErrors >= 2 {
+			addr, err = s.edgeIPs.GetDifferentAddr(thisConnID)
+			if err != nil {
+				return
+			}
+		}
+		err = ServeTunnelLoop(ctx, s, s.config, addr, thisConnID, connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
 	}
 }
 
 // startTunnel starts a new tunnel connection. The resulting error will be sent on
 // s.tunnelErrors.
-func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal *signal.Signal, u uuid.UUID) {
-	err := ServeTunnelLoop(ctx, s.config, s.getEdgeIP(index), uint8(index), connectedSignal, u)
-	s.tunnelErrors <- tunnelError{index: index, err: err}
+func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) {
+	var (
+		addr *net.TCPAddr
+		err  error
+	)
+	defer func() {
+		s.tunnelErrors <- tunnelError{index: index, addr: addr, err: err}
+	}()
+
+	addr, err = s.edgeIPs.GetDifferentAddr(index)
+	if err != nil {
+		return
+	}
+	err = ServeTunnelLoop(ctx, s, s.config, addr, uint8(index), connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
 }
 
 func (s *Supervisor) newConnectedTunnelSignal(index int) *signal.Signal {
@@ -218,38 +300,140 @@ func (s *Supervisor) waitForNextTunnel(index int) bool {
 	return false
 }
 
-func (s *Supervisor) getEdgeIP(index int) *net.TCPAddr {
-	return s.edgeIPs[index%len(s.edgeIPs)]
-}
-
-func (s *Supervisor) resolveEdgeIPs() ([]*net.TCPAddr, error) {
-	// If --edge is specfied, resolve edge server addresses
-	if len(s.config.EdgeAddrs) > 0 {
-		return connection.ResolveAddrs(s.config.EdgeAddrs)
-	}
-	// Otherwise lookup edge server addresses through service discovery
-	return connection.EdgeDiscovery(s.logger)
-}
-
-func (s *Supervisor) refreshEdgeIPs() {
-	if s.resolverC != nil {
-		return
-	}
-	if time.Since(s.lastResolve) < resolveTTL {
-		return
-	}
-	s.resolverC = make(chan resolveResult)
-	go func() {
-		edgeIPs, err := s.resolveEdgeIPs()
-		s.resolverC <- resolveResult{edgeIPs: edgeIPs, err: err}
-	}()
-}
-
 func (s *Supervisor) unusedIPs() bool {
-	return s.nextUnusedEdgeIP < len(s.edgeIPs)
+	return s.edgeIPs.AvailableAddrs() > s.config.HAConnections
 }
 
-func (s *Supervisor) replaceEdgeIP(badIPIndex int) {
-	s.edgeIPs[badIPIndex] = s.edgeIPs[s.nextUnusedEdgeIP]
-	s.nextUnusedEdgeIP++
+func (s *Supervisor) ReconnectToken() ([]byte, error) {
+	s.jwtLock.RLock()
+	defer s.jwtLock.RUnlock()
+	if s.jwt == nil {
+		return nil, errJWTUnset
+	}
+	return s.jwt, nil
+}
+
+func (s *Supervisor) SetReconnectToken(jwt []byte) {
+	s.jwtLock.Lock()
+	defer s.jwtLock.Unlock()
+	s.jwt = jwt
+}
+
+func (s *Supervisor) EventDigest() ([]byte, error) {
+	s.eventDigestLock.RLock()
+	defer s.eventDigestLock.RUnlock()
+	if s.eventDigest == nil {
+		return nil, errEventDigestUnset
+	}
+	return s.eventDigest, nil
+}
+
+func (s *Supervisor) SetEventDigest(eventDigest []byte) {
+	s.eventDigestLock.Lock()
+	defer s.eventDigestLock.Unlock()
+	s.eventDigest = eventDigest
+}
+
+func (s *Supervisor) ConnDigest(connID uint8) ([]byte, error) {
+	s.connDigestLock.RLock()
+	defer s.connDigestLock.RUnlock()
+	digest, ok := s.connDigest[connID]
+	if !ok {
+		return nil, fmt.Errorf("no connection digest for connection %v", connID)
+	}
+	return digest, nil
+}
+
+func (s *Supervisor) SetConnDigest(connID uint8, connDigest []byte) {
+	s.connDigestLock.Lock()
+	defer s.connDigestLock.Unlock()
+	s.connDigest[connID] = connDigest
+}
+
+func (s *Supervisor) refreshAuth(
+	ctx context.Context,
+	backoff *BackoffHandler,
+	authenticate func(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error),
+) (retryTimer <-chan time.Time, err error) {
+	logger := s.config.Logger.WithField("subsystem", subsystemRefreshAuth)
+	authOutcome, err := authenticate(ctx, backoff.Retries())
+	if err != nil {
+		s.config.Metrics.authFail.WithLabelValues(err.Error()).Inc()
+		if duration, ok := backoff.GetBackoffDuration(ctx); ok {
+			logger.WithError(err).Warnf("Retrying in %v", duration)
+			return backoff.BackoffTimer(), nil
+		}
+		return nil, err
+	}
+	// clear backoff timer
+	backoff.SetGracePeriod()
+
+	switch outcome := authOutcome.(type) {
+	case tunnelpogs.AuthSuccess:
+		s.SetReconnectToken(outcome.JWT())
+		s.config.Metrics.authSuccess.Inc()
+		return timeAfter(outcome.RefreshAfter()), nil
+	case tunnelpogs.AuthUnknown:
+		duration := outcome.RefreshAfter()
+		s.config.Metrics.authFail.WithLabelValues(outcome.Error()).Inc()
+		logger.WithError(outcome).Warnf("Retrying in %v", duration)
+		return timeAfter(duration), nil
+	case tunnelpogs.AuthFail:
+		s.config.Metrics.authFail.WithLabelValues(outcome.Error()).Inc()
+		return nil, outcome
+	default:
+		err := fmt.Errorf("Unexpected outcome type %T", authOutcome)
+		s.config.Metrics.authFail.WithLabelValues(err.Error()).Inc()
+		return nil, err
+	}
+}
+
+func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error) {
+	arbitraryEdgeIP, err := s.edgeIPs.GetAddrForRPC()
+	if err != nil {
+		return nil, err
+	}
+
+	edgeConn, err := connection.DialEdge(ctx, dialTimeout, s.config.TlsConfig, arbitraryEdgeIP)
+	if err != nil {
+		return nil, err
+	}
+	defer edgeConn.Close()
+
+	handler := h2mux.MuxedStreamFunc(func(*h2mux.MuxedStream) error {
+		// This callback is invoked by h2mux when the edge initiates a stream.
+		return nil // noop
+	})
+	muxerConfig := s.config.muxerConfig(handler)
+	muxerConfig.Logger = muxerConfig.Logger.WithField("subsystem", subsystemRefreshAuth)
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, muxerConfig, s.config.Metrics.activeStreams)
+	if err != nil {
+		return nil, err
+	}
+	go muxer.Serve(ctx)
+	defer func() {
+		// If we don't wait for the muxer shutdown here, edgeConn.Close() runs before the muxer connections are done,
+		// and the user sees log noise: "error writing data", "connection closed unexpectedly"
+		<-muxer.Shutdown()
+	}()
+
+	tunnelServer, err := connection.NewRPCClient(ctx, muxer, s.logger.WithField("subsystem", subsystemRefreshAuth), openStreamTimeout)
+	if err != nil {
+		return nil, err
+	}
+	defer tunnelServer.Close()
+
+	const arbitraryConnectionID = uint8(0)
+	registrationOptions := s.config.RegistrationOptions(arbitraryConnectionID, edgeConn.LocalAddr().String(), s.cloudflaredUUID)
+	registrationOptions.NumPreviousAttempts = uint8(numPreviousAttempts)
+	authResponse, err := tunnelServer.Authenticate(
+		ctx,
+		s.config.OriginCert,
+		s.config.Hostname,
+		registrationOptions,
+	)
+	if err != nil {
+		return nil, err
+	}
+	return authResponse.Outcome(), nil
 }

origin/supervisor_test.go

@@ -0,0 +1,165 @@
+package origin
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+func testConfig(logger *logrus.Logger) *TunnelConfig {
+	metrics := TunnelMetrics{}
+
+	metrics.authSuccess = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_success",
+			Help:      "Count of successful tunnel authenticate",
+		},
+	)
+
+	metrics.authFail = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_fail",
+			Help:      "Count of tunnel authenticate errors by type",
+		},
+		[]string{"error"},
+	)
+	return &TunnelConfig{Logger: logger, Metrics: &metrics}
+}
+
+func TestRefreshAuthBackoff(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	var wait time.Duration
+	timeAfter = func(d time.Duration) <-chan time.Time {
+		wait = d
+		return time.After(d)
+	}
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return nil, fmt.Errorf("authentication failure")
+	}
+
+	// authentication failures should consume the backoff
+	for i := uint(0); i < backoff.MaxRetries; i++ {
+		retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+		assert.NoError(t, err)
+		assert.NotNil(t, retryChan)
+		assert.Equal(t, (1<<i)*time.Second, wait)
+	}
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.Error(t, err)
+	assert.Nil(t, retryChan)
+
+	// now we actually make contact with the remote server
+	_, _ = s.refreshAuth(context.Background(), backoff, func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
+	})
+
+	// The backoff timer should have been reset. To confirm this, make timeNow
+	// return a value after the backoff timer's grace period
+	timeNow = func() time.Time {
+		expectedGracePeriod := time.Duration(time.Second * 2 << backoff.MaxRetries)
+		return time.Now().Add(expectedGracePeriod * 2)
+	}
+	_, ok := backoff.GetBackoffDuration(context.Background())
+	assert.True(t, ok)
+}
+
+func TestRefreshAuthSuccess(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	var wait time.Duration
+	timeAfter = func(d time.Duration) <-chan time.Time {
+		wait = d
+		return time.After(d)
+	}
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthSuccess([]byte("jwt"), 19), nil
+	}
+
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.NoError(t, err)
+	assert.NotNil(t, retryChan)
+	assert.Equal(t, 19*time.Hour, wait)
+
+	token, err := s.ReconnectToken()
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("jwt"), token)
+}
+
+func TestRefreshAuthUnknown(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	var wait time.Duration
+	timeAfter = func(d time.Duration) <-chan time.Time {
+		wait = d
+		return time.After(d)
+	}
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
+	}
+
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.NoError(t, err)
+	assert.NotNil(t, retryChan)
+	assert.Equal(t, 19*time.Hour, wait)
+
+	token, err := s.ReconnectToken()
+	assert.Equal(t, errJWTUnset, err)
+	assert.Nil(t, token)
+}
+
+func TestRefreshAuthFail(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthFail(errors.New("auth fail")), nil
+	}
+
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.Error(t, err)
+	assert.Nil(t, retryChan)
+
+	token, err := s.ReconnectToken()
+	assert.Equal(t, errJWTUnset, err)
+	assert.Nil(t, token)
+}

origin/tunnel.go

@@ -14,7 +14,15 @@ import (
 	"sync"
 	"time"
 
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/buffer"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/signal"
 	"github.com/cloudflare/cloudflared/streamhandler"
@@ -22,23 +30,25 @@ import (
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/validation"
 	"github.com/cloudflare/cloudflared/websocket"
-
-	raven "github.com/getsentry/raven-go"
-	"github.com/google/uuid"
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-	_ "github.com/prometheus/client_golang/prometheus"
-	log "github.com/sirupsen/logrus"
-	"golang.org/x/sync/errgroup"
-	rpc "zombiezen.com/go/capnproto2/rpc"
 )
 
 const (
 	dialTimeout              = 15 * time.Second
 	openStreamTimeout        = 30 * time.Second
+	muxerTimeout             = 5 * time.Second
 	lbProbeUserAgentPrefix   = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
 	TagHeaderNamePrefix      = "Cf-Warp-Tag-"
 	DuplicateConnectionError = "EDUPCONN"
+	FeatureSerializedHeaders = "serialized_headers"
+	FeatureQuickReconnects   = "quick_reconnects"
+)
+
+type registerRPCName string
+
+const (
+	register  registerRPCName = "register"
+	reconnect registerRPCName = "reconnect"
+	unknown   registerRPCName = "unknown"
 )
 
 type TunnelConfig struct {
@@ -53,6 +63,7 @@ type TunnelConfig struct {
 	HTTPTransport        http.RoundTripper
 	HeartbeatInterval    time.Duration
 	Hostname             string
+	HTTPHostHeader       string
 	IncidentLookup       IncidentLookup
 	IsAutoupdated        bool
 	IsFreeTunnel         bool
@@ -73,14 +84,21 @@ type TunnelConfig struct {
 	WSGI                 bool
 	// OriginUrl may not be used if a user specifies a unix socket.
 	OriginUrl string
+
+	// feature-flag to use new edge reconnect tokens
+	UseReconnectToken bool
+	// feature-flag for using ConnectionDigest
+	UseQuickReconnects bool
 }
 
-type dialError struct {
-	cause error
-}
-
-func (e dialError) Error() string {
-	return e.cause.Error()
+// ReconnectTunnelCredentialManager is invoked by functions in this file to
+// get/set parameters for ReconnectTunnel RPC calls.
+type ReconnectTunnelCredentialManager interface {
+	ReconnectToken() ([]byte, error)
+	EventDigest() ([]byte, error)
+	SetEventDigest(eventDigest []byte)
+	ConnDigest(connID uint8) ([]byte, error)
+	SetConnDigest(connID uint8, connDigest []byte)
 }
 
 type dupConnRegisterTunnelError struct{}
@@ -110,8 +128,8 @@ type clientRegisterTunnelError struct {
 	cause error
 }
 
-func newClientRegisterTunnelError(cause error, counter *prometheus.CounterVec) clientRegisterTunnelError {
-	counter.WithLabelValues(cause.Error()).Inc()
+func newClientRegisterTunnelError(cause error, counter *prometheus.CounterVec, name registerRPCName) clientRegisterTunnelError {
+	counter.WithLabelValues(cause.Error(), string(name)).Inc()
 	return clientRegisterTunnelError{cause: cause}
 }
 
@@ -119,6 +137,18 @@ func (e clientRegisterTunnelError) Error() string {
 	return e.cause.Error()
 }
 
+func (c *TunnelConfig) muxerConfig(handler h2mux.MuxedStreamHandler) h2mux.MuxerConfig {
+	return h2mux.MuxerConfig{
+		Timeout:            muxerTimeout,
+		Handler:            handler,
+		IsClient:           true,
+		HeartbeatInterval:  c.HeartbeatInterval,
+		MaxHeartbeats:      c.MaxHeartbeats,
+		Logger:             c.TransportLogger.WithFields(log.Fields{}),
+		CompressionQuality: h2mux.CompressionSetting(c.CompressionQuality),
+	}
+}
+
 func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
 	policy := tunnelrpc.ExistingTunnelPolicy_balance
 	if c.HAConnections <= 1 && c.LBPool == "" {
@@ -137,21 +167,37 @@ func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP str
 		RunFromTerminal:      c.RunFromTerminal,
 		CompressionQuality:   c.CompressionQuality,
 		UUID:                 uuid.String(),
+		Features:             c.SupportedFeatures(),
 	}
 }
 
-func StartTunnelDaemon(ctx context.Context, config *TunnelConfig, connectedSignal *signal.Signal, cloudflaredID uuid.UUID) error {
-	return NewSupervisor(config).Run(ctx, connectedSignal, cloudflaredID)
+func (c *TunnelConfig) SupportedFeatures() []string {
+	basic := []string{FeatureSerializedHeaders}
+	if c.UseQuickReconnects {
+		basic = append(basic, FeatureQuickReconnects)
+	}
+	return basic
+}
+
+func StartTunnelDaemon(ctx context.Context, config *TunnelConfig, connectedSignal *signal.Signal, cloudflaredID uuid.UUID, reconnectCh chan ReconnectSignal) error {
+	s, err := NewSupervisor(config, cloudflaredID)
+	if err != nil {
+		return err
+	}
+	return s.Run(ctx, connectedSignal, reconnectCh)
 }
 
 func ServeTunnelLoop(ctx context.Context,
+	credentialManager ReconnectTunnelCredentialManager,
 	config *TunnelConfig,
 	addr *net.TCPAddr,
 	connectionID uint8,
 	connectedSignal *signal.Signal,
 	u uuid.UUID,
+	bufferPool *buffer.Pool,
+	reconnectCh chan ReconnectSignal,
 ) error {
-	logger := config.Logger
+	connectionLogger := config.Logger.WithField("connectionID", connectionID)
 	config.Metrics.incrementHaConnections()
 	defer config.Metrics.decrementHaConnections()
 	backoff := BackoffHandler{MaxRetries: config.Retries}
@@ -164,10 +210,21 @@ func ServeTunnelLoop(ctx context.Context,
 	// Ensure the above goroutine will terminate if we return without connecting
 	defer connectedFuse.Fuse(false)
 	for {
-		err, recoverable := ServeTunnel(ctx, config, addr, connectionID, connectedFuse, &backoff, u)
+		err, recoverable := ServeTunnel(
+			ctx,
+			credentialManager,
+			config,
+			connectionLogger,
+			addr, connectionID,
+			connectedFuse,
+			&backoff,
+			u,
+			bufferPool,
+			reconnectCh,
+		)
 		if recoverable {
 			if duration, ok := backoff.GetBackoffDuration(ctx); ok {
-				logger.Infof("Retrying in %s seconds", duration)
+				connectionLogger.Infof("Retrying in %s seconds", duration)
 				backoff.Backoff(ctx)
 				continue
 			}
@@ -178,12 +235,16 @@ func ServeTunnelLoop(ctx context.Context,
 
 func ServeTunnel(
 	ctx context.Context,
+	credentialManager ReconnectTunnelCredentialManager,
 	config *TunnelConfig,
+	logger *log.Entry,
 	addr *net.TCPAddr,
 	connectionID uint8,
 	connectedFuse *h2mux.BooleanFuse,
 	backoff *BackoffHandler,
 	u uuid.UUID,
+	bufferPool *buffer.Pool,
+	reconnectCh chan ReconnectSignal,
 ) (err error, recoverable bool) {
 	// Treat panics as recoverable errors
 	defer func() {
@@ -198,18 +259,17 @@ func ServeTunnel(
 	}()
 
 	connectionTag := uint8ToString(connectionID)
-	logger := config.Logger.WithField("connectionID", connectionTag)
 
 	// additional tags to send other than hostname which is set in cloudflared main package
 	tags := make(map[string]string)
 	tags["ha"] = connectionTag
 
 	// Returns error from parsing the origin URL or handshake errors
-	handler, originLocalIP, err := NewTunnelHandler(ctx, config, addr.String(), connectionID)
+	handler, originLocalIP, err := NewTunnelHandler(ctx, config, addr, connectionID, bufferPool)
 	if err != nil {
-		errLog := config.Logger.WithError(err)
+		errLog := logger.WithError(err)
 		switch err.(type) {
-		case dialError:
+		case connection.DialError:
 			errLog.Error("Unable to dial edge")
 		case h2mux.MuxerHandshakeError:
 			errLog.Error("Handshake failed with edge server")
@@ -222,13 +282,38 @@ func ServeTunnel(
 
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 
-	errGroup.Go(func() error {
-		err := RegisterTunnel(serveCtx, handler.muxer, config, connectionID, originLocalIP, u)
-		if err == nil {
-			connectedFuse.Fuse(true)
-			backoff.SetGracePeriod()
+	errGroup.Go(func() (err error) {
+		defer func() {
+			if err == nil {
+				connectedFuse.Fuse(true)
+				backoff.SetGracePeriod()
+			}
+		}()
+
+		if config.UseReconnectToken && connectedFuse.Value() {
+			token, tokenErr := credentialManager.ReconnectToken()
+			eventDigest, eventDigestErr := credentialManager.EventDigest()
+			// if we have both credentials, we can reconnect
+			if tokenErr == nil && eventDigestErr == nil {
+				var connDigest []byte
+
+				// check if we can use Quick Reconnects
+				if config.UseQuickReconnects {
+					if digest, connDigestErr := credentialManager.ConnDigest(connectionID); connDigestErr == nil {
+						connDigest = digest
+					}
+				}
+				return ReconnectTunnel(serveCtx, token, eventDigest, connDigest, handler.muxer, config, logger, connectionID, originLocalIP, u, credentialManager)
+			}
+			// log errors and proceed to RegisterTunnel
+			if tokenErr != nil {
+				logger.WithError(tokenErr).Error("Couldn't get reconnect token")
+			}
+			if eventDigestErr != nil {
+				logger.WithError(eventDigestErr).Error("Couldn't get event digest")
+			}
 		}
-		return err
+		return RegisterTunnel(serveCtx, credentialManager, handler.muxer, config, logger, connectionID, originLocalIP, u)
 	})
 
 	errGroup.Go(func() error {
@@ -237,7 +322,10 @@ func ServeTunnel(
 			select {
 			case <-serveCtx.Done():
 				// UnregisterTunnel blocks until the RPC call returns
-				err := UnregisterTunnel(handler.muxer, config.GracePeriod, config.TransportLogger)
+				var err error
+				if connectedFuse.Value() {
+					err = UnregisterTunnel(handler.muxer, config.GracePeriod, config.TransportLogger)
+				}
 				handler.muxer.Shutdown()
 				return err
 			case <-updateMetricsTickC:
@@ -246,6 +334,17 @@ func ServeTunnel(
 		}
 	})
 
+	errGroup.Go(func() error {
+		for {
+			select {
+			case reconnect := <-reconnectCh:
+				return &reconnect
+			case <-serveCtx.Done():
+				return nil
+			}
+		}
+	})
+
 	errGroup.Go(func() error {
 		// All routines should stop when muxer finish serving. When muxer is shutdown
 		// gracefully, it doesn't return an error, so we need to return errMuxerShutdown
@@ -259,6 +358,8 @@ func ServeTunnel(
 
 	err = errGroup.Wait()
 	if err != nil {
+		_ = newClientRegisterTunnelError(err, config.Metrics.regFail, unknown)
+
 		switch castedErr := err.(type) {
 		case dupConnRegisterTunnelError:
 			logger.Info("Already connected to this server, selecting a different one")
@@ -273,154 +374,165 @@ func ServeTunnel(
 			return castedErr.cause, !castedErr.permanent
 		case clientRegisterTunnelError:
 			logger.WithError(castedErr.cause).Error("Register tunnel error on client side")
-			raven.CaptureError(castedErr.cause, tags)
 			return err, true
 		case muxerShutdownError:
-			logger.Infof("Muxer shutdown")
+			logger.Info("Muxer shutdown")
+			return err, true
+		case *ReconnectSignal:
+			logger.Warnf("Restarting due to reconnect signal in %d seconds", castedErr.Delay)
+			castedErr.DelayBeforeReconnect()
 			return err, true
 		default:
 			logger.WithError(err).Error("Serve tunnel error")
-			raven.CaptureError(err, tags)
 			return err, true
 		}
 	}
 	return nil, true
 }
 
-func IsRPCStreamResponse(headers []h2mux.Header) bool {
-	if len(headers) != 1 {
-		return false
-	}
-	if headers[0].Name != ":status" || headers[0].Value != "200" {
-		return false
-	}
-	return true
-}
-
 func RegisterTunnel(
 	ctx context.Context,
+	credentialManager ReconnectTunnelCredentialManager,
 	muxer *h2mux.Muxer,
 	config *TunnelConfig,
+	logger *log.Entry,
 	connectionID uint8,
 	originLocalIP string,
 	uuid uuid.UUID,
 ) error {
 	config.TransportLogger.Debug("initiating RPC stream to register")
-	stream, err := openStream(ctx, muxer)
+	tunnelServer, err := connection.NewRPCClient(ctx, muxer, config.TransportLogger.WithField("subsystem", "rpc-register"), openStreamTimeout)
 	if err != nil {
 		// RPC stream open error
-		return newClientRegisterTunnelError(err, config.Metrics.rpcFail)
+		return newClientRegisterTunnelError(err, config.Metrics.rpcFail, register)
 	}
-	if !IsRPCStreamResponse(stream.Headers) {
-		// stream response error
-		return newClientRegisterTunnelError(err, config.Metrics.rpcFail)
-	}
-	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(config.TransportLogger.WithField("subsystem", "rpc-register"), rpc.StreamTransport(stream)),
-		tunnelrpc.ConnLog(config.TransportLogger.WithField("subsystem", "rpc-transport")),
-	)
-	defer conn.Close()
-	ts := tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx)}
+	defer tunnelServer.Close()
 	// Request server info without blocking tunnel registration; must use capnp library directly.
-	tsClient := tunnelrpc.TunnelServer{Client: ts.Client}
-	serverInfoPromise := tsClient.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+	serverInfoPromise := tunnelrpc.TunnelServer{Client: tunnelServer.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
 		return nil
 	})
-	registration, err := ts.RegisterTunnel(
+	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger)
+	registration := tunnelServer.RegisterTunnel(
 		ctx,
 		config.OriginCert,
 		config.Hostname,
 		config.RegistrationOptions(connectionID, originLocalIP, uuid),
 	)
-	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, config.Logger)
-	if err != nil {
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
 		// RegisterTunnel RPC failure
-		return newClientRegisterTunnelError(err, config.Metrics.regFail)
-	}
-	for _, logLine := range registration.LogLines {
-		config.Logger.Info(logLine)
+		return processRegisterTunnelError(registrationErr, config.Metrics, register)
 	}
+	credentialManager.SetEventDigest(registration.EventDigest)
+	return processRegistrationSuccess(config, logger, connectionID, registration, register, credentialManager)
+}
 
-	if regErr := processRegisterTunnelError(registration.Err, registration.PermanentFailure, config.Metrics); regErr != nil {
-		return regErr
+func ReconnectTunnel(
+	ctx context.Context,
+	token []byte,
+	eventDigest, connDigest []byte,
+	muxer *h2mux.Muxer,
+	config *TunnelConfig,
+	logger *log.Entry,
+	connectionID uint8,
+	originLocalIP string,
+	uuid uuid.UUID,
+	credentialManager ReconnectTunnelCredentialManager,
+) error {
+	config.TransportLogger.Debug("initiating RPC stream to reconnect")
+	tunnelServer, err := connection.NewRPCClient(ctx, muxer, config.TransportLogger.WithField("subsystem", "rpc-reconnect"), openStreamTimeout)
+	if err != nil {
+		// RPC stream open error
+		return newClientRegisterTunnelError(err, config.Metrics.rpcFail, reconnect)
+	}
+	defer tunnelServer.Close()
+	// Request server info without blocking tunnel registration; must use capnp library directly.
+	serverInfoPromise := tunnelrpc.TunnelServer{Client: tunnelServer.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+		return nil
+	})
+	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger)
+	registration := tunnelServer.ReconnectTunnel(
+		ctx,
+		token,
+		eventDigest,
+		connDigest,
+		config.Hostname,
+		config.RegistrationOptions(connectionID, originLocalIP, uuid),
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// ReconnectTunnel RPC failure
+		return processRegisterTunnelError(registrationErr, config.Metrics, reconnect)
+	}
+	return processRegistrationSuccess(config, logger, connectionID, registration, reconnect, credentialManager)
+}
+
+func processRegistrationSuccess(
+	config *TunnelConfig,
+	logger *log.Entry,
+	connectionID uint8,
+	registration *tunnelpogs.TunnelRegistration,
+	name registerRPCName,
+	credentialManager ReconnectTunnelCredentialManager,
+) error {
+	for _, logLine := range registration.LogLines {
+		logger.Info(logLine)
 	}
 
 	if registration.TunnelID != "" {
 		config.Metrics.tunnelsHA.AddTunnelID(connectionID, registration.TunnelID)
-		config.Logger.Infof("Each HA connection's tunnel IDs: %v", config.Metrics.tunnelsHA.String())
+		logger.Infof("Each HA connection's tunnel IDs: %v", config.Metrics.tunnelsHA.String())
 	}
 
 	// Print out the user's trial zone URL in a nice box (if they requested and got one)
 	if isTrialTunnel := config.Hostname == ""; isTrialTunnel {
 		if url, err := url.Parse(registration.Url); err == nil {
 			for _, line := range asciiBox(trialZoneMsg(url.String()), 2) {
-				config.Logger.Infoln(line)
+				logger.Infoln(line)
 			}
 		} else {
-			config.Logger.Errorln("Failed to connect tunnel, please try again.")
+			logger.Errorln("Failed to connect tunnel, please try again.")
 			return fmt.Errorf("empty URL in response from Cloudflare edge")
 		}
 	}
 
+	credentialManager.SetConnDigest(connectionID, registration.ConnDigest)
 	config.Metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
 
-	config.Logger.Infof("Route propagating, it may take up to 1 minute for your new route to become functional")
+	logger.Infof("Route propagating, it may take up to 1 minute for your new route to become functional")
+	config.Metrics.regSuccess.WithLabelValues(string(name)).Inc()
 	return nil
 }
 
-func processRegisterTunnelError(err string, permanentFailure bool, metrics *TunnelMetrics) error {
-	if err == "" {
-		metrics.regSuccess.Inc()
-		return nil
-	}
-
-	metrics.regFail.WithLabelValues(err).Inc()
-	if err == DuplicateConnectionError {
+func processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, metrics *TunnelMetrics, name registerRPCName) error {
+	if err.Error() == DuplicateConnectionError {
+		metrics.regFail.WithLabelValues("dup_edge_conn", string(name)).Inc()
 		return dupConnRegisterTunnelError{}
 	}
+	metrics.regFail.WithLabelValues("server_error", string(name)).Inc()
 	return serverRegisterTunnelError{
-		cause:     fmt.Errorf("Server error: %s", err),
-		permanent: permanentFailure,
+		cause:     fmt.Errorf("Server error: %s", err.Error()),
+		permanent: err.IsPermanent(),
 	}
 }
 
 func UnregisterTunnel(muxer *h2mux.Muxer, gracePeriod time.Duration, logger *log.Logger) error {
 	logger.Debug("initiating RPC stream to unregister")
 	ctx := context.Background()
-	stream, err := openStream(ctx, muxer)
+	tunnelServer, err := connection.NewRPCClient(ctx, muxer, logger.WithField("subsystem", "rpc-unregister"), openStreamTimeout)
 	if err != nil {
 		// RPC stream open error
 		return err
 	}
-	if !IsRPCStreamResponse(stream.Headers) {
-		// stream response error
-		return err
-	}
-	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger.WithField("subsystem", "rpc-unregister"), rpc.StreamTransport(stream)),
-		tunnelrpc.ConnLog(logger.WithField("subsystem", "rpc-transport")),
-	)
-	defer conn.Close()
-	ts := tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx)}
-	// gracePeriod is encoded in int64 using capnproto
-	return ts.UnregisterTunnel(ctx, gracePeriod.Nanoseconds())
-}
+	defer tunnelServer.Close()
 
-func openStream(ctx context.Context, muxer *h2mux.Muxer) (*h2mux.MuxedStream, error) {
-	openStreamCtx, cancel := context.WithTimeout(ctx, openStreamTimeout)
-	defer cancel()
-	return muxer.OpenStream(openStreamCtx, []h2mux.Header{
-		{Name: ":method", Value: "RPC"},
-		{Name: ":scheme", Value: "capnp"},
-		{Name: ":path", Value: "*"},
-	}, nil)
+	// gracePeriod is encoded in int64 using capnproto
+	return tunnelServer.UnregisterTunnel(ctx, gracePeriod.Nanoseconds())
 }
 
 func LogServerInfo(
 	promise tunnelrpc.ServerInfo_Promise,
 	connectionID uint8,
 	metrics *TunnelMetrics,
-	logger *log.Logger,
+	logger *log.Entry,
 ) {
 	serverInfoMessage, err := promise.Struct()
 	if err != nil {
@@ -436,36 +548,28 @@ func LogServerInfo(
 	metrics.registerServerLocation(uint8ToString(connectionID), serverInfo.LocationName)
 }
 
-func H1ResponseToH2Response(h1 *http.Response) (h2 []h2mux.Header) {
-	h2 = []h2mux.Header{{Name: ":status", Value: fmt.Sprintf("%d", h1.StatusCode)}}
-	for headerName, headerValues := range h1.Header {
-		for _, headerValue := range headerValues {
-			h2 = append(h2, h2mux.Header{Name: strings.ToLower(headerName), Value: headerValue})
-		}
-	}
-	return
-}
-
 type TunnelHandler struct {
-	originUrl  string
-	muxer      *h2mux.Muxer
-	httpClient http.RoundTripper
-	tlsConfig  *tls.Config
-	tags       []tunnelpogs.Tag
-	metrics    *TunnelMetrics
+	originUrl      string
+	httpHostHeader string
+	muxer          *h2mux.Muxer
+	httpClient     http.RoundTripper
+	tlsConfig      *tls.Config
+	tags           []tunnelpogs.Tag
+	metrics        *TunnelMetrics
 	// connectionID is only used by metrics, and prometheus requires labels to be string
 	connectionID      string
 	logger            *log.Logger
 	noChunkedEncoding bool
-}
 
-var dialer = net.Dialer{DualStack: true}
+	bufferPool *buffer.Pool
+}
 
 // NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
 func NewTunnelHandler(ctx context.Context,
 	config *TunnelConfig,
-	addr string,
+	addr *net.TCPAddr,
 	connectionID uint8,
+	bufferPool *buffer.Pool,
 ) (*TunnelHandler, string, error) {
 	originURL, err := validation.ValidateUrl(config.OriginUrl)
 	if err != nil {
@@ -473,6 +577,7 @@ func NewTunnelHandler(ctx context.Context,
 	}
 	h := &TunnelHandler{
 		originUrl:         originURL,
+		httpHostHeader:    config.HTTPHostHeader,
 		httpClient:        config.HTTPTransport,
 		tlsConfig:         config.ClientTlsConfig,
 		tags:              config.Tags,
@@ -480,41 +585,23 @@ func NewTunnelHandler(ctx context.Context,
 		connectionID:      uint8ToString(connectionID),
 		logger:            config.Logger,
 		noChunkedEncoding: config.NoChunkedEncoding,
+		bufferPool:        bufferPool,
 	}
 	if h.httpClient == nil {
 		h.httpClient = http.DefaultTransport
 	}
-	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
-	dialCtx, dialCancel := context.WithTimeout(ctx, dialTimeout)
-	// TUN-92: enforce a timeout on dial and handshake (as tls.Dial does not support one)
-	plaintextEdgeConn, err := dialer.DialContext(dialCtx, "tcp", addr)
-	dialCancel()
+
+	edgeConn, err := connection.DialEdge(ctx, dialTimeout, config.TlsConfig, addr)
 	if err != nil {
-		return nil, "", dialError{cause: errors.Wrap(err, "DialContext error")}
+		return nil, "", err
 	}
-	edgeConn := tls.Client(plaintextEdgeConn, config.TlsConfig)
-	edgeConn.SetDeadline(time.Now().Add(dialTimeout))
-	err = edgeConn.Handshake()
-	if err != nil {
-		return nil, "", dialError{cause: errors.Wrap(err, "Handshake with edge error")}
-	}
-	// clear the deadline on the conn; h2mux has its own timeouts
-	edgeConn.SetDeadline(time.Time{})
 	// Establish a muxed connection with the edge
 	// Client mux handshake with agent server
-	h.muxer, err = h2mux.Handshake(edgeConn, edgeConn, h2mux.MuxerConfig{
-		Timeout:            5 * time.Second,
-		Handler:            h,
-		IsClient:           true,
-		HeartbeatInterval:  config.HeartbeatInterval,
-		MaxHeartbeats:      config.MaxHeartbeats,
-		Logger:             config.TransportLogger.WithFields(log.Fields{}),
-		CompressionQuality: h2mux.CompressionSetting(config.CompressionQuality),
-	})
+	h.muxer, err = h2mux.Handshake(edgeConn, edgeConn, config.muxerConfig(h), h.metrics.activeStreams)
 	if err != nil {
-		return h, "", errors.New("TLS handshake error")
+		return nil, "", errors.Wrap(err, "h2mux handshake with edge error")
 	}
-	return h, edgeConn.LocalAddr().String(), err
+	return h, edgeConn.LocalAddr().String(), nil
 }
 
 func (h *TunnelHandler) AppendTagHeaders(r *http.Request) {
@@ -529,7 +616,7 @@ func (h *TunnelHandler) ServeStream(stream *h2mux.MuxedStream) error {
 
 	req, reqErr := h.createRequest(stream)
 	if reqErr != nil {
-		h.logError(stream, reqErr)
+		h.writeErrorResponse(stream, reqErr)
 		return reqErr
 	}
 
@@ -545,7 +632,7 @@ func (h *TunnelHandler) ServeStream(stream *h2mux.MuxedStream) error {
 		resp, respErr = h.serveHTTP(stream, req)
 	}
 	if respErr != nil {
-		h.logError(stream, respErr)
+		h.writeErrorResponse(stream, respErr)
 		return respErr
 	}
 	h.logResponseOk(resp, cfRay, lbProbe)
@@ -557,7 +644,7 @@ func (h *TunnelHandler) createRequest(stream *h2mux.MuxedStream) (*http.Request,
 	if err != nil {
 		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
 	}
-	err = streamhandler.H2RequestHeadersToH1Request(stream.Headers, req)
+	err = h2mux.H2RequestHeadersToH1Request(stream.Headers, req)
 	if err != nil {
 		return nil, errors.Wrap(err, "invalid request received")
 	}
@@ -566,18 +653,24 @@ func (h *TunnelHandler) createRequest(stream *h2mux.MuxedStream) (*http.Request,
 }
 
 func (h *TunnelHandler) serveWebsocket(stream *h2mux.MuxedStream, req *http.Request) (*http.Response, error) {
+	if h.httpHostHeader != "" {
+		req.Header.Set("Host", h.httpHostHeader)
+		req.Host = h.httpHostHeader
+	}
+
 	conn, response, err := websocket.ClientConnect(req, h.tlsConfig)
 	if err != nil {
 		return nil, err
 	}
 	defer conn.Close()
-	err = stream.WriteHeaders(H1ResponseToH2Response(response))
+	err = stream.WriteHeaders(h2mux.H1ResponseToH2ResponseHeaders(response))
 	if err != nil {
 		return nil, errors.Wrap(err, "Error writing response header")
 	}
 	// Copy to/from stream to the undelying connection. Use the underlying
 	// connection because cloudflared doesn't operate on the message themselves
 	websocket.Stream(conn.UnderlyingConn(), stream)
+
 	return response, nil
 }
 
@@ -594,13 +687,20 @@ func (h *TunnelHandler) serveHTTP(stream *h2mux.MuxedStream, req *http.Request)
 	// Request origin to keep connection alive to improve performance
 	req.Header.Set("Connection", "keep-alive")
 
+	if h.httpHostHeader != "" {
+		req.Header.Set("Host", h.httpHostHeader)
+		req.Host = h.httpHostHeader
+	}
+
 	response, err := h.httpClient.RoundTrip(req)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error proxying request to origin")
 	}
 	defer response.Body.Close()
 
-	err = stream.WriteHeaders(H1ResponseToH2Response(response))
+	headers := h2mux.H1ResponseToH2ResponseHeaders(response)
+	headers = append(headers, h2mux.CreateResponseMetaHeader(h2mux.ResponseMetaHeaderField, h2mux.ResponseSourceOrigin))
+	err = stream.WriteHeaders(headers)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error writing response header")
 	}
@@ -609,7 +709,9 @@ func (h *TunnelHandler) serveHTTP(stream *h2mux.MuxedStream, req *http.Request)
 	} else {
 		// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
 		// compression generates dictionary on first write
-		io.CopyBuffer(stream, response.Body, make([]byte, 512*1024))
+		buf := h.bufferPool.Get()
+		defer h.bufferPool.Put(buf)
+		io.CopyBuffer(stream, response.Body, buf)
 	}
 	return response, nil
 }
@@ -633,9 +735,12 @@ func (h *TunnelHandler) isEventStream(response *http.Response) bool {
 	return false
 }
 
-func (h *TunnelHandler) logError(stream *h2mux.MuxedStream, err error) {
+func (h *TunnelHandler) writeErrorResponse(stream *h2mux.MuxedStream, err error) {
 	h.logger.WithError(err).Error("HTTP request error")
-	stream.WriteHeaders([]h2mux.Header{{Name: ":status", Value: "502"}})
+	stream.WriteHeaders([]h2mux.Header{
+		{Name: ":status", Value: "502"},
+		h2mux.CreateResponseMetaHeader(h2mux.ResponseMetaHeaderField, h2mux.ResponseSourceCloudflared),
+	})
 	stream.Write([]byte("502 Bad Gateway"))
 	h.metrics.incrementResponses(h.connectionID, "502")
 }

originservice/originservice.go

@@ -12,11 +12,11 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/cloudflare/cloudflared/buffer"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/log"
 	"github.com/cloudflare/cloudflared/websocket"
-
 	"github.com/pkg/errors"
 )
 
@@ -33,6 +33,7 @@ type HTTPService struct {
 	client          http.RoundTripper
 	originURL       *url.URL
 	chunkedEncoding bool
+	bufferPool      *buffer.Pool
 }
 
 func NewHTTPService(transport http.RoundTripper, url *url.URL, chunkedEncoding bool) OriginService {
@@ -40,10 +41,13 @@ func NewHTTPService(transport http.RoundTripper, url *url.URL, chunkedEncoding b
 		client:          transport,
 		originURL:       url,
 		chunkedEncoding: chunkedEncoding,
+		bufferPool:      buffer.NewPool(512 * 1024),
 	}
 }
 
 func (hc *HTTPService) Proxy(stream *h2mux.MuxedStream, req *http.Request) (*http.Response, error) {
+	const responseSourceOrigin = "origin"
+
 	// Support for WSGI Servers by switching transfer encoding from chunked to gzip/deflate
 	if !hc.chunkedEncoding {
 		req.TransferEncoding = []string{"gzip", "deflate"}
@@ -62,7 +66,9 @@ func (hc *HTTPService) Proxy(stream *h2mux.MuxedStream, req *http.Request) (*htt
 	}
 	defer resp.Body.Close()
 
-	err = stream.WriteHeaders(h1ResponseToH2Response(resp))
+	responseHeaders := h1ResponseToH2Response(resp)
+	responseHeaders = append(responseHeaders, h2mux.CreateResponseMetaHeader(h2mux.ResponseMetaHeaderField, responseSourceOrigin))
+	err = stream.WriteHeaders(responseHeaders)
 	if err != nil {
 		return nil, errors.Wrap(err, "error writing response header to HTTP origin")
 	}
@@ -71,7 +77,9 @@ func (hc *HTTPService) Proxy(stream *h2mux.MuxedStream, req *http.Request) (*htt
 	} else {
 		// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
 		// compression generates dictionary on first write
-		io.CopyBuffer(stream, resp.Body, make([]byte, 512*1024))
+		buf := hc.bufferPool.Get()
+		defer hc.bufferPool.Put(buf)
+		io.CopyBuffer(stream, resp.Body, buf)
 	}
 	return resp, nil
 }
@@ -100,7 +108,7 @@ func NewWebSocketService(tlsConfig *tls.Config, url *url.URL) (OriginService, er
 	}
 	shutdownC := make(chan struct{})
 	go func() {
-		websocket.StartProxyServer(log.CreateLogger(), listener, url.String(), shutdownC)
+		websocket.StartProxyServer(log.CreateLogger(), listener, url.String(), shutdownC, websocket.DefaultStreamHandler)
 	}()
 	return &WebsocketService{
 		tlsConfig: tlsConfig,
@@ -142,10 +150,11 @@ func (wsc *WebsocketService) Shutdown() {
 
 // HelloWorldService talks to the hello world example origin
 type HelloWorldService struct {
-	client    http.RoundTripper
-	listener  net.Listener
-	originURL *url.URL
-	shutdownC chan struct{}
+	client     http.RoundTripper
+	listener   net.Listener
+	originURL  *url.URL
+	shutdownC  chan struct{}
+	bufferPool *buffer.Pool
 }
 
 func NewHelloWorldService(transport http.RoundTripper) (OriginService, error) {
@@ -164,7 +173,8 @@ func NewHelloWorldService(transport http.RoundTripper) (OriginService, error) {
 			Scheme: "https",
 			Host:   listener.Addr().String(),
 		},
-		shutdownC: shutdownC,
+		shutdownC:  shutdownC,
+		bufferPool: buffer.NewPool(512 * 1024),
 	}, nil
 }
 
@@ -184,7 +194,9 @@ func (hwc *HelloWorldService) Proxy(stream *h2mux.MuxedStream, req *http.Request
 
 	// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
 	// compression generates dictionary on first write
-	io.CopyBuffer(stream, resp.Body, make([]byte, 512*1024))
+	buf := hwc.bufferPool.Get()
+	defer hwc.bufferPool.Put(buf)
+	io.CopyBuffer(stream, resp.Body, buf)
 
 	return resp, nil
 }

overwatch/app_manager.go

@@ -0,0 +1,53 @@
+package overwatch
+
+// AppManager is the default implementation of overwatch service management
+type AppManager struct {
+	services  map[string]Service
+	errorChan chan error
+}
+
+// NewAppManager creates a new overwatch manager
+func NewAppManager(errorChan chan error) Manager {
+	return &AppManager{services: make(map[string]Service), errorChan: errorChan}
+}
+
+// Add takes in a new service to manage.
+// It stops the service if it already exist in the manager and is running
+// It then starts the newly added service
+func (m *AppManager) Add(service Service) {
+	// check for existing service
+	if currentService, ok := m.services[service.Name()]; ok {
+		if currentService.Hash() == service.Hash() {
+			return // the exact same service, no changes, so move along
+		}
+		currentService.Shutdown() //shutdown the listener since a new one is starting
+	}
+	m.services[service.Name()] = service
+
+	//start the service!
+	go m.serviceRun(service)
+}
+
+// Remove shutdowns the service by name and removes it from its current management list
+func (m *AppManager) Remove(name string) {
+	if currentService, ok := m.services[name]; ok {
+		currentService.Shutdown()
+	}
+	delete(m.services, name)
+}
+
+// Services returns all the current Services being managed
+func (m *AppManager) Services() []Service {
+	values := []Service{}
+	for _, value := range m.services {
+		values = append(values, value)
+	}
+	return values
+}
+
+func (m *AppManager) serviceRun(service Service) {
+	err := service.Run()
+	if err != nil && m.errorChan != nil {
+		m.errorChan <- err
+	}
+}

overwatch/manager.go

@@ -0,0 +1,17 @@
+package overwatch
+
+// Service is the required functions for an object to be managed by the overwatch Manager
+type Service interface {
+	Name() string
+	Type() string
+	Hash() string
+	Shutdown()
+	Run() error
+}
+
+// Manager is based type to manage running services
+type Manager interface {
+	Add(Service)
+	Remove(string)
+	Services() []Service
+}