Release 2019.11.1

Dockerfile

@@ -1,28 +1,12 @@
-# use a builder image for building cloudflare
-FROM golang:1.13.3 as builder
-ENV GO111MODULE=on
-ENV CGO_ENABLED=0
-ENV GOOS=linux
-
+FROM golang:1.12 as builder
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
-
-# copy our sources into the builder image
+RUN apt-get update && apt-get install -y --no-install-recommends upx
+# Run after `apt-get update` to improve rebuild scenarios
 COPY . .
-
-# compile cloudflared
 RUN make cloudflared
+RUN upx --no-progress cloudflared
 
-# ---
-
-# use a distroless base image with glibc
-FROM gcr.io/distroless/base-debian10:nonroot
-
-# copy our compiled binary
-COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
-
-# run as non-privileged user
-USER nonroot
-
-# command / entrypoint of container
+FROM gcr.io/distroless/base
+COPY --from=builder /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
 ENTRYPOINT ["cloudflared", "--no-autoupdate"]
 CMD ["version"]

RELEASE_NOTES

@@ -1,29 +1,3 @@
-2019.12.0
-- 2019-12-11 TUN-2631: only notify that activeStreamMap is closed if ignoreNewStreams=true
-- 2019-12-17 bug(cloudflared): Set the MaxIdleConnsPerHost of http.Transport to proxy-keepalive-connections (#155)
-- 2019-12-17 refactor(docker): optimize Dockerfile (#126)
-- 2019-12-19 Fix timer scheduling for systemd update service (#159)
-- 2019-12-13 TUN-2637: Manage edge IPs in a region-aware manner
-- 2019-12-03 bug(cloudflared): nil pointer deference on h2DictWriter Close() (#154)
-- 2019-12-03 TUN-2608: h2mux.Muxer.Shutdown always returns a non-nil channel
-- 2019-12-04 TUN-2555: origin/supervisor.go calls Authenticate
-- 2019-12-06 TUN-2554: cloudflared calls ReconnectTunnel
-- 2019-11-20 TUN-2575: Constructors + simpler conversions for AuthOutcome
-- 2019-11-22 Fix Docker build failure (#149)
-- 2019-11-21 TUN-2573: Refactor TunnelRegistration into PermanentRegistrationError, RetryableRegistrationError and SuccessfulTunnelRegistration
-- 2019-11-22 TUN-2582: EventDigest field in tunnelrpc
-- 2019-11-22 Fix "happy eyeballs" not being disabled since Golang 1.12 upgrade * The Dialer.DualStack setting is now ignored and deprecated; RFC 6555 Fast Fallback ("Happy Eyeballs") is now enabled by default. To disable, set Dialer.FallbackDelay to a negative value.
-- 2019-11-25 TUN-2591: ReconnectTunnel now sends EventDigest
-- 2019-11-21 TUN-2606: add DialEdge helpers
-- 2019-11-21 TUN-2607: add RPC stream helpers
-
-2019.11.3
-- 2019-11-20 TUN-2562: Update Cloudflare Origin CA RSA root
-
-2019.11.2
-- 2019-11-18 TUN-2567: AuthOutcome can be turned back into AuthResponse
-- 2019-11-18 TUN-2563: Exposes config_version metrics
-
 2019.11.1
 - 2019-11-12 Add db-connect, a SQL over HTTPS server
 - 2019-11-12 TUN-2053: Add a /healthcheck endpoint to the metrics server

cmd/cloudflared/linux_service.go

@@ -73,7 +73,7 @@ ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 64 ]; then s
 Description=Update Argo Tunnel
 
 [Timer]
-OnCalendar=daily
+OnUnitActiveSec=1d
 
 [Install]
 WantedBy=timers.target

cmd/cloudflared/tunnel/cmd.go

@@ -977,12 +977,6 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_INTENT"},
 			Hidden:  true,
 		}),
-		altsrc.NewBoolFlag(&cli.BoolFlag{
-			Name:    "use-reconnect-token",
-			Usage:   "Test reestablishing connections with the new 'reconnect token' flow.",
-			EnvVars: []string{"TUNNEL_USE_RECONNECT_TOKEN"},
-			Hidden:  true,
-		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:    "dial-edge-timeout",
 			Usage:   "Maximum wait time to set up a connection with the edge",
@@ -1050,6 +1044,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Usage:   "Absolute path of directory to save SSH host keys in",
 			EnvVars: []string{"HOST_KEY_PATH"},
 			Hidden:  true,
+
 		}),
 	}
-}
+}

cmd/cloudflared/tunnel/configuration.go

@@ -197,21 +197,17 @@ func prepareTunnelConfig(
 	httpTransport := &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
 		MaxIdleConns:          c.Int("proxy-keepalive-connections"),
-		MaxIdleConnsPerHost:   c.Int("proxy-keepalive-connections"),
 		IdleConnTimeout:       c.Duration("proxy-keepalive-timeout"),
 		TLSHandshakeTimeout:   c.Duration("proxy-tls-timeout"),
 		ExpectContinueTimeout: 1 * time.Second,
 		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: c.IsSet("no-tls-verify")},
 	}
 
-	dialer := &net.Dialer{
+	dialContext := (&net.Dialer{
 		Timeout:   c.Duration("proxy-connect-timeout"),
 		KeepAlive: c.Duration("proxy-tcp-keepalive"),
-	}
-	if c.Bool("proxy-no-happy-eyeballs") {
-		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
-	}
-	dialContext := dialer.DialContext
+		DualStack: !c.Bool("proxy-no-happy-eyeballs"),
+	}).DialContext
 
 	if c.IsSet("unix-socket") {
 		unixSocket, err := config.ValidateUnixSocket(c)
@@ -276,7 +272,6 @@ func prepareTunnelConfig(
 		TlsConfig:            toEdgeTLSConfig,
 		TransportLogger:      transportLogger,
 		UseDeclarativeTunnel: c.Bool("use-declarative-tunnels"),
-		UseReconnectToken:    c.Bool("use-reconnect-token"),
 	}, nil
 }
 

connection/connection.go

@@ -5,25 +5,35 @@ import (
 	"net"
 	"time"
 
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	rpc "zombiezen.com/go/capnproto2/rpc"
 )
 
 const (
 	openStreamTimeout = 30 * time.Second
 )
 
+type dialError struct {
+	cause error
+}
+
+func (e dialError) Error() string {
+	return e.cause.Error()
+}
+
 type Connection struct {
 	id    uuid.UUID
 	muxer *h2mux.Muxer
-	addr  *net.TCPAddr
 }
 
-func newConnection(muxer *h2mux.Muxer, addr *net.TCPAddr) (*Connection, error) {
+func newConnection(muxer *h2mux.Muxer, edgeIP *net.TCPAddr) (*Connection, error) {
 	id, err := uuid.NewRandom()
 	if err != nil {
 		return nil, err
@@ -31,7 +41,6 @@ func newConnection(muxer *h2mux.Muxer, addr *net.TCPAddr) (*Connection, error) {
 	return &Connection{
 		id:    id,
 		muxer: muxer,
-		addr:  addr,
 	}, nil
 }
 
@@ -41,15 +50,32 @@ func (c *Connection) Serve(ctx context.Context) error {
 }
 
 // Connect is used to establish connections with cloudflare's edge network
-func (c *Connection) Connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters, logger *logrus.Entry) (tunnelpogs.ConnectResult, error) {
-	tsClient, err := NewRPCClient(ctx, c.muxer, logger.WithField("rpc", "connect"), openStreamTimeout)
+func (c *Connection) Connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters, logger *logrus.Entry) (pogs.ConnectResult, error) {
+	openStreamCtx, cancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer cancel()
+
+	rpcConn, err := c.newRPConn(openStreamCtx, logger)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot create new RPC connection")
 	}
-	defer tsClient.Close()
+	defer rpcConn.Close()
+
+	tsClient := tunnelpogs.TunnelServer_PogsClient{Client: rpcConn.Bootstrap(ctx)}
+
 	return tsClient.Connect(ctx, parameters)
 }
 
 func (c *Connection) Shutdown() {
 	c.muxer.Shutdown()
 }
+
+func (c *Connection) newRPConn(ctx context.Context, logger *logrus.Entry) (*rpc.Conn, error) {
+	stream, err := c.muxer.OpenRPCStream(ctx)
+	if err != nil {
+		return nil, err
+	}
+	return rpc.NewConn(
+		tunnelrpc.NewTransportLogger(logger.WithField("rpc", "connect"), rpc.StreamTransport(stream)),
+		tunnelrpc.ConnLog(logger.WithField("rpc", "connect")),
+	), nil
+}

connection/dial.go

@@ -1,54 +0,0 @@
-package connection
-
-import (
-	"context"
-	"crypto/tls"
-	"net"
-	"time"
-
-	"github.com/pkg/errors"
-)
-
-// DialEdge makes a TLS connection to a Cloudflare edge node
-func DialEdge(
-	ctx context.Context,
-	timeout time.Duration,
-	tlsConfig *tls.Config,
-	edgeTCPAddr *net.TCPAddr,
-) (net.Conn, error) {
-	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
-	dialCtx, dialCancel := context.WithTimeout(ctx, timeout)
-	defer dialCancel()
-
-	dialer := net.Dialer{}
-	edgeConn, err := dialer.DialContext(dialCtx, "tcp", edgeTCPAddr.String())
-	if err != nil {
-		return nil, newDialError(err, "DialContext error")
-	}
-	tlsEdgeConn := tls.Client(edgeConn, tlsConfig)
-	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
-
-	if err = tlsEdgeConn.Handshake(); err != nil {
-		return nil, newDialError(err, "Handshake with edge error")
-	}
-	// clear the deadline on the conn; h2mux has its own timeouts
-	tlsEdgeConn.SetDeadline(time.Time{})
-	return tlsEdgeConn, nil
-}
-
-// DialError is an error returned from DialEdge
-type DialError struct {
-	cause error
-}
-
-func newDialError(err error, message string) error {
-	return DialError{cause: errors.Wrap(err, message)}
-}
-
-func (e DialError) Error() string {
-	return e.cause.Error()
-}
-
-func (e DialError) Cause() error {
-	return e.cause
-}

connection/discovery.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"math/rand"
 	"net"
 	"sync"
 	"time"
@@ -27,28 +26,8 @@ const (
 
 	// SRV record resolution TTL
 	resolveEdgeAddrTTL = 1 * time.Hour
-
-	subsystemEdgeAddrResolver = "edgeAddrResolver"
 )
 
-// Redeclare network functions so they can be overridden in tests.
-var (
-	netLookupSRV = net.LookupSRV
-	netLookupIP  = net.LookupIP
-)
-
-// If the call to net.LookupSRV fails, try to fall back to DoT from Cloudflare directly.
-//
-// Note: Instead of DoT, we could also have used DoH. Either of these:
-//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
-//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
-// But both of these cases miss out on a key feature from the stdlib:
-//     "The returned records are sorted by priority and randomized by weight within a priority."
-//     (https://golang.org/pkg/net/#Resolver.LookupSRV)
-// Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it.
-// See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552
-var fallbackLookupSRV = lookupSRVWithDOT
-
 var friendlyDNSErrorLines = []string{
 	`Please try the following things to diagnose this issue:`,
 	`  1. ensure that argotunnel.com is returning "origintunneld" service records.`,
@@ -61,26 +40,11 @@ var friendlyDNSErrorLines = []string{
 
 // EdgeServiceDiscoverer is an interface for looking up Cloudflare's edge network addresses
 type EdgeServiceDiscoverer interface {
-	// Addr returns an unused address to connect to cloudflare's edge network.
-	// Before this method returns, the address will be removed from the pool of available addresses,
-	// so the caller can assume they have exclusive access to the address for tunneling purposes.
-	// The caller should remember to put it back via ReplaceAddr or MarkAddrBad.
-	Addr() (*net.TCPAddr, error)
-	// AnyAddr returns an address to connect to cloudflare's edge network.
-	// It may or may not be in active use for a tunnel.
-	// The caller should NOT return it via ReplaceAddr or MarkAddrBad!
-	AnyAddr() (*net.TCPAddr, error)
-	// ReplaceAddr is called when the address is no longer needed, e.g. due to a scaling-down of numHAConnections.
-	// It returns the address to the pool of available addresses.
-	ReplaceAddr(addr *net.TCPAddr)
-	// MarkAddrBad is called when there was a connectivity error for the address.
-	// It marks the address as unused but doesn't return it to the pool of available addresses.
-	MarkAddrBad(addr *net.TCPAddr)
-	// AvailableAddrs returns the number of addresses available for use
-	// (less those that have been marked bad).
-	AvailableAddrs() int
-	// Refresh rediscovers Cloudflare's edge network addresses.
-	// It resets the state of "bad" addresses but not those in active use.
+	// Addr returns an address to connect to cloudflare's edge network
+	Addr() *net.TCPAddr
+	// AvailableAddrs returns the number of unique addresses
+	AvailableAddrs() uint8
+	// Refresh rediscover Cloudflare's edge network addresses
 	Refresh() error
 }
 
@@ -88,31 +52,16 @@ type EdgeServiceDiscoverer interface {
 // It implements EdgeServiceDiscoverer interface
 type EdgeAddrResolver struct {
 	sync.Mutex
-	// HA regions
-	regions []*region
-	// Logger for noteworthy events
-	logger *logrus.Entry
-}
-
-type region struct {
-	// Addresses that we expect will be in active use
+	// Addrs to connect to cloudflare's edge network
 	addrs []*net.TCPAddr
-	// Addresses that are in active use.
-	// This is actually a set of net.TCPAddr's, but we can't make a map like
-	//     map[net.TCPAddr]bool
-	// since net.TCPAddr contains a field of type net.IP and therefore it cannot be used as a map key.
-	// So instead we use map[string]*net.TCPAddr, where the keys are obtained by net.TCPAddr.String().
-	// (We keep the "raw" *net.TCPAddr values for the convenience of AnyAddr(). If that method didn't
-	// exist, we wouldn't strictly need the values, and this could be a map[string]bool.)
-	inUse map[string]*net.TCPAddr
-	// Addresses that were discarded due to a network error.
-	// Not sure what we'll do with these, but it feels good to keep them around for now.
-	bad []*net.TCPAddr
+	// index of the next element to use in addrs
+	nextAddrIndex int
+	logger        *logrus.Entry
 }
 
 func NewEdgeAddrResolver(logger *logrus.Logger) (EdgeServiceDiscoverer, error) {
 	r := &EdgeAddrResolver{
-		logger: logger.WithField("subsystem", subsystemEdgeAddrResolver),
+		logger: logger.WithField("subsystem", " edgeAddrResolver"),
 	}
 	if err := r.Refresh(); err != nil {
 		return nil, err
@@ -120,153 +69,83 @@ func NewEdgeAddrResolver(logger *logrus.Logger) (EdgeServiceDiscoverer, error) {
 	return r, nil
 }
 
-func (r *EdgeAddrResolver) Addr() (*net.TCPAddr, error) {
+func (r *EdgeAddrResolver) Addr() *net.TCPAddr {
 	r.Lock()
 	defer r.Unlock()
-
-	// compute the largest region based on len(addrs)
-	var largestRegion *region
-	{
-		if len(r.regions) == 0 {
-			return nil, errors.New("No HA regions")
-		}
-		largestRegion = r.regions[0]
-		for _, region := range r.regions[1:] {
-			if len(region.addrs) > len(largestRegion.addrs) {
-				largestRegion = region
-			}
-		}
-		if len(largestRegion.addrs) == 0 {
-			return nil, errors.New("No IP address to claim")
-		}
-	}
-
-	var addr *net.TCPAddr
-	addr, largestRegion.addrs = popAddr(largestRegion.addrs)
-	largestRegion.inUse[addr.String()] = addr
-	return addr, nil
+	addr := r.addrs[r.nextAddrIndex]
+	r.nextAddrIndex = (r.nextAddrIndex + 1) % len(r.addrs)
+	return addr
 }
 
-func (r *EdgeAddrResolver) AnyAddr() (*net.TCPAddr, error) {
+func (r *EdgeAddrResolver) AvailableAddrs() uint8 {
 	r.Lock()
 	defer r.Unlock()
-	for _, region := range r.regions {
-		// return an unused addr
-		if len(region.addrs) > 0 {
-			return region.addrs[rand.Intn(len(region.addrs))], nil
-		}
-		// return an addr that's in use
-		for _, addr := range region.inUse {
-			return addr, nil
-		}
-	}
-	return nil, fmt.Errorf("No IP addresses")
-}
-
-func (r *EdgeAddrResolver) ReplaceAddr(addr *net.TCPAddr) {
-	r.Lock()
-	defer r.Unlock()
-	addrString := addr.String()
-	for _, region := range r.regions {
-		if _, ok := region.inUse[addrString]; ok {
-			delete(region.inUse, addrString)
-			region.addrs = append(region.addrs, addr)
-			break
-		}
-	}
-}
-
-func (r *EdgeAddrResolver) MarkAddrBad(addr *net.TCPAddr) {
-	r.Lock()
-	defer r.Unlock()
-	addrString := addr.String()
-	for _, region := range r.regions {
-		if _, ok := region.inUse[addrString]; ok {
-			delete(region.inUse, addrString)
-			region.bad = append(region.bad, addr)
-			break
-		}
-	}
-}
-
-func (r *EdgeAddrResolver) AvailableAddrs() int {
-	r.Lock()
-	defer r.Unlock()
-	result := 0
-	for _, region := range r.regions {
-		result += len(region.addrs)
-	}
-	return result
+	return uint8(len(r.addrs))
 }
 
 func (r *EdgeAddrResolver) Refresh() error {
-	addrLists, err := EdgeDiscovery(r.logger)
+	newAddrs, err := EdgeDiscovery(r.logger)
 	if err != nil {
 		return err
 	}
-
 	r.Lock()
 	defer r.Unlock()
-	inUse := allInUse(r.regions)
-	r.regions = makeHARegions(addrLists, inUse)
+	r.addrs = newAddrs
+	r.nextAddrIndex = 0
 	return nil
 }
 
-// EdgeDiscovery implements HA service discovery lookup.
-func EdgeDiscovery(logger *logrus.Entry) ([][]*net.TCPAddr, error) {
-	_, addrs, err := netLookupSRV(srvService, srvProto, srvName)
+// HA service discovery lookup
+func EdgeDiscovery(logger *logrus.Entry) ([]*net.TCPAddr, error) {
+	_, addrs, err := net.LookupSRV(srvService, srvProto, srvName)
 	if err != nil {
-		_, fallbackAddrs, fallbackErr := fallbackLookupSRV(srvService, srvProto, srvName)
+		// Try to fall back to DoT from Cloudflare directly.
+		//
+		// Note: Instead of DoT, we could also have used DoH. Either of these:
+		//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
+		//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
+		// But both of these cases miss out on a key feature from the stdlib:
+		//     "The returned records are sorted by priority and randomized by weight within a priority."
+		//     (https://golang.org/pkg/net/#Resolver.LookupSRV)
+		// Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it.
+		// See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552
+		r := fallbackResolver(dotServerName, dotServerAddr)
+		ctx, cancel := context.WithTimeout(context.Background(), dotTimeout)
+		defer cancel()
+		_, fallbackAddrs, fallbackErr := r.LookupSRV(ctx, srvService, srvProto, srvName)
 		if fallbackErr != nil || len(fallbackAddrs) == 0 {
 			// use the original DNS error `err` in messages, not `fallbackErr`
 			logger.Errorln("Error looking up Cloudflare edge IPs: the DNS query failed:", err)
 			for _, s := range friendlyDNSErrorLines {
 				logger.Errorln(s)
 			}
-			return nil, errors.Wrapf(err, "Could not lookup srv records on _%v._%v.%v", srvService, srvProto, srvName)
+			return nil, errors.Wrap(err, "Could not lookup srv records on _origintunneld._tcp.argotunnel.com")
 		}
 		// Accept the fallback results and keep going
 		addrs = fallbackAddrs
 	}
-
 	var resolvedIPsPerCNAME [][]*net.TCPAddr
+	var lookupErr error
 	for _, addr := range addrs {
 		ips, err := resolveSRVToTCP(addr)
-		if err != nil {
-			return nil, err
+		if err != nil || len(ips) == 0 {
+			// don't return early, we might be able to resolve other addresses
+			lookupErr = err
+			continue
 		}
 		resolvedIPsPerCNAME = append(resolvedIPsPerCNAME, ips)
 	}
-
-	return resolvedIPsPerCNAME, nil
-}
-
-func lookupSRVWithDOT(service, proto, name string) (cname string, addrs []*net.SRV, err error) {
-	// Inspiration: https://github.com/artyom/dot/blob/master/dot.go
-	r := &net.Resolver{
-		PreferGo: true,
-		Dial: func(ctx context.Context, _ string, _ string) (net.Conn, error) {
-			var dialer net.Dialer
-			conn, err := dialer.DialContext(ctx, "tcp", dotServerAddr)
-			if err != nil {
-				return nil, err
-			}
-			tlsConfig := &tls.Config{ServerName: dotServerName}
-			return tls.Client(conn, tlsConfig), nil
-		},
+	ips := flattenServiceIPs(resolvedIPsPerCNAME)
+	if lookupErr == nil && len(ips) == 0 {
+		return nil, fmt.Errorf("Unknown service discovery error")
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), dotTimeout)
-	defer cancel()
-	return r.LookupSRV(ctx, srvService, srvProto, srvName)
+	return ips, lookupErr
 }
 
 func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
-	ips, err := netLookupIP(srv.Target)
+	ips, err := net.LookupIP(srv.Target)
 	if err != nil {
-		return nil, errors.Wrapf(err, "Couldn't resolve SRV record %v", srv)
-	}
-	if len(ips) == 0 {
-		return nil, fmt.Errorf("SRV record %v had no IPs", srv)
+		return nil, err
 	}
 	addrs := make([]*net.TCPAddr, len(ips))
 	for i, ip := range ips {
@@ -275,6 +154,43 @@ func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
 	return addrs, nil
 }
 
+// FlattenServiceIPs transposes and flattens the input slices such that the
+// first element of the n inner slices are the first n elements of the result.
+func flattenServiceIPs(ipsByService [][]*net.TCPAddr) []*net.TCPAddr {
+	var result []*net.TCPAddr
+	for len(ipsByService) > 0 {
+		filtered := ipsByService[:0]
+		for _, ips := range ipsByService {
+			if len(ips) == 0 {
+				// sanity check
+				continue
+			}
+			result = append(result, ips[0])
+			if len(ips) > 1 {
+				filtered = append(filtered, ips[1:])
+			}
+		}
+		ipsByService = filtered
+	}
+	return result
+}
+
+// Inspiration: https://github.com/artyom/dot/blob/master/dot.go
+func fallbackResolver(serverName, serverAddress string) *net.Resolver {
+	return &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, _ string, _ string) (net.Conn, error) {
+			var dialer net.Dialer
+			conn, err := dialer.DialContext(ctx, "tcp", serverAddress)
+			if err != nil {
+				return nil, err
+			}
+			tlsConfig := &tls.Config{ServerName: serverName}
+			return tls.Client(conn, tlsConfig), nil
+		},
+	}
+}
+
 // EdgeHostnameResolver discovers the addresses of Cloudflare's edge network via a list of server hostnames.
 // It implements EdgeServiceDiscoverer interface, and is used mainly for testing connectivity.
 type EdgeHostnameResolver struct {
@@ -283,20 +199,13 @@ type EdgeHostnameResolver struct {
 	hostnames []string
 	// Addrs to connect to cloudflare's edge network
 	addrs []*net.TCPAddr
-	// Addresses that are in active use.
-	// This is actually a set of net.TCPAddr's. We have to encode the keys
-	// with .String(), since net.TCPAddr contains a field of type net.IP and
-	// therefore it cannot be used as a map key
-	inUse map[string]*net.TCPAddr
-	// Addresses that were discarded due to a network error.
-	// Not sure what we'll do with these, but it feels good to keep them around for now.
-	bad []*net.TCPAddr
+	// index of the next element to use in addrs
+	nextAddrIndex int
 }
 
 func NewEdgeHostnameResolver(edgeHostnames []string) (EdgeServiceDiscoverer, error) {
 	r := &EdgeHostnameResolver{
 		hostnames: edgeHostnames,
-		inUse:     map[string]*net.TCPAddr{},
 	}
 	if err := r.Refresh(); err != nil {
 		return nil, err
@@ -304,49 +213,18 @@ func NewEdgeHostnameResolver(edgeHostnames []string) (EdgeServiceDiscoverer, err
 	return r, nil
 }
 
-func (r *EdgeHostnameResolver) Addr() (*net.TCPAddr, error) {
+func (r *EdgeHostnameResolver) Addr() *net.TCPAddr {
 	r.Lock()
 	defer r.Unlock()
-	if len(r.addrs) == 0 {
-		return nil, errors.New("No IP address to claim")
-	}
-	var addr *net.TCPAddr
-	addr, r.addrs = popAddr(r.addrs)
-	r.inUse[addr.String()] = addr
-	return addr, nil
+	addr := r.addrs[r.nextAddrIndex]
+	r.nextAddrIndex = (r.nextAddrIndex + 1) % len(r.addrs)
+	return addr
 }
 
-func (r *EdgeHostnameResolver) AnyAddr() (*net.TCPAddr, error) {
+func (r *EdgeHostnameResolver) AvailableAddrs() uint8 {
 	r.Lock()
 	defer r.Unlock()
-	// return an unused addr
-	if len(r.addrs) > 0 {
-		return r.addrs[rand.Intn(len(r.addrs))], nil
-	}
-	// return an addr that's in use
-	for _, addr := range r.inUse {
-		return addr, nil
-	}
-	return nil, errors.New("No IP addresses")
-}
-
-func (r *EdgeHostnameResolver) ReplaceAddr(addr *net.TCPAddr) {
-	r.Lock()
-	defer r.Unlock()
-	delete(r.inUse, addr.String())
-	r.addrs = append(r.addrs, addr)
-}
-func (r *EdgeHostnameResolver) MarkAddrBad(addr *net.TCPAddr) {
-	r.Lock()
-	defer r.Unlock()
-	delete(r.inUse, addr.String())
-	r.bad = append(r.bad, addr)
-}
-
-func (r *EdgeHostnameResolver) AvailableAddrs() int {
-	r.Lock()
-	defer r.Unlock()
-	return len(r.addrs)
+	return uint8(len(r.addrs))
 }
 
 func (r *EdgeHostnameResolver) Refresh() error {
@@ -356,14 +234,8 @@ func (r *EdgeHostnameResolver) Refresh() error {
 	}
 	r.Lock()
 	defer r.Unlock()
-	var notInUse []*net.TCPAddr
-	for _, newAddr := range newAddrs {
-		if _, ok := r.inUse[newAddr.String()]; !ok {
-			notInUse = append(notInUse, newAddr)
-		}
-	}
-	r.addrs = notInUse
-	r.bad = nil
+	r.addrs = newAddrs
+	r.nextAddrIndex = 0
 	return nil
 }
 
@@ -380,41 +252,3 @@ func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) {
 	}
 	return tcpAddrs, nil
 }
-
-// Compute total set of IP addresses in use. This is useful if the regions
-// are returned in a different order, or if an IP address is assigned to
-// a different region for some reasion.
-func allInUse(regions []*region) map[string]*net.TCPAddr {
-	result := make(map[string]*net.TCPAddr)
-	for _, region := range regions {
-		for k, v := range region.inUse {
-			result[k] = v
-		}
-	}
-	return result
-}
-
-func makeHARegions(addrLists [][]*net.TCPAddr, inUse map[string]*net.TCPAddr) (regions []*region) {
-	for _, addrList := range addrLists {
-		region := &region{inUse: map[string]*net.TCPAddr{}}
-		for _, addr := range addrList {
-			addrString := addr.String()
-			// No matter what region `addr` used to belong to, it's now a part
-			// of this region, so add it to this region's `inUse` map.
-			if _, ok := inUse[addrString]; ok {
-				region.inUse[addrString] = addr
-			} else {
-				region.addrs = append(region.addrs, addr)
-			}
-		}
-		regions = append(regions, region)
-	}
-	return
-}
-
-func popAddr(addrs []*net.TCPAddr) (*net.TCPAddr, []*net.TCPAddr) {
-	first := addrs[0]
-	addrs[0] = nil // prevent memory leak
-	addrs = addrs[1:]
-	return first, addrs
-}

connection/discovery_test.go

@@ -2,316 +2,62 @@ package connection
 
 import (
 	"net"
-	"sync"
 	"testing"
-	"testing/quick"
-	"time"
 
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 )
 
-func TestEdgeDiscovery(t *testing.T) {
-	mockAddrs := newMockAddrs(19, 2, 5)
-	netLookupSRV = mockNetLookupSRV(mockAddrs)
-	netLookupIP = mockNetLookupIP(mockAddrs)
-
-	expectedAddrSet := map[string]bool{}
-	for _, addrs := range mockAddrs.addrMap {
-		for _, addr := range addrs {
-			expectedAddrSet[addr.String()] = true
-		}
-	}
-
-	addrLists, err := EdgeDiscovery(logrus.New().WithFields(logrus.Fields{}))
-	assert.NoError(t, err)
-	actualAddrSet := map[string]bool{}
-	for _, addrs := range addrLists {
-		for _, addr := range addrs {
-			actualAddrSet[addr.String()] = true
-		}
-	}
-
-	assert.Equal(t, expectedAddrSet, actualAddrSet)
+type mockEdgeServiceDiscoverer struct {
 }
 
-func TestAllInUse(t *testing.T) {
-	for _, testCase := range []struct {
-		regions  []*region
-		expected map[string]*net.TCPAddr
-	}{
-		{
-			regions:  nil,
-			expected: map[string]*net.TCPAddr{},
+func (mr *mockEdgeServiceDiscoverer) Addr() *net.TCPAddr {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("127.0.0.1"),
+		Port: 63102,
+	}
+}
+
+func (mr *mockEdgeServiceDiscoverer) AvailableAddrs() uint8 {
+	return 1
+}
+
+func (mr *mockEdgeServiceDiscoverer) Refresh() error {
+	return nil
+}
+
+func TestFlattenServiceIPs(t *testing.T) {
+	result := flattenServiceIPs([][]*net.TCPAddr{
+		[]*net.TCPAddr{
+			&net.TCPAddr{Port: 1},
+			&net.TCPAddr{Port: 2},
+			&net.TCPAddr{Port: 3},
+			&net.TCPAddr{Port: 4},
 		},
-		{
-			regions: []*region{
-				&region{inUse: map[string]*net.TCPAddr{}},
-				&region{inUse: map[string]*net.TCPAddr{}},
-			},
-			expected: map[string]*net.TCPAddr{},
+		[]*net.TCPAddr{
+			&net.TCPAddr{Port: 10},
+			&net.TCPAddr{Port: 12},
+			&net.TCPAddr{Port: 13},
 		},
-		{
-			regions: []*region{
-				&region{inUse: map[string]*net.TCPAddr{":1": &net.TCPAddr{Port: 1}}},
-				&region{inUse: map[string]*net.TCPAddr{":4": &net.TCPAddr{Port: 4}}},
-			},
-			expected: map[string]*net.TCPAddr{":1": &net.TCPAddr{Port: 1}, ":4": &net.TCPAddr{Port: 4}},
+		[]*net.TCPAddr{
+			&net.TCPAddr{Port: 21},
+			&net.TCPAddr{Port: 22},
+			&net.TCPAddr{Port: 23},
+			&net.TCPAddr{Port: 24},
+			&net.TCPAddr{Port: 25},
 		},
-	} {
-		actual := allInUse(testCase.regions)
-		assert.Equal(t, testCase.expected, actual)
-	}
-}
-
-func TestMakeRegions(t *testing.T) {
-	for _, testCase := range []struct {
-		addrList [][]*net.TCPAddr
-		inUse    map[string]*net.TCPAddr
-		expected []*region
-	}{
-		{
-			addrList: [][]*net.TCPAddr{},
-			expected: nil,
-		},
-		{
-			addrList: [][]*net.TCPAddr{
-				[]*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}},
-			},
-			expected: []*region{
-				&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}}, inUse: map[string]*net.TCPAddr{}},
-			},
-		},
-		{
-			addrList: [][]*net.TCPAddr{
-				[]*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}},
-				[]*net.TCPAddr{&net.TCPAddr{Port: 3}, &net.TCPAddr{Port: 4}},
-			},
-			expected: []*region{
-				&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}}, inUse: map[string]*net.TCPAddr{}},
-				&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 3}, &net.TCPAddr{Port: 4}}, inUse: map[string]*net.TCPAddr{}},
-			},
-		},
-		{
-			addrList: [][]*net.TCPAddr{
-				[]*net.TCPAddr{&net.TCPAddr{Port: 1}, &net.TCPAddr{Port: 2}},
-				[]*net.TCPAddr{&net.TCPAddr{Port: 3}, &net.TCPAddr{Port: 4}},
-			},
-			inUse: map[string]*net.TCPAddr{
-				":1": &net.TCPAddr{Port: 1},
-				":4": &net.TCPAddr{Port: 4},
-			},
-			expected: []*region{
-				&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 2}}, inUse: map[string]*net.TCPAddr{":1": &net.TCPAddr{Port: 1}}},
-				&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 3}}, inUse: map[string]*net.TCPAddr{":4": &net.TCPAddr{Port: 4}}},
-			},
-		},
-	} {
-		actual := makeHARegions(testCase.addrList, testCase.inUse)
-		assert.Equal(t, testCase.expected, actual)
-	}
-}
-
-func assertIsBalanced(t *testing.T, regions []*region) bool {
-	// Compute max(len(region.addrs) for region in regions)
-	// No region should have significantly fewer addresses than this
-	var longestAddrs int
-	{
-		longestAddrs = 0
-		for _, region := range regions {
-			if l := len(region.addrs); l > longestAddrs {
-				longestAddrs = l
-			}
-		}
-	}
-	for _, region := range regions {
-		if len(region.addrs) == longestAddrs || len(region.addrs) == longestAddrs-1 {
-			continue
-		}
-		return assert.Fail(t,
-			"found a region with %v free addrs, while the longest addrs list is %v",
-			len(region.addrs), longestAddrs)
-	}
-	return true
-}
-
-// Various end-to-end tests, run with quickcheck (i.e. the testing/quick package)
-func TestEdgeAddrResolver(t *testing.T) {
-	concurrentReplacement := func(mockAddrs mockAddrs) bool {
-		netLookupSRV = mockNetLookupSRV(mockAddrs)
-		netLookupIP = mockNetLookupIP(mockAddrs)
-
-		resolver, err := NewEdgeAddrResolver(logrus.New())
-		if !assert.NoError(t, err) {
-			return false
-		}
-		assert.Equal(t, mockAddrs.numAddrs, resolver.AvailableAddrs(),
-			"every address should be initially available")
-
-		// Create several goroutines to simulate HA connections that acquire
-		// and replace IP addresses.
-		var wg sync.WaitGroup
-		wg.Add(mockAddrs.numAddrs)
-		for i := 0; i < mockAddrs.numAddrs; i++ {
-			go func() {
-				defer wg.Done()
-				const reconnectionCount = 50
-				for i := 0; i < reconnectionCount; i++ {
-					if resolver.AvailableAddrs() == 0 {
-						err = resolver.Refresh()
-						assert.NoError(t, err)
-					}
-					addr, err := resolver.Addr()
-					if !assert.NoError(t, err) {
-						return
-					}
-					time.Sleep(0) // allow some other goroutine to run
-					resolver.ReplaceAddr(addr)
-					time.Sleep(0) // allow some other goroutine to run
-				}
-			}()
-		}
-		wg.Wait()
-		assert.Equal(t, mockAddrs.numAddrs, resolver.AvailableAddrs(),
-			"every address should be available after replacement")
-		return !t.Failed()
-	}
-
-	badAddrWithRefresh := func(mockAddrs mockAddrs) bool {
-		netLookupSRV = mockNetLookupSRV(mockAddrs)
-		netLookupIP = mockNetLookupIP(mockAddrs)
-
-		resolver, err := NewEdgeAddrResolver(logrus.New())
-		if !assert.NoError(t, err) {
-			return false
-		}
-		assert.Equal(t, mockAddrs.numAddrs, resolver.AvailableAddrs(),
-			"every address should be initially available")
-
-		var addrs []*net.TCPAddr
-		for i := 0; i < mockAddrs.numAddrs; i++ {
-			assert.Equal(t, mockAddrs.numAddrs-i, resolver.AvailableAddrs())
-			addr, err := resolver.Addr()
-			assert.NoError(t, err)
-			addrs = append(addrs, addr)
-		}
-		assert.Equal(t, 0, resolver.AvailableAddrs(), "all addresses should have been taken")
-		_, err = resolver.Addr()
-		assert.Error(t, err)
-
-		anyAddr, err := resolver.AnyAddr()
-		assert.NoError(t, err, "should still be okay to call AnyAddr")
-
-		resolver.MarkAddrBad(anyAddr)
-
-		assert.Equal(t, 0, resolver.AvailableAddrs(), "all addresses should still be used")
-		_, err = resolver.Addr()
-		assert.Error(t, err, "all addresses should still be used")
-
-		err = resolver.Refresh()
-		assert.NoError(t, err, "Refresh() should have worked")
-
-		assert.Equal(t, 1, resolver.AvailableAddrs(),
-			"Refresh() should have reset the state of the 'bad' address")
-		addr, err := resolver.Addr()
-		assert.NoError(t, err)
-		assert.Equal(t, anyAddr, addr)
-
-		_, err = resolver.Addr()
-		assert.Error(t, err, "all addresses should be used again")
-
-		return !t.Failed()
-	}
-
-	assert.NoError(t, quick.Check(concurrentReplacement, nil))
-	assert.NoError(t, quick.Check(badAddrWithRefresh, nil))
-}
-
-// "White-box" test: runs Addr() and checks internal state
-func TestEdgeAddrResolver_Addr(t *testing.T) {
-	e := &EdgeAddrResolver{regions: nil}
-	addr, err := e.Addr()
-	assert.Error(t, err)
-
-	testRegions := func() []*region {
-		return []*region{
-			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}}, inUse: map[string]*net.TCPAddr{":2": &net.TCPAddr{Port: 2}, ":3": &net.TCPAddr{Port: 3}}},
-			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 4}, &net.TCPAddr{Port: 5}}, inUse: map[string]*net.TCPAddr{":6": &net.TCPAddr{Port: 6}}},
-			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 7}, &net.TCPAddr{Port: 8}}, inUse: map[string]*net.TCPAddr{":9": &net.TCPAddr{Port: 9}}},
-		}
-	}
-	e = &EdgeAddrResolver{regions: testRegions()}
-	addr, err = e.Addr()
-	assert.NoError(t, err)
-	assert.Equal(t, &net.TCPAddr{Port: 4}, addr)
-	var expected []*region
-	{
-		expected = testRegions()
-		expected[1].addrs = expected[1].addrs[1:]
-		expected[1].inUse[":4"] = &net.TCPAddr{Port: 4}
-	}
-	assert.Equal(t, expected, e.regions)
-}
-
-// "White-box" test: runs AnyAddr() and checks internal state
-func TestEdgeAddrResolver_AnyAddr(t *testing.T) {
-	e := &EdgeAddrResolver{regions: nil}
-	addr, err := e.AnyAddr()
-	assert.Error(t, err)
-
-	e = &EdgeAddrResolver{regions: []*region{&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}}, inUse: map[string]*net.TCPAddr{":2": &net.TCPAddr{Port: 2}}}}}
-	addr, err = e.AnyAddr()
-	assert.NoError(t, err)
-	assert.Equal(t, &net.TCPAddr{Port: 1}, addr, "should have chosen the inactive address")
-
-	e = &EdgeAddrResolver{regions: []*region{&region{inUse: map[string]*net.TCPAddr{":1": &net.TCPAddr{Port: 1}}}}}
-	addr, err = e.AnyAddr()
-	assert.NoError(t, err)
-	assert.Equal(t, &net.TCPAddr{Port: 1}, addr, "should have chosen an active address rather than nothing")
-}
-
-// "White-box" test: runs ReplaceAddr() and checks internal state
-func TestEdgeAddrResolver_ReplaceAddr(t *testing.T) {
-	e := &EdgeAddrResolver{regions: nil}
-	e.ReplaceAddr(&net.TCPAddr{Port: 1}) // this shouldn't panic, I guess
-
-	testRegions := func() []*region {
-		return []*region{
-			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}}, inUse: map[string]*net.TCPAddr{":2": &net.TCPAddr{Port: 2}, ":3": &net.TCPAddr{Port: 3}}},
-			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 4}, &net.TCPAddr{Port: 5}}, inUse: map[string]*net.TCPAddr{":6": &net.TCPAddr{Port: 6}}},
-			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 7}, &net.TCPAddr{Port: 8}}, inUse: map[string]*net.TCPAddr{":9": &net.TCPAddr{Port: 9}}},
-		}
-	}
-	e = &EdgeAddrResolver{regions: testRegions()}
-	e.ReplaceAddr(&net.TCPAddr{Port: 6})
-	var expected []*region
-	{
-		expected = testRegions()
-		delete(expected[1].inUse, ":6")
-		expected[1].addrs = append(expected[1].addrs, &net.TCPAddr{Port: 6})
-	}
-	assert.Equal(t, expected, e.regions)
-}
-
-// "White-box" test: runs MarkAddrBad() and checks internal state
-func TestEdgeAddrResolver_MarkAddrBad(t *testing.T) {
-	e := &EdgeAddrResolver{regions: nil}
-	e.ReplaceAddr(&net.TCPAddr{Port: 1}) // this shouldn't panic, I guess
-
-	testRegions := func() []*region {
-		return []*region{
-			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 1}}, inUse: map[string]*net.TCPAddr{":2": &net.TCPAddr{Port: 2}, ":3": &net.TCPAddr{Port: 3}}},
-			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 4}, &net.TCPAddr{Port: 5}}, inUse: map[string]*net.TCPAddr{":6": &net.TCPAddr{Port: 6}}},
-			&region{addrs: []*net.TCPAddr{&net.TCPAddr{Port: 7}, &net.TCPAddr{Port: 8}}, inUse: map[string]*net.TCPAddr{":9": &net.TCPAddr{Port: 9}}},
-		}
-	}
-	e = &EdgeAddrResolver{regions: testRegions()}
-	e.MarkAddrBad(&net.TCPAddr{Port: 6})
-	var expected []*region
-	{
-		expected = testRegions()
-		delete(expected[1].inUse, ":6")
-		expected[1].bad = append(expected[1].bad, &net.TCPAddr{Port: 6})
-	}
-	assert.Equal(t, expected, e.regions)
+	})
+	assert.EqualValues(t, []*net.TCPAddr{
+		&net.TCPAddr{Port: 1},
+		&net.TCPAddr{Port: 10},
+		&net.TCPAddr{Port: 21},
+		&net.TCPAddr{Port: 2},
+		&net.TCPAddr{Port: 12},
+		&net.TCPAddr{Port: 22},
+		&net.TCPAddr{Port: 3},
+		&net.TCPAddr{Port: 13},
+		&net.TCPAddr{Port: 23},
+		&net.TCPAddr{Port: 4},
+		&net.TCPAddr{Port: 24},
+		&net.TCPAddr{Port: 25},
+	}, result)
 }

connection/manager.go

@@ -4,18 +4,19 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"net"
 	"sync"
 	"time"
 
-	"github.com/google/uuid"
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/sirupsen/logrus"
-
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/streamhandler"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 const (
@@ -58,12 +59,12 @@ func newMetrics(namespace, subsystem string) *metrics {
 // EdgeManagerConfigurable is the configurable attributes of a EdgeConnectionManager
 type EdgeManagerConfigurable struct {
 	TunnelHostnames []h2mux.TunnelHostname
-	*tunnelpogs.EdgeConnectionConfig
+	*pogs.EdgeConnectionConfig
 }
 
 type CloudflaredConfig struct {
 	CloudflaredID uuid.UUID
-	Tags          []tunnelpogs.Tag
+	Tags          []pogs.Tag
 	BuildInfo     *buildinfo.BuildInfo
 	IntentLabel   string
 }
@@ -126,16 +127,13 @@ func (em *EdgeManager) UpdateConfigurable(newConfigurable *EdgeManagerConfigurab
 	em.state.updateConfigurable(newConfigurable)
 }
 
-func (em *EdgeManager) newConnection(ctx context.Context) *tunnelpogs.ConnectError {
-	edgeTCPAddr, err := em.serviceDiscoverer.Addr()
-	if err != nil {
-		return retryConnection(fmt.Sprintf("edge address discovery error: %v", err))
-	}
-	configurable := em.state.getConfigurable()
-	edgeConn, err := DialEdge(ctx, configurable.Timeout, em.tlsConfig, edgeTCPAddr)
+func (em *EdgeManager) newConnection(ctx context.Context) *pogs.ConnectError {
+	edgeIP := em.serviceDiscoverer.Addr()
+	edgeConn, err := em.dialEdge(ctx, edgeIP)
 	if err != nil {
 		return retryConnection(fmt.Sprintf("dial edge error: %v", err))
 	}
+	configurable := em.state.getConfigurable()
 	// Establish a muxed connection with the edge
 	// Client mux handshake with agent server
 	muxer, err := h2mux.Handshake(edgeConn, edgeConn, h2mux.MuxerConfig{
@@ -150,14 +148,14 @@ func (em *EdgeManager) newConnection(ctx context.Context) *tunnelpogs.ConnectErr
 		retryConnection(fmt.Sprintf("couldn't perform handshake with edge: %v", err))
 	}
 
-	h2muxConn, err := newConnection(muxer, edgeTCPAddr)
+	h2muxConn, err := newConnection(muxer, edgeIP)
 	if err != nil {
 		return retryConnection(fmt.Sprintf("couldn't create h2mux connection: %v", err))
 	}
 
 	go em.serveConn(ctx, h2muxConn)
 
-	connResult, err := h2muxConn.Connect(ctx, &tunnelpogs.ConnectParameters{
+	connResult, err := h2muxConn.Connect(ctx, &pogs.ConnectParameters{
 		CloudflaredID:       em.cloudflaredConfig.CloudflaredID,
 		CloudflaredVersion:  em.cloudflaredConfig.BuildInfo.CloudflaredVersion,
 		NumPreviousAttempts: 0,
@@ -189,7 +187,6 @@ func (em *EdgeManager) closeConnection(ctx context.Context) error {
 		return fmt.Errorf("no connection to close")
 	}
 	conn.Shutdown()
-	// teardown will be handled by EdgeManager.serveConn in another goroutine
 	return nil
 }
 
@@ -197,7 +194,28 @@ func (em *EdgeManager) serveConn(ctx context.Context, conn *Connection) {
 	err := conn.Serve(ctx)
 	em.logger.WithError(err).Warn("Connection closed")
 	em.state.closeConnection(conn)
-	em.serviceDiscoverer.ReplaceAddr(conn.addr)
+}
+
+func (em *EdgeManager) dialEdge(ctx context.Context, edgeIP *net.TCPAddr) (*tls.Conn, error) {
+	timeout := em.state.getConfigurable().Timeout
+	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
+	dialCtx, dialCancel := context.WithTimeout(ctx, timeout)
+	defer dialCancel()
+
+	dialer := net.Dialer{DualStack: true}
+	edgeConn, err := dialer.DialContext(dialCtx, "tcp", edgeIP.String())
+	if err != nil {
+		return nil, dialError{cause: errors.Wrap(err, "DialContext error")}
+	}
+	tlsEdgeConn := tls.Client(edgeConn, em.tlsConfig)
+	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
+
+	if err = tlsEdgeConn.Handshake(); err != nil {
+		return nil, dialError{cause: errors.Wrap(err, "Handshake with edge error")}
+	}
+	// clear the deadline on the conn; h2mux has its own timeouts
+	tlsEdgeConn.SetDeadline(time.Time{})
+	return tlsEdgeConn, nil
 }
 
 func (em *EdgeManager) noRetryMessage() string {
@@ -226,14 +244,14 @@ func newEdgeConnectionManagerState(configurable *EdgeManagerConfigurable, userCr
 	}
 }
 
-func (ems *edgeManagerState) shouldCreateConnection(availableEdgeAddrs int) bool {
+func (ems *edgeManagerState) shouldCreateConnection(availableEdgeAddrs uint8) bool {
 	ems.RLock()
 	defer ems.RUnlock()
-	expectedHAConns := int(ems.configurable.NumHAConnections)
+	expectedHAConns := ems.configurable.NumHAConnections
 	if availableEdgeAddrs < expectedHAConns {
 		expectedHAConns = availableEdgeAddrs
 	}
-	return len(ems.conns) < expectedHAConns
+	return uint8(len(ems.conns)) < expectedHAConns
 }
 
 func (ems *edgeManagerState) shouldReduceConnection() bool {
@@ -290,8 +308,8 @@ func (ems *edgeManagerState) getUserCredential() []byte {
 	return ems.userCredential
 }
 
-func retryConnection(cause string) *tunnelpogs.ConnectError {
-	return &tunnelpogs.ConnectError{
+func retryConnection(cause string) *pogs.ConnectError {
+	return &pogs.ConnectError{
 		Cause:       cause,
 		RetryAfter:  defaultRetryAfter,
 		ShouldRetry: true,

connection/manager_test.go

@@ -4,15 +4,15 @@ import (
 	"testing"
 	"time"
 
-	"github.com/google/uuid"
-	"github.com/sirupsen/logrus"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/stretchr/testify/assert"
 
-
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/streamhandler"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+
+	"github.com/google/uuid"
+	"github.com/sirupsen/logrus"
 )
 
 var (

connection/mocks_for_test.go

@@ -1,118 +0,0 @@
-package connection
-
-import (
-	"fmt"
-	"math"
-	"math/rand"
-	"net"
-	"reflect"
-	"testing/quick"
-)
-
-type mockAddrs struct {
-	// a set of synthetic SRV records
-	addrMap map[net.SRV][]*net.TCPAddr
-	// the total number of addresses, aggregated across addrMap.
-	// For the convenience of test code that would otherwise have to compute
-	// this by hand every time.
-	numAddrs int
-}
-
-func newMockAddrs(port uint16, numRegions uint8, numAddrsPerRegion uint8) mockAddrs {
-	addrMap := make(map[net.SRV][]*net.TCPAddr)
-	numAddrs := 0
-
-	for r := uint8(0); r < numRegions; r++ {
-		var (
-			srv   = net.SRV{Target: fmt.Sprintf("test-region-%v.example.com", r), Port: port}
-			addrs []*net.TCPAddr
-		)
-		for a := uint8(0); a < numAddrsPerRegion; a++ {
-			addrs = append(addrs, &net.TCPAddr{
-				IP:   net.ParseIP(fmt.Sprintf("10.0.%v.%v", r, a)),
-				Port: int(port),
-			})
-		}
-		addrMap[srv] = addrs
-		numAddrs += len(addrs)
-	}
-	return mockAddrs{addrMap: addrMap, numAddrs: numAddrs}
-}
-
-var _ quick.Generator = mockAddrs{}
-
-func (mockAddrs) Generate(rand *rand.Rand, size int) reflect.Value {
-	port := uint16(rand.Intn(math.MaxUint16))
-	numRegions := uint8(1 + rand.Intn(10))
-	numAddrsPerRegion := uint8(1 + rand.Intn(32))
-	result := newMockAddrs(port, numRegions, numAddrsPerRegion)
-	return reflect.ValueOf(result)
-}
-
-// Returns a function compatible with net.LookupSRV that will return the SRV
-// records from mockAddrs.
-func mockNetLookupSRV(
-	m mockAddrs,
-) func(service, proto, name string) (cname string, addrs []*net.SRV, err error) {
-	var addrs []*net.SRV
-	for k := range m.addrMap {
-		addr := k
-		addrs = append(addrs, &addr)
-		// We can't just do
-		//   addrs = append(addrs, &k)
-		// `k` will be reused by subsequent loop iterations,
-		// so all the copies of `&k` would point to the same location.
-	}
-	return func(_, _, _ string) (string, []*net.SRV, error) {
-		return "", addrs, nil
-	}
-}
-
-// Returns a function compatible with net.LookupIP that translates the SRV records
-// from mockAddrs into IP addresses, based on the TCP addresses in mockAddrs.
-func mockNetLookupIP(
-	m mockAddrs,
-) func(host string) ([]net.IP, error) {
-	return func(host string) ([]net.IP, error) {
-		for srv, tcpAddrs := range m.addrMap {
-			if srv.Target != host {
-				continue
-			}
-			result := make([]net.IP, len(tcpAddrs))
-			for i, tcpAddr := range tcpAddrs {
-				result[i] = tcpAddr.IP
-			}
-			return result, nil
-		}
-		return nil, fmt.Errorf("No IPs for %v", host)
-	}
-}
-
-type mockEdgeServiceDiscoverer struct {
-}
-
-func (mr *mockEdgeServiceDiscoverer) Addr() (*net.TCPAddr, error) {
-	return &net.TCPAddr{
-		IP:   net.ParseIP("127.0.0.1"),
-		Port: 63102,
-	}, nil
-}
-
-func (mr *mockEdgeServiceDiscoverer) AnyAddr() (*net.TCPAddr, error) {
-	return &net.TCPAddr{
-		IP:   net.ParseIP("127.0.0.1"),
-		Port: 63102,
-	}, nil
-}
-
-func (mr *mockEdgeServiceDiscoverer) ReplaceAddr(addr *net.TCPAddr) {}
-
-func (mr *mockEdgeServiceDiscoverer) MarkAddrBad(addr *net.TCPAddr) {}
-
-func (mr *mockEdgeServiceDiscoverer) AvailableAddrs() int {
-	return 1
-}
-
-func (mr *mockEdgeServiceDiscoverer) Refresh() error {
-	return nil
-}

connection/rpc.go

@@ -1,49 +0,0 @@
-package connection
-
-import (
-	"context"
-	"fmt"
-	"time"
-
-	"github.com/sirupsen/logrus"
-	rpc "zombiezen.com/go/capnproto2/rpc"
-
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-// NewRPCClient creates and returns a new RPC client, which will communicate
-// using a stream on the given muxer
-func NewRPCClient(
-	ctx context.Context,
-	muxer *h2mux.Muxer,
-	logger *logrus.Entry,
-	openStreamTimeout time.Duration,
-) (client tunnelpogs.TunnelServer_PogsClient, err error) {
-	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
-	defer openStreamCancel()
-	stream, err := muxer.OpenRPCStream(openStreamCtx)
-	if err != nil {
-		return
-	}
-
-	if !isRPCStreamResponse(stream.Headers) {
-		stream.Close()
-		err = fmt.Errorf("rpc: bad response headers: %v", stream.Headers)
-		return
-	}
-
-	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
-		tunnelrpc.ConnLog(logger),
-	)
-	client = tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	return client, nil
-}
-
-func isRPCStreamResponse(headers []h2mux.Header) bool {
-	return len(headers) == 1 &&
-		headers[0].Name == ":status" &&
-		headers[0].Value == "200"
-}

go.mod

@@ -25,10 +25,9 @@ require (
 	github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
-	github.com/google/certificate-transparency-go v1.1.0
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.3
-	github.com/gorilla/websocket v1.4.0
+	github.com/gorilla/websocket v1.2.0
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
@@ -57,6 +56,7 @@ require (
 	golang.org/x/net v0.0.0-20191007182048-72f939374954
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20191008105621-543471e840be
+	golang.org/x/text v0.3.2 // indirect
 	google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd // indirect
 	google.golang.org/grpc v1.24.0 // indirect
 	gopkg.in/coreos/go-oidc.v2 v2.1.0

go.sum

@@ -1,28 +1,18 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw=
-contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
 github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
-github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
-github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
-github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
-github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/aws/aws-sdk-go v1.25.8 h1:n7I+HUUXjun2CsX7JK+1hpRIkZrlKhd3nayeb+Xmavs=
 github.com/aws/aws-sdk-go v1.25.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk=
 github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4=
-github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg=
 github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -34,26 +24,15 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY
 github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80=
 github.com/coredns/coredns v1.2.0 h1:YEI38K2BJYzL/SxO2tZFD727T/C68DqVWkBQjT0sWPU=
 github.com/coredns/coredns v1.2.0/go.mod h1:zASH/MVDgR6XZTbxvOnsZfffS+31vg6Ackf/wo1+AM0=
-github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73 h1:7CNPV0LWRCa1FNmqg700pbXhzvmoaXKyfxWRkjRym7Q=
 github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
-github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a h1:W8b4lQ4tFF21aspRGoBuCNV6V2fFJBF+pm1J6OY8Lys=
 github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0 h1:epsH3lb7KVbXHYk7LYGN5EiE0MxcevHU85CKITJ0wUY=
 github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/equinox-io/equinox v1.2.0 h1:bBS7Ou+Y7Jwgmy8TWSYxEh85WctuFn7FPlgbUzX4DBA=
 github.com/equinox-io/equinox v1.2.0/go.mod h1:6s3HJB0PYUNgs0mxmI8fHdfVl3TQ25ieA/PVfr+eyVo=
 github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 h1:0JZ+dUmQeA8IIVUMzysrX4/AKuQwWhV2dYQuPZdvdSQ=
@@ -66,176 +45,73 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojt
 github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
 github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 h1:E2s37DuLxFhQDg5gKsWoLBOB0n+ZW8s599zru8FJ2/Y=
 github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
-github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/kkTFdWhctH96grJ7qiy7bMdiZcIvPKs=
 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0 h1:gF8ngtda767ddth2SH0YSAhswhz6qUkvyI9EZFYCWJA=
 github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
-github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
-github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
-github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
-github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
-github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
-github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
-github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
-github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
-github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
-github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
-github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
-github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
-github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
-github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
-github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1:zN2lZNZRflqFyxVaTIU61KNKQ9C0055u9CAfpmqUvo4=
 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
-github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
-github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
-github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
-github.com/golangci/go-tools v0.0.0-20190318055746-e32c54105b7c/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
-github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
-github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
-github.com/golangci/golangci-lint v1.17.2-0.20190910081718-bad04bb7378f/go.mod h1:kaqo8l0OZKYPtjNmG4z4HrWLgcYNIJ9B9q3LWri9uLg=
-github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
-github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
-github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
-github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
-github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
-github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
-github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
-github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/certificate-transparency-go v1.1.0 h1:10MlrYzh5wfkToxWI4yJzffsxLfxcEDlOATMx/V9Kzw=
-github.com/google/certificate-transparency-go v1.1.0/go.mod h1:i+Q7XY+ArBveOUT36jiHGfuSK1fHICIg6sUkRxPAbCs=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/monologue v0.0.0-20190606152607-4b11a32b5934/go.mod h1:6NTfaQoUpg5QmPsCUWLR3ig33FHrKXhTtWzF0DVdmuk=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/trillian v1.2.2-0.20190612132142-05461f4df60a/go.mod h1:YPmUVn5NGwgnDUgqlVyFGMTgaWlnSvH7W5p+NdOG8UA=
-github.com/google/trillian-examples v0.0.0-20190603134952-4e75ba15216c/go.mod h1:WgL3XZ3pA8/9cm7yxqWrZE6iZkESB2ItGxy5Fo6k2lk=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
 github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
-github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
-github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
 github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
-github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kshvakov/clickhouse v1.3.11 h1:dtzTJY0fCA+MWkLyuKZaNPkmSwdX4gh8+Klic9NB1Lw=
 github.com/kshvakov/clickhouse v1.3.11/go.mod h1:/SVBAcqF3u7rxQ9sTWCZwf8jzzvxiZGeQvtmSF2BBEc=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
-github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/letsencrypt/pkcs11key v2.0.1-0.20170608213348-396559074696+incompatible/go.mod h1:iGYXKqDXt0cpBthCHdr9ZdsQwyGlYFh/+8xa4WzIQ34=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
-github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
-github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.10 h1:qxFzApOv4WsAL965uUPIsXzAKCZxN2p9UqdhFS4ZW10=
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
-github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
-github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q=
 github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
-github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b h1:/BbY4n99iMazlr2igipph+hj0MwlZIWpcsP8Iy+na+s=
 github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b/go.mod h1:Wb1PlT4DAYSqOEd03MsqkdkXnTxA8v9pKjdpxbqM1kY=
 github.com/miekg/dns v1.1.8 h1:1QYRAKU3lN5cRfLCkPU08hwvLJFhvjP6MqNMmQz6ZVI=
 github.com/miekg/dns v1.1.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
-github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
-github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
-github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
-github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
-github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
-github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
-github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
 github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
@@ -255,191 +131,80 @@ github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLy
 github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
 github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 h1:mZHayPoR0lNmnHyvtYjDeq0zlVHn9K/ZXoy17ylucdo=
 github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5/go.mod h1:GEXHk5HgEKCvEIIrSpFI3ozzG5xOKA2DVlEX/gGnewM=
-github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
-github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
-github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
-github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
-github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
-github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
-github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
-github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
-github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
-github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
-github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
-github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
-github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
 github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/tomasen/realip v0.0.0-20180522021738-f0c99a92ddce/go.mod h1:o8v6yHRoik09Xen7gje4m9ERNah1d1PPsVq1VEx9vE4=
-github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
-github.com/ultraware/funlen v0.0.1/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
-github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
-github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
-github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
-github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
-github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
-github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0 h1:6DtWz8hNS4qbq0OCRPhdBMG9E2qKTSDKlwnP3dmZvuA=
 github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0/go.mod h1:A47W3pdWONaZmXuLZgfKLAVgUY0qvfTRM5vVDKS40S4=
-github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.etcd.io/etcd v3.3.13+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
 golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191007182048-72f939374954 h1:JGZucVF/L/TotR719NbujzadOZ2AgnYlqphQGHDCKaU=
 golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190605220351-eb0b1bdb6ae6/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s=
 google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd h1:84VQPzup3IpKLxuIAZjHMhVjJ8fZ4/i3yUnj3k6fUdw=
 google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
 google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
-gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
 gopkg.in/coreos/go-oidc.v2 v2.1.0 h1:E8PjVFdj/SLDKB0hvb70KTbMbYVHjqztiQdSkIg8E+I=
 gopkg.in/coreos/go-oidc.v2 v2.1.0/go.mod h1:fYaTe2FS96wZZwR17YTDHwG+Mw6fmyqJNxN2eNCGPCI=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
-gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/square/go-jose.v2 v2.4.0 h1:0kXPskUMGAXXWJlP05ktEMOV0vmzFQUWw6d+aZJQU8A=
 gopkg.in/square/go-jose.v2 v2.4.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8 h1:/pLAskKF+d5SawboKd8GB8ew4ClHDbt2c3K9EBFeRGU=
 gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8/go.mod h1:cKXr3E0k4aosgycml1b5z33BVV6hai1Kh7uDgFOkbcs=
-gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
-mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
-mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
-sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
 zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7 h1:CZoOFlTPbKfAShKYrMuUfYbnXexFT1rYRUX1SPnrdE4=
 zombiezen.com/go/capnproto2 v0.0.0-20180616160808-7cfd211c19c7/go.mod h1:TMGa8HWGJkXiq4nHe9Zu/JgRF5oUtg4XizFC+Vexbec=

h2mux/activestreammap.go

@@ -13,28 +13,26 @@ type activeStreamMap struct {
 	sync.RWMutex
 	// streams tracks open streams.
 	streams map[uint32]*MuxedStream
+	// streamsEmpty is a chan that should be closed when no more streams are open.
+	streamsEmpty chan struct{}
 	// nextStreamID is the next ID to use on our side of the connection.
 	// This is odd for clients, even for servers.
 	nextStreamID uint32
 	// maxPeerStreamID is the ID of the most recent stream opened by the peer.
 	maxPeerStreamID uint32
-	// activeStreams is a gauge shared by all muxers of this process to expose the total number of active streams
-	activeStreams prometheus.Gauge
-
 	// ignoreNewStreams is true when the connection is being shut down. New streams
 	// cannot be registered.
 	ignoreNewStreams bool
-	// streamsEmpty is a chan that will be closed when no more streams are open.
-	streamsEmptyChan chan struct{}
-	closeOnce        sync.Once
+	// activeStreams is a gauge shared by all muxers of this process to expose the total number of active streams
+	activeStreams prometheus.Gauge
 }
 
 func newActiveStreamMap(useClientStreamNumbers bool, activeStreams prometheus.Gauge) *activeStreamMap {
 	m := &activeStreamMap{
-		streams:          make(map[uint32]*MuxedStream),
-		streamsEmptyChan: make(chan struct{}),
-		nextStreamID:     1,
-		activeStreams:    activeStreams,
+		streams:       make(map[uint32]*MuxedStream),
+		streamsEmpty:  make(chan struct{}),
+		nextStreamID:  1,
+		activeStreams: activeStreams,
 	}
 	// Client initiated stream uses odd stream ID, server initiated stream uses even stream ID
 	if !useClientStreamNumbers {
@@ -43,13 +41,6 @@ func newActiveStreamMap(useClientStreamNumbers bool, activeStreams prometheus.Ga
 	return m
 }
 
-// This function should be called while `m` is locked.
-func (m *activeStreamMap) notifyStreamsEmpty() {
-	m.closeOnce.Do(func() {
-		close(m.streamsEmptyChan)
-	})
-}
-
 // Len returns the number of active streams.
 func (m *activeStreamMap) Len() int {
 	m.RLock()
@@ -88,29 +79,30 @@ func (m *activeStreamMap) Delete(streamID uint32) {
 		delete(m.streams, streamID)
 		m.activeStreams.Dec()
 	}
-
-	// shutting down, and now the map is empty
-	if m.ignoreNewStreams && len(m.streams) == 0 {
-		m.notifyStreamsEmpty()
+	if len(m.streams) == 0 && m.streamsEmpty != nil {
+		close(m.streamsEmpty)
+		m.streamsEmpty = nil
 	}
 }
 
-// Shutdown blocks new streams from being created.
-// It returns `done`, a channel that is closed once the last stream has closed
-// and `progress`, whether a shutdown was already in progress
-func (m *activeStreamMap) Shutdown() (done <-chan struct{}, alreadyInProgress bool) {
+// Shutdown blocks new streams from being created. It returns a channel that receives an event
+// once the last stream has closed, or nil if a shutdown is in progress.
+func (m *activeStreamMap) Shutdown() <-chan struct{} {
 	m.Lock()
 	defer m.Unlock()
 	if m.ignoreNewStreams {
 		// already shutting down
-		return m.streamsEmptyChan, true
+		return nil
 	}
 	m.ignoreNewStreams = true
+	done := make(chan struct{})
 	if len(m.streams) == 0 {
-		// there are no streams to wait for
-		m.notifyStreamsEmpty()
+		// nothing to shut down
+		close(done)
+		return done
 	}
-	return m.streamsEmptyChan, false
+	m.streamsEmpty = done
+	return done
 }
 
 // AcquireLocalID acquires a new stream ID for a stream you're opening.
@@ -178,5 +170,4 @@ func (m *activeStreamMap) Abort() {
 		stream.Close()
 	}
 	m.ignoreNewStreams = true
-	m.notifyStreamsEmpty()
 }

h2mux/activestreammap_test.go

@@ -1,195 +0,0 @@
-package h2mux
-
-import (
-	"sync"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestShutdown(t *testing.T) {
-	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
-
-	// Add all the streams
-	{
-		var wg sync.WaitGroup
-		wg.Add(numStreams)
-		for i := 0; i < numStreams; i++ {
-			go func(streamID int) {
-				defer wg.Done()
-				stream := &MuxedStream{streamID: uint32(streamID)}
-				ok := m.Set(stream)
-				assert.True(t, ok)
-			}(i)
-		}
-		wg.Wait()
-	}
-	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
-
-	shutdownChan, alreadyInProgress := m.Shutdown()
-	select {
-	case <-shutdownChan:
-		assert.Fail(t, "before Shutdown(), shutdownChan shouldn't be closed")
-	default:
-	}
-	assert.False(t, alreadyInProgress)
-
-	shutdownChan2, alreadyInProgress2 := m.Shutdown()
-	assert.Equal(t, shutdownChan, shutdownChan2, "repeated calls to Shutdown() should return the same channel")
-	assert.True(t, alreadyInProgress2, "repeated calls to Shutdown() should return true for 'in progress'")
-
-	// Delete all the streams
-	{
-		var wg sync.WaitGroup
-		wg.Add(numStreams)
-		for i := 0; i < numStreams; i++ {
-			go func(streamID int) {
-				defer wg.Done()
-				m.Delete(uint32(streamID))
-			}(i)
-		}
-		wg.Wait()
-	}
-	assert.Equal(t, 0, m.Len(), "All the streams should have been deleted")
-
-	select {
-	case <-shutdownChan:
-	default:
-		assert.Fail(t, "After all the streams are deleted, shutdownChan should have been closed")
-	}
-}
-
-func TestEmptyBeforeShutdown(t *testing.T) {
-	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
-
-	// Add all the streams
-	{
-		var wg sync.WaitGroup
-		wg.Add(numStreams)
-		for i := 0; i < numStreams; i++ {
-			go func(streamID int) {
-				defer wg.Done()
-				stream := &MuxedStream{streamID: uint32(streamID)}
-				ok := m.Set(stream)
-				assert.True(t, ok)
-			}(i)
-		}
-		wg.Wait()
-	}
-	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
-
-	// Delete all the streams, bringing m to size 0
-	{
-		var wg sync.WaitGroup
-		wg.Add(numStreams)
-		for i := 0; i < numStreams; i++ {
-			go func(streamID int) {
-				defer wg.Done()
-				m.Delete(uint32(streamID))
-			}(i)
-		}
-		wg.Wait()
-	}
-	assert.Equal(t, 0, m.Len(), "All the streams should have been deleted")
-
-	// Add one stream back
-	const soloStreamID = uint32(0)
-	ok := m.Set(&MuxedStream{streamID: soloStreamID})
-	assert.True(t, ok)
-
-	shutdownChan, alreadyInProgress := m.Shutdown()
-	select {
-	case <-shutdownChan:
-		assert.Fail(t, "before Shutdown(), shutdownChan shouldn't be closed")
-	default:
-	}
-	assert.False(t, alreadyInProgress)
-
-	shutdownChan2, alreadyInProgress2 := m.Shutdown()
-	assert.Equal(t, shutdownChan, shutdownChan2, "repeated calls to Shutdown() should return the same channel")
-	assert.True(t, alreadyInProgress2, "repeated calls to Shutdown() should return true for 'in progress'")
-
-	// Remove the remaining stream
-	m.Delete(soloStreamID)
-
-	select {
-	case <-shutdownChan:
-	default:
-		assert.Fail(t, "After all the streams are deleted, shutdownChan should have been closed")
-	}
-}
-
-type noopBuffer struct {
-	isClosed bool
-}
-
-func (t *noopBuffer) Read(p []byte) (n int, err error)  { return len(p), nil }
-func (t *noopBuffer) Write(p []byte) (n int, err error) { return len(p), nil }
-func (t *noopBuffer) Reset()                            {}
-func (t *noopBuffer) Len() int                          { return 0 }
-func (t *noopBuffer) Close() error                      { t.isClosed = true; return nil }
-func (t *noopBuffer) Closed() bool                      { return t.isClosed }
-
-type noopReadyList struct{}
-
-func (_ *noopReadyList) Signal(streamID uint32) {}
-
-func TestAbort(t *testing.T) {
-	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
-
-	var openedStreams sync.Map
-
-	// Add all the streams
-	{
-		var wg sync.WaitGroup
-		wg.Add(numStreams)
-		for i := 0; i < numStreams; i++ {
-			go func(streamID int) {
-				defer wg.Done()
-				stream := &MuxedStream{
-					streamID:    uint32(streamID),
-					readBuffer:  &noopBuffer{},
-					writeBuffer: &noopBuffer{},
-					readyList:   &noopReadyList{},
-				}
-				ok := m.Set(stream)
-				assert.True(t, ok)
-
-				openedStreams.Store(stream.streamID, stream)
-			}(i)
-		}
-		wg.Wait()
-	}
-	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
-
-	shutdownChan, alreadyInProgress := m.Shutdown()
-	select {
-	case <-shutdownChan:
-		assert.Fail(t, "before Abort(), shutdownChan shouldn't be closed")
-	default:
-	}
-	assert.False(t, alreadyInProgress)
-
-	m.Abort()
-	assert.Equal(t, numStreams, m.Len(), "Abort() shouldn't delete any streams")
-	openedStreams.Range(func(key interface{}, value interface{}) bool {
-		stream := value.(*MuxedStream)
-		readBuffer := stream.readBuffer.(*noopBuffer)
-		writeBuffer := stream.writeBuffer.(*noopBuffer)
-		return assert.True(t, readBuffer.isClosed && writeBuffer.isClosed, "Abort() should have closed all the streams")
-	})
-
-	select {
-	case <-shutdownChan:
-	default:
-		assert.Fail(t, "after Abort(), shutdownChan should have been closed")
-	}
-
-	// multiple aborts shouldn't cause any issues
-	m.Abort()
-	m.Abort()
-	m.Abort()
-}

h2mux/h2_dictionaries.go

@@ -542,10 +542,7 @@ func (w *h2DictWriter) Write(p []byte) (n int, err error) {
 }
 
 func (w *h2DictWriter) Close() error {
-	if w.comp != nil {
-		return w.comp.Close()
-	}
-	return nil
+	return w.comp.Close()
 }
 
 // From http2/hpack

h2mux/h2mux.go

@@ -353,11 +353,9 @@ func (m *Muxer) Serve(ctx context.Context) error {
 }
 
 // Shutdown is called to initiate the "happy path" of muxer termination.
-// It blocks new streams from being created.
-// It returns a channel that is closed when the last stream has been closed.
-func (m *Muxer) Shutdown() <-chan struct{} {
+func (m *Muxer) Shutdown() {
 	m.explicitShutdown.Fuse(true)
-	return m.muxReader.Shutdown()
+	m.muxReader.Shutdown()
 }
 
 // IsUnexpectedTunnelError identifies errors that are expected when shutting down the h2mux tunnel.

h2mux/h2mux_test.go

@@ -55,8 +55,6 @@ func NewDefaultMuxerPair(t assert.TestingT, testName string, f MuxedStreamFunc)
 			DefaultWindowSize:       (1 << 8) - 1,
 			MaxWindowSize:           (1 << 15) - 1,
 			StreamWriteBufferMaxLen: 1024,
-			HeartbeatInterval:       defaultTimeout,
-			MaxHeartbeats:           defaultRetries,
 		},
 		OriginConn: origin,
 		EdgeMuxConfig: MuxerConfig{
@@ -67,8 +65,6 @@ func NewDefaultMuxerPair(t assert.TestingT, testName string, f MuxedStreamFunc)
 			DefaultWindowSize:       (1 << 8) - 1,
 			MaxWindowSize:           (1 << 15) - 1,
 			StreamWriteBufferMaxLen: 1024,
-			HeartbeatInterval:       defaultTimeout,
-			MaxHeartbeats:           defaultRetries,
 		},
 		EdgeConn: edge,
 		doneC:    make(chan struct{}),
@@ -87,8 +83,6 @@ func NewCompressedMuxerPair(t assert.TestingT, testName string, quality Compress
 			Name:               "origin",
 			CompressionQuality: quality,
 			Logger:             log.NewEntry(log.New()),
-			HeartbeatInterval:  defaultTimeout,
-			MaxHeartbeats:      defaultRetries,
 		},
 		OriginConn: origin,
 		EdgeMuxConfig: MuxerConfig{
@@ -97,8 +91,6 @@ func NewCompressedMuxerPair(t assert.TestingT, testName string, quality Compress
 			Name:               "edge",
 			CompressionQuality: quality,
 			Logger:             log.NewEntry(log.New()),
-			HeartbeatInterval:  defaultTimeout,
-			MaxHeartbeats:      defaultRetries,
 		},
 		EdgeConn: edge,
 		doneC:    make(chan struct{}),

h2mux/muxedstream.go

@@ -17,12 +17,6 @@ type ReadWriteClosedCloser interface {
 	Closed() bool
 }
 
-// MuxedStreamDataSignaller is a write-only *ReadyList
-type MuxedStreamDataSignaller interface {
-	// Non-blocking: call this when data is ready to be sent for the given stream ID.
-	Signal(ID uint32)
-}
-
 // MuxedStream is logically an HTTP/2 stream, with an additional buffer for outgoing data.
 type MuxedStream struct {
 	streamID uint32
@@ -61,8 +55,8 @@ type MuxedStream struct {
 	// This is the amount of bytes that are in the peer's receive window
 	// (how much data we can send from this stream).
 	sendWindow uint32
-	// The muxer's readyList
-	readyList MuxedStreamDataSignaller
+	// Reference to the muxer's readyList; signal this for stream data to be sent.
+	readyList *ReadyList
 	// The headers that should be sent, and a flag so we only send them once.
 	headersSent  bool
 	writeHeaders []Header
@@ -94,7 +88,7 @@ func (th TunnelHostname) IsSet() bool {
 	return th != ""
 }
 
-func NewStream(config MuxerConfig, writeHeaders []Header, readyList MuxedStreamDataSignaller, dictionaries h2Dictionaries) *MuxedStream {
+func NewStream(config MuxerConfig, writeHeaders []Header, readyList *ReadyList, dictionaries h2Dictionaries) *MuxedStream {
 	return &MuxedStream{
 		responseHeadersReceived: make(chan struct{}),
 		readBuffer:              NewSharedBuffer(),

h2mux/muxreader.go

@@ -51,12 +51,10 @@ type MuxReader struct {
 	dictionaries h2Dictionaries
 }
 
-// Shutdown blocks new streams from being created.
-// It returns a channel that is closed once the last stream has closed.
-func (r *MuxReader) Shutdown() <-chan struct{} {
-	done, alreadyInProgress := r.streams.Shutdown()
-	if alreadyInProgress {
-		return done
+func (r *MuxReader) Shutdown() {
+	done := r.streams.Shutdown()
+	if done == nil {
+		return
 	}
 	r.sendGoAway(http2.ErrCodeNo)
 	go func() {
@@ -64,7 +62,6 @@ func (r *MuxReader) Shutdown() <-chan struct{} {
 		<-done
 		r.r.Close()
 	}()
-	return done
 }
 
 func (r *MuxReader) run(parentLogger *log.Entry) error {

origin/backoffhandler.go

@@ -92,8 +92,3 @@ func (b BackoffHandler) GetBaseTime() time.Duration {
 	}
 	return b.BaseTime
 }
-
-// Retries returns the number of retries consumed so far.
-func (b *BackoffHandler) Retries() int {
-	return int(b.retries)
-}

origin/supervisor.go

@@ -2,19 +2,16 @@ package origin
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"net"
-	"sync"
 	"time"
 
-	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
 
 	"github.com/cloudflare/cloudflared/connection"
-	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/signal"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+
+	"github.com/google/uuid"
 )
 
 const (
@@ -24,25 +21,13 @@ const (
 	resolveTTL = time.Hour
 	// Interval between registering new tunnels
 	registrationInterval = time.Second
-
-	subsystemRefreshAuth = "refresh_auth"
-	// Maximum exponent for 'Authenticate' exponential backoff
-	refreshAuthMaxBackoff = 10
-	// Waiting time before retrying a failed 'Authenticate' connection
-	refreshAuthRetryDuration = time.Second * 10
-	// Maximum time to make an Authenticate RPC
-	authTokenTimeout = time.Second * 30
-)
-
-var (
-	errJWTUnset         = errors.New("JWT unset")
-	errEventDigestUnset = errors.New("event digest unset")
 )
 
 type Supervisor struct {
-	cloudflaredUUID   uuid.UUID
-	config            *TunnelConfig
-	edgeIPs           connection.EdgeServiceDiscoverer
+	config  *TunnelConfig
+	edgeIPs []*net.TCPAddr
+	// nextUnusedEdgeIP is the index of the next addr k edgeIPs to try
+	nextUnusedEdgeIP  int
 	lastResolve       time.Time
 	resolverC         chan resolveResult
 	tunnelErrors      chan tunnelError
@@ -53,63 +38,36 @@ type Supervisor struct {
 	nextConnectedSignal chan struct{}
 
 	logger *logrus.Entry
-
-	jwtLock *sync.RWMutex
-	jwt     []byte
-
-	eventDigestLock *sync.RWMutex
-	eventDigest     []byte
 }
 
 type resolveResult struct {
-	err error
+	edgeIPs []*net.TCPAddr
+	err     error
 }
 
 type tunnelError struct {
 	index int
-	addr  *net.TCPAddr
 	err   error
 }
 
-func NewSupervisor(config *TunnelConfig, u uuid.UUID) (*Supervisor, error) {
-	edgeIPs, err := connection.NewEdgeAddrResolver(config.Logger)
-	if err != nil {
-		return nil, err
-	}
+func NewSupervisor(config *TunnelConfig) *Supervisor {
 	return &Supervisor{
-		cloudflaredUUID:   u,
 		config:            config,
-		edgeIPs:           edgeIPs,
 		tunnelErrors:      make(chan tunnelError),
 		tunnelsConnecting: map[int]chan struct{}{},
 		logger:            config.Logger.WithField("subsystem", "supervisor"),
-		jwtLock:           &sync.RWMutex{},
-		eventDigestLock:   &sync.RWMutex{},
-	}, nil
+	}
 }
 
-func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal) error {
+func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, u uuid.UUID) error {
 	logger := s.config.Logger
-	if err := s.initialize(ctx, connectedSignal); err != nil {
+	if err := s.initialize(ctx, connectedSignal, u); err != nil {
 		return err
 	}
 	var tunnelsWaiting []int
-	tunnelsActive := s.config.HAConnections
-
 	backoff := BackoffHandler{MaxRetries: s.config.Retries, BaseTime: tunnelRetryDuration, RetryForever: true}
 	var backoffTimer <-chan time.Time
-
-	refreshAuthBackoff := &BackoffHandler{MaxRetries: refreshAuthMaxBackoff, BaseTime: refreshAuthRetryDuration, RetryForever: true}
-	var refreshAuthBackoffTimer <-chan time.Time
-
-	if s.config.UseReconnectToken {
-		if timer, err := s.refreshAuth(ctx, refreshAuthBackoff, s.authenticate); err == nil {
-			refreshAuthBackoffTimer = timer
-		} else {
-			logger.WithError(err).Errorf("initial refreshAuth failed, retrying in %v", refreshAuthRetryDuration)
-			refreshAuthBackoffTimer = time.After(refreshAuthRetryDuration)
-		}
-	}
+	tunnelsActive := s.config.HAConnections
 
 	for {
 		select {
@@ -136,8 +94,8 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal) er
 				// If the error is a dial error, the problem is likely to be network related
 				// try another addr before refreshing since we are likely to get back the
 				// same IPs in the same order. Same problem with duplicate connection error.
-				if s.unusedIPs() && tunnelError.addr != nil {
-					s.edgeIPs.MarkAddrBad(tunnelError.addr)
+				if s.unusedIPs() {
+					s.replaceEdgeIP(tunnelError.index)
 				} else {
 					s.refreshEdgeIPs()
 				}
@@ -146,20 +104,10 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal) er
 		case <-backoffTimer:
 			backoffTimer = nil
 			for _, index := range tunnelsWaiting {
-				go s.startTunnel(ctx, index, s.newConnectedTunnelSignal(index))
+				go s.startTunnel(ctx, index, s.newConnectedTunnelSignal(index), u)
 			}
 			tunnelsActive += len(tunnelsWaiting)
 			tunnelsWaiting = nil
-		// Time to call Authenticate
-		case <-refreshAuthBackoffTimer:
-			newTimer, err := s.refreshAuth(ctx, refreshAuthBackoff, s.authenticate)
-			if err != nil {
-				logger.WithError(err).Error("Authentication failed")
-				// Permanent failure. Leave the `select` without setting the
-				// channel to be non-null, so we'll never hit this case of the `select` again.
-				continue
-			}
-			refreshAuthBackoffTimer = newTimer
 		// Tunnel successfully connected
 		case <-s.nextConnectedSignal:
 			if !s.waitForNextTunnel(s.nextConnectedIndex) && len(tunnelsWaiting) == 0 {
@@ -172,6 +120,7 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal) er
 			s.resolverC = nil
 			if result.err == nil {
 				logger.Debug("Service discovery refresh complete")
+				s.edgeIPs = result.edgeIPs
 			} else {
 				logger.WithError(result.err).Error("Service discovery error")
 			}
@@ -179,27 +128,29 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal) er
 	}
 }
 
-// Returns nil if initialization succeeded, else the initialization error.
-func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal) error {
+func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal, u uuid.UUID) error {
 	logger := s.logger
 
-	err := s.edgeIPs.Refresh()
+	edgeIPs, err := s.resolveEdgeIPs()
+
 	if err != nil {
 		logger.Infof("ResolveEdgeIPs err")
 		return err
 	}
-	s.lastResolve = time.Now()
-	availableAddrs := int(s.edgeIPs.AvailableAddrs())
-	if s.config.HAConnections > availableAddrs {
-		logger.Warnf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
-		s.config.HAConnections = availableAddrs
+	s.edgeIPs = edgeIPs
+	if s.config.HAConnections > len(edgeIPs) {
+		logger.Warnf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, len(edgeIPs))
+		s.config.HAConnections = len(edgeIPs)
 	}
-
-	go s.startFirstTunnel(ctx, connectedSignal)
+	s.lastResolve = time.Now()
+	// check entitlement and version too old error before attempting to register more tunnels
+	s.nextUnusedEdgeIP = s.config.HAConnections
+	go s.startFirstTunnel(ctx, connectedSignal, u)
 	select {
 	case <-ctx.Done():
 		<-s.tunnelErrors
-		return ctx.Err()
+		// Error can't be nil. A nil error signals that initialization succeed
+		return fmt.Errorf("context was canceled")
 	case tunnelError := <-s.tunnelErrors:
 		return tunnelError.err
 	case <-connectedSignal.Wait():
@@ -207,7 +158,7 @@ func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Sig
 	// At least one successful connection, so start the rest
 	for i := 1; i < s.config.HAConnections; i++ {
 		ch := signal.New(make(chan struct{}))
-		go s.startTunnel(ctx, i, ch)
+		go s.startTunnel(ctx, i, ch, u)
 		time.Sleep(registrationInterval)
 	}
 	return nil
@@ -215,60 +166,37 @@ func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Sig
 
 // startTunnel starts the first tunnel connection. The resulting error will be sent on
 // s.tunnelErrors. It will send a signal via connectedSignal if registration succeed
-func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *signal.Signal) {
-	var (
-		addr *net.TCPAddr
-		err  error
-	)
+func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *signal.Signal, u uuid.UUID) {
+	err := ServeTunnelLoop(ctx, s.config, s.getEdgeIP(0), 0, connectedSignal, u)
 	defer func() {
-		s.tunnelErrors <- tunnelError{index: 0, addr: addr, err: err}
+		s.tunnelErrors <- tunnelError{index: 0, err: err}
 	}()
 
-	addr, err = s.edgeIPs.Addr()
-	if err != nil {
-		return
-	}
-
-	err = ServeTunnelLoop(ctx, s, s.config, addr, 0, connectedSignal, s.cloudflaredUUID)
-
 	for s.unusedIPs() {
-		if ctx.Err() != nil {
+		select {
+		case <-ctx.Done():
 			return
+		default:
 		}
 		switch err.(type) {
 		case nil:
 			return
 		// try the next address if it was a dialError(network problem) or
 		// dupConnRegisterTunnelError
-		case connection.DialError, dupConnRegisterTunnelError:
-			s.edgeIPs.MarkAddrBad(addr)
+		case dialError, dupConnRegisterTunnelError:
+			s.replaceEdgeIP(0)
 		default:
 			return
 		}
-		addr, err = s.edgeIPs.Addr()
-		if err != nil {
-			return
-		}
-		err = ServeTunnelLoop(ctx, s, s.config, addr, 0, connectedSignal, s.cloudflaredUUID)
+		err = ServeTunnelLoop(ctx, s.config, s.getEdgeIP(0), 0, connectedSignal, u)
 	}
 }
 
 // startTunnel starts a new tunnel connection. The resulting error will be sent on
 // s.tunnelErrors.
-func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal *signal.Signal) {
-	var (
-		addr *net.TCPAddr
-		err  error
-	)
-	defer func() {
-		s.tunnelErrors <- tunnelError{index: index, addr: addr, err: err}
-	}()
-
-	addr, err = s.edgeIPs.Addr()
-	if err != nil {
-		return
-	}
-	err = ServeTunnelLoop(ctx, s, s.config, addr, uint8(index), connectedSignal, s.cloudflaredUUID)
+func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal *signal.Signal, u uuid.UUID) {
+	err := ServeTunnelLoop(ctx, s.config, s.getEdgeIP(index), uint8(index), connectedSignal, u)
+	s.tunnelErrors <- tunnelError{index: index, err: err}
 }
 
 func (s *Supervisor) newConnectedTunnelSignal(index int) *signal.Signal {
@@ -290,8 +218,17 @@ func (s *Supervisor) waitForNextTunnel(index int) bool {
 	return false
 }
 
-func (s *Supervisor) unusedIPs() bool {
-	return s.edgeIPs.AvailableAddrs() > s.config.HAConnections
+func (s *Supervisor) getEdgeIP(index int) *net.TCPAddr {
+	return s.edgeIPs[index%len(s.edgeIPs)]
+}
+
+func (s *Supervisor) resolveEdgeIPs() ([]*net.TCPAddr, error) {
+	// If --edge is specfied, resolve edge server addresses
+	if len(s.config.EdgeAddrs) > 0 {
+		return connection.ResolveAddrs(s.config.EdgeAddrs)
+	}
+	// Otherwise lookup edge server addresses through service discovery
+	return connection.EdgeDiscovery(s.logger)
 }
 
 func (s *Supervisor) refreshEdgeIPs() {
@@ -303,119 +240,16 @@ func (s *Supervisor) refreshEdgeIPs() {
 	}
 	s.resolverC = make(chan resolveResult)
 	go func() {
-		err := s.edgeIPs.Refresh()
-		s.resolverC <- resolveResult{err: err}
+		edgeIPs, err := s.resolveEdgeIPs()
+		s.resolverC <- resolveResult{edgeIPs: edgeIPs, err: err}
 	}()
 }
 
-func (s *Supervisor) ReconnectToken() ([]byte, error) {
-	s.jwtLock.RLock()
-	defer s.jwtLock.RUnlock()
-	if s.jwt == nil {
-		return nil, errJWTUnset
-	}
-	return s.jwt, nil
+func (s *Supervisor) unusedIPs() bool {
+	return s.nextUnusedEdgeIP < len(s.edgeIPs)
 }
 
-func (s *Supervisor) SetReconnectToken(jwt []byte) {
-	s.jwtLock.Lock()
-	defer s.jwtLock.Unlock()
-	s.jwt = jwt
-}
-
-func (s *Supervisor) EventDigest() ([]byte, error) {
-	s.eventDigestLock.RLock()
-	defer s.eventDigestLock.RUnlock()
-	if s.eventDigest == nil {
-		return nil, errEventDigestUnset
-	}
-	return s.eventDigest, nil
-}
-
-func (s *Supervisor) SetEventDigest(eventDigest []byte) {
-	s.eventDigestLock.Lock()
-	defer s.eventDigestLock.Unlock()
-	s.eventDigest = eventDigest
-}
-
-func (s *Supervisor) refreshAuth(
-	ctx context.Context,
-	backoff *BackoffHandler,
-	authenticate func(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error),
-) (retryTimer <-chan time.Time, err error) {
-	logger := s.config.Logger.WithField("subsystem", subsystemRefreshAuth)
-	authOutcome, err := authenticate(ctx, backoff.Retries())
-	if err != nil {
-		if duration, ok := backoff.GetBackoffDuration(ctx); ok {
-			logger.WithError(err).Warnf("Retrying in %v", duration)
-			return backoff.BackoffTimer(), nil
-		}
-		return nil, err
-	}
-	// clear backoff timer
-	backoff.SetGracePeriod()
-
-	switch outcome := authOutcome.(type) {
-	case tunnelpogs.AuthSuccess:
-		s.SetReconnectToken(outcome.JWT())
-		return timeAfter(outcome.RefreshAfter()), nil
-	case tunnelpogs.AuthUnknown:
-		duration := outcome.RefreshAfter()
-		logger.WithError(outcome).Warnf("Retrying in %v", duration)
-		return timeAfter(duration), nil
-	case tunnelpogs.AuthFail:
-		return nil, outcome
-	default:
-		return nil, fmt.Errorf("Unexpected outcome type %T", authOutcome)
-	}
-}
-
-func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error) {
-	arbitraryEdgeIP, err := s.edgeIPs.AnyAddr()
-	if err != nil {
-		return nil, err
-	}
-
-	edgeConn, err := connection.DialEdge(ctx, dialTimeout, s.config.TlsConfig, arbitraryEdgeIP)
-	if err != nil {
-		return nil, err
-	}
-	defer edgeConn.Close()
-
-	handler := h2mux.MuxedStreamFunc(func(*h2mux.MuxedStream) error {
-		// This callback is invoked by h2mux when the edge initiates a stream.
-		return nil // noop
-	})
-	muxerConfig := s.config.muxerConfig(handler)
-	muxerConfig.Logger = muxerConfig.Logger.WithField("subsystem", subsystemRefreshAuth)
-	muxer, err := h2mux.Handshake(edgeConn, edgeConn, muxerConfig, s.config.Metrics.activeStreams)
-	if err != nil {
-		return nil, err
-	}
-	go muxer.Serve(ctx)
-	defer func() {
-		// If we don't wait for the muxer shutdown here, edgeConn.Close() runs before the muxer connections are done,
-		// and the user sees log noise: "error writing data", "connection closed unexpectedly"
-		<-muxer.Shutdown()
-	}()
-
-	tunnelServer, err := connection.NewRPCClient(ctx, muxer, s.logger.WithField("subsystem", subsystemRefreshAuth), openStreamTimeout)
-	if err != nil {
-		return nil, err
-	}
-	defer tunnelServer.Close()
-
-	const arbitraryConnectionID = uint8(0)
-	registrationOptions := s.config.RegistrationOptions(arbitraryConnectionID, edgeConn.LocalAddr().String(), s.cloudflaredUUID)
-	registrationOptions.NumPreviousAttempts = uint8(numPreviousAttempts)
-	authResponse, err := tunnelServer.Authenticate(
-		ctx,
-		s.config.OriginCert,
-		s.config.Hostname,
-		registrationOptions,
-	)
-	if err != nil {
-		return nil, err
-	}
-	return authResponse.Outcome(), nil
+func (s *Supervisor) replaceEdgeIP(badIPIndex int) {
+	s.edgeIPs[badIPIndex] = s.edgeIPs[s.nextUnusedEdgeIP]
+	s.nextUnusedEdgeIP++
 }

origin/supervisor_test.go

@@ -1,140 +0,0 @@
-package origin
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/assert"
-
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-)
-
-func TestRefreshAuthBackoff(t *testing.T) {
-	logger := logrus.New()
-	logger.Level = logrus.ErrorLevel
-
-	var wait time.Duration
-	timeAfter = func(d time.Duration) <-chan time.Time {
-		wait = d
-		return time.After(d)
-	}
-
-	s, err := NewSupervisor(&TunnelConfig{Logger: logger}, uuid.New())
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-	backoff := &BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return nil, fmt.Errorf("authentication failure")
-	}
-
-	// authentication failures should consume the backoff
-	for i := uint(0); i < backoff.MaxRetries; i++ {
-		retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
-		assert.NoError(t, err)
-		assert.NotNil(t, retryChan)
-		assert.Equal(t, (1<<i)*time.Second, wait)
-	}
-	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
-	assert.Error(t, err)
-	assert.Nil(t, retryChan)
-
-	// now we actually make contact with the remote server
-	_, _ = s.refreshAuth(context.Background(), backoff, func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
-	})
-
-	// The backoff timer should have been reset. To confirm this, make timeNow
-	// return a value after the backoff timer's grace period
-	timeNow = func() time.Time {
-		expectedGracePeriod := time.Duration(time.Second * 2 << backoff.MaxRetries)
-		return time.Now().Add(expectedGracePeriod * 2)
-	}
-	_, ok := backoff.GetBackoffDuration(context.Background())
-	assert.True(t, ok)
-}
-
-func TestRefreshAuthSuccess(t *testing.T) {
-	logger := logrus.New()
-	logger.Level = logrus.ErrorLevel
-
-	var wait time.Duration
-	timeAfter = func(d time.Duration) <-chan time.Time {
-		wait = d
-		return time.After(d)
-	}
-
-	s, err := NewSupervisor(&TunnelConfig{Logger: logger}, uuid.New())
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-	backoff := &BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthSuccess([]byte("jwt"), 19), nil
-	}
-
-	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
-	assert.NoError(t, err)
-	assert.NotNil(t, retryChan)
-	assert.Equal(t, 19*time.Hour, wait)
-
-	token, err := s.ReconnectToken()
-	assert.NoError(t, err)
-	assert.Equal(t, []byte("jwt"), token)
-}
-
-func TestRefreshAuthUnknown(t *testing.T) {
-	logger := logrus.New()
-	logger.Level = logrus.ErrorLevel
-
-	var wait time.Duration
-	timeAfter = func(d time.Duration) <-chan time.Time {
-		wait = d
-		return time.After(d)
-	}
-
-	s, err := NewSupervisor(&TunnelConfig{Logger: logger}, uuid.New())
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-	backoff := &BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
-	}
-
-	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
-	assert.NoError(t, err)
-	assert.NotNil(t, retryChan)
-	assert.Equal(t, 19*time.Hour, wait)
-
-	token, err := s.ReconnectToken()
-	assert.Equal(t, errJWTUnset, err)
-	assert.Nil(t, token)
-}
-
-func TestRefreshAuthFail(t *testing.T) {
-	logger := logrus.New()
-	logger.Level = logrus.ErrorLevel
-
-	s, err := NewSupervisor(&TunnelConfig{Logger: logger}, uuid.New())
-	if !assert.NoError(t, err) {
-		t.FailNow()
-	}
-	backoff := &BackoffHandler{MaxRetries: 3}
-	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
-		return tunnelpogs.NewAuthFail(errors.New("auth fail")), nil
-	}
-
-	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
-	assert.Error(t, err)
-	assert.Nil(t, retryChan)
-
-	token, err := s.ReconnectToken()
-	assert.Equal(t, errJWTUnset, err)
-	assert.Nil(t, token)
-}

origin/tunnel.go

@@ -14,14 +14,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/google/uuid"
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-	log "github.com/sirupsen/logrus"
-	"golang.org/x/sync/errgroup"
-
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
-	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/signal"
 	"github.com/cloudflare/cloudflared/streamhandler"
@@ -29,12 +22,19 @@ import (
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/validation"
 	"github.com/cloudflare/cloudflared/websocket"
+
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	_ "github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
+	rpc "zombiezen.com/go/capnproto2/rpc"
 )
 
 const (
 	dialTimeout              = 15 * time.Second
 	openStreamTimeout        = 30 * time.Second
-	muxerTimeout             = 5 * time.Second
 	lbProbeUserAgentPrefix   = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
 	TagHeaderNamePrefix      = "Cf-Warp-Tag-"
 	DuplicateConnectionError = "EDUPCONN"
@@ -73,17 +73,14 @@ type TunnelConfig struct {
 	WSGI                 bool
 	// OriginUrl may not be used if a user specifies a unix socket.
 	OriginUrl string
-
-	// feature-flag to use new edge reconnect tokens
-	UseReconnectToken bool
 }
 
-// ReconnectTunnelCredentialManager is invoked by functions in this file to
-// get/set parameters for ReconnectTunnel RPC calls.
-type ReconnectTunnelCredentialManager interface {
-	ReconnectToken() ([]byte, error)
-	EventDigest() ([]byte, error)
-	SetEventDigest(eventDigest []byte)
+type dialError struct {
+	cause error
+}
+
+func (e dialError) Error() string {
+	return e.cause.Error()
 }
 
 type dupConnRegisterTunnelError struct{}
@@ -122,18 +119,6 @@ func (e clientRegisterTunnelError) Error() string {
 	return e.cause.Error()
 }
 
-func (c *TunnelConfig) muxerConfig(handler h2mux.MuxedStreamHandler) h2mux.MuxerConfig {
-	return h2mux.MuxerConfig{
-		Timeout:            muxerTimeout,
-		Handler:            handler,
-		IsClient:           true,
-		HeartbeatInterval:  c.HeartbeatInterval,
-		MaxHeartbeats:      c.MaxHeartbeats,
-		Logger:             c.TransportLogger.WithFields(log.Fields{}),
-		CompressionQuality: h2mux.CompressionSetting(c.CompressionQuality),
-	}
-}
-
 func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
 	policy := tunnelrpc.ExistingTunnelPolicy_balance
 	if c.HAConnections <= 1 && c.LBPool == "" {
@@ -156,15 +141,10 @@ func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP str
 }
 
 func StartTunnelDaemon(ctx context.Context, config *TunnelConfig, connectedSignal *signal.Signal, cloudflaredID uuid.UUID) error {
-	s, err := NewSupervisor(config, cloudflaredID)
-	if err != nil {
-		return err
-	}
-	return s.Run(ctx, connectedSignal)
+	return NewSupervisor(config).Run(ctx, connectedSignal, cloudflaredID)
 }
 
 func ServeTunnelLoop(ctx context.Context,
-	credentialManager ReconnectTunnelCredentialManager,
 	config *TunnelConfig,
 	addr *net.TCPAddr,
 	connectionID uint8,
@@ -186,7 +166,6 @@ func ServeTunnelLoop(ctx context.Context,
 	for {
 		err, recoverable := ServeTunnel(
 			ctx,
-			credentialManager,
 			config,
 			connectionLogger,
 			addr, connectionID,
@@ -207,7 +186,6 @@ func ServeTunnelLoop(ctx context.Context,
 
 func ServeTunnel(
 	ctx context.Context,
-	credentialManager ReconnectTunnelCredentialManager,
 	config *TunnelConfig,
 	logger *log.Entry,
 	addr *net.TCPAddr,
@@ -235,11 +213,11 @@ func ServeTunnel(
 	tags["ha"] = connectionTag
 
 	// Returns error from parsing the origin URL or handshake errors
-	handler, originLocalIP, err := NewTunnelHandler(ctx, config, addr, connectionID)
+	handler, originLocalIP, err := NewTunnelHandler(ctx, config, addr.String(), connectionID)
 	if err != nil {
 		errLog := logger.WithError(err)
 		switch err.(type) {
-		case connection.DialError:
+		case dialError:
 			errLog.Error("Unable to dial edge")
 		case h2mux.MuxerHandshakeError:
 			errLog.Error("Handshake failed with edge server")
@@ -252,30 +230,13 @@ func ServeTunnel(
 
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 
-	errGroup.Go(func() (err error) {
-		defer func() {
-			if err == nil {
-				connectedFuse.Fuse(true)
-				backoff.SetGracePeriod()
-			}
-		}()
-
-		if config.UseReconnectToken && connectedFuse.Value() {
-			token, tokenErr := credentialManager.ReconnectToken()
-			eventDigest, eventDigestErr := credentialManager.EventDigest()
-			// if we have both credentials, we can reconnect
-			if tokenErr == nil && eventDigestErr == nil {
-				return ReconnectTunnel(ctx, token, eventDigest, handler.muxer, config, logger, connectionID, originLocalIP, u)
-			}
-			// log errors and proceed to RegisterTunnel
-			if tokenErr != nil {
-				logger.WithError(tokenErr).Error("Couldn't get reconnect token")
-			}
-			if eventDigestErr != nil {
-				logger.WithError(eventDigestErr).Error("Couldn't get event digest")
-			}
+	errGroup.Go(func() error {
+		err := RegisterTunnel(serveCtx, handler.muxer, config, logger, connectionID, originLocalIP, u)
+		if err == nil {
+			connectedFuse.Fuse(true)
+			backoff.SetGracePeriod()
 		}
-		return RegisterTunnel(serveCtx, credentialManager, handler.muxer, config, logger, connectionID, originLocalIP, u)
+		return err
 	})
 
 	errGroup.Go(func() error {
@@ -334,9 +295,18 @@ func ServeTunnel(
 	return nil, true
 }
 
+func IsRPCStreamResponse(headers []h2mux.Header) bool {
+	if len(headers) != 1 {
+		return false
+	}
+	if headers[0].Name != ":status" || headers[0].Value != "200" {
+		return false
+	}
+	return true
+}
+
 func RegisterTunnel(
 	ctx context.Context,
-	credentialManager ReconnectTunnelCredentialManager,
 	muxer *h2mux.Muxer,
 	config *TunnelConfig,
 	logger *log.Entry,
@@ -345,73 +315,45 @@ func RegisterTunnel(
 	uuid uuid.UUID,
 ) error {
 	config.TransportLogger.Debug("initiating RPC stream to register")
-	tunnelServer, err := connection.NewRPCClient(ctx, muxer, config.TransportLogger.WithField("subsystem", "rpc-register"), openStreamTimeout)
+	stream, err := openStream(ctx, muxer)
 	if err != nil {
 		// RPC stream open error
 		return newClientRegisterTunnelError(err, config.Metrics.rpcFail)
 	}
-	defer tunnelServer.Close()
+	if !IsRPCStreamResponse(stream.Headers) {
+		// stream response error
+		return newClientRegisterTunnelError(err, config.Metrics.rpcFail)
+	}
+	conn := rpc.NewConn(
+		tunnelrpc.NewTransportLogger(config.TransportLogger.WithField("subsystem", "rpc-register"), rpc.StreamTransport(stream)),
+		tunnelrpc.ConnLog(config.TransportLogger.WithField("subsystem", "rpc-transport")),
+	)
+	defer conn.Close()
+	ts := tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx)}
 	// Request server info without blocking tunnel registration; must use capnp library directly.
-	serverInfoPromise := tunnelrpc.TunnelServer{Client: tunnelServer.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+	tsClient := tunnelrpc.TunnelServer{Client: ts.Client}
+	serverInfoPromise := tsClient.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
 		return nil
 	})
-	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger)
-	registration := tunnelServer.RegisterTunnel(
+	registration, err := ts.RegisterTunnel(
 		ctx,
 		config.OriginCert,
 		config.Hostname,
 		config.RegistrationOptions(connectionID, originLocalIP, uuid),
 	)
-	if registrationErr := registration.DeserializeError(); registrationErr != nil {
-		// RegisterTunnel RPC failure
-		return processRegisterTunnelError(registrationErr, config.Metrics)
-	}
-	credentialManager.SetEventDigest(registration.EventDigest)
-	return processRegistrationSuccess(config, logger, connectionID, registration)
-}
-
-func ReconnectTunnel(
-	ctx context.Context,
-	token []byte,
-	eventDigest []byte,
-	muxer *h2mux.Muxer,
-	config *TunnelConfig,
-	logger *log.Entry,
-	connectionID uint8,
-	originLocalIP string,
-	uuid uuid.UUID,
-) error {
-	config.TransportLogger.Debug("initiating RPC stream to reconnect")
-	tunnelServer, err := connection.NewRPCClient(ctx, muxer, config.TransportLogger.WithField("subsystem", "rpc-reconnect"), openStreamTimeout)
-	if err != nil {
-		// RPC stream open error
-		return newClientRegisterTunnelError(err, config.Metrics.rpcFail)
-	}
-	defer tunnelServer.Close()
-	// Request server info without blocking tunnel registration; must use capnp library directly.
-	serverInfoPromise := tunnelrpc.TunnelServer{Client: tunnelServer.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
-		return nil
-	})
 	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger)
-	registration := tunnelServer.ReconnectTunnel(
-		ctx,
-		token,
-		eventDigest,
-		config.Hostname,
-		config.RegistrationOptions(connectionID, originLocalIP, uuid),
-	)
-	if registrationErr := registration.DeserializeError(); registrationErr != nil {
-		// ReconnectTunnel RPC failure
-		return processRegisterTunnelError(registrationErr, config.Metrics)
+	if err != nil {
+		// RegisterTunnel RPC failure
+		return newClientRegisterTunnelError(err, config.Metrics.regFail)
 	}
-	return processRegistrationSuccess(config, logger, connectionID, registration)
-}
-
-func processRegistrationSuccess(config *TunnelConfig, logger *log.Entry, connectionID uint8, registration *tunnelpogs.TunnelRegistration) error {
 	for _, logLine := range registration.LogLines {
 		logger.Info(logLine)
 	}
 
+	if regErr := processRegisterTunnelError(registration.Err, registration.PermanentFailure, config.Metrics); regErr != nil {
+		return regErr
+	}
+
 	if registration.TunnelID != "" {
 		config.Metrics.tunnelsHA.AddTunnelID(connectionID, registration.TunnelID)
 		logger.Infof("Each HA connection's tunnel IDs: %v", config.Metrics.tunnelsHA.String())
@@ -432,32 +374,55 @@ func processRegistrationSuccess(config *TunnelConfig, logger *log.Entry, connect
 	config.Metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
 
 	logger.Infof("Route propagating, it may take up to 1 minute for your new route to become functional")
-	config.Metrics.regSuccess.Inc()
 	return nil
 }
 
-func processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, metrics *TunnelMetrics) error {
-	if err.Error() == DuplicateConnectionError {
-		metrics.regFail.WithLabelValues("dup_edge_conn").Inc()
+func processRegisterTunnelError(err string, permanentFailure bool, metrics *TunnelMetrics) error {
+	if err == "" {
+		metrics.regSuccess.Inc()
+		return nil
+	}
+
+	metrics.regFail.WithLabelValues(err).Inc()
+	if err == DuplicateConnectionError {
 		return dupConnRegisterTunnelError{}
 	}
-	metrics.regFail.WithLabelValues("server_error").Inc()
 	return serverRegisterTunnelError{
-		cause:     fmt.Errorf("Server error: %s", err.Error()),
-		permanent: err.IsPermanent(),
+		cause:     fmt.Errorf("Server error: %s", err),
+		permanent: permanentFailure,
 	}
 }
 
 func UnregisterTunnel(muxer *h2mux.Muxer, gracePeriod time.Duration, logger *log.Logger) error {
 	logger.Debug("initiating RPC stream to unregister")
 	ctx := context.Background()
-	tunnelServer, err := connection.NewRPCClient(ctx, muxer, logger.WithField("subsystem", "rpc-unregister"), openStreamTimeout)
+	stream, err := openStream(ctx, muxer)
 	if err != nil {
 		// RPC stream open error
 		return err
 	}
+	if !IsRPCStreamResponse(stream.Headers) {
+		// stream response error
+		return err
+	}
+	conn := rpc.NewConn(
+		tunnelrpc.NewTransportLogger(logger.WithField("subsystem", "rpc-unregister"), rpc.StreamTransport(stream)),
+		tunnelrpc.ConnLog(logger.WithField("subsystem", "rpc-transport")),
+	)
+	defer conn.Close()
+	ts := tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx)}
 	// gracePeriod is encoded in int64 using capnproto
-	return tunnelServer.UnregisterTunnel(ctx, gracePeriod.Nanoseconds())
+	return ts.UnregisterTunnel(ctx, gracePeriod.Nanoseconds())
+}
+
+func openStream(ctx context.Context, muxer *h2mux.Muxer) (*h2mux.MuxedStream, error) {
+	openStreamCtx, cancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer cancel()
+	return muxer.OpenStream(openStreamCtx, []h2mux.Header{
+		{Name: ":method", Value: "RPC"},
+		{Name: ":scheme", Value: "capnp"},
+		{Name: ":path", Value: "*"},
+	}, nil)
 }
 
 func LogServerInfo(
@@ -504,12 +469,12 @@ type TunnelHandler struct {
 	noChunkedEncoding bool
 }
 
-var dialer = net.Dialer{}
+var dialer = net.Dialer{DualStack: true}
 
 // NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
 func NewTunnelHandler(ctx context.Context,
 	config *TunnelConfig,
-	addr *net.TCPAddr,
+	addr string,
 	connectionID uint8,
 ) (*TunnelHandler, string, error) {
 	originURL, err := validation.ValidateUrl(config.OriginUrl)
@@ -530,18 +495,37 @@ func NewTunnelHandler(ctx context.Context,
 	if h.httpClient == nil {
 		h.httpClient = http.DefaultTransport
 	}
-
-	edgeConn, err := connection.DialEdge(ctx, dialTimeout, config.TlsConfig, addr)
+	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
+	dialCtx, dialCancel := context.WithTimeout(ctx, dialTimeout)
+	// TUN-92: enforce a timeout on dial and handshake (as tls.Dial does not support one)
+	plaintextEdgeConn, err := dialer.DialContext(dialCtx, "tcp", addr)
+	dialCancel()
 	if err != nil {
-		return nil, "", err
+		return nil, "", dialError{cause: errors.Wrap(err, "DialContext error")}
 	}
+	edgeConn := tls.Client(plaintextEdgeConn, config.TlsConfig)
+	edgeConn.SetDeadline(time.Now().Add(dialTimeout))
+	err = edgeConn.Handshake()
+	if err != nil {
+		return nil, "", dialError{cause: errors.Wrap(err, "Handshake with edge error")}
+	}
+	// clear the deadline on the conn; h2mux has its own timeouts
+	edgeConn.SetDeadline(time.Time{})
 	// Establish a muxed connection with the edge
 	// Client mux handshake with agent server
-	h.muxer, err = h2mux.Handshake(edgeConn, edgeConn, config.muxerConfig(h), h.metrics.activeStreams)
+	h.muxer, err = h2mux.Handshake(edgeConn, edgeConn, h2mux.MuxerConfig{
+		Timeout:            5 * time.Second,
+		Handler:            h,
+		IsClient:           true,
+		HeartbeatInterval:  config.HeartbeatInterval,
+		MaxHeartbeats:      config.MaxHeartbeats,
+		Logger:             config.TransportLogger.WithFields(log.Fields{}),
+		CompressionQuality: h2mux.CompressionSetting(config.CompressionQuality),
+	}, h.metrics.activeStreams)
 	if err != nil {
-		return nil, "", errors.Wrap(err, "Handshake with edge error")
+		return h, "", errors.New("TLS handshake error")
 	}
-	return h, edgeConn.LocalAddr().String(), nil
+	return h, edgeConn.LocalAddr().String(), err
 }
 
 func (h *TunnelHandler) AppendTagHeaders(r *http.Request) {

supervisor/supervisor.go

@@ -16,7 +16,6 @@ import (
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/streamhandler"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
 
@@ -29,27 +28,6 @@ type Supervisor struct {
 	useConfigResultChan chan<- *pogs.UseConfigurationResult
 	state               *state
 	logger              *logrus.Entry
-	metrics             metrics
-}
-
-type metrics struct {
-	configVersion prometheus.Gauge
-}
-
-func newMetrics() metrics {
-	configVersion := prometheus.NewGauge(prometheus.GaugeOpts{
-		Namespace: "supervisor",
-		Subsystem: "supervisor",
-		Name:      "config_version",
-		Help:      "Latest configuration version received from Cloudflare",
-	},
-	)
-	prometheus.MustRegister(
-		configVersion,
-	)
-	return metrics{
-		configVersion: configVersion,
-	}
 }
 
 func NewSupervisor(
@@ -92,7 +70,6 @@ func NewSupervisor(
 		useConfigResultChan: useConfigResultChan,
 		state:               newState(defaultClientConfig),
 		logger:              logger.WithField("subsystem", "supervisor"),
-		metrics:             newMetrics(),
 	}, nil
 }
 
@@ -154,7 +131,6 @@ func (s *Supervisor) notifySubsystemsNewConfig(newConfig *pogs.ClientConfig) *po
 			Success: true,
 		}
 	}
-	s.metrics.configVersion.Set(float64(newConfig.Version))
 
 	s.state.updateConfig(newConfig)
 	var tunnelHostnames []h2mux.TunnelHostname

tlsconfig/cloudflare_ca.go

@@ -26,28 +26,28 @@ mcifak4CQsr+DH4pn5SJD7JxtCG3YGswW8QZsw==
 -----END CERTIFICATE-----
 Issuer: C=US, O=CloudFlare, Inc., OU=CloudFlare Origin SSL Certificate Authority, L=San Francisco, ST=California
 -----BEGIN CERTIFICATE-----
-MIIEADCCAuigAwIBAgIID+rOSdTGfGcwDQYJKoZIhvcNAQELBQAwgYsxCzAJBgNV
-BAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTQwMgYDVQQLEytDbG91
-ZEZsYXJlIE9yaWdpbiBTU0wgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MRYwFAYDVQQH
-Ew1TYW4gRnJhbmNpc2NvMRMwEQYDVQQIEwpDYWxpZm9ybmlhMB4XDTE5MDgyMzIx
-MDgwMFoXDTI5MDgxNTE3MDAwMFowgYsxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBD
-bG91ZEZsYXJlLCBJbmMuMTQwMgYDVQQLEytDbG91ZEZsYXJlIE9yaWdpbiBTU0wg
-Q2VydGlmaWNhdGUgQXV0aG9yaXR5MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRMw
-EQYDVQQIEwpDYWxpZm9ybmlhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
-AQEAwEiVZ/UoQpHmFsHvk5isBxRehukP8DG9JhFev3WZtG76WoTthvLJFRKFCHXm
-V6Z5/66Z4S09mgsUuFwvJzMnE6Ej6yIsYNCb9r9QORa8BdhrkNn6kdTly3mdnykb
-OomnwbUfLlExVgNdlP0XoRoeMwbQ4598foiHblO2B/LKuNfJzAMfS7oZe34b+vLB
-yrP/1bgCSLdc1AxQc1AC0EsQQhgcyTJNgnG4va1c7ogPlwKyhbDyZ4e59N5lbYPJ
-SmXI/cAe3jXj1FBLJZkwnoDKe0v13xeF+nF32smSH0qB7aJX2tBMW4TWtFPmzs5I
-lwrFSySWAdwYdgxw180yKU0dvwIDAQABo2YwZDAOBgNVHQ8BAf8EBAMCAQYwEgYD
-VR0TAQH/BAgwBgEB/wIBAjAdBgNVHQ4EFgQUJOhTV118NECHqeuU27rhFnj8KaQw
-HwYDVR0jBBgwFoAUJOhTV118NECHqeuU27rhFnj8KaQwDQYJKoZIhvcNAQELBQAD
-ggEBAHwOf9Ur1l0Ar5vFE6PNrZWrDfQIMyEfdgSKofCdTckbqXNTiXdgbHs+TWoQ
-wAB0pfJDAHJDXOTCWRyTeXOseeOi5Btj5CnEuw3P0oXqdqevM1/+uWp0CM35zgZ8
-VD4aITxity0djzE6Qnx3Syzz+ZkoBgTnNum7d9A66/V636x4vTeqbZFBr9erJzgz
-hhurjcoacvRNhnjtDRM0dPeiCJ50CP3wEYuvUzDHUaowOsnLCjQIkWbR7Ni6KEIk
-MOz2U0OBSif3FTkhCgZWQKOOLo1P42jHC3ssUZAtVNXrCk3fw9/E15k8NPkBazZ6
-0iykLhH1trywrKRMVw67F44IE8Y=
+MIID/DCCAuagAwIBAgIID+rOSdTGfGcwCwYJKoZIhvcNAQELMIGLMQswCQYDVQQG
+EwJVUzEZMBcGA1UEChMQQ2xvdWRGbGFyZSwgSW5jLjE0MDIGA1UECxMrQ2xvdWRG
+bGFyZSBPcmlnaW4gU1NMIENlcnRpZmljYXRlIEF1dGhvcml0eTEWMBQGA1UEBxMN
+U2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5pYTAeFw0xNDExMTMyMDM4
+NTBaFw0xOTExMTQwMTQzNTBaMIGLMQswCQYDVQQGEwJVUzEZMBcGA1UEChMQQ2xv
+dWRGbGFyZSwgSW5jLjE0MDIGA1UECxMrQ2xvdWRGbGFyZSBPcmlnaW4gU1NMIENl
+cnRpZmljYXRlIEF1dGhvcml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEG
+A1UECBMKQ2FsaWZvcm5pYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
+AMBIlWf1KEKR5hbB75OYrAcUXobpD/AxvSYRXr91mbRu+lqE7YbyyRUShQh15lem
+ef+umeEtPZoLFLhcLyczJxOhI+siLGDQm/a/UDkWvAXYa5DZ+pHU5ct5nZ8pGzqJ
+p8G1Hy5RMVYDXZT9F6EaHjMG0OOffH6Ih25TtgfyyrjXycwDH0u6GXt+G/rywcqz
+/9W4Aki3XNQMUHNQAtBLEEIYHMkyTYJxuL2tXO6ID5cCsoWw8meHufTeZW2DyUpl
+yP3AHt4149RQSyWZMJ6AyntL9d8Xhfpxd9rJkh9Kge2iV9rQTFuE1rRT5s7OSJcK
+xUsklgHcGHYMcNfNMilNHb8CAwEAAaNmMGQwDgYDVR0PAQH/BAQDAgAGMBIGA1Ud
+EwEB/wQIMAYBAf8CAQIwHQYDVR0OBBYEFCToU1ddfDRAh6nrlNu64RZ4/CmkMB8G
+A1UdIwQYMBaAFCToU1ddfDRAh6nrlNu64RZ4/CmkMAsGCSqGSIb3DQEBCwOCAQEA
+cQDBVAoRrhhsGegsSFsv1w8v27zzHKaJNv6ffLGIRvXK8VKKK0gKXh2zQtN9SnaD
+gYNe7Pr4C3I8ooYKRJJWLsmEHdGdnYYmj0OJfGrfQf6MLIc/11bQhLepZTxdhFYh
+QGgDl6gRmb8aDwk7Q92BPvek5nMzaWlP82ixavvYI+okoSY8pwdcVKobx6rWzMWz
+ZEC9M6H3F0dDYE23XcCFIdgNSAmmGyXPBstOe0aAJXwJTxOEPn36VWr0PKIQJy5Y
+4o1wpMpqCOIwWc8J9REV/REzN6Z1LXImdUgXIXOwrz56gKUJzPejtBQyIGj0mveX
+Fu6q54beR89jDc+oABmOgg==
 -----END CERTIFICATE-----
 Issuer: C=US, O=CloudFlare, Inc., OU=Origin Pull, L=San Francisco, ST=California, CN=origin-pull.cloudflare.net
 -----BEGIN CERTIFICATE-----

tunnelrpc/pogs/auth_outcome.go

@@ -1,7 +1,7 @@
 package pogs
 
 import (
-	"errors"
+	"fmt"
 	"time"
 )
 
@@ -18,21 +18,21 @@ type AuthenticateResponse struct {
 
 // Outcome turns the deserialized response of Authenticate into a programmer-friendly sum type.
 func (ar AuthenticateResponse) Outcome() AuthOutcome {
-	// If the user's authentication was unsuccessful, the server will return an error explaining why.
-	// cloudflared should fatal with this error.
-	if ar.PermanentErr != "" {
-		return NewAuthFail(errors.New(ar.PermanentErr))
-	}
-
 	// If there was a network error, then cloudflared should retry later,
 	// because origintunneld couldn't prove whether auth was correct or not.
 	if ar.RetryableErr != "" {
-		return NewAuthUnknown(errors.New(ar.RetryableErr), ar.HoursUntilRefresh)
+		return &AuthUnknown{Err: fmt.Errorf(ar.RetryableErr), HoursUntilRefresh: ar.HoursUntilRefresh}
+	}
+
+	// If the user's authentication was unsuccessful, the server will return an error explaining why.
+	// cloudflared should fatal with this error.
+	if ar.PermanentErr != "" {
+		return &AuthFail{Err: fmt.Errorf(ar.PermanentErr)}
 	}
 
 	// If auth succeeded, return the token and refresh it when instructed.
-	if len(ar.Jwt) > 0 {
-		return NewAuthSuccess(ar.Jwt, ar.HoursUntilRefresh)
+	if ar.PermanentErr == "" && len(ar.Jwt) > 0 {
+		return &AuthSuccess{Jwt: ar.Jwt, HoursUntilRefresh: ar.HoursUntilRefresh}
 	}
 
 	// Otherwise the state got messed up.
@@ -43,90 +43,41 @@ func (ar AuthenticateResponse) Outcome() AuthOutcome {
 //go-sumtype:decl AuthOutcome
 type AuthOutcome interface {
 	isAuthOutcome()
-	// Serialize into an AuthenticateResponse which can be sent via Capnp
-	Serialize() AuthenticateResponse
 }
 
 // AuthSuccess means the backend successfully authenticated this cloudflared.
 type AuthSuccess struct {
-	jwt               []byte
-	hoursUntilRefresh uint8
-}
-
-func NewAuthSuccess(jwt []byte, hoursUntilRefresh uint8) AuthSuccess {
-	return AuthSuccess{jwt: jwt, hoursUntilRefresh: hoursUntilRefresh}
-}
-
-func (ao AuthSuccess) JWT() []byte {
-	return ao.jwt
+	Jwt               []byte
+	HoursUntilRefresh uint8
 }
 
 // RefreshAfter is how long cloudflared should wait before rerunning Authenticate.
-func (ao AuthSuccess) RefreshAfter() time.Duration {
-	return hoursToTime(ao.hoursUntilRefresh)
+func (ao *AuthSuccess) RefreshAfter() time.Duration {
+	return hoursToTime(ao.HoursUntilRefresh)
 }
 
-// Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao AuthSuccess) Serialize() AuthenticateResponse {
-	return AuthenticateResponse{
-		Jwt:               ao.jwt,
-		HoursUntilRefresh: ao.hoursUntilRefresh,
-	}
-}
-
-func (ao AuthSuccess) isAuthOutcome() {}
+func (ao *AuthSuccess) isAuthOutcome() {}
 
 // AuthFail means this cloudflared has the wrong auth and should exit.
 type AuthFail struct {
-	err error
+	Err error
 }
 
-func NewAuthFail(err error) AuthFail {
-	return AuthFail{err: err}
-}
-
-func (ao AuthFail) Error() string {
-	return ao.err.Error()
-}
-
-// Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao AuthFail) Serialize() AuthenticateResponse {
-	return AuthenticateResponse{
-		PermanentErr: ao.err.Error(),
-	}
-}
-
-func (ao AuthFail) isAuthOutcome() {}
+func (ao *AuthFail) isAuthOutcome() {}
 
 // AuthUnknown means the backend couldn't finish checking authentication. Try again later.
 type AuthUnknown struct {
-	err               error
-	hoursUntilRefresh uint8
-}
-
-func NewAuthUnknown(err error, hoursUntilRefresh uint8) AuthUnknown {
-	return AuthUnknown{err: err, hoursUntilRefresh: hoursUntilRefresh}
-}
-
-func (ao AuthUnknown) Error() string {
-	return ao.err.Error()
+	Err               error
+	HoursUntilRefresh uint8
 }
 
 // RefreshAfter is how long cloudflared should wait before rerunning Authenticate.
-func (ao AuthUnknown) RefreshAfter() time.Duration {
-	return hoursToTime(ao.hoursUntilRefresh)
+func (ao *AuthUnknown) RefreshAfter() time.Duration {
+	return hoursToTime(ao.HoursUntilRefresh)
 }
 
-// Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao AuthUnknown) Serialize() AuthenticateResponse {
-	return AuthenticateResponse{
-		RetryableErr:      ao.err.Error(),
-		HoursUntilRefresh: ao.hoursUntilRefresh,
-	}
-}
-
-func (ao AuthUnknown) isAuthOutcome() {}
+func (ao *AuthUnknown) isAuthOutcome() {}
 
 func hoursToTime(hours uint8) time.Duration {
 	return time.Duration(hours) * time.Hour
-}
+}

tunnelrpc/pogs/auth_test.go

@@ -31,15 +31,15 @@ func TestAuthenticateResponseOutcome(t *testing.T) {
 	}{
 		{"success",
 			fields{Jwt: []byte("asdf"), HoursUntilRefresh: 6},
-			AuthSuccess{jwt: []byte("asdf"), hoursUntilRefresh: 6},
+			&AuthSuccess{Jwt: []byte("asdf"), HoursUntilRefresh: 6},
 		},
 		{"fail",
 			fields{PermanentErr: "bad creds"},
-			AuthFail{err: fmt.Errorf("bad creds")},
+			&AuthFail{Err: fmt.Errorf("bad creds")},
 		},
 		{"error",
 			fields{RetryableErr: "bad conn", HoursUntilRefresh: 6},
-			AuthUnknown{err: fmt.Errorf("bad conn"), hoursUntilRefresh: 6},
+			&AuthUnknown{Err: fmt.Errorf("bad conn"), HoursUntilRefresh: 6},
 		},
 		{"nil (no fields are set)",
 			fields{},
@@ -58,38 +58,13 @@ func TestAuthenticateResponseOutcome(t *testing.T) {
 				Jwt:               tt.fields.Jwt,
 				HoursUntilRefresh: tt.fields.HoursUntilRefresh,
 			}
-			got := ar.Outcome()
-			if !reflect.DeepEqual(got, tt.want) {
+			if got := ar.Outcome(); !reflect.DeepEqual(got, tt.want) {
 				t.Errorf("AuthenticateResponse.Outcome() = %T, want %v", got, tt.want)
 			}
-			if got != nil && !reflect.DeepEqual(got.Serialize(), ar) {
-				t.Errorf(".Outcome() and .Serialize() should be inverses but weren't. Expected %v, got %v", ar, got.Serialize())
-			}
 		})
 	}
 }
 
-func TestAuthSuccess(t *testing.T) {
-	input := NewAuthSuccess([]byte("asdf"), 6)
-	output, ok := input.Serialize().Outcome().(AuthSuccess)
-	assert.True(t, ok)
-	assert.Equal(t, input, output)
-}
-
-func TestAuthUnknown(t *testing.T) {
-	input := NewAuthUnknown(fmt.Errorf("pdx unreachable"), 6)
-	output, ok := input.Serialize().Outcome().(AuthUnknown)
-	assert.True(t, ok)
-	assert.Equal(t, input, output)
-}
-
-func TestAuthFail(t *testing.T) {
-	input := NewAuthFail(fmt.Errorf("wrong creds"))
-	output, ok := input.Serialize().Outcome().(AuthFail)
-	assert.True(t, ok)
-	assert.Equal(t, input, output)
-}
-
 func TestWhenToRefresh(t *testing.T) {
 	expected := 4 * time.Hour
 	actual := hoursToTime(4)

tunnelrpc/pogs/config.go

@@ -197,14 +197,11 @@ func (hc *HTTPOriginConfig) Service() (originservice.OriginService, error) {
 		return nil, err
 	}
 
-	dialer := &net.Dialer{
+	dialContext := (&net.Dialer{
 		Timeout:   hc.ProxyConnectionTimeout,
 		KeepAlive: hc.TCPKeepAlive,
-	}
-	if !hc.DialDualStack {
-		dialer.FallbackDelay = -1
-	}
-	dialContext := dialer.DialContext
+		DualStack: hc.DialDualStack,
+	}).DialContext
 	transport := &http.Transport{
 		Proxy:       http.ProxyFromEnvironment,
 		DialContext: dialContext,
@@ -273,6 +270,7 @@ func (*HelloWorldOriginConfig) Service() (originservice.OriginService, error) {
 		DialContext: (&net.Dialer{
 			Timeout:   30 * time.Second,
 			KeepAlive: 30 * time.Second,
+			DualStack: true,
 		}).DialContext,
 		TLSClientConfig: &tls.Config{
 			RootCAs: rootCAs,

tunnelrpc/pogs/reconnect_tunnel.go

@@ -12,10 +12,6 @@ func (i TunnelServer_PogsImpl) ReconnectTunnel(p tunnelrpc.TunnelServer_reconnec
 	if err != nil {
 		return err
 	}
-	eventDigest, err := p.Params.EventDigest()
-	if err != nil {
-		return err
-	}
 	hostname, err := p.Params.Hostname()
 	if err != nil {
 		return err
@@ -29,7 +25,7 @@ func (i TunnelServer_PogsImpl) ReconnectTunnel(p tunnelrpc.TunnelServer_reconnec
 		return err
 	}
 	server.Ack(p.Options)
-	registration, err := i.impl.ReconnectTunnel(p.Ctx, jwt, eventDigest, hostname, pogsOptions)
+	registration, err := i.impl.ReconnectTunnel(p.Ctx, jwt, hostname, pogsOptions)
 	if err != nil {
 		return err
 	}
@@ -42,21 +38,16 @@ func (i TunnelServer_PogsImpl) ReconnectTunnel(p tunnelrpc.TunnelServer_reconnec
 
 func (c TunnelServer_PogsClient) ReconnectTunnel(
 	ctx context.Context,
-	jwt,
-	eventDigest []byte,
+	jwt []byte,
 	hostname string,
 	options *RegistrationOptions,
-) *TunnelRegistration {
+) (*TunnelRegistration, error) {
 	client := tunnelrpc.TunnelServer{Client: c.Client}
 	promise := client.ReconnectTunnel(ctx, func(p tunnelrpc.TunnelServer_reconnectTunnel_Params) error {
 		err := p.SetJwt(jwt)
 		if err != nil {
 			return err
 		}
-		err = p.SetEventDigest(eventDigest)
-		if err != nil {
-			return err
-		}
 		err = p.SetHostname(hostname)
 		if err != nil {
 			return err
@@ -73,11 +64,7 @@ func (c TunnelServer_PogsClient) ReconnectTunnel(
 	})
 	retval, err := promise.Result().Struct()
 	if err != nil {
-		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
+		return nil, err
 	}
-	registration, err := UnmarshalTunnelRegistration(retval)
-	if err != nil {
-		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
-	}
-	return registration
+	return UnmarshalTunnelRegistration(retval)
 }

tunnelrpc/pogs/tunnelrpc.go

@@ -9,16 +9,13 @@ import (
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 
+	log "github.com/sirupsen/logrus"
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/pogs"
 	"zombiezen.com/go/capnproto2/rpc"
 	"zombiezen.com/go/capnproto2/server"
 )
 
-const (
-	defaultRetryAfterSeconds = 15
-)
-
 type Authentication struct {
 	Key         string
 	Email       string
@@ -36,114 +33,14 @@ func UnmarshalAuthentication(s tunnelrpc.Authentication) (*Authentication, error
 }
 
 type TunnelRegistration struct {
-	SuccessfulTunnelRegistration
 	Err               string
+	Url               string
+	LogLines          []string
 	PermanentFailure  bool
+	TunnelID          string `capnp:"tunnelID"`
 	RetryAfterSeconds uint16
 }
 
-type SuccessfulTunnelRegistration struct {
-	Url         string
-	LogLines    []string
-	TunnelID    string `capnp:"tunnelID"`
-	EventDigest []byte
-}
-
-func NewSuccessfulTunnelRegistration(
-	url string,
-	logLines []string,
-	tunnelID string,
-	eventDigest []byte,
-) *TunnelRegistration {
-	// Marshal nil will result in an error
-	if logLines == nil {
-		logLines = []string{}
-	}
-	return &TunnelRegistration{
-		SuccessfulTunnelRegistration: SuccessfulTunnelRegistration{
-			Url:         url,
-			LogLines:    logLines,
-			TunnelID:    tunnelID,
-			EventDigest: eventDigest,
-		},
-	}
-}
-
-// Not calling this function Error() to avoid confusion with implementing error interface
-func (tr TunnelRegistration) DeserializeError() TunnelRegistrationError {
-	if tr.Err != "" {
-		err := fmt.Errorf(tr.Err)
-		if tr.PermanentFailure {
-			return NewPermanentRegistrationError(err)
-		}
-		retryAfterSeconds := tr.RetryAfterSeconds
-		if retryAfterSeconds < defaultRetryAfterSeconds {
-			retryAfterSeconds = defaultRetryAfterSeconds
-		}
-		return NewRetryableRegistrationError(err, retryAfterSeconds)
-	}
-	return nil
-}
-
-type TunnelRegistrationError interface {
-	error
-	Serialize() *TunnelRegistration
-	IsPermanent() bool
-}
-
-type PermanentRegistrationError struct {
-	err string
-}
-
-func NewPermanentRegistrationError(err error) TunnelRegistrationError {
-	return &PermanentRegistrationError{
-		err: err.Error(),
-	}
-}
-
-func (pre *PermanentRegistrationError) Error() string {
-	return pre.err
-}
-
-func (pre *PermanentRegistrationError) Serialize() *TunnelRegistration {
-	return &TunnelRegistration{
-		Err:              pre.err,
-		PermanentFailure: true,
-	}
-}
-
-func (*PermanentRegistrationError) IsPermanent() bool {
-	return true
-}
-
-type RetryableRegistrationError struct {
-	err               string
-	retryAfterSeconds uint16
-}
-
-func NewRetryableRegistrationError(err error, retryAfterSeconds uint16) TunnelRegistrationError {
-	return &RetryableRegistrationError{
-		err:               err.Error(),
-		retryAfterSeconds: retryAfterSeconds,
-	}
-}
-
-func (rre *RetryableRegistrationError) Error() string {
-	return rre.err
-}
-
-func (rre *RetryableRegistrationError) Serialize() *TunnelRegistration {
-	return &TunnelRegistration{
-		Err:               rre.err,
-		PermanentFailure:  false,
-		RetryAfterSeconds: rre.retryAfterSeconds,
-	}
-}
-
-func (*RetryableRegistrationError) IsPermanent() bool {
-	return false
-}
-
 func MarshalTunnelRegistration(s tunnelrpc.TunnelRegistration, p *TunnelRegistration) error {
 	return pogs.Insert(tunnelrpc.TunnelRegistration_TypeID, s.Struct, p)
 }
@@ -428,12 +325,12 @@ func UnmarshalConnectParameters(s tunnelrpc.CapnpConnectParameters) (*ConnectPar
 }
 
 type TunnelServer interface {
-	RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) *TunnelRegistration
+	RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error)
 	GetServerInfo(ctx context.Context) (*ServerInfo, error)
 	UnregisterTunnel(ctx context.Context, gracePeriodNanoSec int64) error
 	Connect(ctx context.Context, parameters *ConnectParameters) (ConnectResult, error)
 	Authenticate(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*AuthenticateResponse, error)
-	ReconnectTunnel(ctx context.Context, jwt, eventDigest []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error)
+	ReconnectTunnel(ctx context.Context, jwt []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error)
 }
 
 func TunnelServer_ServerToClient(s TunnelServer) tunnelrpc.TunnelServer {
@@ -462,12 +359,15 @@ func (i TunnelServer_PogsImpl) RegisterTunnel(p tunnelrpc.TunnelServer_registerT
 		return err
 	}
 	server.Ack(p.Options)
-	registration := i.impl.RegisterTunnel(p.Ctx, originCert, hostname, pogsOptions)
-
+	registration, err := i.impl.RegisterTunnel(p.Ctx, originCert, hostname, pogsOptions)
+	if err != nil {
+		return err
+	}
 	result, err := p.Results.NewResult()
 	if err != nil {
 		return err
 	}
+	log.Info(registration.TunnelID)
 	return MarshalTunnelRegistration(result, registration)
 }
 
@@ -520,7 +420,7 @@ func (c TunnelServer_PogsClient) Close() error {
 	return c.Conn.Close()
 }
 
-func (c TunnelServer_PogsClient) RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) *TunnelRegistration {
+func (c TunnelServer_PogsClient) RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error) {
 	client := tunnelrpc.TunnelServer{Client: c.Client}
 	promise := client.RegisterTunnel(ctx, func(p tunnelrpc.TunnelServer_registerTunnel_Params) error {
 		err := p.SetOriginCert(originCert)
@@ -543,13 +443,9 @@ func (c TunnelServer_PogsClient) RegisterTunnel(ctx context.Context, originCert
 	})
 	retval, err := promise.Result().Struct()
 	if err != nil {
-		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
+		return nil, err
 	}
-	registration, err := UnmarshalTunnelRegistration(retval)
-	if err != nil {
-		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
-	}
-	return registration
+	return UnmarshalTunnelRegistration(retval)
 }
 
 func (c TunnelServer_PogsClient) GetServerInfo(ctx context.Context) (*ServerInfo, error) {

tunnelrpc/pogs/tunnelrpc_test.go

@@ -1,7 +1,6 @@
 package pogs
 
 import (
-	"fmt"
 	"reflect"
 	"testing"
 	"time"
@@ -12,30 +11,16 @@ import (
 	capnp "zombiezen.com/go/capnproto2"
 )
 
-const (
-	testURL               = "tunnel.example.com"
-	testTunnelID          = "asdfghjkl;"
-	testRetryAfterSeconds = 19
-)
-
-var (
-	testErr         = fmt.Errorf("Invalid credential")
-	testLogLines    = []string{"all", "working"}
-	testEventDigest = []byte("asdf")
-)
-
-// *PermanentRegistrationError implements TunnelRegistrationError
-var _ TunnelRegistrationError = (*PermanentRegistrationError)(nil)
-
-// *RetryableRegistrationError implements TunnelRegistrationError
-var _ TunnelRegistrationError = (*RetryableRegistrationError)(nil)
-
 func TestTunnelRegistration(t *testing.T) {
 	testCases := []*TunnelRegistration{
-		NewSuccessfulTunnelRegistration(testURL, testLogLines, testTunnelID, testEventDigest),
-		NewSuccessfulTunnelRegistration(testURL, nil, testTunnelID, testEventDigest),
-		NewPermanentRegistrationError(testErr).Serialize(),
-		NewRetryableRegistrationError(testErr, testRetryAfterSeconds).Serialize(),
+		&TunnelRegistration{
+			Err:               "it broke",
+			Url:               "asdf.cftunnel.com",
+			LogLines:          []string{"it", "was", "broken"},
+			PermanentFailure:  true,
+			TunnelID:          "asdfghjkl;",
+			RetryAfterSeconds: 19,
+		},
 	}
 	for i, testCase := range testCases {
 		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))

tunnelrpc/tunnelrpc.capnp

@@ -21,8 +21,6 @@ struct TunnelRegistration {
     tunnelID @4 :Text;
     # How long should this connection wait to retry in seconds, if the error wasn't permanent
     retryAfterSeconds @5 :UInt16;
-    # A unique ID used to reconnect this tunnel.
-    eventDigest @6 :Data;
 }
 
 struct RegistrationOptions {
@@ -293,7 +291,7 @@ interface TunnelServer {
     unregisterTunnel @2 (gracePeriodNanoSec :Int64) -> ();
     connect @3 (parameters :CapnpConnectParameters) -> (result :ConnectResult);
     authenticate @4 (originCert :Data, hostname :Text, options :RegistrationOptions) -> (result :AuthenticateResponse);
-    reconnectTunnel @5 (jwt :Data, eventDigest :Data, hostname :Text, options :RegistrationOptions) -> (result :TunnelRegistration);
+    reconnectTunnel @5 (jwt :Data, hostname :Text, options :RegistrationOptions) -> (result :TunnelRegistration);
 }
 
 interface ClientService {

tunnelrpc/tunnelrpc.capnp.go

@@ -125,12 +125,12 @@ type TunnelRegistration struct{ capnp.Struct }
 const TunnelRegistration_TypeID = 0xf41a0f001ad49e46
 
 func NewTunnelRegistration(s *capnp.Segment) (TunnelRegistration, error) {
-	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 5})
+	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4})
 	return TunnelRegistration{st}, err
 }
 
 func NewRootTunnelRegistration(s *capnp.Segment) (TunnelRegistration, error) {
-	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 5})
+	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4})
 	return TunnelRegistration{st}, err
 }
 
@@ -242,26 +242,12 @@ func (s TunnelRegistration) SetRetryAfterSeconds(v uint16) {
 	s.Struct.SetUint16(2, v)
 }
 
-func (s TunnelRegistration) EventDigest() ([]byte, error) {
-	p, err := s.Struct.Ptr(4)
-	return []byte(p.Data()), err
-}
-
-func (s TunnelRegistration) HasEventDigest() bool {
-	p, err := s.Struct.Ptr(4)
-	return p.IsValid() || err != nil
-}
-
-func (s TunnelRegistration) SetEventDigest(v []byte) error {
-	return s.Struct.SetData(4, v)
-}
-
 // TunnelRegistration_List is a list of TunnelRegistration.
 type TunnelRegistration_List struct{ capnp.List }
 
 // NewTunnelRegistration creates a new list of TunnelRegistration.
 func NewTunnelRegistration_List(s *capnp.Segment, sz int32) (TunnelRegistration_List, error) {
-	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 5}, sz)
+	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4}, sz)
 	return TunnelRegistration_List{l}, err
 }
 
@@ -2890,7 +2876,7 @@ func (c TunnelServer) ReconnectTunnel(ctx context.Context, params func(TunnelSer
 		Options: capnp.NewCallOptions(opts),
 	}
 	if params != nil {
-		call.ParamsSize = capnp.ObjectSize{DataSize: 0, PointerCount: 4}
+		call.ParamsSize = capnp.ObjectSize{DataSize: 0, PointerCount: 3}
 		call.ParamsFunc = func(s capnp.Struct) error { return params(TunnelServer_reconnectTunnel_Params{Struct: s}) }
 	}
 	return TunnelServer_reconnectTunnel_Results_Promise{Pipeline: capnp.NewPipeline(c.Client.Call(call))}
@@ -3888,12 +3874,12 @@ type TunnelServer_reconnectTunnel_Params struct{ capnp.Struct }
 const TunnelServer_reconnectTunnel_Params_TypeID = 0xa353a3556df74984
 
 func NewTunnelServer_reconnectTunnel_Params(s *capnp.Segment) (TunnelServer_reconnectTunnel_Params, error) {
-	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 4})
+	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 3})
 	return TunnelServer_reconnectTunnel_Params{st}, err
 }
 
 func NewRootTunnelServer_reconnectTunnel_Params(s *capnp.Segment) (TunnelServer_reconnectTunnel_Params, error) {
-	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 4})
+	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 3})
 	return TunnelServer_reconnectTunnel_Params{st}, err
 }
 
@@ -3921,51 +3907,37 @@ func (s TunnelServer_reconnectTunnel_Params) SetJwt(v []byte) error {
 	return s.Struct.SetData(0, v)
 }
 
-func (s TunnelServer_reconnectTunnel_Params) EventDigest() ([]byte, error) {
-	p, err := s.Struct.Ptr(1)
-	return []byte(p.Data()), err
-}
-
-func (s TunnelServer_reconnectTunnel_Params) HasEventDigest() bool {
-	p, err := s.Struct.Ptr(1)
-	return p.IsValid() || err != nil
-}
-
-func (s TunnelServer_reconnectTunnel_Params) SetEventDigest(v []byte) error {
-	return s.Struct.SetData(1, v)
-}
-
 func (s TunnelServer_reconnectTunnel_Params) Hostname() (string, error) {
-	p, err := s.Struct.Ptr(2)
+	p, err := s.Struct.Ptr(1)
 	return p.Text(), err
 }
 
 func (s TunnelServer_reconnectTunnel_Params) HasHostname() bool {
-	p, err := s.Struct.Ptr(2)
+	p, err := s.Struct.Ptr(1)
 	return p.IsValid() || err != nil
 }
 
 func (s TunnelServer_reconnectTunnel_Params) HostnameBytes() ([]byte, error) {
-	p, err := s.Struct.Ptr(2)
+	p, err := s.Struct.Ptr(1)
 	return p.TextBytes(), err
 }
 
 func (s TunnelServer_reconnectTunnel_Params) SetHostname(v string) error {
-	return s.Struct.SetText(2, v)
+	return s.Struct.SetText(1, v)
 }
 
 func (s TunnelServer_reconnectTunnel_Params) Options() (RegistrationOptions, error) {
-	p, err := s.Struct.Ptr(3)
+	p, err := s.Struct.Ptr(2)
 	return RegistrationOptions{Struct: p.Struct()}, err
 }
 
 func (s TunnelServer_reconnectTunnel_Params) HasOptions() bool {
-	p, err := s.Struct.Ptr(3)
+	p, err := s.Struct.Ptr(2)
 	return p.IsValid() || err != nil
 }
 
 func (s TunnelServer_reconnectTunnel_Params) SetOptions(v RegistrationOptions) error {
-	return s.Struct.SetPtr(3, v.Struct.ToPtr())
+	return s.Struct.SetPtr(2, v.Struct.ToPtr())
 }
 
 // NewOptions sets the options field to a newly
@@ -3975,7 +3947,7 @@ func (s TunnelServer_reconnectTunnel_Params) NewOptions() (RegistrationOptions,
 	if err != nil {
 		return RegistrationOptions{}, err
 	}
-	err = s.Struct.SetPtr(3, ss.Struct.ToPtr())
+	err = s.Struct.SetPtr(2, ss.Struct.ToPtr())
 	return ss, err
 }
 
@@ -3984,7 +3956,7 @@ type TunnelServer_reconnectTunnel_Params_List struct{ capnp.List }
 
 // NewTunnelServer_reconnectTunnel_Params creates a new list of TunnelServer_reconnectTunnel_Params.
 func NewTunnelServer_reconnectTunnel_Params_List(s *capnp.Segment, sz int32) (TunnelServer_reconnectTunnel_Params_List, error) {
-	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 4}, sz)
+	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 3}, sz)
 	return TunnelServer_reconnectTunnel_Params_List{l}, err
 }
 
@@ -4010,7 +3982,7 @@ func (p TunnelServer_reconnectTunnel_Params_Promise) Struct() (TunnelServer_reco
 }
 
 func (p TunnelServer_reconnectTunnel_Params_Promise) Options() RegistrationOptions_Promise {
-	return RegistrationOptions_Promise{Pipeline: p.Pipeline.GetPipeline(3)}
+	return RegistrationOptions_Promise{Pipeline: p.Pipeline.GetPipeline(2)}
 }
 
 type TunnelServer_reconnectTunnel_Results struct{ capnp.Struct }
@@ -4330,253 +4302,249 @@ func (p ClientService_useConfiguration_Results_Promise) Result() UseConfiguratio
 	return UseConfigurationResult_Promise{Pipeline: p.Pipeline.GetPipeline(0)}
 }
 
-const schema_db8274f9144abc7e = "x\xda\xccZ}\x90\x14ez\x7f\x9e\xeeY\x9a\x85]" +
-	"f:=\x96+\x02#[\x12\x85\x13\xa2\"\x89\xb7I" +
-	"n\xf6\x03\xb8]\x8e\x8f\xe9\x9d\x05\xbd\x95K\xd1;\xf3" +
-	"\xeen/=\xddCw\x0f\xb0\x1b8\x84\xc2xl\xe4" +
-	"\x04OR\xe2\xe1\x95\xa0\xc4\x8fp9\xf0\xa0\xa2\x06-" +
-	"M\xee\"\xe6\x8ex\\ %\x17\xadS\xd1J\x9d\xa5" +
-	"eP)c\xca\xb3SO\x7f\xef\xec\xba\x80I\xaa\xee" +
-	"\x1f\x98z\xfay?\x9e\xaf\xdf\xf3\xf1\xee\x8dC\x93\x9a" +
-	"\xb9\x9bj\xaeK\x02\xc8\x87k&8l\xce/\x87\x1e" +
-	"\x9e\xf5\x8f\xdb@\x9e\x8a\xe8|\xfb\xf8\x92\xf4\xa7\xf6\xb6" +
-	"\x7f\x87\x1a^\x00\x98?(\x0c\xa1\xb4S\x10\x00\xa4\x1d" +
-	"\xc2\x7f\x00:5\xbf\x7f\xe6\x8d\xf2\x1b\xc2v\x10\xa7\xc6" +
-	"\x999b.M\\\x82\xd2\xd6\x89\xc4\xbcy\xe2\x06@" +
-	"\xe7OJ\xaf\x1c\xf8\xc3=?#f.b\x06\x9c\xff" +
-	"\xce\xc4!\x94>u9/L\\\x01\xe8|t\x7f\xc3" +
-	"\xdf\xec\xff\x97\x13w\x81x\x1d\x82\x7fv}\xed\xaf\x10" +
-	"P\x9aY\xfb#@\xe7_o\xdc\xf4\xf6\x9a\x8fv\x7f" +
-	"g\xe4\xb9\x09\xe2{\xb1v\x18\xa5\xb3\xb5\x02\xf0\xceC" +
-	"w\xa4\xff\x19\x1f\xfed7\x88\xd7\xd36H\x9f\x8f\xd5" +
-	"N\xe2\x00\xa5\x93\xb5Y@\xe7\x95\x1b\x8e?\xbb\xeb\xc7" +
-	"w\x7f\x1f\xe4\xeb\x10\xc1[\xff~\xed\x7f\xd398\x89" +
-	"\x18\xce?\xfa\x95\xc4\x0f_\xf9\xbd\x1f\xb8\x0c\xce\xc1S" +
-	"\xb7=\xb5\xeb\xc7\xd7\xbc\x0b+9\x01\x13\x00\xf3gO" +
-	"2\x89w\xc1$\xd2\xc5\xfd\xaf>\xb7\xbc\xb4\xfb\xc1\x03" +
-	"\xde\xa5\xdd\xbd\xae\x98\xccq\x90p\xb6w|RZ\xf9" +
-	"H\xfe\x11_\x1c\xf7S\xed\xe4\x0fi\xe9\xf4\xc9\xb4t" +
-	"\xc1\xaf\xdeY\xb1\xec\xa9\xde\xc7|\x06\xf7\xa2\x9fN~" +
-	"\x8a\x18j\xeb\xe8\x1e'6\xa4\xeei\xf9\xa3{\x1f\xab" +
-	"\xb6J\x0dq\xce\xad\x1bFiQ\x1d\xfdl\xa9\xbb\x0d" +
-	"\x01\x9d\xe1\xaf>\xb7\xea\xa3\xbf\xb0\x9e\x04y.&\x9c" +
-	"\x9f\xec8\xb7~\xf6\x13\xbd/\xb9\xd7\xe6\x01\xe6?S" +
-	"\xffK\xda\xfad=\xa9\xb2\xfe\xef\xe7,\xbf\xf7\xed\xa5" +
-	"Gh\xeb\x98Y\xbcK\x94\xa64\xa1\xb4y\x0aYf" +
-	"p\x0aq\xff\xe2\x86U\xcf?\x7f\xb8\xefH\xf5E\\" +
-	"\x8b_\x95\\\x82\xd2\xdc$q\xcfN\x12\xf7\x15\x1d\xf8" +
-	"\xda\x0b7%\xfe.n\xc7\xd7\x93\xef\xd2\xe1\xe7]\x86" +
-	";>;\xf6\x0f\x8b>8\xfdL\xdcB\xbbS\x1cY" +
-	"\xe8`\x8a\x04\xef\x1e\xc6\xd2kM\xcd\xcf\x83|=\xa2" +
-	"3\xb0g\x93\xdd\xfe\xc0N\x07V\xa2\x80\x1c\xc0\xfc\x93" +
-	"\xa9!\xda\xecl\x8a\xfck\xfa\xfb\xad\xf5\xfa\x07\xdb^" +
-	"\xa8rF\xf7\xd4\x05\xe2\x12\x94:D\xba\xda\"\xf1G" +
-	"\x80\x9f<y\xf7\xae\x8es\x0b_\x92\xa7b\xa2Z\xe8" +
-	"7\xc5!\x94.\x10\xef\xfc\xf3b\x86\xf4\x19j\xb0\x8a" +
-	"\xdd\x95z\xa64\x80\xd2\x02\x89~\xde$\xb9\xecK\xee" +
-	"\xf8\xde}5\xef|\xef\xa5j\x95R\xe0\xcc\xff\xd3\xb4" +
-	"\x89\x92\x9c\xa6\x9f\xcb\xd2\xbf\xe6\x00\x9d\xa9\x87\xff\xf8o" +
-	"[\x8bg\x7f6F\x10I\x9b\xaf\xfcP\xdaq%\xfd" +
-	"\xba\xebJ\x92\xf1\xdc\xdc#\x7f\xfe\x9b\x9d\xa7N\xc7=" +
-	"\xe5\xcd+]\x8f\xbdp%)\xec\xee\xaf\x0c\x0e-\x9f" +
-	"5|\xa6\xda@.\xe7\x15\x0d\xc3(\xcdmp\x0d\xd4" +
-	"@\xdbq\xef(W\xdd\xf9o_{-\xe6\xb3{\x1b" +
-	"\xdeBH8\xcbW\xdd1P\xbb\xf9\xdc\xb9\xf8A;" +
-	"\x1b\\\xd3\xedo\xa0\x83\x8e\x8a\xf7I\xc7\xf7\xff\xf5\xdb" +
-	"t\x90P\xad\xee\x17\x1b\xbaQ:C\x07\xcd?\xd5\xf0" +
-	"\x18\x09\x19\xc6\xceX\x8es\xf2\xea&\x94^\xbf\x9a\xee" +
-	"u\xf6j\xba\xd7\x825-l\xf5\xad\xb7\xbf\x0b\xe2T" +
-	"~\x04T\\5\xad\x09\xa5\xd9\xd3h\xd1\xaciw\xa3" +
-	"4w\xba\x00\xe0|\xb7\xaf\xfb\xe5\xf3m\xfb\xff\xb3z" +
-	"s/\x08\xa77\xa14k\xbak\xaa\xe9\xae}\xe6\xdf" +
-	"\xf4\x97\xef\xefy\xa4\xed\xfc\xa8\xddK3ZQ\xda<" +
-	"\xc3u\xf7\x19_\x97\x9e\x98\xe1n\xfe\xed\x85+\xbe\xda" +
-	"\xf8\xe2\x87qM\xec\x9e\xe1F\xef\xc1\x19\xa4\x89\xde[" +
-	"\xdf\xfb\xfa\xac\xef\xfe\xd3\x87U\xf6s\x19\x7f:c\x0e" +
-	"Jg\xdc\x1dO\x11\xf3\x07\x8b\x7fpzjr\xea\xc7" +
-	"c\xc5\xf1\x85\x19\x03(\xd5f\xe8gM\xe6^\xba\xe8" +
-	"\xedo=\xb8!\xfb\xfd\x8f?!\xb9\xf8*\x9c{\xe6" +
-	"\x9an\x94N^C;\xbf|\x0d\xc5\xd2\xd2Cg\xbf" +
-	"\xd6\xbf\xe7\xc4\xa7c\"we\xe66\x94v\xcct\x1d" +
-	"i&A\xce_\x09\xfb\xce\xdd\xf9\xeb?\xfb,.U" +
-	"\xa9\xf1-\x92jk#I\xb5\xe9\x83\xbd\xed\xf7\xae>" +
-	"\xf4y\x9ca\x7f\xe3\xb3\xc4p\xc4e\x08\x83q,O" +
-	";\xd5\xd8\x8a\xd2\x9b\x8dt\xde\xeb\x8dY\x98\xeb\xd8\x15" +
-	"]g\x9aYN\x14\xfe \xf8Y\x98WP\xcaz\xb9" +
-	"\xa9\xa5b\xf73\xddV\x0b\x8a\xcd:Y\xd6*\x1b\xba" +
-	"\xc5r\x88r\x8aO\x00$\x10@T\x06\x00\xe45<" +
-	"\xca\x1a\x87\"b\x9a\xd0ZT\x89\xd8\xcf\xa3ls(" +
-	"r\\\x9a\x10A\\\xd7\x08 k<\xca\x1b9D>" +
-	"Mx'V\xee\x03\x907\xf2(o\xe7\xd0)3\xb3" +
-	"\xa4\xe8L\x87\xa4\xbd\xc84\xb1\x0e8\xac\x03tLf" +
-	"\x9b\x83J\x8f\x06I\x16#\x0b\x03\x1bl\xac\x07\x0e\xeb" +
-	"\x01\x9d~\xa3bZ+u\x1bU\xad\x93\xf5\x9a\xcc\xc2" +
-	"~\x9c\x00\x1cN\x00\x1cO\xbc6C\xd7Y\xc1\xceW" +
-	"\x0a\x05fY\x00$\xd9\xc4P\xb2\xd9\x0f\x02\xc87\xf0" +
-	"(\xdf\x1a\x93l\x01Iv\x0b\x8fr3\x87\x8e\xc5\xcc" +
-	"\xf5\xcc\\j`A\xb1UC_\xae\xf0%\x16^\xbb" +
-	"\xa0\xa9L\xb7\xdb\x0cH\xea\xbdj\x1f\xa6\xa2P\x00\xc4" +
-	"\xd4\xf8\x17[\xb4Q\xb5lU\xef\xebr\xe9\xd9\x9c\xa1" +
-	"\xa9\x85A\xba]\x9d\xab\xc9\xe9M\xb4\x87xE7\x00" +
-	"r\xa2\xd8\x0a\x90U\xfbt\xc3dNQ\xb5\x0a$\x14" +
-	"\xf0\x05{K\x8f\xa2)z\x81\x85\x07M\x18}\x90w" +
-	"@\xde\x95c\x9e\x12\xb3\xf6\xb59\xc5T\xf8\x92%\xd7" +
-	"\x85\xfaX\xd4\x0d /\xe4Q\xce\xc5\xf4\xb1l\x09\x80" +
-	"\xbc\x94G\xf9\xf6\x98\xa5W\xb6\x02\xc89\x1e\xe5\xd5\x1c" +
-	":\x86\xa9\xf6\xa9z\x1b\x03\xde\x8c\x1b\xcc\xb2u\xa5\xc4" +
-	"\x00 P\xd8\x16\xa3LJ\xb40\x15\xa1t\x95\xa6j" +
-	"F\x0b\xd0\xce4\xcd\xb8\xcd0\xb5\xe2\x0a\xef\x1c\x83\xb4" +
-	"\xed\x9a2\\&\x8cay\xd78$\xb7Z`\xf3*" +
-	"\x16\xf3\xd6UL\xd7\x90\xd7v2\xab\xa2\xd9\x16\x80\x9c" +
-	"\x08\xc5\xafo\x02\x90'\xf2(\xa79\xcc\x9a.\x03\xa6" +
-	"\"P\xaf\xba\xea\xc5t]\xd1M\xd6\xa7Z63=" +
-	"\xf2\xb5YRx\xc9\x8a\x1fH\xfe\x97\xe2Q\x9e\xc6\xa1" +
-	"\xd3g*\x05\x96c&\xaaFq\xb9\xa2\x1by\x9e\x15" +
-	"\xb0\x068\xac\x19\xdf\x93\x16+\xaa\xc6\x8a\x9et\xf3\x0a" +
-	"\x19\xf7\x7f\x8a\xde:\xc7\xf1\xc2\xb7;\x0a\xdfz\xfc\xdc" +
-	"\xf1\xe3w(\x8a\xdfz\xee\xb7\xce\xe8\x00\xae\xe7?s" +
-	"\xfc\x10\xa6\x88\xb0y\x94\xef\xa4\x88\xa8\x94I\xa7\x16\xf0" +
-	"\x86\x89\xa9\x08%}\xed\xb0b\x1fiZ\x87,+\x90" +
-	"\xa21\x15d{\x8fA(\x1a\xfd\x98\x8aJ\x19\x7f\x99" +
-	"\xc9\xd63\xd3b9H\x9a\xc6\xc6ALEY\xbfJ" +
-	"\xebS.W\xeb\x81\xa1\xc3U\xe3\xaf7Y\xc1\x83\x0c" +
-	"\x7fy.\xe3\x19-\x06\x87\xa4\xa3\xd5<\xca\xfd\xb1 " +
-	"a=\x00r\x91G\xb9\x1c\x0b\x92\xd2\x92H\x9b\"\x1f" +
-	"\xe0!EN\x99Gy\x137\x12\xe1\xd8z\xa6\xdb\x0b" +
-	"\xd5>\x10\x98\xf5\xff\x10F#\xa4\xf4e\x0c\xa5\x8b\xb9" +
-	"$yK\x1d\x8fr\x03\xc15}e6\x056\x9d\x16" +
-	"\x16\xc2\x17?\xad\x8d\xfe\xf5\xc17\xe7\xefb\xfa\xf8\xdb" +
-	"\x10\x1e\xb6\x97\x0e{\x80G\xf9\xd1\x98*\xf7\x9b\x00\xf2" +
-	"\xc3<\xca\x878D_\x93O\x1c\x00\x90\x0f\xf1(?" +
-	"M\x9a\xe4<M\x1e\x9bCm\x13\x8f\xf2\xcf9\x14\x13" +
-	"|\x9a\xba\x02\xf1e\x0a\xa9\x9f\xf3(\xbf\xca\xa1X\x93" +
-	"Hc\x0d\x80x\x86\xacs\x9aG\xf9\x8d/B\xab\x82" +
-	"fT\x8a\xbd\x9a\x02\x19\x93\x15;\x16\x86t\xbdR\xca" +
-	"\x99l\xbd\x8aF\xc5j\xb1mV\x12\xca\xb6\x15$\x9e" +
-	"\xa4\xad\xf4Y8\x050\xc7#\xa6\xa2R\x12\x90\x88\xe1" +
-	"\x9eh\xb2\xe2*fZ*o\xe8a\xeePu\x9b\xe9" +
-	"\xf6R\x05\x84\x1e\xa6\x85\xd4q\xb0\xa5\xd3\x8f\x10\x8a\x0f" +
-	"?\xd8\x8d\x08\x0f\xb1\x8f`|\x9a\xe3\xf8J\\D\xba" +
-	"i\xe6Q^\xca\xe1t\xfc\x9c\xc8\xa4\xc7\x8eN\x00\xb9" +
-	"\x9dG\xb9\x8b\xc3\xe9\xdco\x89L\x9a\x94\xbb#4O" +
-	"\xf6\xdbv\x19SQ\x89\xe9\x1b{\x03\xeb\xb1\x8c\xc2Z" +
-	"\x06H\xa0\x18\xd6;\xfe\xd7~\x1f\xa4\x81\xd7\x8a\x98\x8a" +
-	"Z\xc4*O\xe1\xbf(Cg\xa9\x1e0L7\x01F" +
-	"\xe9\xe8\xe6H\x88\xc0;:\xba#\x09D\xae\xd9\x13K" +
-	"\xee\x89\xee\x9f)(\x15\x8b\x8d,-Zzm\xe0\x99" +
-	"\x19\xa2\xa9\xd5oT\xb4b'\x03\xc16\x07\x11\x81C" +
-	"\x1c\x1fc\x17\x1a\xed1\xc5{n<v\xda\x0c\xb3f" +
-	"w<k\xfa\xea_I\xea\xef\xf2P\xc2\xd1\x08\xa5\xf4" +
-	"v\x03x\xcb\x0e\xaf\xeb\x11s\x86\xeb\x9c\x02p(\x00" +
-	":\x95\xb2e\x9bL)\x01\x86\xdeF\xfcS.#\x19" +
-	"U\x81bNI\xbaq\xff\xbb\x94\xfa/?\x87{\xf9" +
-	"tD\x06?\x10K\xa8\x05\x7f5\xba\xcb\xdb\x0c]\xb8" +
-	"\xec*\xcdG0/\x87\xcc\xf3k\x02* \x83\xe4:" +
-	"\x9b\x92\xc1\xb5<\xca7\xc6\x93\xeb\\R\xd1\xf5<\xca" +
-	"\xb7p(0\x93\xf2d\xd8\xe9{\x87n\xb1\xbc\x8a\x14" +
-	"S\xd1\x18\xe7\xe2\xd7\x89\x15\xeb\xaa\xa1\x8fr\xc3\xc6(" +
-	"\\B\x13v\xdc\x1c\xb3k`\xc2e=\x91]\x85\xb5" +
-	"l0\xb0R\x86\x95\x145B#\xdf\xb8- |#" +
-	"\xe2\x19\xb7\xa8\xf5\x93\xbf\x97\xfa\xb3\x9e\xb5\xe8\x92\xe9\xf0" +
-	"\x92\x9b\x87\x01\xe4;y\x94\xef\x89]r\x07\xf5\x08\xf7" +
-	"\xf0(?\x10\xbb\xe4\x1eR\xe2.\x1e\xe5}\xb1\xec\xb9" +
-	"\x97\x0c\xbc\x8fG\xf9q\x0e1\xe1A\xfeA\x82\xfc\xc7" +
-	"y\x94\x8fr.`\xb7\xb7\xb4\x19:\xfa\x97\xb0\x00\xc2" +
-	">\xa1\x9f)\xa6\xdd\xc3\x14\xb4;t\x9b\x99\xeb\x15\xd4" +
-	"\x02H\xd8b\xab%fT\xec\x10\"J\xcaF\xb7\xb0" +
-	"\xc2b\xbb\xb7JPl\x0bk\x81\xc3Z\x8aH\x8b\x99" +
-	"m&+\"YC\xd1r\x0ao\xf7_\x8a\x82F\x82" +
-	"xr\x0c\xf5PY\xb6\x89G\xf9;\x04%\x18\x9b&" +
-	"\x89w\x0d\x00\xe7\"\x09\xc9\xbc\xae5^Zp^\x9a" +
-	"\x8b\xb7ZnB\x9c\x00 n%\xedl\xe7Q\xde\xc5" +
-	"\x05Wk7 \xebEh\xb5\xa9\xfdVf\x0b\xa1\xa6" +
-	"\xca\"y\xfdzAEC\xefr\x15\x85\x91\xa6\x0aF" +
-	"\xa9l\x92+\xab\x86.W\x14M\xe5\xed\xc1p\xe1\xb8" +
-	"\xba H\xf2ByE9\xe3\x1a\x8b\x94qK\xa0\x0c" +
-	"\xe9[\xb8\x04 \xbf\x1ay\xcc\xf7c\xe4.\x12\xc3V" +
-	"\x80\xfc\x1a\xa2k\x18y\x8c\xa4\xe2T\x80|\x91\xe8e" +
-	"\x0c;P\xa9\x84O\x02\xe4\xcbD\xde\x84Q\xa9 \x0d" +
-	"\xba\xdbo$\xfav\x8c\xaa\x05i+\xce\x01\xc8o\"" +
-	"\xfa\x03D\x9f\xc0\xb9\x9a\x94\xf6\xe0\x00@\xfe~\xa2?" +
-	"Lt\xa1&M\xed\xb6\xf4\x10\x9a\x00\xf9}D\x7f\x9c" +
-	"\xe8\x13\x1b\xd28\x11@:\xe8\xd2\x1f%\xfaa\xa2\xd7" +
-	"^\x95\xc6Z\x00\xe9\x87\xb8\x0d \x7f\x88\xe8O\x13}" +
-	"\x12\xa6q\x12\x80t\x0c\x1f\x04\xc8?M\xf4\x9f\x10}" +
-	"\xf2\x844N\x06\x90^t\xefs\x9c\xe8'\x88^\x97" +
-	"Hc\x1d\x80\xf4S<\x00\x90?A\xf4\xd3\x18\xe2]" +
-	"G1\x0e\xbb\xe4njTv\xf0\x86\x15\x9a\x9c\xf9\x1d" +
-	"(z9!g$\xa9\x05\xc5d4*\x06\xc4$\xa0" +
-	"S6\x0cm\xf9H8\xbfX\xe5\xe3\xbb\x0b$\x0d\xbd" +
-	"\xa3\x18\xc6\x9f\xe7dK\x0d\xc8\x14\x14\xad\xa3\x1c\xd5B" +
-	"VK\xc56*e\xc8\x14\x15\x9b\x15\xc3\x84lV\xf4" +
-	"\xc5\xa6Q\xeaBf\x96T]\xd1 \xfc2\x9e\xcf%" +
-	"+\x15\xb5\x18\xee=n\x01\x17\xba'W\xed\x9e\x99r" +
-	"S\x97\xd2W5-\x98\x13a}\x08]so\x8e\xa0" +
-	">\x19\x0f\xa9\xcczE\xab\xb0K\xa9\xec\xc6\xed?:" +
-	"\xb3^\xffr\xb165\x98m]\xbc4_Y\x95F" +
-	"\xbd\xe46j4\xd2\x1a\x09\x1b\xcaj\xfa\xe3\x92v." +
-	"J`\x81Iz\xfd6\x142\xb4w\xcc9\xc2\xe1\xa3" +
-	"\xef\x1c\x97\xaa\x89>f{\xbf:\xf4^\x83r\xbd\xa0" +
-	"\x94\xac/\xb9\xba\x93Y\xc9K\xd1b4N\xbcx2" +
-	"n\xef\xea\xcaE\x13\x09\xdeC\xf2\x1bC\xf0j\xc1N" +
-	"\x80|3E\xe7R\x0cu(u\xb8 \xd2N\xe4." +
-	"\x8cJXIv\xc1\"G\xf4\xd5\x1859\xd27\xdd" +
-	" \x8f00\xd1\xe2\x81\x17s\xb7\x0f\xb1N\xacA\x0f" +
-	"\xbcJ\xee\xfe\x1a\xd17\xc6\xc1\xab\x82\xc3#\xc0N\xe0" +
-	"=\xf0\xda\xea\x82\xcev\xa2\xefr\xc1+\xe1\x81\xd7N" +
-	"|\x0a \xbf\x8b\xe8\xfb\\\xf0\xaa\xf1\xc0k/>;" +
-	"\x02\xec&M\xf0\xc0\xeb\xa0\xcb\xff8\xd1\x8f\xba\xe0\xd5" +
-	"\xea\x81\xd7\x11\x17\xec\x0e\x13\xfd8\x81T\xc5\xd4\xf2\xb6" +
-	"\xa9\xea\x80}Ql\x14\xca\xdf`\xac\xdc\x02IM]" +
-	"\xcf\xc2\xc4RT\x15maE\xd1 \x93\xb7\x95\xc2\xda" +
-	"\xa8N\xd7\xacvE/Z\xd8\xaf\xace\x94\x8e\x84x" +
-	"\xe2\xb65k\x153\xd5^\xc0\xa8\xb2\x0f\x0b\x99d\xce" +
-	"0\xaa\xeb\x1b\xb7@d\xa6\x87p\xe1\xb7\x92\xb2\xb1\xa3" +
-	"\xa8\xb16\x0c\xca\x19^\x8f\xd2\xa1J_\x0c]G\xaf" +
-	"\xc6\xe8R3#\x8b\x87\xb2\xdf+\x04EHW\xb6\xaa" +
-	"\xba`\x1b\xcb\xac`\xb7\x19\xa8\xdb\xaa^a\xa36(" +
-	"\xf4W\xf4\xb5\xac\xb8\x08\xf5\x82QT\xf5>\x18\xd5\xa4" +
-	"\xf0_4\x08\x8aU]n4c\xec!M\x9c\xdd\x04" +
-	"\x9c\x0b]TC\x88MQ\xab\x9f-\xb8\xab\xb2&S" +
-	"\xacX\x97:\xcei\xfe\xe0\xd2\x0b2\xaf\xad\xaf\x01\x08" +
-	"_\x9d0\x98\xdc\x8bG\x86\x80\x13\x9f\x100z\xf0\xc0" +
-	"\xe0}C|\xc8\x04N\xdc# \x17\xbe\x06b\xf0\x92" +
-	"'\xee\x18\x06N\xbcK@>|\xa1\xc3`,.\x0e" +
-	"\xb6\x02'\x96\x04L\x84\xaf\x95\x18\xcc\xd4E\x85\xea\xa4" +
-	"o\x0aX\x13>\xfda\xf0p#.\xdb\x06\x9c\xb8H" +
-	"p\x82v\x08\xb2\x9e\x18\xcd\xe8\x04\x80\x01\x19\x172\x9a" +
-	"\xd1\x09FI\x18\xb4M\x00\xcd\xb8\xc5\x87\xe7ft\x82" +
-	"a*$\x0b\x8a\xcd\x9a\xa9\xd7\xf4>\xa2\x0f\xde\xd0\x8c" +
-	"\xf1!%\xffE\x0d\xce\xd8\x85rkT\xcc\x05\x00\xbc" +
-	"u8\xaa\xe5\xc2\xa6r\xe7\x93\xf1:\xd9\x9f\x8d\xec\xdd" +
-	"\xe6OV\x8e\xc6f#G\xa8x>\xca\xa3\xfc\x0b." +
-	"\xaa\x0c\x02\x9f\x0e\xe6zh\x98A\x97;\xcex\xcf\xf7" +
-	"|\xbf\x86\xad\x1e\xf29E\xa3\xdf\xadq\xd1\xdb\xca\x82" +
-	"(\x1d\xc4'\x7fSb\x93?\x0c\xfakaD\xf6\x88" +
-	"\xcf\x01\xa7\\\xa4Y\x8bw\x8bn:K\xb8.\x19\xbc" +
-	"sb\xf0$-\x8a\xe4Z\xf5\x82\x13t\x94\x18\xe4B" +
-	"\xa82\xd9e\xb6\xd5\x9d,\xf3\xbfI\xd6c8\x88w" +
-	"N\x92<\xd2\x13(\xdcw 6\xa7\xd3\x0c\xbf#L" +
-	".\x8f\x17\xf5\xe3\xe8\xca\xbbpP\x82'i1\xed?" +
-	"-\xdc\xffX\xa3?\\;\x1e+v\x9ei\xf4\x1d\xe8" +
-	"\x85X\x9f\xf6\xdc\x12\x00\xf9\xb87q\x0b\x1e}\xce\x90" +
-	"\xa3\xbe\xca\xa3\xfcv\xcc\xfd\xde$\xc67x\x94\xdf\x8b" +
-	"\xf2\x95\xf8\x1b\xeaY\xde\xe3Q\xfe/JV\x09\xafg" +
-	"\xb9@\xfd\xe9\xc7<v\xa2\xdf?\x07/B\x153B" +
-	"o\xcd\xe8[\xaa\xea\xcc\xa2\xb2\xb4j*\x12<3\xa1" +
-	"M\x98X1\x09\xd8G\x02h\xc7\xc2X5\x1b\x0e\x89" +
-	"\x90\x99y\x8a\xe1\"Z\xe1\xf0e\xec\xb1\xec8\xaa\xcd" +
-	"\xfb\x81\xe4\xc5\x91_\x17\xc4\xba\xf4\x03\xb1\x01V\xa0X" +
-	"\xf9Y\x7f0\xb4&\xa6\xd8o\xf5D\x83f\x02\x1bc" +
-	"e\xb9\xa8\xa0\xcd\x16\x9bl]\x85\x09za0\xeaV" +
-	"\xa9_+X+\xb1L\x15\xf4b\x93e\xd7UX\x9c" +
-	"!x\\\x00A5\x8a\xa3^\x15\xc6\xa8\x12oc=" +
-	"y\xa3\xb0\x96\xd9#\x1e]\xaa\x1e\x06;\xa3\x97\x85\xf0" +
-	"]\xb03\xfe.\xe8C\xd4\xba\x81h\xe6\x1dB\xd4\xe0" +
-	"p\xd4\xea\x8e]\x16\xfc\xdfd\xf2/\xf56FE\xb1" +
-	"p)\x05c\xf8';_r\x02\x7f\xa9\xf5}\xf4\xe2" +
-	"{\x99S+\x08q\x03c\x7f\xd1A\x87p\xfe\xe6\xff" +
-	"\x13\x00\x00\xff\xff<\x08\xe4z"
+const schema_db8274f9144abc7e = "x\xda\xc4Z}\x90\x14ez\x7f\x9e\xee\x19\x9a\x85]" +
+	"f\xda^\x8b=\x04\x16(\x8d\xc2\x09Q\xd1D7\xf1" +
+	"f?\x80\xdb\xe5\xf8\x98\xdeYPWL\xd1\xcc\xbc\xbb" +
+	"\xdb\xd0\xd3=t\xf7\x00K\xf0\x10\x0a\xe3\xb9\x91\x13<" +
+	"I\x89\x07W\x80\x1a?\xc2\xe5\xc0\x83\x8a\x1a\xce:\x13" +
+	"\x8d\x98\xbb\x8b\xde\x05R\xc7\xc5\xaa3\"\x95\xd2\x0ae" +
+	"\xd4\xb3,Sj\xa7\x9e\xb7?wv]\xc0\\U\xfe" +
+	"\x81\xa9\xa7\x9f\xf7\xe3\xf9\xfa=\x1f\xef^\xd7;\xa1U" +
+	"\xb8>}u\x06@=\x92\x1e\xe7\xb19\xbf\xda|\xe0" +
+	"\xaa\x7f\xdc\x0e\xea\x14D\xef\xdb'\x167~\xean\xff" +
+	"wH\x8b\x12\xc0\xfcAi3*;%\x09@\xb9_" +
+	"\xfaO@/\xfd\x07\xa7\xdf\xaa\xbc%\xed\x00yJ\x92" +
+	"Y \xe6\xf2\xf8\xc5\xa8l\x1bO\xccw\x8f\xdf\x08\xe8" +
+	"\xfdi\xf9\xf5C\x7f\xb4\xe7g\xc4,\xc4\xcc\x80\xf3\xcf" +
+	"\x8d\xdf\x8c\xca\xa7\x9c\xf3\xe3\xf1\xcb\x01\xbd\x8f\x1en\xfa" +
+	"\x9b\x83\xffr\xf2^\x90\xafF\x08\xcen\xa8\xfb\x0d\x02" +
+	"*3\xeb~\x04\xe8\xfd\xebu[\xdeY\xfd\xd1\xee\xef" +
+	"\x0c?7E|/\xd5\x0d\xa1r\xa6N\x02\xd1\xdb\x7f" +
+	"g\xe3?\xe3\x81Ov\x83|\x0dm\x83\xf4\xf9x\xdd" +
+	"\x04\x01P\xf9E]\x0e\xd0{\xfd\xda\x13/\xec\xfa\xf1" +
+	"}\xdf\x07\xf5jD\xf0\xd7\x9f\xaf\xfb\x1f:\x07'\x10" +
+	"\xc3\x07\x8f\x7f=\xf5\xc3\xd7/\xfb\x01g\xf0\x9e\xf8\xe5" +
+	"m\xcf\xee\xfa\xf1\x8c\xf7`\x85 a\x0a`\xfe\xec\x09" +
+	"6\xf1\xde4\x81t\xf1\xf0\xaf\x7f\xb2\xac\xbc\xfb\xd1C" +
+	"\xfe\xa5\xf9^\x97O\x14\x04Hy;\xba>)\xafx" +
+	"\xac\xf0XR\x9c\xba\x89\x1f\xd2\xd2i\x13I\x9c\x9b~" +
+	"sn\xf9\xd2g\xfb\x9e\x0c\x18\xf8E\x9f\x9f\xf8,1" +
+	"\xbc6\x91\xeeqrc\xf6\x81\xb6?~\xf0\xc9Z\xab" +
+	"\xa4\xf9\x8d'\x0e\xa1\x92\xae\xa7\x9fX\x7f\x1b\x02zC" +
+	"\xb7\xfcd\xe5G\x7f\xe1<\x03\xea\\Ly/\xdf\x7f" +
+	"v\xc3\xec\xa7\xfb^\xe5\xd7\x16\xc9\x8c\x0d\xbf\xa2\xadw" +
+	"6\xd0\xd9\x0d\x7f?g\xd9\x83\xef,9J['\xcc" +
+	"\xe2_b\xee\xa4\x16Tn\x9dD\x96\xb9e\x12q\xbf" +
+	"q\xed\xca\x17_<\xd2\x7f\xb4\xf6\"\xdc\xe2g&-" +
+	"F\xe5<\xe7~\x97s_\xde\x85o\xfe\xf4\xfa\xd4\xdf" +
+	"%\x05\xdf\x9fy\x8f\x0e?\x9a!\x86;?;\xfe\x0f" +
+	"\x0b\xdf?\xf5|\xd2BjV \x0b\xb1,\x09\xde;" +
+	"\x84\xe57[Z_\x04\xf5\x1aDo\xed\x9e-n\xe7" +
+	"#;=X\x81\x12\x0a\x00\xf3wf7\xd3f{\xb3" +
+	"\xe4_\xd3\xce\xb77\x98\xefo\xffi\x8d3\xf2S?" +
+	"\xce.F\xa5N\xa6\xab\xa5\xe5\x1f\x01~\xf2\xcc}\xbb" +
+	"\xba\xce.xU\x9d\x82\xa9Z\xa1\x0f\xca\x9bQ9N" +
+	"\xbc\xf3\x8f\xca\xcd\xa4\xcfH\x835\xec\\\xea\xff\xb8l" +
+	"-*\x1f_F??\xb8\x8c\xb3/\xbe\xf3{\x0f\xa5" +
+	"\xcf}\xef\xd5Z\x95R\xe0\xcc\xff\\\xb1Q\x91\x1b\xb9" +
+	"K7\xfeV\x00\xf4\xa6\x1c\xf9\x93\xbfm/\x9d\xf9\xd9" +
+	"(A\xa4\xdc:\xf9C\xa5k2\xfdZ8\x99d<" +
+	";\xf7\xe8\x9f\xbf\xbb\xf3\x97\xa7\x92\x9erp2\xf7\xd8" +
+	"\xe3\x93Ia\xf7}}p\xf3\xb2\xab\x86N\xd7\x1a\x88" +
+	"s\x9e\x9e<\x84\xcay\xbe\xdd\xbb|;\xe1\x9c\xf6\xb5" +
+	"{\xfe\xed\x1bo&|\xf6\x8e\xa6\xb7\x11R\xde\xb2\x95" +
+	"w\xae\xad\xbb\xfb\xec\xd9\xe4AK\x9b\xb8\xe9\xb4&:" +
+	"\xe8\x98\xfc\x90r\xe2\xe0_\xbfC\x07I\xb5\xea\xde\xd6" +
+	"\xd4\x8b\xca\x9e&\xfa\xb9\xbb\xe9I\x122\x8a\x9d\xd1\x1c" +
+	"g\xe7\x15-\xa8\xec\xbf\x82\xee\xb5\xf7\x0a\xba\xd7M\xab" +
+	"\xdb\xd8\xaa\x9bo\x7f\x0f\xe4)\xe20\xa88C\x9c\xef" +
+	"\x12\xe7\xfcsW\xdc\x87\xca\xf9\xa9\x12\x80\xf7\xdd\xfe\xde" +
+	"\xd7>\xe88\xf8\xdf\xb5\x9bs\x81NOmA\xe5\xdc" +
+	"Tn\xaa\xa9\xdc>\xf3\xaf\xff\xcb\xf3{\x1e\xeb\xf8`" +
+	"\xc4\xees\xa7\xb7\xa3r\xebt\xee\xee\xd3\xbf\xa9\xe8\xd3" +
+	"\xf9\xe6\xdf^\xb0\xfc\x96Y/}\x98\xd4\x84:\x9dG" +
+	"/\x9bN\x9a\xe8\xbb\xf9\xbf\xbey\xd5w\xff\xe9\xc3\x1a" +
+	"\xfbq\xc6{\xa7\xcfAe\x0f\xdfq71\xbf\xbf\xe8" +
+	"\x07\xa7\xa6d\xa6\xfcn\xb4\x8b\x1e\x9f\xbe\x16\x95\xd7\x88" +
+	"w\xfe+\xd3y\x1c\xdf\xfe\xf6\xa3\x1bs\xdf\xff\xdd'" +
+	"$\x97X\x83s\xb7\xcc\xe8Ee\xe9\x0c\xda\xb9k\x06" +
+	"\xc5\xd2\x92\xc3g\xbe1\xb0\xe7\xe4\xa7\xa3\"\xf7\xc73" +
+	"\xb6\xa3R7\x93\xfb\xffLB\xab\xbf\x92\xf6\x9d\xbd\xe7" +
+	"\xb7\x7f\xf6YR\xaa\xf33\xdf\xe6\xd07\x8b\xa4\xda\xf2" +
+	"\xfe\xde\xce\x07W\x1d\xfe\"\xc90s\xd6\x0b\xc4p=" +
+	"g\x88\x82q4OSg\xb5\xa3\xa2\xcd\xa2\xf3\xee\x9a" +
+	"\x95\x83\xb9\x9e[5Mf\xd8\x95T\xf1\x0f\xc3\x9f\xc5" +
+	"yE\xadbVZ\xda\xaa\xee\x003]\xbd\xa8\xb9\xac" +
+	"\x9b\xe5\x9c\x8ae:,\x8f\xa8f\xc5\x14@\x0a\x01d" +
+	"m-\x80\xbaZD\xd5\x10PFl$\xb4\x96u\"" +
+	"\x0e\x88\xa8\xba\x02\xca\x82\xd0H\x88 \xaf\x9f\x05\xa0\x1a" +
+	"\"\xaa\x9b\x04D\xb1\x91\xf0N\xae>\x04\xa0n\x12Q" +
+	"\xdd!\xa0WavY3\x99\x09\x19w\xa1mc=" +
+	"\x08X\x0f\xe8\xd9\xcc\xb5\x07\xb55\x06dX\x82,\xad" +
+	"\xdd\xe8b\x03\x08\xd8\x00\xe8\x0dXU\xdbYa\xba\xa8" +
+	"\x1b\xdd\xac\xcff\x0e\x0e\xe08\x10p\x1c\xe0X\xe2u" +
+	"X\xa6\xc9\x8an\xa1Z,2\xc7\x01 \xc9\xc6G\x92" +
+	"\xcd~\x14@\xbdVD\xf5\xe6\x84d7\x91d7\x8a" +
+	"\xa8\xb6\x0a\xe89\xcc\xde\xc0\xec%\x16\x165W\xb7\xcc" +
+	"e\x9aXf\xd1\xb5\x8b\x86\xceL\xb7\xc3\x82\x8c\xd9\xa7" +
+	"\xf7c6\x0e\x05@\xcc\x8e}\xb1\x85\x9bt\xc7\xd5\xcd" +
+	"\xfe\x1eN\xcf\xe5-C/\x0e\xd2\xed\xea\xb9&\xa7\xb5" +
+	"\xd0\x1e\xf2\xe5\xbd\x00(\xc8r;@N\xef7-\x9b" +
+	"y%\xdd)\x92P \x16\xdd\xadk4C3\x8b," +
+	":h\xdc\xc8\x83\xfc\x03\x0a\\\x8eyZ\xc2\xdaW\xe6" +
+	"5[\x13\xcb\x8eZ\x1f\xe9ca/\x80\xba@D5" +
+	"\x9f\xd0\xc7\xd2\xc5\x00\xea\x12\x11\xd5\xdb\x13\x96^\xd1\x0e" +
+	"\xa0\xe6ETW\x09\xe8Y\xb6\xde\xaf\x9b\x1d\x0cD;" +
+	"i0\xc75\xb52\x03\x80Pa[\xad\x0a)\xd1\xc1" +
+	"l\x8c\xd25\x9aJ\x8f\x14\xa0\x93\x19\x86u\x9be\x1b" +
+	"\xa5\xe5\xfe9\x16i\x9b\x9b2Z&\x8dbyn\x1c" +
+	"\x92[/\xb2yU\x87\xf9\xeb\xaa67\xe4\x95\xdd\xcc" +
+	"\xa9\x1a\xae\x03\xa0\xa6\"\xf1\x1bZ\x00\xd4\xf1\"\xaa\x8d" +
+	"\x02\xe6l\xce\x80\xd9\x18\xd4k\xaez!]WM\x9b" +
+	"\xf5\xeb\x8e\xcbl\x9f|e\x8e\x14^v\x92\x07\x92\xff" +
+	"eET\xa7\x0a\xe8\xf5\xdbZ\x91\xe5\x99\x8d\xbaUZ" +
+	"\xa6\x99VAdEL\x83\x80\xe9\xb1=i\x91\xa6\x1b" +
+	"\xac\xe4K7\xaf\xd8\xcc\xff\xa7\xe8\xad\xf7<?|{" +
+	"\xe3\xf0m\xc0/\xbc ~7\xc7\xf1\xdb |\xee\x8d" +
+	"\x0c\xe0\x06\xf13/\x08a\x8a\x08WD\xf5\x1e\x8a\x88" +
+	"j\x85t\xea\x80h\xd9\x98\x8dQ2\xd0\x0e+\xf5\x93" +
+	"\xa6M\xc8\xb1\")\x1a\xb3a\xb6\xf7\x19\xa4\x925\x80" +
+	"\xd9\xb8\x94\x09\x96\xd9l\x03\xb3\x1d\x96\x87\x8cmm\x1a" +
+	"\xc4l\x9c\xf5k\xb4>\xe9R\xb5\x1e\x1a:Z5\xf6" +
+	"z\x9b\x15}\xc8\x08\x96\xe7\x9b}\xa3%\x82\x84t\xd4" +
+	"*\xa2\xba$\x11$]\x14$\x9d\"\xaa=\x89 Q" +
+	"\xdb\xe3\xc8\xa9\x01\xb3\xdfSl\x0c\xbbzp\xf1\xe8\xca" +
+	"\x09?#\x17\xa8\x17Qm\"\x0c\xa6\xaf\xcc\xa5h\xa5" +
+	"\xd3\xa2\xea\xf6\xc2\xa7u\xd0\xbf\x01\xa2\xe6\x83]\xec\x00" +
+	"T\x9b\xa2\xc3\xf6\xd2a\x8f\x88\xa8>\x9e\xd0\xcfA\x1b" +
+	"@= \xa2zX@\x0c\xd4\xf3\xf4!\x00\xf5\xb0\x88" +
+	"\xeas\x02\xca\xa2\xe0\xfb\xda\xf19\xd4\x0b\x89\xa8\xfe\\" +
+	"@9%6R\xa9/\xbfFq\xf2s\x11\xd5_\x0b" +
+	"(\xa7S\x8d\x98\x06\x90O\xaf\x01PO\x89\xa8\xbe\xf5" +
+	"e\x10T4\xacj\xa9\xcf\xd0\xa0\xd9f\xa5\xae\x05\x11" +
+	"\xdd\xac\x96\xf36\xdb\xa0\xa3Uu\xda\\\x97\x95\xa5\x8a" +
+	"\xeb\x84\xd9$\xe3j\xfd\x0eN\x02\xcc\x8b\x88\xd9\xb8>" +
+	"\x04$b\xb4'\xda\xac\xb4\x92\xd9\x8e.Zf\x94\x10" +
+	"t\xd3e\xa6\xbbD\x03i\x0d3\"\xea\x18\x80\xd1\x1d" +
+	"\xb8=9}\x10\xc1V\x0cr\xd8On7\xd5\xf3\x02" +
+	"%.\x9c\x13{\xde4\xfc\x82\xc8\xa4\xc7\xae\xee\xd8\xf7" +
+	"\xa6\x09\x9f\x13\x994\xa9\xf6\xc6\x10\x9d\x19p\xdd\x0af" +
+	"\xe3\xba10\xf6F\xb6\xc6\xb1\x8a\xeb\x18 !]T" +
+	"\xc4\x04_\x07\x02\xe4\x05\xd1(a6\xee\xfbj<E" +
+	"\xfc\xb2\xb4\x9b\xa3$o\xd9<\xab\xc5\xe1sC,D" +
+	"\x14=\xbd\xc9\xe8i\x0d\xa2gM|\xff\xe6\xa2Vu" +
+	"\xd8\xf0z\xa1\xad\xcf\x05\x91\xd9\x11D:\x03V\xd5(" +
+	"u3\x90\\{\x10\x11\x04\xc4\xb1\x81s\x81\xd5\x99P" +
+	"\xbc\xef\xc6\xa3\xe7\xc2(\x15\xf6&Sa\xa0\xfe\x15\xa4" +
+	"\xfe\x1e\x11\xd5\x8a\x80\x9eA\xd0cvZ :nt" +
+	"]\x9f\x98\xb7\xb8sJ \xa0\x04\xe8U+\x8ek3" +
+	"\xad\x0c\x18y\x1b\xf1O\xba\x84\x0cS\x83ty-S" +
+	"\x0bU\xff\xef\xf9\xfc\xd2\x13\xb3\x9f$\x87\xa5\xe5C\x89" +
+	",Y\x0cV#_\xdea\x99\xd2%\x97^\x01\x82\xf9" +
+	"\x89a^\x90\xe8\xa9*\x0c3\xe6lB\xf8+ET" +
+	"\xafKf\xcc\xb9\xa4\xa2kDTo\x14Pb6%" +
+	"\xbf\xa8}\xf7\x0f\xdd\xea\xf8e&f\xe3\xd9\xcc\x85\xaf" +
+	"\x93\xa8\xc0u\xcb\x1c\xe1\x86\xa3f\x9b\x1b\x12v\x0dM" +
+	"\xb8tM\"\xdb\xacc\x83\xa1\x95\x9aYY\xd3c4" +
+	"\x0a\x8c\xdb\x06\xd2\xb7b\x9e1+\xd5 \xa3\xfb\xf9<" +
+	"\xe7[\x8b.\xd9\x18]\xf2\xee!\x00\xf5\x1e\x11\xd5\x07" +
+	"\x12\x97\xbc\x9f\x0a\xff\x07DT\x1fI\\r\x0f)q" +
+	"\x97\x88\xea>\xc2\xfc\xa0E\xd8K\x06\xde'\xa2\xfa\x94" +
+	"\x80\x98\xf2!\xff\x09\x82\xfc\xa7DT\x8f\x09\x1c\xb0;" +
+	"\xdb:,\x13\x83K8\x00Q\xf1?\xc04\xdb]\xc3" +
+	"4t\xbbL\x97\xd9\x1b44BH\xd8\xea\xeaef" +
+	"U\xdd\x08\"\xca\xda&^-a\xa9\xd3_%i\xae" +
+	"\x83u `\x1dE\xa4\xc3\xec\x0e\x9b\x95\x90\xac\xa1\x19" +
+	"yMt\x07.FA\xc3A<3\x8az\xa8\xd6\xda" +
+	"\"\xa2\xfa\x1d\x82\x12L\x8c\x88\xe4{\xd7\x82\xc0\x91\x84" +
+	"d^\xdf\x1eW_<!\xa6k\xfa'\x9e\x10\xc7\x01" +
+	"\xc8\xdbH;;DTw\x09\xe1\xd5:-\xc8\xf9\x11" +
+	"Zk\xea\xa0?\xd9J\xa8\xa9\xb3X\xde\xa0^\xd0\xd1" +
+	"2{\xb8\xa20\xd6T\xd1*Wlre\xdd2\xd5" +
+	"\xaaf\xe8\xa2;\x18-\x1cS\x17\x04I~(/\xaf" +
+	"4sc\x912n\x0c\x95\xa1\xdc\x85\x8b\x01\x0a\xabP" +
+	"\xc4\xc2\x00\xc6\xee\xa20l\x07(\xac&\xba\x81\xb1\xc7" +
+	"(:N\x01(\x94\x88^\xc1\xa8\xadT\xca\xf8\x0c@" +
+	"\xa1B\xe4-\x18\x97\x0a\xca \xdf~\x13\xd1w`\\" +
+	"-(\xdbp\x0e@a\x0b\xd1\x1f!\xfa8\x81kR" +
+	"\xd9\x83k\x01\x0a\x0f\x13\xfd\x00\xd1\xa5t#\xf5\xd0\xca" +
+	"~\xb4\x01\x0a\xfb\x88\xfe\x14\xd1\xc775\xe2x\x00\xe5" +
+	"\x09N\x7f\x9c\xe8G\x88^\xf7\xb5F\xac\x03P~\x88" +
+	"\xdb\x01\x0a\x87\x89\xfe\x1c\xd1'`#N\x00P\x8e\xe3" +
+	"\xa3\x00\x85\xe7\x88\xfe2\xd1'\x8ek\xc4\x89\x00\xcaK" +
+	"\xfc>'\x88~\x92\xe8\xf5\xa9F\xac\x07P^\xc1C" +
+	"\x00\x85\x93D?\x85\x11\xdeu\x95\x92\xb0K\xee\xa6\xc7" +
+	"e\x87h9\x91\xc9Y\xd0V\xa2\x9f\x13\xf2V\x86\xfa" +
+	"J\xcc\xc4\xf3_@\xcc\x00z\x15\xcb2\x96\x0d\x87\xf3" +
+	"\x0bU>\x81\xbb@\xc62\xbbJQ\xfc\xf9N\xb6\xc4" +
+	"\x82\xe6\xa2ftU\xe2Z\xc8i\xab\xbaV\xb5\x02\xcd" +
+	"%\xcde\xa5(!\xdbUs\x91m\x95{\x90\xd9e" +
+	"\xdd\xd4\x0c\x88\xbe\x8c\xe5s\x99jU/E{\x8fY" +
+	"\xc0E\xee)\xd4\xbags\xa5\xa5G\xeb\xaf\x19\x01\xcc" +
+	"\x89\xb1>\x82\xae\xb97\xc4P\x9fI\x86T\xf3\x06\xcd" +
+	"\xa8\xb2\x8b\xa9\xec\xc6l*\xbas~Sr\xa1\xde3" +
+	"\x1cX]\xb84_Q\x93F\xfd\xe46b\xde\xd1\x1e" +
+	"\x0b\x1b\xc9j\x073\x90N!N`\xa1I\xfa\x82\xde" +
+	"\x12\x9ai\xef\x84sD\x13\xc5\xc09.V\x13\xfd\xcc" +
+	"\xf5\x7fu\x99}\x16\xe5zI+;_qu7s" +
+	"2\x17\xa3\xc5xFx\xe1d\xdc\xd9\xd3\x93\x8f\xc7\x0c" +
+	"\xa2\x8f\xe4\xd7E\xe0\xd5\x86\xdd\x00\x85V\x8a\xce%\x18" +
+	"\xe9P\xe9\xe2 \xd2I\xe4\x1e\x8cKXE\xe5`\x91" +
+	"'\xfa*\x8c\x9b\x1c\xe5\x0e\x1e\xe41\x06\xa6\xda|\xf0" +
+	"b|\xfb\x08\xeb\xe44\xfa\xe0U\xe6\xfb\x1bD\xdf\x94" +
+	"\x04\xaf*\x0e\x0d\x03;I\xf4\xc1k\x1b\x07\x9d\x1dD" +
+	"\xdf\xc5\xc1+\xe5\x83\xd7N|\x16\xa0\xb0\x8b\xe8\xfb8" +
+	"x\xa5}\xf0\xda\x8b/\x0c\x03\xbb\x09\xe3|\xf0z\x82" +
+	"\xf3?E\xf4c\x1c\xbc\xda}\xf0:\xca\xc1\xee\x08\xd1" +
+	"O\x10HUm\xa3\xe0\xda\xba\x09\xd8\x1f\xc7F\xb1\xf2" +
+	"-\xc6*m\x901\xf4\x0d,J,%]3\x16T" +
+	"5\x03\x9a\x0b\xaeV\\\x17\xd7\xe9\x86\xd3\xa9\x99%\x07" +
+	"\x07\xb4u\x8c\xd2\x91\x94L\xdc\xae\xe1\xacd\xb6\xde\x07" +
+	"\x18W\xf6Q!\x93\xc9[Vm}\xc3\x0bDf\xfb" +
+	"\x08\x17}+k\x9b\xbaJ\x06\xeb\xc0\xb0\x9c\x11\xcd8" +
+	"\x1d\xea\xf4\xc52M\xf4k\x8c\x1e\xbdyx\xf1P\x09" +
+	"z\x85\xb0\x08\xe9\xc9\xd5T\x17lS\x85\x15\xdd\x0e\x0b" +
+	"MW7\xabl\xc4\x06\xc5\x81\xaa\xb9\x8e\x95\x16\xa2Y" +
+	"\xb4J\xba\xd9\x0f#\x9a\x14\xf1\xcb\xa6;\x89\xaa\x8bG" +
+	"3&^\xc7\xe4\xd9- p\xe8\xa2\x1aBn\x89[" +
+	"\xfd\\\x91\xaf\xca\xd9Ls\x12]\xea\x18\xa7\x05\xd3H" +
+	"?\xc8\xfc\xb6>\x0d\x10=%a8\x8e\x97\x8fn\x06" +
+	"A~Z\xc2\xf8\x15\x03\xc3G\x0by\xbf\x0d\x82\xbcG" +
+	"B!z\xe2\xc3\xf0yN\xbe\x7f\x08\x04\xf9^\x09\xc5" +
+	"\xe8\xd9\x0d\xc3Y\xb7<\xd8\x0e\x82\\\x960\x15=A" +
+	"b8(\x975\xaa\x93\xee\x900\x1d\xbd\xe7a\xf8\x1a" +
+	"#/\xdd\x0e\x82\xbcP\xf2\xc2v\x08r\xbe\x18\xad\xe8" +
+	"\x85\x80\x01\xcd\x1c2Z\xd1\x0b\xe7C\x18\xb6M\x00\xad" +
+	"\xb85\x80\xe7V\xf4\xc2\x09)d\x8a\x9a\xcbZ\xa9\xd7" +
+	"\xf4?b\x00\xde\xd0\x8a\xc9\xc9\xa3\xf8e\x0d\xce\xe8\x85" +
+	"r{\\\xcc\x85\x00\xbcm(\xae\xe5\xa2\xa6r\xe73" +
+	"\xc9:9\x98\x8d\xec\xdd\x1eLV\x8e%f#G\xa9" +
+	"x>&\xa2\xfa\x86\x10W\x06\xa1O\x87\xc3:\xb4\xec" +
+	"\xb0\xcb\x1dcf\x17x~P\xc3\xd6N\xee\xbc\x925" +
+	"\xc0k\\\xf4\xb7r N\x07\xc9q\xde\xa4\xc48\x0f" +
+	"\xc3\xfeZ\x1a\x96=\x92\xc3\xbdI\x17h\xd6\x92\xdd\"" +
+	"Og)\xee\x92\xe1\xe3%\x86\xef\xcc\xb2L\xae\xd5 " +
+	"yaG\x89a.\x84\x1a\x93]b[\xdd\xcd\x9a\xff" +
+	"/\xc9z\x14\x07\xf1\xcf\xc9\x90G\xfa\x02E\xfb\xaeM" +
+	"\xcc\xe9\x0c+\xe8\x083\xcb\x92E\xfd\x18\xba\xf2/\x1c" +
+	"\x96\xe0\x19Z\\3\x9a\xa3f\xf2a\x11\xd5\x03\x89b" +
+	"g\xff\xac\xc4\xbc.\xec\xd3\x0e.\x0e\xe6u'\xe2\x97" +
+	"\x9c\xe7\xc9QO\x88\xa8\x9eL\xb8\xdf+\xc4\xf8\xb2\xef" +
+	"~a\xbe\x92\x7fA=\xcb\x1b\"\xaao\x06\xadr\xf8" +
+	"\xa2S\xb5c\xa06\xac\xfe%\xba\xc9\x1c\xaa@k\x06" +
+	" \xe13\x11\xba\x04\x7fU\x9b0|8Vv-H" +
+	"\x14\xae\xd1<\x08\x99]\xa0p-\xa1\x13\xcdY\xc6\xd0" +
+	"W!\x88\x0e?8\x82d\x9fh\xbd\x0f%\xa6R\xa1" +
+	"\xb6\xd4\x17\x82i\xcf\xea\x84\xb6\xee\xa2\xd6{\x95\x88\xea" +
+	"\x80\xc0\x11\xc4ZQ)i\xe8\xb2E6[_e\x92" +
+	"Y\x1c\x8c[Pj\xc2\x8a\xce\x0a\xacPY\xbc\xc8f" +
+	"\xb9\xf5U\x96d\x08\x9f\x01@\xd2\xad\xd2\x88\xf9\xff(" +
+	"\xa5\xdfmlM\xc1*\xaec\xee\xb0\xe7\x91\x9a'\xbc" +
+	"\xee\xf8\x0d z\xc1\xebN\xbe\xe0\x05\xb8\xb3\x9e<\xb0" +
+	"\"\xa2\xba%\x81;\x83Cq\xff:z\xae\xff\xfd\xa4" +
+	"\xe7\xaf\xf4\x8aE\x95\xaet1U`\xf4\xc75_q" +
+	"\xac~\xb1E{\xfc6{\x89\xa3(\x88\xc0\x00\x13\x7f" +
+	"{A\x87\x08\xc1\xe6\xff\x1b\x00\x00\xff\xff\x99N\xd0U"
 
 func init() {
 	schemas.Register(schema_db8274f9144abc7e,

validation/validation.go

@@ -190,11 +190,10 @@ func ValidateHTTPService(originURL string, hostname string, transport http.Round
 		_, secondErr := client.Do(secondRequest)
 		if secondErr == nil { // Worked this time--advise the user to switch protocols
 			return errors.Errorf(
-				"%s doesn't seem to work over %s, but does seem to work over %s. Reason: %v. Consider changing the origin URL to %s",
+				"%s doesn't seem to work over %s, but does seem to work over %s. Consider changing the origin URL to %s",
 				parsedURL.Host,
 				oldScheme,
 				parsedURL.Scheme,
-				initialErr,
 				parsedURL,
 			)
 		}