Release 2020.4.0

Previously it was only being set after RegisterTunnel.

Dockerfile

@@ -1,12 +1,28 @@
-FROM golang:1.12 as builder
-WORKDIR /go/src/github.com/cloudflare/cloudflared/
-RUN apt-get update && apt-get install -y --no-install-recommends upx
-# Run after `apt-get update` to improve rebuild scenarios
-COPY . .
-RUN make cloudflared
-RUN upx --no-progress cloudflared
+# use a builder image for building cloudflare
+FROM golang:1.13.3 as builder
+ENV GO111MODULE=on
+ENV CGO_ENABLED=0
+ENV GOOS=linux
 
-FROM gcr.io/distroless/base
-COPY --from=builder /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+WORKDIR /go/src/github.com/cloudflare/cloudflared/
+
+# copy our sources into the builder image
+COPY . .
+
+# compile cloudflared
+RUN make cloudflared
+
+# ---
+
+# use a distroless base image with glibc
+FROM gcr.io/distroless/base-debian10:nonroot
+
+# copy our compiled binary
+COPY --from=builder --chown=nonroot /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/
+
+# run as non-privileged user
+USER nonroot
+
+# command / entrypoint of container
 ENTRYPOINT ["cloudflared", "--no-autoupdate"]
 CMD ["version"]

RELEASE_NOTES

@@ -1,3 +1,84 @@
+2020.4.0
+- 2020-04-10 TUN-2881: Parameterize response meta information header name in the generating function
+- 2020-04-11 TUN-2894: ResponseMetaHeader should be public
+- 2020-04-09 TUN-2880: Return metadata about source of the response from cloudflared
+- 2020-04-04 ARES-899: Fixes DoH client as system resolver. Fixes #91
+- 2020-03-31 AUTH-2394 added socks5 proxy
+- 2020-02-24 AUTH-2235 GetTokenIfExists now parses JWT payload for json expiry field to detect if the cached access token is expired
+
+2020.3.2
+- 2020-03-31 TUN-2854: Quick Reconnects should be an optional supported feature
+- 2020-03-30 TUN-2850: Tunnel stripping Cloudflare headers
+
+2020.3.1
+- 2020-03-27 TUN-2846: Trigger debug reconnects from stdin commands, not SIGUSR1
+
+2020.3.0
+- 2020-03-23 AUTH-2394 fixed header for websockets. Added TCP alias
+- 2020-03-10 TUN-2797: Fix panic in SetConnDigest by making mutexes values.
+- 2020-03-13 TUN-2807: cloudflared hello-world shouldn't assume it's my first tunnel
+- 2020-03-13 TUN-2756: Set connection digest after reconnect.
+- 2020-03-16 TUN-2812: Tunnel proxies and RPCs can share an edge address
+- 2020-03-18 TUN-2816: cloudflared metrics server should be more discoverable
+- 2020-03-19 TUN-2820: Serialized headers for Websockets
+- 2020-03-19 TUN-2819: cloudflared should close its connections when a signal is sent
+- 2020-03-19 TUN-2823: Bugfix. cloudflared would hang forever if error occurred.
+- 2020-03-10 TUN-2796: Implement HTTP2 CONTINUATION headers correctly
+- 2020-03-02 TUN-2779: update sample HTML pages
+- 2020-03-04 TUN-2785: Use reconnect token by default
+- 2020-03-05 TUN-2754: Add ConnDigest to cloudflared and its RPCs
+- 2020-03-06 TUN-2755: ReconnectTunnel RPC now transmits ConnectionDigest
+- 2020-03-06 TUN-2761: Use the new header management functions in cloudflared
+- 2020-03-06 TUN-2788: cloudflared should store one ConnDigest per HA connection
+- 2020-02-26 TUN-2767: Test for large headers
+- 2020-02-28 do not terminate tunnel if origin is not reachable on start-up (#177)
+- 2020-02-28 TUN-2776: Add header serialization feature in cloudflared
+- 2020-02-21 TUN-2748: Insecure randomness vulnerability in github.com/miekg/dns
+
+2020.2.1
+- 2020-02-20 TUN-2745: Rename existing header management functions
+- 2020-02-21 TUN-2746: Add the new header management functions
+- 2020-02-25 perf(cloudflared): reuse memory from buffer pool to get better throughput (#161)
+- 2020-02-25 Tweak HTTP host header. Fixes #107 (#168)
+- 2020-02-25 TUN-2765: Add list of features to tunnelrpc
+- 2020-02-19 TUN-2725: Specify in code that --edge is for internal testing only
+- 2020-02-19 TUN-2703: Muxer.Serve terminates when its context is Done
+- 2020-02-09 TUN-2717: Function to serialize/deserialize HTTP headers
+- 2020-02-05 TUN-2714: New edge discovery. Connections try to reconnect to the same edge IP.
+
+2020.2.0
+- 2020-01-30 TUN-2651: Fix panic in h2mux reader when a stream error is encountered
+- 2020-01-27 TUN-2645: Revert "TUN-2645: Turn on reconnect tokens"
+- 2020-01-28 TUN-2693: Metrics for ReconnectTunnel
+- 2020-01-28 TUN-2696: Add unknown registerRPCName
+- 2020-01-28 TUN-2699: Metrics for Authenticate RPCs
+- 2020-01-28 TUN-2690: cloudflared reconnect uses wrong context
+- 2020-01-29 TUN-2707: Inconsistent cardinality in tunnel error metrics
+- 2020-01-13 TUN-2645: Turn on reconnect tokens
+- 2019-12-23 TUN-2646: Make --edge flag work again for local development
+
+2019.12.0
+- 2019-12-11 TUN-2631: only notify that activeStreamMap is closed if ignoreNewStreams=true
+- 2019-12-17 bug(cloudflared): Set the MaxIdleConnsPerHost of http.Transport to proxy-keepalive-connections (#155)
+- 2019-12-17 refactor(docker): optimize Dockerfile (#126)
+- 2019-12-19 Fix timer scheduling for systemd update service (#159)
+- 2019-12-13 TUN-2637: Manage edge IPs in a region-aware manner
+- 2019-12-03 bug(cloudflared): nil pointer deference on h2DictWriter Close() (#154)
+- 2019-12-03 TUN-2608: h2mux.Muxer.Shutdown always returns a non-nil channel
+- 2019-12-04 TUN-2555: origin/supervisor.go calls Authenticate
+- 2019-12-06 TUN-2554: cloudflared calls ReconnectTunnel
+- 2019-11-20 TUN-2575: Constructors + simpler conversions for AuthOutcome
+- 2019-11-22 Fix Docker build failure (#149)
+- 2019-11-21 TUN-2573: Refactor TunnelRegistration into PermanentRegistrationError, RetryableRegistrationError and SuccessfulTunnelRegistration
+- 2019-11-22 TUN-2582: EventDigest field in tunnelrpc
+- 2019-11-22 Fix "happy eyeballs" not being disabled since Golang 1.12 upgrade * The Dialer.DualStack setting is now ignored and deprecated; RFC 6555 Fast Fallback ("Happy Eyeballs") is now enabled by default. To disable, set Dialer.FallbackDelay to a negative value.
+- 2019-11-25 TUN-2591: ReconnectTunnel now sends EventDigest
+- 2019-11-21 TUN-2606: add DialEdge helpers
+- 2019-11-21 TUN-2607: add RPC stream helpers
+
+2019.11.3
+- 2019-11-20 TUN-2562: Update Cloudflare Origin CA RSA root
+
 2019.11.2
 - 2019-11-18 TUN-2567: AuthOutcome can be turned back into AuthResponse
 - 2019-11-18 TUN-2563: Exposes config_version metrics

buffer/pool.go

@@ -0,0 +1,29 @@
+package buffer
+
+import (
+	"sync"
+)
+
+type Pool struct {
+	// A Pool must not be copied after first use.
+	// https://golang.org/pkg/sync/#Pool
+	buffers sync.Pool
+}
+
+func NewPool(bufferSize int) *Pool {
+	return &Pool{
+		buffers: sync.Pool{
+			New: func() interface{} {
+				return make([]byte, bufferSize)
+			},
+		},
+	}
+}
+
+func (p *Pool) Get() []byte {
+	return p.buffers.Get().([]byte)
+}
+
+func (p *Pool) Put(buf []byte) {
+	p.buffers.Put(buf)
+}

carrier/carrier.go

@@ -11,9 +11,7 @@ import (
 	"strings"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
-	cloudflaredWebsocket "github.com/cloudflare/cloudflared/websocket"
-	"github.com/gorilla/websocket"
-	"github.com/sirupsen/logrus"
+	"github.com/pkg/errors"
 )
 
 type StartOptions struct {
@@ -21,6 +19,15 @@ type StartOptions struct {
 	Headers   http.Header
 }
 
+// Connection wraps up all the needed functions to forward over the tunnel
+type Connection interface {
+	// ServeStream is used to forward data from the client to the edge
+	ServeStream(*StartOptions, io.ReadWriter) error
+
+	// StartServer is used to listen for incoming connections from the edge to the origin
+	StartServer(net.Listener, string, <-chan struct{}) error
+}
+
 // StdinoutStream is empty struct for wrapping stdin/stdout
 // into a single ReadWriter
 type StdinoutStream struct {
@@ -44,29 +51,27 @@ func closeRespBody(resp *http.Response) {
 	}
 }
 
-// StartClient will copy the data from stdin/stdout over a WebSocket connection
-// to the edge (originURL)
-func StartClient(logger *logrus.Logger, stream io.ReadWriter, options *StartOptions) error {
-	return serveStream(logger, stream, options)
-}
-
-// StartServer will setup a listener on a specified address/port and then
+// StartForwarder will setup a listener on a specified address/port and then
 // forward connections to the origin by calling `Serve()`.
-func StartServer(logger *logrus.Logger, address string, shutdownC <-chan struct{}, options *StartOptions) error {
+func StartForwarder(conn Connection, address string, shutdownC <-chan struct{}, options *StartOptions) error {
 	listener, err := net.Listen("tcp", address)
 	if err != nil {
-		logger.WithError(err).Error("failed to start forwarding server")
-		return err
+		return errors.Wrap(err, "failed to start forwarding server")
 	}
-	logger.Info("Started listening on ", address)
-	return Serve(logger, listener, shutdownC, options)
+	return Serve(conn, listener, shutdownC, options)
+}
+
+// StartClient will copy the data from stdin/stdout over a WebSocket connection
+// to the edge (originURL)
+func StartClient(conn Connection, stream io.ReadWriter, options *StartOptions) error {
+	return serveStream(conn, stream, options)
 }
 
 // Serve accepts incoming connections on the specified net.Listener.
 // Each connection is handled in a new goroutine: its data is copied over a
 // WebSocket connection to the edge (originURL).
 // `Serve` always closes `listener`.
-func Serve(logger *logrus.Logger, listener net.Listener, shutdownC <-chan struct{}, options *StartOptions) error {
+func Serve(remoteConn Connection, listener net.Listener, shutdownC <-chan struct{}, options *StartOptions) error {
 	defer listener.Close()
 	for {
 		select {
@@ -77,53 +82,20 @@ func Serve(logger *logrus.Logger, listener net.Listener, shutdownC <-chan struct
 			if err != nil {
 				return err
 			}
-			go serveConnection(logger, conn, options)
+			go serveConnection(remoteConn, conn, options)
 		}
 	}
 }
 
 // serveConnection handles connections for the Serve() call
-func serveConnection(logger *logrus.Logger, c net.Conn, options *StartOptions) {
+func serveConnection(remoteConn Connection, c net.Conn, options *StartOptions) {
 	defer c.Close()
-	serveStream(logger, c, options)
+	serveStream(remoteConn, c, options)
 }
 
 // serveStream will serve the data over the WebSocket stream
-func serveStream(logger *logrus.Logger, conn io.ReadWriter, options *StartOptions) error {
-	wsConn, err := createWebsocketStream(options)
-	if err != nil {
-		logger.WithError(err).Errorf("failed to connect to %s\n", options.OriginURL)
-		return err
-	}
-	defer wsConn.Close()
-
-	cloudflaredWebsocket.Stream(wsConn, conn)
-
-	return nil
-}
-
-// createWebsocketStream will create a WebSocket connection to stream data over
-// It also handles redirects from Access and will present that flow if
-// the token is not present on the request
-func createWebsocketStream(options *StartOptions) (*cloudflaredWebsocket.Conn, error) {
-	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	req.Header = options.Headers
-
-	wsConn, resp, err := cloudflaredWebsocket.ClientConnect(req, nil)
-	defer closeRespBody(resp)
-	if err != nil && IsAccessResponse(resp) {
-		wsConn, err = createAccessAuthenticatedStream(options)
-		if err != nil {
-			return nil, err
-		}
-	} else if err != nil {
-		return nil, err
-	}
-
-	return &cloudflaredWebsocket.Conn{Conn: wsConn}, nil
+func serveStream(remoteConn Connection, conn io.ReadWriter, options *StartOptions) error {
+	return remoteConn.ServeStream(options, conn)
 }
 
 // IsAccessResponse checks the http Response to see if the url location
@@ -144,49 +116,7 @@ func IsAccessResponse(resp *http.Response) bool {
 	return false
 }
 
-// createAccessAuthenticatedStream will try load a token from storage and make
-// a connection with the token set on the request. If it still get redirect,
-// this probably means the token in storage is invalid (expired/revoked). If that
-// happens it deletes the token and runs the connection again, so the user can
-// login again and generate a new one.
-func createAccessAuthenticatedStream(options *StartOptions) (*websocket.Conn, error) {
-	wsConn, resp, err := createAccessWebSocketStream(options)
-	defer closeRespBody(resp)
-	if err == nil {
-		return wsConn, nil
-	}
-
-	if !IsAccessResponse(resp) {
-		return nil, err
-	}
-
-	// Access Token is invalid for some reason. Go through regen flow
-	originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
-	if err != nil {
-		return nil, err
-	}
-	if err := token.RemoveTokenIfExists(originReq.URL); err != nil {
-		return nil, err
-	}
-	wsConn, resp, err = createAccessWebSocketStream(options)
-	defer closeRespBody(resp)
-	if err != nil {
-		return nil, err
-	}
-
-	return wsConn, nil
-}
-
-// createAccessWebSocketStream builds an Access request and makes a connection
-func createAccessWebSocketStream(options *StartOptions) (*websocket.Conn, *http.Response, error) {
-	req, err := BuildAccessRequest(options)
-	if err != nil {
-		return nil, nil, err
-	}
-	return cloudflaredWebsocket.ClientConnect(req, nil)
-}
-
-// buildAccessRequest builds an HTTP request with the Access token set
+// BuildAccessRequest builds an HTTP request with the Access token set
 func BuildAccessRequest(options *StartOptions) (*http.Request, error) {
 	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
 	if err != nil {

carrier/carrier_test.go

@@ -44,6 +44,7 @@ func (s *testStreamer) Write(p []byte) (int, error) {
 func TestStartClient(t *testing.T) {
 	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
 	logger := logrus.New()
+	wsConn := NewWSConnection(logger, false)
 	ts := newTestWebSocketServer()
 	defer ts.Close()
 
@@ -52,7 +53,7 @@ func TestStartClient(t *testing.T) {
 		OriginURL: "http://" + ts.Listener.Addr().String(),
 		Headers:   nil,
 	}
-	err := StartClient(logger, buf, options)
+	err := StartClient(wsConn, buf, options)
 	assert.NoError(t, err)
 	buf.Write([]byte(message))
 
@@ -69,6 +70,7 @@ func TestStartServer(t *testing.T) {
 	message := "Good morning Austin! Time for another sunny day in the great state of Texas."
 	logger := logrus.New()
 	shutdownC := make(chan struct{})
+	wsConn := NewWSConnection(logger, false)
 	ts := newTestWebSocketServer()
 	defer ts.Close()
 	options := &StartOptions{
@@ -77,7 +79,7 @@ func TestStartServer(t *testing.T) {
 	}
 
 	go func() {
-		err := Serve(logger, listener, shutdownC, options)
+		err := Serve(wsConn, listener, shutdownC, options)
 		if err != nil {
 			t.Fatalf("Error running server: %v", err)
 		}

carrier/websocket.go

@@ -0,0 +1,137 @@
+package carrier
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/token"
+	"github.com/cloudflare/cloudflared/socks"
+	cfwebsocket "github.com/cloudflare/cloudflared/websocket"
+	"github.com/gorilla/websocket"
+	"github.com/sirupsen/logrus"
+)
+
+// Websocket is used to carry data via WS binary frames over the tunnel from client to the origin
+// This implements the functions for glider proxy (sock5) and the carrier interface
+type Websocket struct {
+	logger  *logrus.Logger
+	isSocks bool
+}
+
+type wsdialer struct {
+	conn *cfwebsocket.Conn
+}
+
+func (d *wsdialer) Dial(address string) (io.ReadWriteCloser, *socks.AddrSpec, error) {
+	local, ok := d.conn.LocalAddr().(*net.TCPAddr)
+	if !ok {
+		return nil, nil, fmt.Errorf("not a tcp connection")
+	}
+
+	addr := socks.AddrSpec{IP: local.IP, Port: local.Port}
+	return d.conn, &addr, nil
+}
+
+// NewWSConnection returns a new connection object
+func NewWSConnection(logger *logrus.Logger, isSocks bool) Connection {
+	return &Websocket{
+		logger:  logger,
+		isSocks: isSocks,
+	}
+}
+
+// ServeStream will create a Websocket client stream connection to the edge
+// it blocks and writes the raw data from conn over the tunnel
+func (ws *Websocket) ServeStream(options *StartOptions, conn io.ReadWriter) error {
+	wsConn, err := createWebsocketStream(options)
+	if err != nil {
+		ws.logger.WithError(err).Errorf("failed to connect to %s", options.OriginURL)
+		return err
+	}
+	defer wsConn.Close()
+
+	if ws.isSocks {
+		dialer := &wsdialer{conn: wsConn}
+		requestHandler := socks.NewRequestHandler(dialer)
+		socksServer := socks.NewConnectionHandler(requestHandler)
+
+		socksServer.Serve(conn)
+	} else {
+		cfwebsocket.Stream(wsConn, conn)
+	}
+	return nil
+}
+
+// StartServer creates a Websocket server to listen for connections.
+// This is used on the origin (tunnel) side to take data from the muxer and send it to the origin
+func (ws *Websocket) StartServer(listener net.Listener, remote string, shutdownC <-chan struct{}) error {
+	return cfwebsocket.StartProxyServer(ws.logger, listener, remote, shutdownC, cfwebsocket.DefaultStreamHandler)
+}
+
+// createWebsocketStream will create a WebSocket connection to stream data over
+// It also handles redirects from Access and will present that flow if
+// the token is not present on the request
+func createWebsocketStream(options *StartOptions) (*cfwebsocket.Conn, error) {
+	req, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header = options.Headers
+
+	wsConn, resp, err := cfwebsocket.ClientConnect(req, nil)
+	defer closeRespBody(resp)
+	if err != nil && IsAccessResponse(resp) {
+		wsConn, err = createAccessAuthenticatedStream(options)
+		if err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+
+	return &cfwebsocket.Conn{Conn: wsConn}, nil
+}
+
+// createAccessAuthenticatedStream will try load a token from storage and make
+// a connection with the token set on the request. If it still get redirect,
+// this probably means the token in storage is invalid (expired/revoked). If that
+// happens it deletes the token and runs the connection again, so the user can
+// login again and generate a new one.
+func createAccessAuthenticatedStream(options *StartOptions) (*websocket.Conn, error) {
+	wsConn, resp, err := createAccessWebSocketStream(options)
+	defer closeRespBody(resp)
+	if err == nil {
+		return wsConn, nil
+	}
+
+	if !IsAccessResponse(resp) {
+		return nil, err
+	}
+
+	// Access Token is invalid for some reason. Go through regen flow
+	originReq, err := http.NewRequest(http.MethodGet, options.OriginURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	if err := token.RemoveTokenIfExists(originReq.URL); err != nil {
+		return nil, err
+	}
+	wsConn, resp, err = createAccessWebSocketStream(options)
+	defer closeRespBody(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	return wsConn, nil
+}
+
+// createAccessWebSocketStream builds an Access request and makes a connection
+func createAccessWebSocketStream(options *StartOptions) (*websocket.Conn, *http.Response, error) {
+	req, err := BuildAccessRequest(options)
+	if err != nil {
+		return nil, nil, err
+	}
+	return cfwebsocket.ClientConnect(req, nil)
+}

cmd/cloudflared/access/carrier.go

@@ -44,6 +44,9 @@ func ssh(c *cli.Context) error {
 		Headers:   headers,
 	}
 
+	// we could add a cmd line variable for this bool if we want the SOCK5 server to be on the client side
+	wsConn := carrier.NewWSConnection(logger, false)
+
 	if c.NArg() > 0 || c.IsSet(sshURLFlag) {
 		localForwarder, err := config.ValidateUrl(c)
 		if err != nil {
@@ -55,10 +58,12 @@ func ssh(c *cli.Context) error {
 			logger.WithError(err).Error("Error validating origin URL")
 			return errors.Wrap(err, "error validating origin URL")
 		}
-		return carrier.StartServer(logger, forwarder.Host, shutdownC, options)
+
+		logger.Infof("Start Websocket listener on: %s", forwarder.Host)
+		return carrier.StartForwarder(wsConn, forwarder.Host, shutdownC, options)
 	}
 
-	return carrier.StartClient(logger, &carrier.StdinoutStream{}, options)
+	return carrier.StartClient(wsConn, &carrier.StdinoutStream{}, options)
 }
 
 func buildRequestHeaders(values []string) http.Header {

cmd/cloudflared/access/cmd.go

@@ -119,7 +119,7 @@ func Commands() []*cli.Command {
 				{
 					Name:        "ssh",
 					Action:      ssh,
-					Aliases:     []string{"rdp"},
+					Aliases:     []string{"rdp", "tcp"},
 					Usage:       "",
 					ArgsUsage:   "",
 					Description: `The ssh subcommand sends data over a proxy to the Cloudflare edge.`,

cmd/cloudflared/linux_service.go

@@ -73,7 +73,7 @@ ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 64 ]; then s
 Description=Update Argo Tunnel
 
 [Timer]
-OnUnitActiveSec=1d
+OnCalendar=daily
 
 [Install]
 WantedBy=timers.target

cmd/cloudflared/token/token.go

@@ -2,12 +2,14 @@ package token
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/url"
 	"os"
 	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/path"
@@ -34,6 +36,21 @@ type signalHandler struct {
 	signals    []os.Signal
 }
 
+type jwtPayload struct {
+	Aud   []string `json:"aud"`
+	Email string   `json:"email"`
+	Exp   int      `json:"exp"`
+	Iat   int      `json:"iat"`
+	Nbf   int      `json:"nbf"`
+	Iss   string   `json:"iss"`
+	Type  string   `json:"type"`
+	Subt  string   `json:"sub"`
+}
+
+func (p jwtPayload) isExpired() bool {
+	return int(time.Now().Unix()) > p.Exp
+}
+
 func (s *signalHandler) register(handler func()) {
 	s.sigChannel = make(chan os.Signal, 1)
 	signal.Notify(s.sigChannel, s.signals...)
@@ -147,7 +164,7 @@ func FetchToken(appURL *url.URL) (string, error) {
 	return string(token), nil
 }
 
-// GetTokenIfExists will return the token from local storage if it exists
+// GetTokenIfExists will return the token from local storage if it exists and not expired
 func GetTokenIfExists(url *url.URL) (string, error) {
 	path, err := path.GenerateFilePathFromURL(url, keyName)
 	if err != nil {
@@ -162,6 +179,17 @@ func GetTokenIfExists(url *url.URL) (string, error) {
 		return "", err
 	}
 
+	var payload jwtPayload
+	err = json.Unmarshal(token.Payload, &payload)
+	if err != nil {
+		return "", err
+	}
+
+	if payload.isExpired() {
+		err := os.Remove(path)
+		return "", err
+	}
+
 	return token.Encode(), nil
 }
 

cmd/cloudflared/tunnel/cmd.go

@@ -1,6 +1,7 @@
 package tunnel
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"io/ioutil"
@@ -23,6 +24,7 @@ import (
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/signal"
+	"github.com/cloudflare/cloudflared/socks"
 	"github.com/cloudflare/cloudflared/sshlog"
 	"github.com/cloudflare/cloudflared/sshserver"
 	"github.com/cloudflare/cloudflared/supervisor"
@@ -76,6 +78,9 @@ const (
 	// hostKeyPath is the path of the dir to save SSH host keys too
 	hostKeyPath = "host-key-path"
 
+	// socks5Flag is to enable the socks server to deframe
+	socks5Flag = "socks5"
+
 	noIntentMsg = "The --intent argument is required. Cloudflared looks up an Intent to determine what configuration to use (i.e. which tunnels to start). If you don't have any Intents yet, you can use a placeholder Intent Label for now. Then, when you make an Intent with that label, cloudflared will get notified and open the tunnels you specified in that Intent."
 )
 
@@ -133,6 +138,12 @@ func Commands() []*cli.Command {
 					Value:   cli.NewStringSlice("https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"),
 					EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
 				},
+				&cli.StringSliceFlag{
+					Name:    "bootstrap",
+					Usage:   "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",
+					Value:   cli.NewStringSlice("https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"),
+					EnvVars: []string{"TUNNEL_DNS_BOOTSTRAP"},
+				},
 			},
 			ArgsUsage: " ", // can't be the empty string or we get the default output
 			Hidden:    false,
@@ -389,7 +400,18 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			errC <- websocket.StartProxyServer(logger, listener, host, shutdownC)
+			streamHandler := websocket.DefaultStreamHandler
+			if c.IsSet(socks5Flag) {
+				logger.Info("SOCKS5 server started")
+				streamHandler = func(wsConn *websocket.Conn, remoteConn net.Conn) {
+					dialer := socks.NewConnDialer(remoteConn)
+					requestHandler := socks.NewRequestHandler(dialer)
+					socksServer := socks.NewConnectionHandler(requestHandler)
+
+					socksServer.Serve(wsConn)
+				}
+			}
+			errC <- websocket.StartProxyServer(logger, listener, host, shutdownC, streamHandler)
 		}()
 		c.Set("url", "http://"+listener.Addr().String())
 	}
@@ -399,10 +421,16 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 		return err
 	}
 
+	reconnectCh := make(chan struct{}, 1)
+	if c.IsSet("stdin-control") {
+		logger.Warn("Enabling control through stdin")
+		go stdinControl(reconnectCh)
+	}
+
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		errC <- origin.StartTunnelDaemon(ctx, tunnelConfig, connectedSignal, cloudflaredID)
+		errC <- origin.StartTunnelDaemon(ctx, tunnelConfig, connectedSignal, cloudflaredID, reconnectCh)
 	}()
 
 	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"))
@@ -596,6 +624,8 @@ func hostnameFromURI(uri string) string {
 		return addPortIfMissing(u, 22)
 	case "rdp":
 		return addPortIfMissing(u, 3389)
+	case "tcp":
+		return addPortIfMissing(u, 7864) // just a random port since there isn't a default in this case
 	}
 	return ""
 }
@@ -690,7 +720,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
 			Name:    "edge",
-			Usage:   "Address of the Cloudflare tunnel server.",
+			Usage:   "Address of the Cloudflare tunnel server. Only works in Cloudflare's internal testing environment.",
 			EnvVars: []string{"TUNNEL_EDGE"},
 			Hidden:  true,
 		}),
@@ -939,6 +969,13 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
 			Hidden:  shouldHide,
 		}),
+		altsrc.NewStringSliceFlag(&cli.StringSliceFlag{
+			Name:    "proxy-dns-bootstrap",
+			Usage:   "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",
+			Value:   cli.NewStringSlice("https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"),
+			EnvVars: []string{"TUNNEL_DNS_BOOTSTRAP"},
+			Hidden:  shouldHide,
+		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:    "grace-period",
 			Usage:   "Duration to accept new requests after cloudflared receives first SIGINT/SIGTERM. A second SIGINT/SIGTERM will force cloudflared to shutdown immediately.",
@@ -977,6 +1014,19 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"TUNNEL_INTENT"},
 			Hidden:  true,
 		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "use-reconnect-token",
+			Usage:   "Test reestablishing connections with the new 'reconnect token' flow.",
+			Value:   true,
+			EnvVars: []string{"TUNNEL_USE_RECONNECT_TOKEN"},
+			Hidden:  true,
+		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "use-quick-reconnects",
+			Usage:   "Test reestablishing connections with the new 'connection digest' flow.",
+			EnvVars: []string{"TUNNEL_USE_QUICK_RECONNECTS"},
+			Hidden:  true,
+		}),
 		altsrc.NewDurationFlag(&cli.DurationFlag{
 			Name:    "dial-edge-timeout",
 			Usage:   "Maximum wait time to set up a connection with the edge",
@@ -1044,7 +1094,36 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Usage:   "Absolute path of directory to save SSH host keys in",
 			EnvVars: []string{"HOST_KEY_PATH"},
 			Hidden:  true,
-
+		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    "stdin-control",
+			Usage:   "Control the process using commands sent through stdin",
+			EnvVars: []string{"STDIN-CONTROL"},
+			Hidden:  true,
+			Value:   false,
+		}),
+		altsrc.NewBoolFlag(&cli.BoolFlag{
+			Name:    socks5Flag,
+			Usage:   "specify if this tunnel is running as a SOCK5 Server",
+			EnvVars: []string{"TUNNEL_SOCKS"},
+			Value:   false,
+			Hidden:  false,
 		}),
 	}
-}
+}
+
+func stdinControl(reconnectCh chan struct{}) {
+	for {
+		scanner := bufio.NewScanner(os.Stdin)
+		for scanner.Scan() {
+			command := scanner.Text()
+
+			switch command {
+			case "reconnect":
+				reconnectCh <- struct{}{}
+			default:
+				logger.Warn("Unknown command: ", command)
+			}
+		}
+	}
+}

cmd/cloudflared/tunnel/configuration.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/tlsconfig"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -197,17 +197,21 @@ func prepareTunnelConfig(
 	httpTransport := &http.Transport{
 		Proxy:                 http.ProxyFromEnvironment,
 		MaxIdleConns:          c.Int("proxy-keepalive-connections"),
+		MaxIdleConnsPerHost:   c.Int("proxy-keepalive-connections"),
 		IdleConnTimeout:       c.Duration("proxy-keepalive-timeout"),
 		TLSHandshakeTimeout:   c.Duration("proxy-tls-timeout"),
 		ExpectContinueTimeout: 1 * time.Second,
 		TLSClientConfig:       &tls.Config{RootCAs: originCertPool, InsecureSkipVerify: c.IsSet("no-tls-verify")},
 	}
 
-	dialContext := (&net.Dialer{
+	dialer := &net.Dialer{
 		Timeout:   c.Duration("proxy-connect-timeout"),
 		KeepAlive: c.Duration("proxy-tcp-keepalive"),
-		DualStack: !c.Bool("proxy-no-happy-eyeballs"),
-	}).DialContext
+	}
+	if c.Bool("proxy-no-happy-eyeballs") {
+		dialer.FallbackDelay = -1 // As of Golang 1.12, a negative delay disables "happy eyeballs"
+	}
+	dialContext := dialer.DialContext
 
 	if c.IsSet("unix-socket") {
 		unixSocket, err := config.ValidateUnixSocket(c)
@@ -233,7 +237,6 @@ func prepareTunnelConfig(
 	err = validation.ValidateHTTPService(originURL, hostname, httpTransport)
 	if err != nil {
 		logger.WithError(err).Error("unable to connect to the origin")
-		return nil, errors.Wrap(err, "unable to connect to the origin")
 	}
 
 	toEdgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c)
@@ -272,16 +275,18 @@ func prepareTunnelConfig(
 		TlsConfig:            toEdgeTLSConfig,
 		TransportLogger:      transportLogger,
 		UseDeclarativeTunnel: c.Bool("use-declarative-tunnels"),
+		UseReconnectToken:    c.Bool("use-reconnect-token"),
+		UseQuickReconnects:   c.Bool("use-quick-reconnects"),
 	}, nil
 }
 
-func serviceDiscoverer(c *cli.Context, logger *logrus.Logger) (connection.EdgeServiceDiscoverer, error) {
+func serviceDiscoverer(c *cli.Context, logger *logrus.Logger) (*edgediscovery.Edge, error) {
 	// If --edge is specfied, resolve edge server addresses
 	if len(c.StringSlice("edge")) > 0 {
-		return connection.NewEdgeHostnameResolver(c.StringSlice("edge"))
+		return edgediscovery.StaticEdge(logger, c.StringSlice("edge"))
 	}
 	// Otherwise lookup edge server addresses through service discovery
-	return connection.NewEdgeAddrResolver(logger)
+	return edgediscovery.ResolveEdge(logger)
 }
 
 func isRunningFromTerminal() bool {

cmd/cloudflared/tunnel/server.go

@@ -14,7 +14,7 @@ func runDNSProxyServer(c *cli.Context, dnsReadySignal, shutdownC chan struct{})
 		logger.Errorf("The 'proxy-dns-port' must be a valid port number in <1, 65535> range.")
 		return errors.New("The 'proxy-dns-port' must be a valid port number in <1, 65535> range.")
 	}
-	listener, err := tunneldns.CreateListener(c.String("proxy-dns-address"), uint16(port), c.StringSlice("proxy-dns-upstream"))
+	listener, err := tunneldns.CreateListener(c.String("proxy-dns-address"), uint16(port), c.StringSlice("proxy-dns-upstream"), c.StringSlice("proxy-dns-bootstrap"))
 	if err != nil {
 		close(dnsReadySignal)
 		listener.Stop()

connection/connection.go

@@ -5,35 +5,27 @@ import (
 	"net"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/tunnelrpc"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
-	rpc "zombiezen.com/go/capnproto2/rpc"
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
 	openStreamTimeout = 30 * time.Second
 )
 
-type dialError struct {
-	cause error
-}
-
-func (e dialError) Error() string {
-	return e.cause.Error()
-}
-
 type Connection struct {
-	id    uuid.UUID
-	muxer *h2mux.Muxer
+	id          uuid.UUID
+	muxer       *h2mux.Muxer
+	addr        *net.TCPAddr
+	isLongLived bool
+	longLivedID int
 }
 
-func newConnection(muxer *h2mux.Muxer, edgeIP *net.TCPAddr) (*Connection, error) {
+func newConnection(muxer *h2mux.Muxer, addr *net.TCPAddr) (*Connection, error) {
 	id, err := uuid.NewRandom()
 	if err != nil {
 		return nil, err
@@ -41,6 +33,7 @@ func newConnection(muxer *h2mux.Muxer, edgeIP *net.TCPAddr) (*Connection, error)
 	return &Connection{
 		id:    id,
 		muxer: muxer,
+		addr:  addr,
 	}, nil
 }
 
@@ -50,32 +43,15 @@ func (c *Connection) Serve(ctx context.Context) error {
 }
 
 // Connect is used to establish connections with cloudflare's edge network
-func (c *Connection) Connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters, logger *logrus.Entry) (pogs.ConnectResult, error) {
-	openStreamCtx, cancel := context.WithTimeout(ctx, openStreamTimeout)
-	defer cancel()
-
-	rpcConn, err := c.newRPConn(openStreamCtx, logger)
+func (c *Connection) Connect(ctx context.Context, parameters *tunnelpogs.ConnectParameters, logger *logrus.Entry) (tunnelpogs.ConnectResult, error) {
+	tsClient, err := NewRPCClient(ctx, c.muxer, logger.WithField("rpc", "connect"), openStreamTimeout)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot create new RPC connection")
 	}
-	defer rpcConn.Close()
-
-	tsClient := tunnelpogs.TunnelServer_PogsClient{Client: rpcConn.Bootstrap(ctx)}
-
+	defer tsClient.Close()
 	return tsClient.Connect(ctx, parameters)
 }
 
 func (c *Connection) Shutdown() {
 	c.muxer.Shutdown()
 }
-
-func (c *Connection) newRPConn(ctx context.Context, logger *logrus.Entry) (*rpc.Conn, error) {
-	stream, err := c.muxer.OpenRPCStream(ctx)
-	if err != nil {
-		return nil, err
-	}
-	return rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger.WithField("rpc", "connect"), rpc.StreamTransport(stream)),
-		tunnelrpc.ConnLog(logger.WithField("rpc", "connect")),
-	), nil
-}

connection/dial.go

@@ -0,0 +1,54 @@
+package connection
+
+import (
+	"context"
+	"crypto/tls"
+	"net"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// DialEdge makes a TLS connection to a Cloudflare edge node
+func DialEdge(
+	ctx context.Context,
+	timeout time.Duration,
+	tlsConfig *tls.Config,
+	edgeTCPAddr *net.TCPAddr,
+) (net.Conn, error) {
+	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
+	dialCtx, dialCancel := context.WithTimeout(ctx, timeout)
+	defer dialCancel()
+
+	dialer := net.Dialer{}
+	edgeConn, err := dialer.DialContext(dialCtx, "tcp", edgeTCPAddr.String())
+	if err != nil {
+		return nil, newDialError(err, "DialContext error")
+	}
+	tlsEdgeConn := tls.Client(edgeConn, tlsConfig)
+	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
+
+	if err = tlsEdgeConn.Handshake(); err != nil {
+		return nil, newDialError(err, "TLS handshake with edge error")
+	}
+	// clear the deadline on the conn; h2mux has its own timeouts
+	tlsEdgeConn.SetDeadline(time.Time{})
+	return tlsEdgeConn, nil
+}
+
+// DialError is an error returned from DialEdge
+type DialError struct {
+	cause error
+}
+
+func newDialError(err error, message string) error {
+	return DialError{cause: errors.Wrap(err, message)}
+}
+
+func (e DialError) Error() string {
+	return e.cause.Error()
+}
+
+func (e DialError) Cause() error {
+	return e.cause
+}

connection/discovery.go

@@ -1,254 +0,0 @@
-package connection
-
-import (
-	"context"
-	"crypto/tls"
-	"fmt"
-	"net"
-	"sync"
-	"time"
-
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-)
-
-const (
-	// Used to discover HA origintunneld servers
-	srvService = "origintunneld"
-	srvProto   = "tcp"
-	srvName    = "argotunnel.com"
-
-	// Used to fallback to DoT when we can't use the default resolver to
-	// discover HA origintunneld servers (GitHub issue #75).
-	dotServerName = "cloudflare-dns.com"
-	dotServerAddr = "1.1.1.1:853"
-	dotTimeout    = time.Duration(15 * time.Second)
-
-	// SRV record resolution TTL
-	resolveEdgeAddrTTL = 1 * time.Hour
-)
-
-var friendlyDNSErrorLines = []string{
-	`Please try the following things to diagnose this issue:`,
-	`  1. ensure that argotunnel.com is returning "origintunneld" service records.`,
-	`     Run your system's equivalent of: dig srv _origintunneld._tcp.argotunnel.com`,
-	`  2. ensure that your DNS resolver is not returning compressed SRV records.`,
-	`     See GitHub issue https://github.com/golang/go/issues/27546`,
-	`     For example, you could use Cloudflare's 1.1.1.1 as your resolver:`,
-	`     https://developers.cloudflare.com/1.1.1.1/setting-up-1.1.1.1/`,
-}
-
-// EdgeServiceDiscoverer is an interface for looking up Cloudflare's edge network addresses
-type EdgeServiceDiscoverer interface {
-	// Addr returns an address to connect to cloudflare's edge network
-	Addr() *net.TCPAddr
-	// AvailableAddrs returns the number of unique addresses
-	AvailableAddrs() uint8
-	// Refresh rediscover Cloudflare's edge network addresses
-	Refresh() error
-}
-
-// EdgeAddrResolver discovers the addresses of Cloudflare's edge network through SRV record.
-// It implements EdgeServiceDiscoverer interface
-type EdgeAddrResolver struct {
-	sync.Mutex
-	// Addrs to connect to cloudflare's edge network
-	addrs []*net.TCPAddr
-	// index of the next element to use in addrs
-	nextAddrIndex int
-	logger        *logrus.Entry
-}
-
-func NewEdgeAddrResolver(logger *logrus.Logger) (EdgeServiceDiscoverer, error) {
-	r := &EdgeAddrResolver{
-		logger: logger.WithField("subsystem", " edgeAddrResolver"),
-	}
-	if err := r.Refresh(); err != nil {
-		return nil, err
-	}
-	return r, nil
-}
-
-func (r *EdgeAddrResolver) Addr() *net.TCPAddr {
-	r.Lock()
-	defer r.Unlock()
-	addr := r.addrs[r.nextAddrIndex]
-	r.nextAddrIndex = (r.nextAddrIndex + 1) % len(r.addrs)
-	return addr
-}
-
-func (r *EdgeAddrResolver) AvailableAddrs() uint8 {
-	r.Lock()
-	defer r.Unlock()
-	return uint8(len(r.addrs))
-}
-
-func (r *EdgeAddrResolver) Refresh() error {
-	newAddrs, err := EdgeDiscovery(r.logger)
-	if err != nil {
-		return err
-	}
-	r.Lock()
-	defer r.Unlock()
-	r.addrs = newAddrs
-	r.nextAddrIndex = 0
-	return nil
-}
-
-// HA service discovery lookup
-func EdgeDiscovery(logger *logrus.Entry) ([]*net.TCPAddr, error) {
-	_, addrs, err := net.LookupSRV(srvService, srvProto, srvName)
-	if err != nil {
-		// Try to fall back to DoT from Cloudflare directly.
-		//
-		// Note: Instead of DoT, we could also have used DoH. Either of these:
-		//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
-		//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
-		// But both of these cases miss out on a key feature from the stdlib:
-		//     "The returned records are sorted by priority and randomized by weight within a priority."
-		//     (https://golang.org/pkg/net/#Resolver.LookupSRV)
-		// Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it.
-		// See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552
-		r := fallbackResolver(dotServerName, dotServerAddr)
-		ctx, cancel := context.WithTimeout(context.Background(), dotTimeout)
-		defer cancel()
-		_, fallbackAddrs, fallbackErr := r.LookupSRV(ctx, srvService, srvProto, srvName)
-		if fallbackErr != nil || len(fallbackAddrs) == 0 {
-			// use the original DNS error `err` in messages, not `fallbackErr`
-			logger.Errorln("Error looking up Cloudflare edge IPs: the DNS query failed:", err)
-			for _, s := range friendlyDNSErrorLines {
-				logger.Errorln(s)
-			}
-			return nil, errors.Wrap(err, "Could not lookup srv records on _origintunneld._tcp.argotunnel.com")
-		}
-		// Accept the fallback results and keep going
-		addrs = fallbackAddrs
-	}
-	var resolvedIPsPerCNAME [][]*net.TCPAddr
-	var lookupErr error
-	for _, addr := range addrs {
-		ips, err := resolveSRVToTCP(addr)
-		if err != nil || len(ips) == 0 {
-			// don't return early, we might be able to resolve other addresses
-			lookupErr = err
-			continue
-		}
-		resolvedIPsPerCNAME = append(resolvedIPsPerCNAME, ips)
-	}
-	ips := flattenServiceIPs(resolvedIPsPerCNAME)
-	if lookupErr == nil && len(ips) == 0 {
-		return nil, fmt.Errorf("Unknown service discovery error")
-	}
-	return ips, lookupErr
-}
-
-func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
-	ips, err := net.LookupIP(srv.Target)
-	if err != nil {
-		return nil, err
-	}
-	addrs := make([]*net.TCPAddr, len(ips))
-	for i, ip := range ips {
-		addrs[i] = &net.TCPAddr{IP: ip, Port: int(srv.Port)}
-	}
-	return addrs, nil
-}
-
-// FlattenServiceIPs transposes and flattens the input slices such that the
-// first element of the n inner slices are the first n elements of the result.
-func flattenServiceIPs(ipsByService [][]*net.TCPAddr) []*net.TCPAddr {
-	var result []*net.TCPAddr
-	for len(ipsByService) > 0 {
-		filtered := ipsByService[:0]
-		for _, ips := range ipsByService {
-			if len(ips) == 0 {
-				// sanity check
-				continue
-			}
-			result = append(result, ips[0])
-			if len(ips) > 1 {
-				filtered = append(filtered, ips[1:])
-			}
-		}
-		ipsByService = filtered
-	}
-	return result
-}
-
-// Inspiration: https://github.com/artyom/dot/blob/master/dot.go
-func fallbackResolver(serverName, serverAddress string) *net.Resolver {
-	return &net.Resolver{
-		PreferGo: true,
-		Dial: func(ctx context.Context, _ string, _ string) (net.Conn, error) {
-			var dialer net.Dialer
-			conn, err := dialer.DialContext(ctx, "tcp", serverAddress)
-			if err != nil {
-				return nil, err
-			}
-			tlsConfig := &tls.Config{ServerName: serverName}
-			return tls.Client(conn, tlsConfig), nil
-		},
-	}
-}
-
-// EdgeHostnameResolver discovers the addresses of Cloudflare's edge network via a list of server hostnames.
-// It implements EdgeServiceDiscoverer interface, and is used mainly for testing connectivity.
-type EdgeHostnameResolver struct {
-	sync.Mutex
-	// hostnames of edge servers
-	hostnames []string
-	// Addrs to connect to cloudflare's edge network
-	addrs []*net.TCPAddr
-	// index of the next element to use in addrs
-	nextAddrIndex int
-}
-
-func NewEdgeHostnameResolver(edgeHostnames []string) (EdgeServiceDiscoverer, error) {
-	r := &EdgeHostnameResolver{
-		hostnames: edgeHostnames,
-	}
-	if err := r.Refresh(); err != nil {
-		return nil, err
-	}
-	return r, nil
-}
-
-func (r *EdgeHostnameResolver) Addr() *net.TCPAddr {
-	r.Lock()
-	defer r.Unlock()
-	addr := r.addrs[r.nextAddrIndex]
-	r.nextAddrIndex = (r.nextAddrIndex + 1) % len(r.addrs)
-	return addr
-}
-
-func (r *EdgeHostnameResolver) AvailableAddrs() uint8 {
-	r.Lock()
-	defer r.Unlock()
-	return uint8(len(r.addrs))
-}
-
-func (r *EdgeHostnameResolver) Refresh() error {
-	newAddrs, err := ResolveAddrs(r.hostnames)
-	if err != nil {
-		return err
-	}
-	r.Lock()
-	defer r.Unlock()
-	r.addrs = newAddrs
-	r.nextAddrIndex = 0
-	return nil
-}
-
-// Resolve TCP address given a list of addresses. Address can be a hostname, however, it will return at most one
-// of the hostname's IP addresses
-func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) {
-	var tcpAddrs []*net.TCPAddr
-	for _, addr := range addrs {
-		tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
-		if err != nil {
-			return nil, err
-		}
-		tcpAddrs = append(tcpAddrs, tcpAddr)
-	}
-	return tcpAddrs, nil
-}

connection/discovery_test.go

@@ -1,63 +0,0 @@
-package connection
-
-import (
-	"net"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-type mockEdgeServiceDiscoverer struct {
-}
-
-func (mr *mockEdgeServiceDiscoverer) Addr() *net.TCPAddr {
-	return &net.TCPAddr{
-		IP:   net.ParseIP("127.0.0.1"),
-		Port: 63102,
-	}
-}
-
-func (mr *mockEdgeServiceDiscoverer) AvailableAddrs() uint8 {
-	return 1
-}
-
-func (mr *mockEdgeServiceDiscoverer) Refresh() error {
-	return nil
-}
-
-func TestFlattenServiceIPs(t *testing.T) {
-	result := flattenServiceIPs([][]*net.TCPAddr{
-		[]*net.TCPAddr{
-			&net.TCPAddr{Port: 1},
-			&net.TCPAddr{Port: 2},
-			&net.TCPAddr{Port: 3},
-			&net.TCPAddr{Port: 4},
-		},
-		[]*net.TCPAddr{
-			&net.TCPAddr{Port: 10},
-			&net.TCPAddr{Port: 12},
-			&net.TCPAddr{Port: 13},
-		},
-		[]*net.TCPAddr{
-			&net.TCPAddr{Port: 21},
-			&net.TCPAddr{Port: 22},
-			&net.TCPAddr{Port: 23},
-			&net.TCPAddr{Port: 24},
-			&net.TCPAddr{Port: 25},
-		},
-	})
-	assert.EqualValues(t, []*net.TCPAddr{
-		&net.TCPAddr{Port: 1},
-		&net.TCPAddr{Port: 10},
-		&net.TCPAddr{Port: 21},
-		&net.TCPAddr{Port: 2},
-		&net.TCPAddr{Port: 12},
-		&net.TCPAddr{Port: 22},
-		&net.TCPAddr{Port: 3},
-		&net.TCPAddr{Port: 13},
-		&net.TCPAddr{Port: 23},
-		&net.TCPAddr{Port: 4},
-		&net.TCPAddr{Port: 24},
-		&net.TCPAddr{Port: 25},
-	}, result)
-}

connection/manager.go

@@ -4,19 +4,19 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"net"
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/streamhandler"
-	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/prometheus/client_golang/prometheus"
-
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/streamhandler"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
@@ -36,7 +36,7 @@ type EdgeManager struct {
 	// cloudflaredConfig is the cloudflared configuration that is determined when the process first starts
 	cloudflaredConfig *CloudflaredConfig
 	// serviceDiscoverer returns the next edge addr to connect to
-	serviceDiscoverer EdgeServiceDiscoverer
+	serviceDiscoverer *edgediscovery.Edge
 	// state is attributes of ConnectionManager that can change during runtime.
 	state *edgeManagerState
 
@@ -59,12 +59,12 @@ func newMetrics(namespace, subsystem string) *metrics {
 // EdgeManagerConfigurable is the configurable attributes of a EdgeConnectionManager
 type EdgeManagerConfigurable struct {
 	TunnelHostnames []h2mux.TunnelHostname
-	*pogs.EdgeConnectionConfig
+	*tunnelpogs.EdgeConnectionConfig
 }
 
 type CloudflaredConfig struct {
 	CloudflaredID uuid.UUID
-	Tags          []pogs.Tag
+	Tags          []tunnelpogs.Tag
 	BuildInfo     *buildinfo.BuildInfo
 	IntentLabel   string
 }
@@ -74,7 +74,7 @@ func NewEdgeManager(
 	edgeConnMgrConfigurable *EdgeManagerConfigurable,
 	userCredential []byte,
 	tlsConfig *tls.Config,
-	serviceDiscoverer EdgeServiceDiscoverer,
+	serviceDiscoverer *edgediscovery.Edge,
 	cloudflaredConfig *CloudflaredConfig,
 	logger *logrus.Logger,
 ) *EdgeManager {
@@ -92,27 +92,29 @@ func NewEdgeManager(
 func (em *EdgeManager) Run(ctx context.Context) error {
 	defer em.shutdown()
 
-	resolveEdgeIPTicker := time.Tick(resolveEdgeAddrTTL)
+	// Currently, declarative tunnels don't have any concept of a stable connection
+	// Each edge connection is transient and when it dies, it is replaced by a different one,
+	// not restarted.
+	// So in the future we should really change this so that n connections are stored individually
+	connIndex := 0
 	for {
 		select {
 		case <-ctx.Done():
 			return errors.Wrap(ctx.Err(), "EdgeConnectionManager terminated")
-		case <-resolveEdgeIPTicker:
-			if err := em.serviceDiscoverer.Refresh(); err != nil {
-				em.logger.WithError(err).Warn("Cannot refresh Cloudflare edge addresses")
-			}
 		default:
 			time.Sleep(1 * time.Second)
 		}
 		// Create/delete connection one at a time, so we don't need to adjust for connections that are being created/deleted
 		// in shouldCreateConnection or shouldReduceConnection calculation
 		if em.state.shouldCreateConnection(em.serviceDiscoverer.AvailableAddrs()) {
-			if connErr := em.newConnection(ctx); connErr != nil {
+			if connErr := em.newConnection(ctx, connIndex); connErr != nil {
 				if !connErr.ShouldRetry {
 					em.logger.WithError(connErr).Error(em.noRetryMessage())
 					return connErr
 				}
 				em.logger.WithError(connErr).Error("cannot create new connection")
+			} else {
+				connIndex++
 			}
 		} else if em.state.shouldReduceConnection() {
 			if err := em.closeConnection(ctx); err != nil {
@@ -127,13 +129,16 @@ func (em *EdgeManager) UpdateConfigurable(newConfigurable *EdgeManagerConfigurab
 	em.state.updateConfigurable(newConfigurable)
 }
 
-func (em *EdgeManager) newConnection(ctx context.Context) *pogs.ConnectError {
-	edgeIP := em.serviceDiscoverer.Addr()
-	edgeConn, err := em.dialEdge(ctx, edgeIP)
+func (em *EdgeManager) newConnection(ctx context.Context, index int) *tunnelpogs.ConnectError {
+	edgeTCPAddr, err := em.serviceDiscoverer.GetAddr(index)
+	if err != nil {
+		return retryConnection(fmt.Sprintf("edge address discovery error: %v", err))
+	}
+	configurable := em.state.getConfigurable()
+	edgeConn, err := DialEdge(ctx, configurable.Timeout, em.tlsConfig, edgeTCPAddr)
 	if err != nil {
 		return retryConnection(fmt.Sprintf("dial edge error: %v", err))
 	}
-	configurable := em.state.getConfigurable()
 	// Establish a muxed connection with the edge
 	// Client mux handshake with agent server
 	muxer, err := h2mux.Handshake(edgeConn, edgeConn, h2mux.MuxerConfig{
@@ -148,14 +153,14 @@ func (em *EdgeManager) newConnection(ctx context.Context) *pogs.ConnectError {
 		retryConnection(fmt.Sprintf("couldn't perform handshake with edge: %v", err))
 	}
 
-	h2muxConn, err := newConnection(muxer, edgeIP)
+	h2muxConn, err := newConnection(muxer, edgeTCPAddr)
 	if err != nil {
 		return retryConnection(fmt.Sprintf("couldn't create h2mux connection: %v", err))
 	}
 
 	go em.serveConn(ctx, h2muxConn)
 
-	connResult, err := h2muxConn.Connect(ctx, &pogs.ConnectParameters{
+	connResult, err := h2muxConn.Connect(ctx, &tunnelpogs.ConnectParameters{
 		CloudflaredID:       em.cloudflaredConfig.CloudflaredID,
 		CloudflaredVersion:  em.cloudflaredConfig.BuildInfo.CloudflaredVersion,
 		NumPreviousAttempts: 0,
@@ -187,6 +192,7 @@ func (em *EdgeManager) closeConnection(ctx context.Context) error {
 		return fmt.Errorf("no connection to close")
 	}
 	conn.Shutdown()
+	// teardown will be handled by EdgeManager.serveConn in another goroutine
 	return nil
 }
 
@@ -194,28 +200,7 @@ func (em *EdgeManager) serveConn(ctx context.Context, conn *Connection) {
 	err := conn.Serve(ctx)
 	em.logger.WithError(err).Warn("Connection closed")
 	em.state.closeConnection(conn)
-}
-
-func (em *EdgeManager) dialEdge(ctx context.Context, edgeIP *net.TCPAddr) (*tls.Conn, error) {
-	timeout := em.state.getConfigurable().Timeout
-	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
-	dialCtx, dialCancel := context.WithTimeout(ctx, timeout)
-	defer dialCancel()
-
-	dialer := net.Dialer{DualStack: true}
-	edgeConn, err := dialer.DialContext(dialCtx, "tcp", edgeIP.String())
-	if err != nil {
-		return nil, dialError{cause: errors.Wrap(err, "DialContext error")}
-	}
-	tlsEdgeConn := tls.Client(edgeConn, em.tlsConfig)
-	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
-
-	if err = tlsEdgeConn.Handshake(); err != nil {
-		return nil, dialError{cause: errors.Wrap(err, "Handshake with edge error")}
-	}
-	// clear the deadline on the conn; h2mux has its own timeouts
-	tlsEdgeConn.SetDeadline(time.Time{})
-	return tlsEdgeConn, nil
+	em.serviceDiscoverer.GiveBack(conn.addr)
 }
 
 func (em *EdgeManager) noRetryMessage() string {
@@ -244,14 +229,14 @@ func newEdgeConnectionManagerState(configurable *EdgeManagerConfigurable, userCr
 	}
 }
 
-func (ems *edgeManagerState) shouldCreateConnection(availableEdgeAddrs uint8) bool {
+func (ems *edgeManagerState) shouldCreateConnection(availableEdgeAddrs int) bool {
 	ems.RLock()
 	defer ems.RUnlock()
-	expectedHAConns := ems.configurable.NumHAConnections
+	expectedHAConns := int(ems.configurable.NumHAConnections)
 	if availableEdgeAddrs < expectedHAConns {
 		expectedHAConns = availableEdgeAddrs
 	}
-	return uint8(len(ems.conns)) < expectedHAConns
+	return len(ems.conns) < expectedHAConns
 }
 
 func (ems *edgeManagerState) shouldReduceConnection() bool {
@@ -308,8 +293,8 @@ func (ems *edgeManagerState) getUserCredential() []byte {
 	return ems.userCredential
 }
 
-func retryConnection(cause string) *pogs.ConnectError {
-	return &pogs.ConnectError{
+func retryConnection(cause string) *tunnelpogs.ConnectError {
+	return &tunnelpogs.ConnectError{
 		Cause:       cause,
 		RetryAfter:  defaultRetryAfter,
 		ShouldRetry: true,

connection/manager_test.go

@@ -1,18 +1,19 @@
 package connection
 
 import (
+	"net"
 	"testing"
 	"time"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/google/uuid"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/streamhandler"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-
-	"github.com/google/uuid"
-	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -48,14 +49,15 @@ func mockEdgeManager() *EdgeManager {
 	newConfigChan := make(chan<- *pogs.ClientConfig)
 	useConfigResultChan := make(<-chan *pogs.UseConfigurationResult)
 	logger := logrus.New()
+	edge := edgediscovery.MockEdge(logger, []*net.TCPAddr{})
 	return NewEdgeManager(
 		streamhandler.NewStreamHandler(newConfigChan, useConfigResultChan, logger),
 		configurable,
 		[]byte{},
 		nil,
-		&mockEdgeServiceDiscoverer{},
+		edge,
 		cloudflaredConfig,
-		logrus.New(),
+		logger,
 	)
 }
 

connection/mocks_for_test.go

@@ -0,0 +1,118 @@
+package connection
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+	"net"
+	"reflect"
+	"testing/quick"
+)
+
+type mockAddrs struct {
+	// a set of synthetic SRV records
+	addrMap map[net.SRV][]*net.TCPAddr
+	// the total number of addresses, aggregated across addrMap.
+	// For the convenience of test code that would otherwise have to compute
+	// this by hand every time.
+	numAddrs int
+}
+
+func newMockAddrs(port uint16, numRegions uint8, numAddrsPerRegion uint8) mockAddrs {
+	addrMap := make(map[net.SRV][]*net.TCPAddr)
+	numAddrs := 0
+
+	for r := uint8(0); r < numRegions; r++ {
+		var (
+			srv   = net.SRV{Target: fmt.Sprintf("test-region-%v.example.com", r), Port: port}
+			addrs []*net.TCPAddr
+		)
+		for a := uint8(0); a < numAddrsPerRegion; a++ {
+			addrs = append(addrs, &net.TCPAddr{
+				IP:   net.ParseIP(fmt.Sprintf("10.0.%v.%v", r, a)),
+				Port: int(port),
+			})
+		}
+		addrMap[srv] = addrs
+		numAddrs += len(addrs)
+	}
+	return mockAddrs{addrMap: addrMap, numAddrs: numAddrs}
+}
+
+var _ quick.Generator = mockAddrs{}
+
+func (mockAddrs) Generate(rand *rand.Rand, size int) reflect.Value {
+	port := uint16(rand.Intn(math.MaxUint16))
+	numRegions := uint8(1 + rand.Intn(10))
+	numAddrsPerRegion := uint8(1 + rand.Intn(32))
+	result := newMockAddrs(port, numRegions, numAddrsPerRegion)
+	return reflect.ValueOf(result)
+}
+
+// Returns a function compatible with net.LookupSRV that will return the SRV
+// records from mockAddrs.
+func mockNetLookupSRV(
+	m mockAddrs,
+) func(service, proto, name string) (cname string, addrs []*net.SRV, err error) {
+	var addrs []*net.SRV
+	for k := range m.addrMap {
+		addr := k
+		addrs = append(addrs, &addr)
+		// We can't just do
+		//   addrs = append(addrs, &k)
+		// `k` will be reused by subsequent loop iterations,
+		// so all the copies of `&k` would point to the same location.
+	}
+	return func(_, _, _ string) (string, []*net.SRV, error) {
+		return "", addrs, nil
+	}
+}
+
+// Returns a function compatible with net.LookupIP that translates the SRV records
+// from mockAddrs into IP addresses, based on the TCP addresses in mockAddrs.
+func mockNetLookupIP(
+	m mockAddrs,
+) func(host string) ([]net.IP, error) {
+	return func(host string) ([]net.IP, error) {
+		for srv, tcpAddrs := range m.addrMap {
+			if srv.Target != host {
+				continue
+			}
+			result := make([]net.IP, len(tcpAddrs))
+			for i, tcpAddr := range tcpAddrs {
+				result[i] = tcpAddr.IP
+			}
+			return result, nil
+		}
+		return nil, fmt.Errorf("No IPs for %v", host)
+	}
+}
+
+type mockEdgeServiceDiscoverer struct {
+}
+
+func (mr *mockEdgeServiceDiscoverer) Addr() (*net.TCPAddr, error) {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("127.0.0.1"),
+		Port: 63102,
+	}, nil
+}
+
+func (mr *mockEdgeServiceDiscoverer) AnyAddr() (*net.TCPAddr, error) {
+	return &net.TCPAddr{
+		IP:   net.ParseIP("127.0.0.1"),
+		Port: 63102,
+	}, nil
+}
+
+func (mr *mockEdgeServiceDiscoverer) ReplaceAddr(addr *net.TCPAddr) {}
+
+func (mr *mockEdgeServiceDiscoverer) MarkAddrBad(addr *net.TCPAddr) {}
+
+func (mr *mockEdgeServiceDiscoverer) AvailableAddrs() int {
+	return 1
+}
+
+func (mr *mockEdgeServiceDiscoverer) Refresh() error {
+	return nil
+}

connection/rpc.go

@@ -0,0 +1,49 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	rpc "zombiezen.com/go/capnproto2/rpc"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/tunnelrpc"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+// NewRPCClient creates and returns a new RPC client, which will communicate
+// using a stream on the given muxer
+func NewRPCClient(
+	ctx context.Context,
+	muxer *h2mux.Muxer,
+	logger *logrus.Entry,
+	openStreamTimeout time.Duration,
+) (client tunnelpogs.TunnelServer_PogsClient, err error) {
+	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer openStreamCancel()
+	stream, err := muxer.OpenRPCStream(openStreamCtx)
+	if err != nil {
+		return
+	}
+
+	if !isRPCStreamResponse(stream.Headers) {
+		stream.Close()
+		err = fmt.Errorf("rpc: bad response headers: %v", stream.Headers)
+		return
+	}
+
+	conn := rpc.NewConn(
+		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
+		tunnelrpc.ConnLog(logger),
+	)
+	client = tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
+	return client, nil
+}
+
+func isRPCStreamResponse(headers []h2mux.Header) bool {
+	return len(headers) == 1 &&
+		headers[0].Name == ":status" &&
+		headers[0].Value == "200"
+}

edgediscovery/allregions/discovery.go

@@ -0,0 +1,135 @@
+package allregions
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	// Used to discover HA origintunneld servers
+	srvService = "origintunneld"
+	srvProto   = "tcp"
+	srvName    = "argotunnel.com"
+
+	// Used to fallback to DoT when we can't use the default resolver to
+	// discover HA origintunneld servers (GitHub issue #75).
+	dotServerName = "cloudflare-dns.com"
+	dotServerAddr = "1.1.1.1:853"
+	dotTimeout    = time.Duration(15 * time.Second)
+
+	// SRV record resolution TTL
+	resolveEdgeAddrTTL = 1 * time.Hour
+
+	subsystemEdgeAddrResolver = "edgeAddrResolver"
+)
+
+// Redeclare network functions so they can be overridden in tests.
+var (
+	netLookupSRV = net.LookupSRV
+	netLookupIP  = net.LookupIP
+)
+
+// If the call to net.LookupSRV fails, try to fall back to DoT from Cloudflare directly.
+//
+// Note: Instead of DoT, we could also have used DoH. Either of these:
+//     - directly via the JSON API (https://1.1.1.1/dns-query?ct=application/dns-json&name=_origintunneld._tcp.argotunnel.com&type=srv)
+//     - indirectly via `tunneldns.NewUpstreamHTTPS()`
+// But both of these cases miss out on a key feature from the stdlib:
+//     "The returned records are sorted by priority and randomized by weight within a priority."
+//     (https://golang.org/pkg/net/#Resolver.LookupSRV)
+// Does this matter? I don't know. It may someday. Let's use DoT so we don't need to worry about it.
+// See also: Go feature request for stdlib-supported DoH: https://github.com/golang/go/issues/27552
+var fallbackLookupSRV = lookupSRVWithDOT
+
+var friendlyDNSErrorLines = []string{
+	`Please try the following things to diagnose this issue:`,
+	`  1. ensure that argotunnel.com is returning "origintunneld" service records.`,
+	`     Run your system's equivalent of: dig srv _origintunneld._tcp.argotunnel.com`,
+	`  2. ensure that your DNS resolver is not returning compressed SRV records.`,
+	`     See GitHub issue https://github.com/golang/go/issues/27546`,
+	`     For example, you could use Cloudflare's 1.1.1.1 as your resolver:`,
+	`     https://developers.cloudflare.com/1.1.1.1/setting-up-1.1.1.1/`,
+}
+
+// EdgeDiscovery implements HA service discovery lookup.
+func edgeDiscovery(logger *logrus.Entry) ([][]*net.TCPAddr, error) {
+	_, addrs, err := netLookupSRV(srvService, srvProto, srvName)
+	if err != nil {
+		_, fallbackAddrs, fallbackErr := fallbackLookupSRV(srvService, srvProto, srvName)
+		if fallbackErr != nil || len(fallbackAddrs) == 0 {
+			// use the original DNS error `err` in messages, not `fallbackErr`
+			logger.Errorln("Error looking up Cloudflare edge IPs: the DNS query failed:", err)
+			for _, s := range friendlyDNSErrorLines {
+				logger.Errorln(s)
+			}
+			return nil, errors.Wrapf(err, "Could not lookup srv records on _%v._%v.%v", srvService, srvProto, srvName)
+		}
+		// Accept the fallback results and keep going
+		addrs = fallbackAddrs
+	}
+
+	var resolvedIPsPerCNAME [][]*net.TCPAddr
+	for _, addr := range addrs {
+		ips, err := resolveSRVToTCP(addr)
+		if err != nil {
+			return nil, err
+		}
+		resolvedIPsPerCNAME = append(resolvedIPsPerCNAME, ips)
+	}
+
+	return resolvedIPsPerCNAME, nil
+}
+
+func lookupSRVWithDOT(service, proto, name string) (cname string, addrs []*net.SRV, err error) {
+	// Inspiration: https://github.com/artyom/dot/blob/master/dot.go
+	r := &net.Resolver{
+		PreferGo: true,
+		Dial: func(ctx context.Context, _ string, _ string) (net.Conn, error) {
+			var dialer net.Dialer
+			conn, err := dialer.DialContext(ctx, "tcp", dotServerAddr)
+			if err != nil {
+				return nil, err
+			}
+			tlsConfig := &tls.Config{ServerName: dotServerName}
+			return tls.Client(conn, tlsConfig), nil
+		},
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), dotTimeout)
+	defer cancel()
+	return r.LookupSRV(ctx, srvService, srvProto, srvName)
+}
+
+func resolveSRVToTCP(srv *net.SRV) ([]*net.TCPAddr, error) {
+	ips, err := netLookupIP(srv.Target)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Couldn't resolve SRV record %v", srv)
+	}
+	if len(ips) == 0 {
+		return nil, fmt.Errorf("SRV record %v had no IPs", srv)
+	}
+	addrs := make([]*net.TCPAddr, len(ips))
+	for i, ip := range ips {
+		addrs[i] = &net.TCPAddr{IP: ip, Port: int(srv.Port)}
+	}
+	return addrs, nil
+}
+
+// ResolveAddrs resolves TCP address given a list of addresses. Address can be a hostname, however, it will return at most one
+// of the hostname's IP addresses
+func ResolveAddrs(addrs []string) ([]*net.TCPAddr, error) {
+	var tcpAddrs []*net.TCPAddr
+	for _, addr := range addrs {
+		tcpAddr, err := net.ResolveTCPAddr("tcp", addr)
+		if err != nil {
+			return nil, err
+		}
+		tcpAddrs = append(tcpAddrs, tcpAddr)
+	}
+	return tcpAddrs, nil
+}

edgediscovery/allregions/discovery_test.go

@@ -0,0 +1,32 @@
+package allregions
+
+import (
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEdgeDiscovery(t *testing.T) {
+	mockAddrs := newMockAddrs(19, 2, 5)
+	netLookupSRV = mockNetLookupSRV(mockAddrs)
+	netLookupIP = mockNetLookupIP(mockAddrs)
+
+	expectedAddrSet := map[string]bool{}
+	for _, addrs := range mockAddrs.addrMap {
+		for _, addr := range addrs {
+			expectedAddrSet[addr.String()] = true
+		}
+	}
+
+	addrLists, err := edgeDiscovery(logrus.New().WithFields(logrus.Fields{}))
+	assert.NoError(t, err)
+	actualAddrSet := map[string]bool{}
+	for _, addrs := range addrLists {
+		for _, addr := range addrs {
+			actualAddrSet[addr.String()] = true
+		}
+	}
+
+	assert.Equal(t, expectedAddrSet, actualAddrSet)
+}

edgediscovery/allregions/mocks_for_test.go

@@ -0,0 +1,89 @@
+package allregions
+
+import (
+	"fmt"
+	"math"
+	"math/rand"
+	"net"
+	"reflect"
+	"testing/quick"
+)
+
+type mockAddrs struct {
+	// a set of synthetic SRV records
+	addrMap map[net.SRV][]*net.TCPAddr
+	// the total number of addresses, aggregated across addrMap.
+	// For the convenience of test code that would otherwise have to compute
+	// this by hand every time.
+	numAddrs int
+}
+
+func newMockAddrs(port uint16, numRegions uint8, numAddrsPerRegion uint8) mockAddrs {
+	addrMap := make(map[net.SRV][]*net.TCPAddr)
+	numAddrs := 0
+
+	for r := uint8(0); r < numRegions; r++ {
+		var (
+			srv   = net.SRV{Target: fmt.Sprintf("test-region-%v.example.com", r), Port: port}
+			addrs []*net.TCPAddr
+		)
+		for a := uint8(0); a < numAddrsPerRegion; a++ {
+			addrs = append(addrs, &net.TCPAddr{
+				IP:   net.ParseIP(fmt.Sprintf("10.0.%v.%v", r, a)),
+				Port: int(port),
+			})
+		}
+		addrMap[srv] = addrs
+		numAddrs += len(addrs)
+	}
+	return mockAddrs{addrMap: addrMap, numAddrs: numAddrs}
+}
+
+var _ quick.Generator = mockAddrs{}
+
+func (mockAddrs) Generate(rand *rand.Rand, size int) reflect.Value {
+	port := uint16(rand.Intn(math.MaxUint16))
+	numRegions := uint8(1 + rand.Intn(10))
+	numAddrsPerRegion := uint8(1 + rand.Intn(32))
+	result := newMockAddrs(port, numRegions, numAddrsPerRegion)
+	return reflect.ValueOf(result)
+}
+
+// Returns a function compatible with net.LookupSRV that will return the SRV
+// records from mockAddrs.
+func mockNetLookupSRV(
+	m mockAddrs,
+) func(service, proto, name string) (cname string, addrs []*net.SRV, err error) {
+	var addrs []*net.SRV
+	for k := range m.addrMap {
+		addr := k
+		addrs = append(addrs, &addr)
+		// We can't just do
+		//   addrs = append(addrs, &k)
+		// `k` will be reused by subsequent loop iterations,
+		// so all the copies of `&k` would point to the same location.
+	}
+	return func(_, _, _ string) (string, []*net.SRV, error) {
+		return "", addrs, nil
+	}
+}
+
+// Returns a function compatible with net.LookupIP that translates the SRV records
+// from mockAddrs into IP addresses, based on the TCP addresses in mockAddrs.
+func mockNetLookupIP(
+	m mockAddrs,
+) func(host string) ([]net.IP, error) {
+	return func(host string) ([]net.IP, error) {
+		for srv, tcpAddrs := range m.addrMap {
+			if srv.Target != host {
+				continue
+			}
+			result := make([]net.IP, len(tcpAddrs))
+			for i, tcpAddr := range tcpAddrs {
+				result[i] = tcpAddr.IP
+			}
+			return result, nil
+		}
+		return nil, fmt.Errorf("No IPs for %v", host)
+	}
+}

edgediscovery/allregions/region.go

@@ -0,0 +1,78 @@
+package allregions
+
+import (
+	"net"
+)
+
+// Region contains cloudflared edge addresses. The edge is partitioned into several regions for
+// redundancy purposes.
+type Region struct {
+	connFor map[*net.TCPAddr]UsedBy
+}
+
+// NewRegion creates a region with the given addresses, which are all unused.
+func NewRegion(addrs []*net.TCPAddr) Region {
+	// The zero value of UsedBy is Unused(), so we can just initialize the map's values with their
+	// zero values.
+	m := make(map[*net.TCPAddr]UsedBy)
+	for _, addr := range addrs {
+		m[addr] = Unused()
+	}
+	return Region{connFor: m}
+}
+
+// AddrUsedBy finds the address used by the given connection in this region.
+// Returns nil if the connection isn't using any IP.
+func (r *Region) AddrUsedBy(connID int) *net.TCPAddr {
+	for addr, used := range r.connFor {
+		if used.Used && used.ConnID == connID {
+			return addr
+		}
+	}
+	return nil
+}
+
+// AvailableAddrs counts how many unused addresses this region contains.
+func (r Region) AvailableAddrs() int {
+	n := 0
+	for _, usedby := range r.connFor {
+		if !usedby.Used {
+			n++
+		}
+	}
+	return n
+}
+
+// GetUnusedIP returns a random unused address in this region.
+// Returns nil if all addresses are in use.
+func (r Region) GetUnusedIP(excluding *net.TCPAddr) *net.TCPAddr {
+	for addr, usedby := range r.connFor {
+		if !usedby.Used && addr != excluding {
+			return addr
+		}
+	}
+	return nil
+}
+
+// Use the address, assigning it to a proxy connection.
+func (r Region) Use(addr *net.TCPAddr, connID int) {
+	r.connFor[addr] = InUse(connID)
+}
+
+// GetAnyAddress returns an arbitrary address from the region.
+func (r Region) GetAnyAddress() *net.TCPAddr {
+	for addr := range r.connFor {
+		return addr
+	}
+	return nil
+}
+
+// GiveBack the address, ensuring it is no longer assigned to an IP.
+// Returns true if the address is in this region.
+func (r Region) GiveBack(addr *net.TCPAddr) (ok bool) {
+	if _, ok := r.connFor[addr]; !ok {
+		return false
+	}
+	r.connFor[addr] = Unused()
+	return true
+}

edgediscovery/allregions/region_test.go

@@ -0,0 +1,287 @@
+package allregions
+
+import (
+	"fmt"
+	"net"
+	"reflect"
+	"testing"
+)
+
+func TestRegion_New(t *testing.T) {
+	r := NewRegion([]*net.TCPAddr{&addr0, &addr1, &addr2})
+	fmt.Println(r.connFor)
+	if r.AvailableAddrs() != 3 {
+		t.Errorf("r.AvailableAddrs() == %v but want 3", r.AvailableAddrs())
+	}
+}
+
+func TestRegion_AddrUsedBy(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	type args struct {
+		connID int
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   *net.TCPAddr
+	}{
+		{
+			name: "happy trivial test",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+			}},
+			args: args{connID: 0},
+			want: &addr0,
+		},
+		{
+			name: "sad trivial test",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+			}},
+			args: args{connID: 1},
+			want: nil,
+		},
+		{
+			name: "sad test",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			args: args{connID: 3},
+			want: nil,
+		},
+		{
+			name: "happy test",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			args: args{connID: 1},
+			want: &addr1,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := &Region{
+				connFor: tt.fields.connFor,
+			}
+			if got := r.AddrUsedBy(tt.args.connID); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Region.AddrUsedBy() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestRegion_AvailableAddrs(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   int
+	}{
+		{
+			name: "contains addresses",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			}},
+			want: 1,
+		},
+		{
+			name: "all free",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: Unused(),
+				&addr1: Unused(),
+				&addr2: Unused(),
+			}},
+			want: 3,
+		},
+		{
+			name: "all used",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			want: 0,
+		},
+		{
+			name:   "empty",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{}},
+			want:   0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Region{
+				connFor: tt.fields.connFor,
+			}
+			if got := r.AvailableAddrs(); got != tt.want {
+				t.Errorf("Region.AvailableAddrs() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestRegion_GetUnusedIP(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	type args struct {
+		excluding *net.TCPAddr
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   *net.TCPAddr
+	}{
+		{
+			name: "happy test with excluding set",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: Unused(),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			}},
+			args: args{excluding: &addr0},
+			want: &addr1,
+		},
+		{
+			name: "happy test with no excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: Unused(),
+				&addr2: InUse(2),
+			}},
+			args: args{excluding: nil},
+			want: &addr1,
+		},
+		{
+			name: "sad test with no excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(0),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			args: args{excluding: nil},
+			want: nil,
+		},
+		{
+			name: "sad test with excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: Unused(),
+				&addr1: InUse(1),
+				&addr2: InUse(2),
+			}},
+			args: args{excluding: &addr0},
+			want: nil,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Region{
+				connFor: tt.fields.connFor,
+			}
+			if got := r.GetUnusedIP(tt.args.excluding); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Region.GetUnusedIP() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestRegion_GiveBack(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	type args struct {
+		addr *net.TCPAddr
+	}
+	tests := []struct {
+		name           string
+		fields         fields
+		args           args
+		wantOk         bool
+		availableAfter int
+	}{
+		{
+			name: "sad test with excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr1: InUse(1),
+			}},
+			args:           args{addr: &addr1},
+			wantOk:         true,
+			availableAfter: 1,
+		},
+		{
+			name: "sad test with excluding",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr1: InUse(1),
+			}},
+			args:           args{addr: &addr2},
+			wantOk:         false,
+			availableAfter: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Region{
+				connFor: tt.fields.connFor,
+			}
+			if gotOk := r.GiveBack(tt.args.addr); gotOk != tt.wantOk {
+				t.Errorf("Region.GiveBack() = %v, want %v", gotOk, tt.wantOk)
+			}
+			if tt.availableAfter != r.AvailableAddrs() {
+				t.Errorf("Region.AvailableAddrs() = %v, want %v", r.AvailableAddrs(), tt.availableAfter)
+			}
+		})
+	}
+}
+
+func TestRegion_GetAnyAddress(t *testing.T) {
+	type fields struct {
+		connFor map[*net.TCPAddr]UsedBy
+	}
+	tests := []struct {
+		name    string
+		fields  fields
+		wantNil bool
+	}{
+		{
+			name:    "Sad test -- GetAnyAddress should only fail if the region is empty",
+			fields:  fields{connFor: map[*net.TCPAddr]UsedBy{}},
+			wantNil: true,
+		},
+		{
+			name: "Happy test (all addresses unused)",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: Unused(),
+			}},
+			wantNil: false,
+		},
+		{
+			name: "Happy test (GetAnyAddress can still return addresses used by proxy conns)",
+			fields: fields{connFor: map[*net.TCPAddr]UsedBy{
+				&addr0: InUse(2),
+			}},
+			wantNil: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Region{
+				connFor: tt.fields.connFor,
+			}
+			if got := r.GetAnyAddress(); tt.wantNil != (got == nil) {
+				t.Errorf("Region.GetAnyAddress() = %v, but should it return nil? %v", got, tt.wantNil)
+			}
+		})
+	}
+}

edgediscovery/allregions/regions.go

@@ -0,0 +1,118 @@
+package allregions
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/sirupsen/logrus"
+)
+
+// Regions stores Cloudflare edge network IPs, partitioned into two regions.
+// This is NOT thread-safe. Users of this package should use it with a lock.
+type Regions struct {
+	region1 Region
+	region2 Region
+}
+
+// ------------------------------------
+// Constructors
+// ------------------------------------
+
+// ResolveEdge resolves the Cloudflare edge, returning all regions discovered.
+func ResolveEdge(logger *logrus.Entry) (*Regions, error) {
+	addrLists, err := edgeDiscovery(logger)
+	if err != nil {
+		return nil, err
+	}
+	if len(addrLists) < 2 {
+		return nil, fmt.Errorf("expected at least 2 Cloudflare Regions regions, but SRV only returned %v", len(addrLists))
+	}
+	return &Regions{
+		region1: NewRegion(addrLists[0]),
+		region2: NewRegion(addrLists[1]),
+	}, nil
+}
+
+// StaticEdge creates a list of edge addresses from the list of hostnames.
+// Mainly used for testing connectivity.
+func StaticEdge(hostnames []string) (*Regions, error) {
+	addrs, err := ResolveAddrs(hostnames)
+	if err != nil {
+		return nil, err
+	}
+	return NewNoResolve(addrs), nil
+}
+
+// NewNoResolve doesn't resolve the edge. Instead it just uses the given addresses.
+// You probably only need this for testing.
+func NewNoResolve(addrs []*net.TCPAddr) *Regions {
+	region1 := make([]*net.TCPAddr, 0)
+	region2 := make([]*net.TCPAddr, 0)
+	for i, v := range addrs {
+		if i%2 == 0 {
+			region1 = append(region1, v)
+		} else {
+			region2 = append(region2, v)
+		}
+	}
+
+	return &Regions{
+		region1: NewRegion(region1),
+		region2: NewRegion(region2),
+	}
+}
+
+// ------------------------------------
+// Methods
+// ------------------------------------
+
+// GetAnyAddress returns an arbitrary address from the larger region.
+func (rs *Regions) GetAnyAddress() *net.TCPAddr {
+	if addr := rs.region1.GetAnyAddress(); addr != nil {
+		return addr
+	}
+	return rs.region2.GetAnyAddress()
+}
+
+// AddrUsedBy finds the address used by the given connection.
+// Returns nil if the connection isn't using an address.
+func (rs *Regions) AddrUsedBy(connID int) *net.TCPAddr {
+	if addr := rs.region1.AddrUsedBy(connID); addr != nil {
+		return addr
+	}
+	return rs.region2.AddrUsedBy(connID)
+}
+
+// GetUnusedAddr gets an unused addr from the edge, excluding the given addr. Prefer to use addresses
+// evenly across both regions.
+func (rs *Regions) GetUnusedAddr(excluding *net.TCPAddr, connID int) *net.TCPAddr {
+	var addr *net.TCPAddr
+	if rs.region1.AvailableAddrs() > rs.region2.AvailableAddrs() {
+		addr = rs.region1.GetUnusedIP(excluding)
+		rs.region1.Use(addr, connID)
+	} else {
+		addr = rs.region2.GetUnusedIP(excluding)
+		rs.region2.Use(addr, connID)
+	}
+
+	if addr == nil {
+		return nil
+	}
+
+	// Mark the address as used and return it
+	return addr
+}
+
+// AvailableAddrs returns how many edge addresses aren't used.
+func (rs *Regions) AvailableAddrs() int {
+	return rs.region1.AvailableAddrs() + rs.region2.AvailableAddrs()
+}
+
+// GiveBack the address so that other connections can use it.
+// Returns true if the address is in this edge.
+func (rs *Regions) GiveBack(addr *net.TCPAddr) bool {
+	if found := rs.region1.GiveBack(addr); found {
+		return found
+	}
+	return rs.region2.GiveBack(addr)
+}

edgediscovery/allregions/regions_test.go

@@ -0,0 +1,140 @@
+package allregions
+
+import (
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	addr0 = net.TCPAddr{
+		IP:   net.ParseIP("123.4.5.0"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr1 = net.TCPAddr{
+		IP:   net.ParseIP("123.4.5.1"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr2 = net.TCPAddr{
+		IP:   net.ParseIP("123.4.5.2"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr3 = net.TCPAddr{
+		IP:   net.ParseIP("123.4.5.3"),
+		Port: 8000,
+		Zone: "",
+	}
+)
+
+func makeRegions() Regions {
+	r1 := NewRegion([]*net.TCPAddr{&addr0, &addr1})
+	r2 := NewRegion([]*net.TCPAddr{&addr2, &addr3})
+	return Regions{region1: r1, region2: r2}
+}
+
+func TestRegions_AddrUsedBy(t *testing.T) {
+	rs := makeRegions()
+	addr1 := rs.GetUnusedAddr(nil, 1)
+	assert.Equal(t, addr1, rs.AddrUsedBy(1))
+	addr2 := rs.GetUnusedAddr(nil, 2)
+	assert.Equal(t, addr2, rs.AddrUsedBy(2))
+	addr3 := rs.GetUnusedAddr(nil, 3)
+	assert.Equal(t, addr3, rs.AddrUsedBy(3))
+}
+
+func TestRegions_Giveback_Region1(t *testing.T) {
+	rs := makeRegions()
+	rs.region1.Use(&addr0, 0)
+	rs.region1.Use(&addr1, 1)
+	rs.region2.Use(&addr2, 2)
+	rs.region2.Use(&addr3, 3)
+
+	assert.Equal(t, 0, rs.AvailableAddrs())
+
+	rs.GiveBack(&addr0)
+	assert.Equal(t, &addr0, rs.GetUnusedAddr(nil, 3))
+}
+func TestRegions_Giveback_Region2(t *testing.T) {
+	rs := makeRegions()
+	rs.region1.Use(&addr0, 0)
+	rs.region1.Use(&addr1, 1)
+	rs.region2.Use(&addr2, 2)
+	rs.region2.Use(&addr3, 3)
+
+	assert.Equal(t, 0, rs.AvailableAddrs())
+
+	rs.GiveBack(&addr2)
+	assert.Equal(t, &addr2, rs.GetUnusedAddr(nil, 2))
+}
+
+func TestRegions_GetUnusedAddr_OneAddrLeft(t *testing.T) {
+	rs := makeRegions()
+
+	rs.region1.Use(&addr0, 0)
+	rs.region1.Use(&addr1, 1)
+	rs.region2.Use(&addr2, 2)
+
+	assert.Equal(t, 1, rs.AvailableAddrs())
+	assert.Equal(t, &addr3, rs.GetUnusedAddr(nil, 3))
+}
+
+func TestRegions_GetUnusedAddr_Excluding_Region1(t *testing.T) {
+	rs := makeRegions()
+
+	rs.region1.Use(&addr0, 0)
+	rs.region1.Use(&addr1, 1)
+
+	assert.Equal(t, 2, rs.AvailableAddrs())
+	assert.Equal(t, &addr3, rs.GetUnusedAddr(&addr2, 3))
+}
+
+func TestRegions_GetUnusedAddr_Excluding_Region2(t *testing.T) {
+	rs := makeRegions()
+
+	rs.region2.Use(&addr2, 0)
+	rs.region2.Use(&addr3, 1)
+
+	assert.Equal(t, 2, rs.AvailableAddrs())
+	assert.Equal(t, &addr1, rs.GetUnusedAddr(&addr0, 1))
+}
+
+func TestNewNoResolveBalancesRegions(t *testing.T) {
+	type args struct {
+		addrs []*net.TCPAddr
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{
+			name: "one address",
+			args: args{addrs: []*net.TCPAddr{&addr0}},
+		},
+		{
+			name: "two addresses",
+			args: args{addrs: []*net.TCPAddr{&addr0, &addr1}},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			regions := NewNoResolve(tt.args.addrs)
+			RegionsIsBalanced(t, regions)
+		})
+	}
+}
+
+func RegionsIsBalanced(t *testing.T, rs *Regions) {
+	delta := rs.region1.AvailableAddrs() - rs.region2.AvailableAddrs()
+	assert.True(t, abs(delta) <= 1)
+}
+
+func abs(x int) int {
+	if x >= 0 {
+		return x
+	}
+	return -x
+}

edgediscovery/allregions/usedby.go

@@ -0,0 +1,14 @@
+package allregions
+
+type UsedBy struct {
+	ConnID int
+	Used   bool
+}
+
+func InUse(connID int) UsedBy {
+	return UsedBy{ConnID: connID, Used: true}
+}
+
+func Unused() UsedBy {
+	return UsedBy{}
+}

edgediscovery/edgediscovery.go

@@ -0,0 +1,143 @@
+package edgediscovery
+
+import (
+	"fmt"
+	"net"
+	"sync"
+
+	"github.com/cloudflare/cloudflared/edgediscovery/allregions"
+
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	subsystem = "edgediscovery"
+)
+
+var errNoAddressesLeft = fmt.Errorf("There are no free edge addresses left")
+
+// Edge finds addresses on the Cloudflare edge and hands them out to connections.
+type Edge struct {
+	regions *allregions.Regions
+	sync.Mutex
+	logger *logrus.Entry
+}
+
+// ------------------------------------
+// Constructors
+// ------------------------------------
+
+// ResolveEdge runs the initial discovery of the Cloudflare edge, finding Addrs that can be allocated
+// to connections.
+func ResolveEdge(l *logrus.Logger) (*Edge, error) {
+	logger := l.WithField("subsystem", subsystem)
+	regions, err := allregions.ResolveEdge(logger)
+	if err != nil {
+		return new(Edge), err
+	}
+	return &Edge{
+		logger:  logger,
+		regions: regions,
+	}, nil
+}
+
+// StaticEdge creates a list of edge addresses from the list of hostnames. Mainly used for testing connectivity.
+func StaticEdge(l *logrus.Logger, hostnames []string) (*Edge, error) {
+	logger := l.WithField("subsystem", subsystem)
+	regions, err := allregions.StaticEdge(hostnames)
+	if err != nil {
+		return new(Edge), err
+	}
+	return &Edge{
+		logger:  logger,
+		regions: regions,
+	}, nil
+}
+
+// MockEdge creates a Cloudflare Edge from arbitrary TCP addresses. Used for testing.
+func MockEdge(l *logrus.Logger, addrs []*net.TCPAddr) *Edge {
+	logger := l.WithField("subsystem", subsystem)
+	regions := allregions.NewNoResolve(addrs)
+	return &Edge{
+		logger:  logger,
+		regions: regions,
+	}
+}
+
+// ------------------------------------
+// Methods
+// ------------------------------------
+
+// GetAddrForRPC gives this connection an edge Addr.
+func (ed *Edge) GetAddrForRPC() (*net.TCPAddr, error) {
+	ed.Lock()
+	defer ed.Unlock()
+	addr := ed.regions.GetAnyAddress()
+	if addr == nil {
+		return nil, errNoAddressesLeft
+	}
+	return addr, nil
+}
+
+// GetAddr gives this proxy connection an edge Addr. Prefer Addrs this connection has already used.
+func (ed *Edge) GetAddr(connID int) (*net.TCPAddr, error) {
+	ed.Lock()
+	defer ed.Unlock()
+	logger := ed.logger.WithFields(logrus.Fields{
+		"connID":   connID,
+		"function": "GetAddr",
+	})
+
+	// If this connection has already used an edge addr, return it.
+	if addr := ed.regions.AddrUsedBy(connID); addr != nil {
+		logger.Debug("Returning same address back to proxy connection")
+		return addr, nil
+	}
+
+	// Otherwise, give it an unused one
+	addr := ed.regions.GetUnusedAddr(nil, connID)
+	if addr == nil {
+		logger.Debug("No addresses left to give proxy connection")
+		return nil, errNoAddressesLeft
+	}
+	logger.Debug("Giving connection its new address")
+	return addr, nil
+}
+
+// GetDifferentAddr gives back the proxy connection's edge Addr and uses a new one.
+func (ed *Edge) GetDifferentAddr(connID int) (*net.TCPAddr, error) {
+	ed.Lock()
+	defer ed.Unlock()
+	logger := ed.logger.WithFields(logrus.Fields{
+		"connID":   connID,
+		"function": "GetDifferentAddr",
+	})
+
+	oldAddr := ed.regions.AddrUsedBy(connID)
+	if oldAddr != nil {
+		ed.regions.GiveBack(oldAddr)
+	}
+	addr := ed.regions.GetUnusedAddr(oldAddr, connID)
+	if addr == nil {
+		logger.Debug("No addresses left to give proxy connection")
+		return nil, errNoAddressesLeft
+	}
+	logger.Debug("Giving connection its new address")
+	return addr, nil
+}
+
+// AvailableAddrs returns how many unused addresses there are left.
+func (ed *Edge) AvailableAddrs() int {
+	ed.Lock()
+	defer ed.Unlock()
+	return ed.regions.AvailableAddrs()
+}
+
+// GiveBack the address so that other connections can use it.
+// Returns true if the address is in this edge.
+func (ed *Edge) GiveBack(addr *net.TCPAddr) bool {
+	ed.Lock()
+	defer ed.Unlock()
+	ed.logger.WithField("function", "GiveBack").Debug("Address now unused")
+	return ed.regions.GiveBack(addr)
+}

edgediscovery/edgediscovery_test.go

@@ -0,0 +1,148 @@
+package edgediscovery
+
+import (
+	"net"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	addr0 = net.TCPAddr{
+		IP:   net.ParseIP("123.0.0.0"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr1 = net.TCPAddr{
+		IP:   net.ParseIP("123.0.0.1"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr2 = net.TCPAddr{
+		IP:   net.ParseIP("123.0.0.2"),
+		Port: 8000,
+		Zone: "",
+	}
+	addr3 = net.TCPAddr{
+		IP:   net.ParseIP("123.0.0.3"),
+		Port: 8000,
+		Zone: "",
+	}
+)
+
+func TestGiveBack(t *testing.T) {
+	l := logrus.New()
+	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+
+	// Give this connection an address
+	assert.Equal(t, 4, edge.AvailableAddrs())
+	const connID = 0
+	addr, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+	assert.Equal(t, 3, edge.AvailableAddrs())
+
+	// Get it back
+	edge.GiveBack(addr)
+	assert.Equal(t, 4, edge.AvailableAddrs())
+}
+
+func TestRPCAndProxyShareSingleEdgeIP(t *testing.T) {
+	l := logrus.New()
+
+	// Make an edge with a single IP
+	edge := MockEdge(l, []*net.TCPAddr{&addr0})
+	tunnelConnID := 0
+
+	// Use the IP for a tunnel
+	addrTunnel, err := edge.GetAddr(tunnelConnID)
+	assert.NoError(t, err)
+
+	// Ensure the IP can be used for RPC too
+	addrRPC, err := edge.GetAddrForRPC()
+	assert.NoError(t, err)
+	assert.Equal(t, addrTunnel, addrRPC)
+}
+
+func TestGetAddrForRPC(t *testing.T) {
+	l := logrus.New()
+	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+
+	// Get a connection
+	assert.Equal(t, 4, edge.AvailableAddrs())
+	addr, err := edge.GetAddrForRPC()
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+
+	// Using an address for RPC shouldn't consume it
+	assert.Equal(t, 4, edge.AvailableAddrs())
+
+	// Get it back
+	edge.GiveBack(addr)
+	assert.Equal(t, 4, edge.AvailableAddrs())
+}
+
+func TestOnlyOneAddrLeft(t *testing.T) {
+	l := logrus.New()
+
+	// Make an edge with only one address
+	edge := MockEdge(l, []*net.TCPAddr{&addr0})
+
+	// Use the only address
+	const connID = 0
+	addr, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+
+	// If that edge address is "bad", there's no alternative address.
+	_, err = edge.GetDifferentAddr(connID)
+	assert.Error(t, err)
+}
+
+func TestNoAddrsLeft(t *testing.T) {
+	l := logrus.New()
+
+	// Make an edge with no addresses
+	edge := MockEdge(l, []*net.TCPAddr{})
+
+	_, err := edge.GetAddr(2)
+	assert.Error(t, err)
+	_, err = edge.GetAddrForRPC()
+	assert.Error(t, err)
+}
+
+func TestGetAddr(t *testing.T) {
+	l := logrus.New()
+	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+
+	// Give this connection an address
+	const connID = 0
+	addr, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+
+	// If the same connection requests another address, it should get the same one.
+	addr2, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.Equal(t, addr, addr2)
+}
+
+func TestGetDifferentAddr(t *testing.T) {
+	l := logrus.New()
+	edge := MockEdge(l, []*net.TCPAddr{&addr0, &addr1, &addr2, &addr3})
+
+	// Give this connection an address
+	assert.Equal(t, 4, edge.AvailableAddrs())
+	const connID = 0
+	addr, err := edge.GetAddr(connID)
+	assert.NoError(t, err)
+	assert.NotNil(t, addr)
+	assert.Equal(t, 3, edge.AvailableAddrs())
+
+	// If the same connection requests another address, it should get the same one.
+	addr2, err := edge.GetDifferentAddr(connID)
+	assert.NoError(t, err)
+	assert.NotEqual(t, addr, addr2)
+	assert.Equal(t, 3, edge.AvailableAddrs())
+}

go.mod

@@ -27,17 +27,19 @@ require (
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/mux v1.7.3
-	github.com/gorilla/websocket v1.2.0
+	github.com/gorilla/websocket v1.4.0
 	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/kr/pretty v0.1.0 // indirect
 	github.com/kshvakov/clickhouse v1.3.11
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lib/pq v1.2.0
 	github.com/mattn/go-colorable v0.1.4
 	github.com/mattn/go-isatty v0.0.10 // indirect
 	github.com/mattn/go-sqlite3 v1.11.0
 	github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b // indirect
-	github.com/miekg/dns v1.1.8
+	github.com/miekg/dns v1.1.27
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/opentracing/opentracing-go v1.1.0 // indirect
 	github.com/philhofer/fwd v1.0.0 // indirect
@@ -52,13 +54,16 @@ require (
 	github.com/stretchr/testify v1.3.0
 	github.com/tinylib/msgp v1.1.0 // indirect
 	github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0
-	golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc
-	golang.org/x/net v0.0.0-20191007182048-72f939374954
+	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550
+	golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582
+	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20191008105621-543471e840be
+	golang.org/x/sys v0.0.0-20191020212454-3e7259c5e7c2
 	golang.org/x/text v0.3.2 // indirect
+	google.golang.org/appengine v1.5.0 // indirect
 	google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd // indirect
 	google.golang.org/grpc v1.24.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/coreos/go-oidc.v2 v2.1.0
 	gopkg.in/square/go-jose.v2 v2.4.0 // indirect
 	gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8

go.sum

@@ -1,4 +1,5 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
@@ -73,8 +74,8 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gorilla/websocket v1.2.0 h1:VJtLvh6VQym50czpZzx07z/kw9EgAxI3x1ZB8taTMQQ=
-github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
@@ -86,10 +87,17 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kshvakov/clickhouse v1.3.11 h1:dtzTJY0fCA+MWkLyuKZaNPkmSwdX4gh8+Klic9NB1Lw=
 github.com/kshvakov/clickhouse v1.3.11/go.mod h1:/SVBAcqF3u7rxQ9sTWCZwf8jzzvxiZGeQvtmSF2BBEc=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
+github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@@ -105,8 +113,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b h1:/BbY4n99iMazlr2igipph+hj0MwlZIWpcsP8Iy+na+s=
 github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b/go.mod h1:Wb1PlT4DAYSqOEd03MsqkdkXnTxA8v9pKjdpxbqM1kY=
-github.com/miekg/dns v1.1.8 h1:1QYRAKU3lN5cRfLCkPU08hwvLJFhvjP6MqNMmQz6ZVI=
-github.com/miekg/dns v1.1.8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM=
+github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -146,23 +154,29 @@ github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0 h1:6DtWz8hNS4qbq0OCRPhdBM
 github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0/go.mod h1:A47W3pdWONaZmXuLZgfKLAVgUY0qvfTRM5vVDKS40S4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc h1:c0o/qxkaO2LF5t6fQrT4b5hzyggAkLLlCUjqfRxd8Q4=
-golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191007182048-72f939374954 h1:JGZucVF/L/TotR719NbujzadOZ2AgnYlqphQGHDCKaU=
-golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582 h1:p9xBe/w/OzkeYVKm234g55gMdD1nSIooTir5kV11kfA=
+golang.org/x/net v0.0.0-20191014212845-da9a3fd4c582/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -173,8 +187,11 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191020212454-3e7259c5e7c2 h1:nq114VpM8lsSlP+lyUbANecYHYiFcSNFtqcBlxRV+gA=
+golang.org/x/sys v0.0.0-20191020212454-3e7259c5e7c2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -183,9 +200,13 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd h1:84VQPzup3IpKLxuIAZjHMhVjJ8fZ4/i3yUnj3k6fUdw=
 google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
@@ -195,6 +216,8 @@ google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRn
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/coreos/go-oidc.v2 v2.1.0 h1:E8PjVFdj/SLDKB0hvb70KTbMbYVHjqztiQdSkIg8E+I=
 gopkg.in/coreos/go-oidc.v2 v2.1.0/go.mod h1:fYaTe2FS96wZZwR17YTDHwG+Mw6fmyqJNxN2eNCGPCI=
 gopkg.in/square/go-jose.v2 v2.4.0 h1:0kXPskUMGAXXWJlP05ktEMOV0vmzFQUWw6d+aZJQU8A=

h2mux/activestreammap.go

@@ -13,26 +13,28 @@ type activeStreamMap struct {
 	sync.RWMutex
 	// streams tracks open streams.
 	streams map[uint32]*MuxedStream
-	// streamsEmpty is a chan that should be closed when no more streams are open.
-	streamsEmpty chan struct{}
 	// nextStreamID is the next ID to use on our side of the connection.
 	// This is odd for clients, even for servers.
 	nextStreamID uint32
 	// maxPeerStreamID is the ID of the most recent stream opened by the peer.
 	maxPeerStreamID uint32
+	// activeStreams is a gauge shared by all muxers of this process to expose the total number of active streams
+	activeStreams prometheus.Gauge
+
 	// ignoreNewStreams is true when the connection is being shut down. New streams
 	// cannot be registered.
 	ignoreNewStreams bool
-	// activeStreams is a gauge shared by all muxers of this process to expose the total number of active streams
-	activeStreams prometheus.Gauge
+	// streamsEmpty is a chan that will be closed when no more streams are open.
+	streamsEmptyChan chan struct{}
+	closeOnce        sync.Once
 }
 
 func newActiveStreamMap(useClientStreamNumbers bool, activeStreams prometheus.Gauge) *activeStreamMap {
 	m := &activeStreamMap{
-		streams:       make(map[uint32]*MuxedStream),
-		streamsEmpty:  make(chan struct{}),
-		nextStreamID:  1,
-		activeStreams: activeStreams,
+		streams:          make(map[uint32]*MuxedStream),
+		streamsEmptyChan: make(chan struct{}),
+		nextStreamID:     1,
+		activeStreams:    activeStreams,
 	}
 	// Client initiated stream uses odd stream ID, server initiated stream uses even stream ID
 	if !useClientStreamNumbers {
@@ -41,6 +43,13 @@ func newActiveStreamMap(useClientStreamNumbers bool, activeStreams prometheus.Ga
 	return m
 }
 
+// This function should be called while `m` is locked.
+func (m *activeStreamMap) notifyStreamsEmpty() {
+	m.closeOnce.Do(func() {
+		close(m.streamsEmptyChan)
+	})
+}
+
 // Len returns the number of active streams.
 func (m *activeStreamMap) Len() int {
 	m.RLock()
@@ -79,30 +88,29 @@ func (m *activeStreamMap) Delete(streamID uint32) {
 		delete(m.streams, streamID)
 		m.activeStreams.Dec()
 	}
-	if len(m.streams) == 0 && m.streamsEmpty != nil {
-		close(m.streamsEmpty)
-		m.streamsEmpty = nil
+
+	// shutting down, and now the map is empty
+	if m.ignoreNewStreams && len(m.streams) == 0 {
+		m.notifyStreamsEmpty()
 	}
 }
 
-// Shutdown blocks new streams from being created. It returns a channel that receives an event
-// once the last stream has closed, or nil if a shutdown is in progress.
-func (m *activeStreamMap) Shutdown() <-chan struct{} {
+// Shutdown blocks new streams from being created.
+// It returns `done`, a channel that is closed once the last stream has closed
+// and `progress`, whether a shutdown was already in progress
+func (m *activeStreamMap) Shutdown() (done <-chan struct{}, alreadyInProgress bool) {
 	m.Lock()
 	defer m.Unlock()
 	if m.ignoreNewStreams {
 		// already shutting down
-		return nil
+		return m.streamsEmptyChan, true
 	}
 	m.ignoreNewStreams = true
-	done := make(chan struct{})
 	if len(m.streams) == 0 {
-		// nothing to shut down
-		close(done)
-		return done
+		// there are no streams to wait for
+		m.notifyStreamsEmpty()
 	}
-	m.streamsEmpty = done
-	return done
+	return m.streamsEmptyChan, false
 }
 
 // AcquireLocalID acquires a new stream ID for a stream you're opening.
@@ -170,4 +178,5 @@ func (m *activeStreamMap) Abort() {
 		stream.Close()
 	}
 	m.ignoreNewStreams = true
+	m.notifyStreamsEmpty()
 }

h2mux/activestreammap_test.go

@@ -0,0 +1,195 @@
+package h2mux
+
+import (
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestShutdown(t *testing.T) {
+	const numStreams = 1000
+	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+
+	// Add all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				stream := &MuxedStream{streamID: uint32(streamID)}
+				ok := m.Set(stream)
+				assert.True(t, ok)
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
+
+	shutdownChan, alreadyInProgress := m.Shutdown()
+	select {
+	case <-shutdownChan:
+		assert.Fail(t, "before Shutdown(), shutdownChan shouldn't be closed")
+	default:
+	}
+	assert.False(t, alreadyInProgress)
+
+	shutdownChan2, alreadyInProgress2 := m.Shutdown()
+	assert.Equal(t, shutdownChan, shutdownChan2, "repeated calls to Shutdown() should return the same channel")
+	assert.True(t, alreadyInProgress2, "repeated calls to Shutdown() should return true for 'in progress'")
+
+	// Delete all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				m.Delete(uint32(streamID))
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, 0, m.Len(), "All the streams should have been deleted")
+
+	select {
+	case <-shutdownChan:
+	default:
+		assert.Fail(t, "After all the streams are deleted, shutdownChan should have been closed")
+	}
+}
+
+func TestEmptyBeforeShutdown(t *testing.T) {
+	const numStreams = 1000
+	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+
+	// Add all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				stream := &MuxedStream{streamID: uint32(streamID)}
+				ok := m.Set(stream)
+				assert.True(t, ok)
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
+
+	// Delete all the streams, bringing m to size 0
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				m.Delete(uint32(streamID))
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, 0, m.Len(), "All the streams should have been deleted")
+
+	// Add one stream back
+	const soloStreamID = uint32(0)
+	ok := m.Set(&MuxedStream{streamID: soloStreamID})
+	assert.True(t, ok)
+
+	shutdownChan, alreadyInProgress := m.Shutdown()
+	select {
+	case <-shutdownChan:
+		assert.Fail(t, "before Shutdown(), shutdownChan shouldn't be closed")
+	default:
+	}
+	assert.False(t, alreadyInProgress)
+
+	shutdownChan2, alreadyInProgress2 := m.Shutdown()
+	assert.Equal(t, shutdownChan, shutdownChan2, "repeated calls to Shutdown() should return the same channel")
+	assert.True(t, alreadyInProgress2, "repeated calls to Shutdown() should return true for 'in progress'")
+
+	// Remove the remaining stream
+	m.Delete(soloStreamID)
+
+	select {
+	case <-shutdownChan:
+	default:
+		assert.Fail(t, "After all the streams are deleted, shutdownChan should have been closed")
+	}
+}
+
+type noopBuffer struct {
+	isClosed bool
+}
+
+func (t *noopBuffer) Read(p []byte) (n int, err error)  { return len(p), nil }
+func (t *noopBuffer) Write(p []byte) (n int, err error) { return len(p), nil }
+func (t *noopBuffer) Reset()                            {}
+func (t *noopBuffer) Len() int                          { return 0 }
+func (t *noopBuffer) Close() error                      { t.isClosed = true; return nil }
+func (t *noopBuffer) Closed() bool                      { return t.isClosed }
+
+type noopReadyList struct{}
+
+func (_ *noopReadyList) Signal(streamID uint32) {}
+
+func TestAbort(t *testing.T) {
+	const numStreams = 1000
+	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+
+	var openedStreams sync.Map
+
+	// Add all the streams
+	{
+		var wg sync.WaitGroup
+		wg.Add(numStreams)
+		for i := 0; i < numStreams; i++ {
+			go func(streamID int) {
+				defer wg.Done()
+				stream := &MuxedStream{
+					streamID:    uint32(streamID),
+					readBuffer:  &noopBuffer{},
+					writeBuffer: &noopBuffer{},
+					readyList:   &noopReadyList{},
+				}
+				ok := m.Set(stream)
+				assert.True(t, ok)
+
+				openedStreams.Store(stream.streamID, stream)
+			}(i)
+		}
+		wg.Wait()
+	}
+	assert.Equal(t, numStreams, m.Len(), "All the streams should have been added")
+
+	shutdownChan, alreadyInProgress := m.Shutdown()
+	select {
+	case <-shutdownChan:
+		assert.Fail(t, "before Abort(), shutdownChan shouldn't be closed")
+	default:
+	}
+	assert.False(t, alreadyInProgress)
+
+	m.Abort()
+	assert.Equal(t, numStreams, m.Len(), "Abort() shouldn't delete any streams")
+	openedStreams.Range(func(key interface{}, value interface{}) bool {
+		stream := value.(*MuxedStream)
+		readBuffer := stream.readBuffer.(*noopBuffer)
+		writeBuffer := stream.writeBuffer.(*noopBuffer)
+		return assert.True(t, readBuffer.isClosed && writeBuffer.isClosed, "Abort() should have closed all the streams")
+	})
+
+	select {
+	case <-shutdownChan:
+	default:
+		assert.Fail(t, "after Abort(), shutdownChan should have been closed")
+	}
+
+	// multiple aborts shouldn't cause any issues
+	m.Abort()
+	m.Abort()
+	m.Abort()
+}

h2mux/h2_dictionaries.go

@@ -542,7 +542,10 @@ func (w *h2DictWriter) Write(p []byte) (n int, err error) {
 }
 
 func (w *h2DictWriter) Close() error {
-	return w.comp.Close()
+	if w.comp != nil {
+		return w.comp.Close()
+	}
+	return nil
 }
 
 // From http2/hpack

h2mux/h2mux.go

@@ -90,10 +90,6 @@ type Muxer struct {
 	compressionQuality CompressionPreset
 }
 
-type Header struct {
-	Name, Value string
-}
-
 func RPCHeaders() []Header {
 	return []Header{
 		{Name: ":method", Value: "RPC"},
@@ -325,24 +321,51 @@ func joinErrorsWithTimeout(errChan <-chan error, receiveCount int, timeout time.
 func (m *Muxer) Serve(ctx context.Context) error {
 	errGroup, _ := errgroup.WithContext(ctx)
 	errGroup.Go(func() error {
-		err := m.muxReader.run(m.config.Logger)
-		m.explicitShutdown.Fuse(false)
-		m.r.Close()
-		m.abort()
-		return err
+		ch := make(chan error)
+		go func() {
+			err := m.muxReader.run(m.config.Logger)
+			m.explicitShutdown.Fuse(false)
+			m.r.Close()
+			m.abort()
+			ch <- err
+		}()
+		select {
+		case err := <-ch:
+			return err
+		case <-ctx.Done():
+			return ctx.Err()
+		}
 	})
 
 	errGroup.Go(func() error {
-		err := m.muxWriter.run(m.config.Logger)
-		m.explicitShutdown.Fuse(false)
-		m.w.Close()
-		m.abort()
-		return err
+		ch := make(chan error)
+		go func() {
+			err := m.muxWriter.run(m.config.Logger)
+			m.explicitShutdown.Fuse(false)
+			m.w.Close()
+			m.abort()
+			ch <- err
+		}()
+		select {
+		case err := <-ch:
+			return err
+		case <-ctx.Done():
+			return ctx.Err()
+		}
 	})
 
 	errGroup.Go(func() error {
-		err := m.muxMetricsUpdater.run(m.config.Logger)
-		return err
+		ch := make(chan error)
+		go func() {
+			err := m.muxMetricsUpdater.run(m.config.Logger)
+			ch <- err
+		}()
+		select {
+		case err := <-ch:
+			return err
+		case <-ctx.Done():
+			return ctx.Err()
+		}
 	})
 
 	err := errGroup.Wait()
@@ -353,9 +376,11 @@ func (m *Muxer) Serve(ctx context.Context) error {
 }
 
 // Shutdown is called to initiate the "happy path" of muxer termination.
-func (m *Muxer) Shutdown() {
+// It blocks new streams from being created.
+// It returns a channel that is closed when the last stream has been closed.
+func (m *Muxer) Shutdown() <-chan struct{} {
 	m.explicitShutdown.Fuse(true)
-	m.muxReader.Shutdown()
+	return m.muxReader.Shutdown()
 }
 
 // IsUnexpectedTunnelError identifies errors that are expected when shutting down the h2mux tunnel.

h2mux/h2mux_test.go

@@ -55,6 +55,8 @@ func NewDefaultMuxerPair(t assert.TestingT, testName string, f MuxedStreamFunc)
 			DefaultWindowSize:       (1 << 8) - 1,
 			MaxWindowSize:           (1 << 15) - 1,
 			StreamWriteBufferMaxLen: 1024,
+			HeartbeatInterval:       defaultTimeout,
+			MaxHeartbeats:           defaultRetries,
 		},
 		OriginConn: origin,
 		EdgeMuxConfig: MuxerConfig{
@@ -65,6 +67,8 @@ func NewDefaultMuxerPair(t assert.TestingT, testName string, f MuxedStreamFunc)
 			DefaultWindowSize:       (1 << 8) - 1,
 			MaxWindowSize:           (1 << 15) - 1,
 			StreamWriteBufferMaxLen: 1024,
+			HeartbeatInterval:       defaultTimeout,
+			MaxHeartbeats:           defaultRetries,
 		},
 		EdgeConn: edge,
 		doneC:    make(chan struct{}),
@@ -83,6 +87,8 @@ func NewCompressedMuxerPair(t assert.TestingT, testName string, quality Compress
 			Name:               "origin",
 			CompressionQuality: quality,
 			Logger:             log.NewEntry(log.New()),
+			HeartbeatInterval:  defaultTimeout,
+			MaxHeartbeats:      defaultRetries,
 		},
 		OriginConn: origin,
 		EdgeMuxConfig: MuxerConfig{
@@ -91,6 +97,8 @@ func NewCompressedMuxerPair(t assert.TestingT, testName string, quality Compress
 			Name:               "edge",
 			CompressionQuality: quality,
 			Logger:             log.NewEntry(log.New()),
+			HeartbeatInterval:  defaultTimeout,
+			MaxHeartbeats:      defaultRetries,
 		},
 		EdgeConn: edge,
 		doneC:    make(chan struct{}),

h2mux/header.go

@@ -0,0 +1,249 @@
+package h2mux
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+type Header struct {
+	Name, Value string
+}
+
+var headerEncoding = base64.RawStdEncoding
+
+const (
+	RequestUserHeadersField  = "cf-cloudflared-request-headers"
+	ResponseUserHeadersField = "cf-cloudflared-response-headers"
+
+	ResponseMetaHeaderField   = "cf-cloudflared-response-meta"
+	ResponseSourceCloudflared = "cloudflared"
+	ResponseSourceOrigin      = "origin"
+)
+
+// H2RequestHeadersToH1Request converts the HTTP/2 headers coming from origintunneld
+// to an HTTP/1 Request object destined for the local origin web service.
+// This operation includes conversion of the pseudo-headers into their closest
+// HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3
+func H2RequestHeadersToH1Request(h2 []Header, h1 *http.Request) error {
+	for _, header := range h2 {
+		if !IsControlHeader(header.Name) {
+			continue
+		}
+
+		switch strings.ToLower(header.Name) {
+		case ":method":
+			h1.Method = header.Value
+		case ":scheme":
+			// noop - use the preexisting scheme from h1.URL
+		case ":authority":
+			// Otherwise the host header will be based on the origin URL
+			h1.Host = header.Value
+		case ":path":
+			// We don't want to be an "opinionated" proxy, so ideally we would use :path as-is.
+			// However, this HTTP/1 Request object belongs to the Go standard library,
+			// whose URL package makes some opinionated decisions about the encoding of
+			// URL characters: see the docs of https://godoc.org/net/url#URL,
+			// in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath,
+			// which is always used when computing url.URL.String(), whether we'd like it or not.
+			//
+			// Well, not *always*. We could circumvent this by using url.URL.Opaque. But
+			// that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque
+			// is treated differently when HTTP_PROXY is set!
+			// See https://github.com/golang/go/issues/5684#issuecomment-66080888
+			//
+			// This means we are subject to the behavior of net/url's function `shouldEscape`
+			// (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101
+
+			if header.Value == "*" {
+				h1.URL.Path = "*"
+				continue
+			}
+			// Due to the behavior of validation.ValidateUrl, h1.URL may
+			// already have a partial value, with or without a trailing slash.
+			base := h1.URL.String()
+			base = strings.TrimRight(base, "/")
+			// But we know :path begins with '/', because we handled '*' above - see RFC7540
+			requestURL, err := url.Parse(base + header.Value)
+			if err != nil {
+				return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value))
+			}
+			h1.URL = requestURL
+		case "content-length":
+			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
+			if err != nil {
+				return fmt.Errorf("unparseable content length")
+			}
+			h1.ContentLength = contentLength
+		case RequestUserHeadersField:
+			// Do not forward the serialized headers to the origin -- deserialize them, and ditch the serialized version
+			// Find and parse user headers serialized into a single one
+			userHeaders, err := ParseUserHeaders(RequestUserHeadersField, h2)
+			if err != nil {
+				return errors.Wrap(err, "Unable to parse user headers")
+			}
+			for _, userHeader := range userHeaders {
+				h1.Header.Add(http.CanonicalHeaderKey(userHeader.Name), userHeader.Value)
+			}
+		default:
+			// All other control headers shall just be proxied transparently
+			h1.Header.Add(http.CanonicalHeaderKey(header.Name), header.Value)
+		}
+	}
+
+	return nil
+}
+
+func ParseUserHeaders(headerNameToParseFrom string, headers []Header) ([]Header, error) {
+	for _, header := range headers {
+		if header.Name == headerNameToParseFrom {
+			return DeserializeHeaders(header.Value)
+		}
+	}
+
+	return nil, fmt.Errorf("%v header not found", RequestUserHeadersField)
+}
+
+func IsControlHeader(headerName string) bool {
+	headerName = strings.ToLower(headerName)
+
+	return headerName == "content-length" ||
+		headerName == "connection" || headerName == "upgrade" || // Websocket headers
+		strings.HasPrefix(headerName, ":") ||
+		strings.HasPrefix(headerName, "cf-")
+}
+
+// IsWebsocketClientHeader returns true if the header name is required by the client to upgrade properly
+func IsWebsocketClientHeader(headerName string) bool {
+	headerName = strings.ToLower(headerName)
+
+	return headerName == "sec-websocket-accept" ||
+		headerName == "connection" ||
+		headerName == "upgrade"
+}
+
+func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
+	h2 = []Header{
+		{Name: ":status", Value: strconv.Itoa(h1.StatusCode)},
+	}
+	userHeaders := http.Header{}
+	for header, values := range h1.Header {
+		for _, value := range values {
+			if strings.ToLower(header) == "content-length" {
+				// This header has meaning in HTTP/2 and will be used by the edge,
+				// so it should be sent as an HTTP/2 response header.
+
+				// Since these are http2 headers, they're required to be lowercase
+				h2 = append(h2, Header{Name: strings.ToLower(header), Value: value})
+			} else if !IsControlHeader(header) || IsWebsocketClientHeader(header) {
+				// User headers, on the other hand, must all be serialized so that
+				// HTTP/2 header validation won't be applied to HTTP/1 header values
+				if _, ok := userHeaders[header]; ok {
+					userHeaders[header] = append(userHeaders[header], value)
+				} else {
+					userHeaders[header] = []string{value}
+				}
+			}
+		}
+	}
+
+	// Perform user header serialization and set them in the single header
+	h2 = append(h2, CreateSerializedHeaders(ResponseUserHeadersField, userHeaders)...)
+
+	return h2
+}
+
+// Serialize HTTP1.x headers by base64-encoding each header name and value,
+// and then joining them in the format of [key:value;]
+func SerializeHeaders(h1Headers http.Header) string {
+	var serializedHeaders []string
+	for headerName, headerValues := range h1Headers {
+		for _, headerValue := range headerValues {
+			encodedName := make([]byte, headerEncoding.EncodedLen(len(headerName)))
+			headerEncoding.Encode(encodedName, []byte(headerName))
+
+			encodedValue := make([]byte, headerEncoding.EncodedLen(len(headerValue)))
+			headerEncoding.Encode(encodedValue, []byte(headerValue))
+
+			serializedHeaders = append(
+				serializedHeaders,
+				strings.Join(
+					[]string{string(encodedName), string(encodedValue)},
+					":",
+				),
+			)
+		}
+	}
+
+	return strings.Join(serializedHeaders, ";")
+}
+
+// Deserialize headers serialized by `SerializeHeader`
+func DeserializeHeaders(serializedHeaders string) ([]Header, error) {
+	const unableToDeserializeErr = "Unable to deserialize headers"
+
+	var deserialized []Header
+	for _, serializedPair := range strings.Split(serializedHeaders, ";") {
+		if len(serializedPair) == 0 {
+			continue
+		}
+
+		serializedHeaderParts := strings.Split(serializedPair, ":")
+		if len(serializedHeaderParts) != 2 {
+			return nil, errors.New(unableToDeserializeErr)
+		}
+
+		serializedName := serializedHeaderParts[0]
+		serializedValue := serializedHeaderParts[1]
+		deserializedName := make([]byte, headerEncoding.DecodedLen(len(serializedName)))
+		deserializedValue := make([]byte, headerEncoding.DecodedLen(len(serializedValue)))
+
+		if _, err := headerEncoding.Decode(deserializedName, []byte(serializedName)); err != nil {
+			return nil, errors.Wrap(err, unableToDeserializeErr)
+		}
+		if _, err := headerEncoding.Decode(deserializedValue, []byte(serializedValue)); err != nil {
+			return nil, errors.Wrap(err, unableToDeserializeErr)
+		}
+
+		deserialized = append(deserialized, Header{
+			Name:  string(deserializedName),
+			Value: string(deserializedValue),
+		})
+	}
+
+	return deserialized, nil
+}
+
+func CreateSerializedHeaders(headersField string, headers ...http.Header) []Header {
+	var serializedHeaderChunks []string
+	for _, headerChunk := range headers {
+		serializedHeaderChunks = append(serializedHeaderChunks, SerializeHeaders(headerChunk))
+	}
+
+	return []Header{{
+		headersField,
+		strings.Join(serializedHeaderChunks, ";"),
+	}}
+}
+
+type ResponseMetaHeader struct {
+	Source string `json:"src"`
+}
+
+func CreateResponseMetaHeader(headerName, source string) Header {
+	jsonResponseMetaHeader, err := json.Marshal(ResponseMetaHeader{Source: source})
+	if err != nil {
+		panic(err)
+	}
+
+	return Header{
+		Name:  headerName,
+		Value: string(jsonResponseMetaHeader),
+	}
+}

h2mux/header_test.go

@@ -0,0 +1,652 @@
+package h2mux
+
+import (
+	"fmt"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"reflect"
+	"regexp"
+	"sort"
+	"strings"
+	"testing"
+	"testing/quick"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type ByName []Header
+
+func (a ByName) Len() int      { return len(a) }
+func (a ByName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a ByName) Less(i, j int) bool {
+	if a[i].Name == a[j].Name {
+		return a[i].Value < a[j].Value
+	}
+
+	return a[i].Name < a[j].Name
+}
+
+func TestH2RequestHeadersToH1Request_RegularHeaders(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	mockHeaders := http.Header{
+		"Mock header 1": {"Mock value 1"},
+		"Mock header 2": {"Mock value 2"},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(CreateSerializedHeaders(RequestUserHeadersField, mockHeaders), request)
+
+	assert.True(t, reflect.DeepEqual(mockHeaders, request.Header))
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_NoHeaders(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	headersConversionErr := H2RequestHeadersToH1Request(
+		[]Header{{
+			RequestUserHeadersField,
+			SerializeHeaders(http.Header{}),
+		}},
+		request,
+	)
+
+	assert.True(t, reflect.DeepEqual(http.Header{}, request.Header))
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_InvalidHostPath(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	mockRequestHeaders := []Header{
+		{Name: ":path", Value: "//bad_path/"},
+		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
+
+	assert.Equal(t, http.Header{
+		"Mock header": []string{"Mock value"},
+	}, request.Header)
+
+	assert.Equal(t, "http://example.com//bad_path/", request.URL.String())
+
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_HostPathWithQuery(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
+	assert.NoError(t, err)
+
+	mockRequestHeaders := []Header{
+		{Name: ":path", Value: "/?query=mock%20value"},
+		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
+
+	assert.Equal(t, http.Header{
+		"Mock header": []string{"Mock value"},
+	}, request.Header)
+
+	assert.Equal(t, "http://example.com/?query=mock%20value", request.URL.String())
+
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_HostPathWithURLEncoding(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
+	assert.NoError(t, err)
+
+	mockRequestHeaders := []Header{
+		{Name: ":path", Value: "/mock%20path"},
+		{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+	}
+
+	headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
+
+	assert.Equal(t, http.Header{
+		"Mock header": []string{"Mock value"},
+	}, request.Header)
+
+	assert.Equal(t, "http://example.com/mock%20path", request.URL.String())
+
+	assert.NoError(t, headersConversionErr)
+}
+
+func TestH2RequestHeadersToH1Request_WeirdURLs(t *testing.T) {
+	type testCase struct {
+		path string
+		want string
+	}
+	testCases := []testCase{
+		{
+			path: "",
+			want: "",
+		},
+		{
+			path: "/",
+			want: "/",
+		},
+		{
+			path: "//",
+			want: "//",
+		},
+		{
+			path: "/test",
+			want: "/test",
+		},
+		{
+			path: "//test",
+			want: "//test",
+		},
+		{
+			// https://github.com/cloudflare/cloudflared/issues/81
+			path: "//test/",
+			want: "//test/",
+		},
+		{
+			path: "/%2Ftest",
+			want: "/%2Ftest",
+		},
+		{
+			path: "//%20test",
+			want: "//%20test",
+		},
+		{
+			// https://github.com/cloudflare/cloudflared/issues/124
+			path: "/test?get=somthing%20a",
+			want: "/test?get=somthing%20a",
+		},
+		{
+			path: "/%20",
+			want: "/%20",
+		},
+		{
+			// stdlib's EscapedPath() will always percent-encode ' '
+			path: "/ ",
+			want: "/%20",
+		},
+		{
+			path: "/ a ",
+			want: "/%20a%20",
+		},
+		{
+			path: "/a%20b",
+			want: "/a%20b",
+		},
+		{
+			path: "/foo/bar;param?query#frag",
+			want: "/foo/bar;param?query#frag",
+		},
+		{
+			// stdlib's EscapedPath() will always percent-encode non-ASCII chars
+			path: "/a␠b",
+			want: "/a%E2%90%A0b",
+		},
+		{
+			path: "/a-umlaut-ä",
+			want: "/a-umlaut-%C3%A4",
+		},
+		{
+			path: "/a-umlaut-%C3%A4",
+			want: "/a-umlaut-%C3%A4",
+		},
+		{
+			path: "/a-umlaut-%c3%a4",
+			want: "/a-umlaut-%c3%a4",
+		},
+		{
+			// here the second '#' is treated as part of the fragment
+			path: "/a#b#c",
+			want: "/a#b%23c",
+		},
+		{
+			path: "/a#b␠c",
+			want: "/a#b%E2%90%A0c",
+		},
+		{
+			path: "/a#b%20c",
+			want: "/a#b%20c",
+		},
+		{
+			path: "/a#b c",
+			want: "/a#b%20c",
+		},
+		{
+			// stdlib's EscapedPath() will always percent-encode '\'
+			path: "/\\",
+			want: "/%5C",
+		},
+		{
+			path: "/a\\",
+			want: "/a%5C",
+		},
+		{
+			path: "/a,b.c.",
+			want: "/a,b.c.",
+		},
+		{
+			path: "/.",
+			want: "/.",
+		},
+		{
+			// stdlib's EscapedPath() will always percent-encode '`'
+			path: "/a`",
+			want: "/a%60",
+		},
+		{
+			path: "/a[0]",
+			want: "/a[0]",
+		},
+		{
+			path: "/?a[0]=5 &b[]=",
+			want: "/?a[0]=5 &b[]=",
+		},
+		{
+			path: "/?a=%22b%20%22",
+			want: "/?a=%22b%20%22",
+		},
+	}
+
+	for index, testCase := range testCases {
+		requestURL := "https://example.com"
+
+		request, err := http.NewRequest(http.MethodGet, requestURL, nil)
+		assert.NoError(t, err)
+
+		mockRequestHeaders := []Header{
+			{Name: ":path", Value: testCase.path},
+			{Name: RequestUserHeadersField, Value: SerializeHeaders(http.Header{"Mock header": {"Mock value"}})},
+		}
+
+		headersConversionErr := H2RequestHeadersToH1Request(mockRequestHeaders, request)
+		assert.NoError(t, headersConversionErr)
+
+		assert.Equal(t,
+			http.Header{
+				"Mock header": []string{"Mock value"},
+			},
+			request.Header)
+
+		assert.Equal(t,
+			"https://example.com"+testCase.want,
+			request.URL.String(),
+			"Failed URL index: %v %#v", index, testCase)
+	}
+}
+
+func TestH2RequestHeadersToH1Request_QuickCheck(t *testing.T) {
+	config := &quick.Config{
+		Values: func(args []reflect.Value, rand *rand.Rand) {
+			args[0] = reflect.ValueOf(randomHTTP2Path(t, rand))
+		},
+	}
+
+	type testOrigin struct {
+		url string
+
+		expectedScheme   string
+		expectedBasePath string
+	}
+	testOrigins := []testOrigin{
+		{
+			url:              "http://origin.hostname.example.com:8080",
+			expectedScheme:   "http",
+			expectedBasePath: "http://origin.hostname.example.com:8080",
+		},
+		{
+			url:              "http://origin.hostname.example.com:8080/",
+			expectedScheme:   "http",
+			expectedBasePath: "http://origin.hostname.example.com:8080",
+		},
+		{
+			url:              "http://origin.hostname.example.com:8080/api",
+			expectedScheme:   "http",
+			expectedBasePath: "http://origin.hostname.example.com:8080/api",
+		},
+		{
+			url:              "http://origin.hostname.example.com:8080/api/",
+			expectedScheme:   "http",
+			expectedBasePath: "http://origin.hostname.example.com:8080/api",
+		},
+		{
+			url:              "https://origin.hostname.example.com:8080/api",
+			expectedScheme:   "https",
+			expectedBasePath: "https://origin.hostname.example.com:8080/api",
+		},
+	}
+
+	// use multiple schemes to demonstrate that the URL is based on the
+	// origin's scheme, not the :scheme header
+	for _, testScheme := range []string{"http", "https"} {
+		for _, testOrigin := range testOrigins {
+			assertion := func(testPath string) bool {
+				const expectedMethod = "POST"
+				const expectedHostname = "request.hostname.example.com"
+
+				h2 := []Header{
+					{Name: ":method", Value: expectedMethod},
+					{Name: ":scheme", Value: testScheme},
+					{Name: ":authority", Value: expectedHostname},
+					{Name: ":path", Value: testPath},
+					{Name: RequestUserHeadersField, Value: ""},
+				}
+				h1, err := http.NewRequest("GET", testOrigin.url, nil)
+				require.NoError(t, err)
+
+				err = H2RequestHeadersToH1Request(h2, h1)
+				return assert.NoError(t, err) &&
+					assert.Equal(t, expectedMethod, h1.Method) &&
+					assert.Equal(t, expectedHostname, h1.Host) &&
+					assert.Equal(t, testOrigin.expectedScheme, h1.URL.Scheme) &&
+					assert.Equal(t, testOrigin.expectedBasePath+testPath, h1.URL.String())
+			}
+			err := quick.Check(assertion, config)
+			assert.NoError(t, err)
+		}
+	}
+}
+
+func randomASCIIPrintableChar(rand *rand.Rand) int {
+	// smallest printable ASCII char is 32, largest is 126
+	const startPrintable = 32
+	const endPrintable = 127
+	return startPrintable + rand.Intn(endPrintable-startPrintable)
+}
+
+// randomASCIIText generates an ASCII string, some of whose characters may be
+// percent-encoded. Its "logical length" (ignoring percent-encoding) is
+// between 1 and `maxLength`.
+func randomASCIIText(rand *rand.Rand, minLength int, maxLength int) string {
+	length := minLength + rand.Intn(maxLength)
+	result := ""
+	for i := 0; i < length; i++ {
+		c := randomASCIIPrintableChar(rand)
+
+		// 1/4 chance of using percent encoding when not necessary
+		if c == '%' || rand.Intn(4) == 0 {
+			result += fmt.Sprintf("%%%02X", c)
+		} else {
+			result += string(c)
+		}
+	}
+	return result
+}
+
+// Calls `randomASCIIText` and ensures the result is a valid URL path,
+// i.e. one that can pass unchanged through url.URL.String()
+func randomHTTP1Path(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
+	text := randomASCIIText(rand, minLength, maxLength)
+	re, err := regexp.Compile("[^/;,]*")
+	require.NoError(t, err)
+	return "/" + re.ReplaceAllStringFunc(text, url.PathEscape)
+}
+
+// Calls `randomASCIIText` and ensures the result is a valid URL query,
+// i.e. one that can pass unchanged through url.URL.String()
+func randomHTTP1Query(rand *rand.Rand, minLength int, maxLength int) string {
+	text := randomASCIIText(rand, minLength, maxLength)
+	return "?" + strings.ReplaceAll(text, "#", "%23")
+}
+
+// Calls `randomASCIIText` and ensures the result is a valid URL fragment,
+// i.e. one that can pass unchanged through url.URL.String()
+func randomHTTP1Fragment(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
+	text := randomASCIIText(rand, minLength, maxLength)
+	u, err := url.Parse("#" + text)
+	require.NoError(t, err)
+	return u.String()
+}
+
+// Assemble a random :path pseudoheader that is legal by Go stdlib standards
+// (i.e. all characters will satisfy "net/url".shouldEscape for their respective locations)
+func randomHTTP2Path(t *testing.T, rand *rand.Rand) string {
+	result := randomHTTP1Path(t, rand, 1, 64)
+	if rand.Intn(2) == 1 {
+		result += randomHTTP1Query(rand, 1, 32)
+	}
+	if rand.Intn(2) == 1 {
+		result += randomHTTP1Fragment(t, rand, 1, 16)
+	}
+	return result
+}
+
+func stdlibHeaderToH2muxHeader(headers http.Header) (h2muxHeaders []Header) {
+	for name, values := range headers {
+		for _, value := range values {
+			h2muxHeaders = append(h2muxHeaders, Header{name, value})
+		}
+	}
+
+	return h2muxHeaders
+}
+
+func TestSerializeHeaders(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	mockHeaders := http.Header{
+		"Mock-Header-One":        {"Mock header one value", "three"},
+		"Mock-Header-Two-Long":   {"Mock header two value\nlong"},
+		":;":                     {":;", ";:"},
+		":":                      {":"},
+		";":                      {";"},
+		";;":                     {";;"},
+		"Empty values":           {"", ""},
+		"":                       {"Empty key"},
+		"control\tcharacter\b\n": {"value\n\b\t"},
+		";\v:":                   {":\v;"},
+	}
+
+	for header, values := range mockHeaders {
+		for _, value := range values {
+			// Note that Golang's http library is opinionated;
+			// at this point every header name will be title-cased in order to comply with the HTTP RFC
+			// This means our proxy is not completely transparent when it comes to proxying headers
+			request.Header.Add(header, value)
+		}
+	}
+
+	serializedHeaders := SerializeHeaders(request.Header)
+
+	// Sanity check: the headers serialized to something that's not an empty string
+	assert.NotEqual(t, "", serializedHeaders)
+
+	// Deserialize back, and ensure we get the same set of headers
+	deserializedHeaders, err := DeserializeHeaders(serializedHeaders)
+	assert.NoError(t, err)
+
+	assert.Equal(t, 13, len(deserializedHeaders))
+	h2muxExpectedHeaders := stdlibHeaderToH2muxHeader(mockHeaders)
+
+	sort.Sort(ByName(deserializedHeaders))
+	sort.Sort(ByName(h2muxExpectedHeaders))
+
+	assert.True(
+		t,
+		reflect.DeepEqual(h2muxExpectedHeaders, deserializedHeaders),
+		fmt.Sprintf("got = %#v, want = %#v\n", deserializedHeaders, h2muxExpectedHeaders),
+	)
+}
+
+func TestSerializeNoHeaders(t *testing.T) {
+	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
+	assert.NoError(t, err)
+
+	serializedHeaders := SerializeHeaders(request.Header)
+	deserializedHeaders, err := DeserializeHeaders(serializedHeaders)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, len(deserializedHeaders))
+}
+
+func TestDeserializeMalformed(t *testing.T) {
+	var err error
+
+	malformedData := []string{
+		"malformed data",
+		"bW9jawo=",                   // "mock"
+		"bW9jawo=:ZGF0YQo=:bW9jawo=", // "mock:data:mock"
+		"::",
+	}
+
+	for _, malformedValue := range malformedData {
+		_, err = DeserializeHeaders(malformedValue)
+		assert.Error(t, err)
+	}
+}
+
+func TestParseHeaders(t *testing.T) {
+	mockUserHeadersToSerialize := http.Header{
+		"Mock-Header-One":   {"1", "1.5"},
+		"Mock-Header-Two":   {"2"},
+		"Mock-Header-Three": {"3"},
+	}
+
+	mockHeaders := []Header{
+		{Name: "One", Value: "1"},
+		{Name: "Cf-Two", Value: "cf-value-1"},
+		{Name: "Cf-Two", Value: "cf-value-2"},
+		{Name: RequestUserHeadersField, Value: SerializeHeaders(mockUserHeadersToSerialize)},
+	}
+
+	expectedHeaders := []Header{
+		{Name: "Mock-Header-One", Value: "1"},
+		{Name: "Mock-Header-One", Value: "1.5"},
+		{Name: "Mock-Header-Two", Value: "2"},
+		{Name: "Mock-Header-Three", Value: "3"},
+	}
+	parsedHeaders, err := ParseUserHeaders(RequestUserHeadersField, mockHeaders)
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, expectedHeaders, parsedHeaders)
+}
+
+func TestParseHeadersNoSerializedHeader(t *testing.T) {
+	mockHeaders := []Header{
+		{Name: "One", Value: "1"},
+		{Name: "Cf-Two", Value: "cf-value-1"},
+		{Name: "Cf-Two", Value: "cf-value-2"},
+	}
+
+	_, err := ParseUserHeaders(RequestUserHeadersField, mockHeaders)
+	assert.EqualError(t, err, fmt.Sprintf("%s header not found", RequestUserHeadersField))
+}
+
+func TestIsControlHeader(t *testing.T) {
+	controlHeaders := []string{
+		// Anything that begins with cf-
+		"cf-sample-header",
+		"CF-SAMPLE-HEADER",
+		"Cf-Sample-Header",
+
+		// Any http2 pseudoheader
+		":sample-pseudo-header",
+
+		// content-length is a special case, it has to be there
+		// for some requests to work (per the HTTP2 spec)
+		"content-length",
+	}
+
+	for _, header := range controlHeaders {
+		assert.True(t, IsControlHeader(header))
+	}
+}
+
+func TestIsNotControlHeader(t *testing.T) {
+	notControlHeaders := []string{
+		"Mock-header",
+		"Another-sample-header",
+	}
+
+	for _, header := range notControlHeaders {
+		assert.False(t, IsControlHeader(header))
+	}
+}
+
+func TestH1ResponseToH2ResponseHeaders(t *testing.T) {
+	mockHeaders := http.Header{
+		"User-header-one": {""},
+		"User-header-two": {"1", "2"},
+		"cf-header":       {"cf-value"},
+		"Content-Length":  {"123"},
+	}
+	mockResponse := http.Response{
+		StatusCode: 200,
+		Header:     mockHeaders,
+	}
+
+	headers := H1ResponseToH2ResponseHeaders(&mockResponse)
+
+	serializedHeadersIndex := -1
+	for i, header := range headers {
+		if header.Name == ResponseUserHeadersField {
+			serializedHeadersIndex = i
+			break
+		}
+	}
+	assert.NotEqual(t, -1, serializedHeadersIndex)
+	actualControlHeaders := append(
+		headers[:serializedHeadersIndex],
+		headers[serializedHeadersIndex+1:]...,
+	)
+	expectedControlHeaders := []Header{
+		{Name: ":status", Value: "200"},
+		{Name: "content-length", Value: "123"},
+	}
+
+	assert.ElementsMatch(t, expectedControlHeaders, actualControlHeaders)
+
+	actualUserHeaders, err := DeserializeHeaders(headers[serializedHeadersIndex].Value)
+	expectedUserHeaders := []Header{
+		{Name: "User-header-one", Value: ""},
+		{Name: "User-header-two", Value: "1"},
+		{Name: "User-header-two", Value: "2"},
+	}
+	assert.NoError(t, err)
+	assert.ElementsMatch(t, expectedUserHeaders, actualUserHeaders)
+}
+
+// The purpose of this test is to check that our code and the http.Header
+// implementation don't throw validation errors about header size
+func TestHeaderSize(t *testing.T) {
+	largeValue := randSeq(5 * 1024 * 1024) // 5Mb
+	largeHeaders := http.Header{
+		"User-header": {largeValue},
+	}
+	mockResponse := http.Response{
+		StatusCode: 200,
+		Header:     largeHeaders,
+	}
+
+	serializedHeaders := H1ResponseToH2ResponseHeaders(&mockResponse)
+	request, err := http.NewRequest(http.MethodGet, "https://example.com/", nil)
+	assert.NoError(t, err)
+	for _, header := range serializedHeaders {
+		request.Header.Set(header.Name, header.Value)
+	}
+
+	for _, header := range serializedHeaders {
+		if header.Name != ResponseUserHeadersField {
+			continue
+		}
+
+		deserializedHeaders, err := DeserializeHeaders(header.Value)
+		assert.NoError(t, err)
+		assert.Equal(t, largeValue, deserializedHeaders[0].Value)
+	}
+}
+
+func randSeq(n int) string {
+	randomizer := rand.New(rand.NewSource(17))
+	var letters = []rune(":;,+/=abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = letters[randomizer.Intn(len(letters))]
+	}
+	return string(b)
+}

h2mux/muxedstream.go

@@ -17,6 +17,12 @@ type ReadWriteClosedCloser interface {
 	Closed() bool
 }
 
+// MuxedStreamDataSignaller is a write-only *ReadyList
+type MuxedStreamDataSignaller interface {
+	// Non-blocking: call this when data is ready to be sent for the given stream ID.
+	Signal(ID uint32)
+}
+
 // MuxedStream is logically an HTTP/2 stream, with an additional buffer for outgoing data.
 type MuxedStream struct {
 	streamID uint32
@@ -55,8 +61,8 @@ type MuxedStream struct {
 	// This is the amount of bytes that are in the peer's receive window
 	// (how much data we can send from this stream).
 	sendWindow uint32
-	// Reference to the muxer's readyList; signal this for stream data to be sent.
-	readyList *ReadyList
+	// The muxer's readyList
+	readyList MuxedStreamDataSignaller
 	// The headers that should be sent, and a flag so we only send them once.
 	headersSent  bool
 	writeHeaders []Header
@@ -88,7 +94,7 @@ func (th TunnelHostname) IsSet() bool {
 	return th != ""
 }
 
-func NewStream(config MuxerConfig, writeHeaders []Header, readyList *ReadyList, dictionaries h2Dictionaries) *MuxedStream {
+func NewStream(config MuxerConfig, writeHeaders []Header, readyList MuxedStreamDataSignaller, dictionaries h2Dictionaries) *MuxedStream {
 	return &MuxedStream{
 		responseHeadersReceived: make(chan struct{}),
 		readBuffer:              NewSharedBuffer(),

h2mux/muxreader.go

@@ -51,10 +51,12 @@ type MuxReader struct {
 	dictionaries h2Dictionaries
 }
 
-func (r *MuxReader) Shutdown() {
-	done := r.streams.Shutdown()
-	if done == nil {
-		return
+// Shutdown blocks new streams from being created.
+// It returns a channel that is closed once the last stream has closed.
+func (r *MuxReader) Shutdown() <-chan struct{} {
+	done, alreadyInProgress := r.streams.Shutdown()
+	if alreadyInProgress {
+		return done
 	}
 	r.sendGoAway(http2.ErrCodeNo)
 	go func() {
@@ -62,6 +64,7 @@ func (r *MuxReader) Shutdown() {
 		<-done
 		r.r.Close()
 	}()
+	return done
 }
 
 func (r *MuxReader) run(parentLogger *log.Entry) error {
@@ -94,7 +97,12 @@ func (r *MuxReader) run(parentLogger *log.Entry) error {
 			switch e := err.(type) {
 			case http2.StreamError:
 				errLogger.Warn("stream error")
-				r.streamError(e.StreamID, e.Code)
+				// Ideally we wouldn't return here, since that aborts the muxer.
+				// We should communicate the error to the relevant MuxedStream
+				// data structure, so that callers of MuxedStream.Read() and
+				// MuxedStream.Write() would see it. Then we could `continue`
+				// and keep the muxer going.
+				return r.streamError(e.StreamID, e.Code)
 			case http2.ConnectionError:
 				errLogger.Warn("connection error")
 				return r.connectionError(err)

h2mux/muxwriter.go

@@ -250,28 +250,52 @@ func (w *MuxWriter) encodeHeaders(headers []Header) ([]byte, error) {
 // writeHeaders writes a block of encoded headers, splitting it into multiple frames if necessary.
 func (w *MuxWriter) writeHeaders(streamID uint32, headers []Header) error {
 	encodedHeaders, err := w.encodeHeaders(headers)
-	if err != nil {
+	if err != nil || len(encodedHeaders) == 0 {
 		return err
 	}
+
 	blockSize := int(w.maxFrameSize)
-	endHeaders := len(encodedHeaders) == 0
-	for !endHeaders && err == nil {
-		blockFragment := encodedHeaders
-		if len(encodedHeaders) > blockSize {
-			blockFragment = blockFragment[:blockSize]
-			encodedHeaders = encodedHeaders[blockSize:]
-			// Send CONTINUATION frame if the headers can't be fit into 1 frame
-			err = w.f.WriteContinuation(streamID, endHeaders, blockFragment)
-		} else {
-			endHeaders = true
-			err = w.f.WriteHeaders(http2.HeadersFrameParam{
-				StreamID:      streamID,
-				EndHeaders:    endHeaders,
-				BlockFragment: blockFragment,
-			})
+	// CONTINUATION is unnecessary; the headers fit within the blockSize
+	if len(encodedHeaders) < blockSize {
+		return w.f.WriteHeaders(http2.HeadersFrameParam{
+			StreamID:      streamID,
+			EndHeaders:    true,
+			BlockFragment: encodedHeaders,
+		})
+	}
+
+	choppedHeaders := chopEncodedHeaders(encodedHeaders, blockSize)
+	// len(choppedHeaders) is at least 2
+	if err := w.f.WriteHeaders(http2.HeadersFrameParam{StreamID: streamID, EndHeaders: false, BlockFragment: choppedHeaders[0]}); err != nil {
+		return err
+	}
+	for i := 1; i < len(choppedHeaders)-1; i++ {
+		if err := w.f.WriteContinuation(streamID, false, choppedHeaders[i]); err != nil {
+			return err
 		}
 	}
-	return err
+	if err := w.f.WriteContinuation(streamID, true, choppedHeaders[len(choppedHeaders)-1]); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Partition a slice of bytes into `len(slice) / blockSize` slices of length `blockSize`
+func chopEncodedHeaders(headers []byte, chunkSize int) [][]byte {
+	var divided [][]byte
+
+	for i := 0; i < len(headers); i += chunkSize {
+		end := i + chunkSize
+
+		if end > len(headers) {
+			end = len(headers)
+		}
+
+		divided = append(divided, headers[i:end])
+	}
+
+	return divided
 }
 
 func (w *MuxWriter) writeUseDictionary(dictRequest useDictRequest) error {

h2mux/muxwriter_test.go

@@ -0,0 +1,26 @@
+package h2mux
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestChopEncodedHeaders(t *testing.T) {
+	mockEncodedHeaders := make([]byte, 5)
+	for i := range mockEncodedHeaders {
+		mockEncodedHeaders[i] = byte(i)
+	}
+	chopped := chopEncodedHeaders(mockEncodedHeaders, 4)
+
+	assert.Equal(t, 2, len(chopped))
+	assert.Equal(t, []byte{0, 1, 2, 3}, chopped[0])
+	assert.Equal(t, []byte{4}, chopped[1])
+}
+
+func TestChopEncodedEmptyHeaders(t *testing.T) {
+	mockEncodedHeaders := make([]byte, 0)
+	chopped := chopEncodedHeaders(mockEncodedHeaders, 3)
+
+	assert.Equal(t, 0, len(chopped))
+}

h2mux/sample/index.html

@@ -80,7 +80,7 @@ ghost.init({
     <link rel="alternate" type="application/rss+xml" title="Cloudflare Blog" href="https://blog.cloudflare.com/rss/" />
     <meta name="msvalidate.01" content="CF295E1604697F9CAD18B5A232E871F6" />
 <meta class="swiftype" name="language" data-type="string" content="en" />
-<script src="https://s3-us-west-1.amazonaws.com/cf-ghost-assets-hotfix/js/index.js"></script>
+<script src="https://blog.cloudflare.com/assets/js/index.js"></script>
 <script type="text/javascript" src="//cdn.bizible.com/scripts/bizible.js" async=""></script>
 <script>
 var trackRecruitingLink = function(role, url) {
@@ -128,7 +128,7 @@ HTMLAttrToAdd.setAttribute("lang", "en");
         overflow-y: scroll;
     }
 </style>
-<link href="https://s3-us-west-1.amazonaws.com/cf-ghost-assets-hotfix/css/screen.css" rel="stylesheet">
+<link href="https://blog.cloudflare.com/assets/css/screen.css" rel="stylesheet">
 <link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.8.1/themes/prism.min.css" rel="stylesheet">
 
     <style>

h2mux/sample/index1.html

@@ -113,7 +113,7 @@ ghost.init({
     <link rel="alternate" type="application/rss+xml" title="Cloudflare Blog" href="https://blog.cloudflare.com/rss/" />
     <meta name="msvalidate.01" content="CF295E1604697F9CAD18B5A232E871F6" />
 <meta class="swiftype" name="language" data-type="string" content="en" />
-<script src="https://s3-us-west-1.amazonaws.com/cf-ghost-assets-hotfix/js/index.js"></script>
+<script src="https://blog.cloudflare.com/assets/js/index.js"></script>
 <script type="text/javascript" src="//cdn.bizible.com/scripts/bizible.js" async=""></script>
 <script>
 var trackRecruitingLink = function(role, url) {
@@ -161,7 +161,7 @@ HTMLAttrToAdd.setAttribute("lang", "en");
         overflow-y: scroll;
     }
 </style>
-<link href="https://s3-us-west-1.amazonaws.com/cf-ghost-assets-hotfix/css/screen.css" rel="stylesheet">
+<link href="https://blog.cloudflare.com/assets/css/screen.css" rel="stylesheet">
 <link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.8.1/themes/prism.min.css" rel="stylesheet">
 
     <style>

h2mux/sample/index2.html

@@ -109,7 +109,7 @@ ghost.init({
     <link rel="alternate" type="application/rss+xml" title="Cloudflare Blog" href="https://blog.cloudflare.com/rss/" />
     <meta name="msvalidate.01" content="CF295E1604697F9CAD18B5A232E871F6" />
 <meta class="swiftype" name="language" data-type="string" content="en" />
-<script src="https://s3-us-west-1.amazonaws.com/cf-ghost-assets-hotfix/js/index.js"></script>
+<script src="https://blog.cloudflare.com/assets/js/index.js"></script>
 <script type="text/javascript" src="//cdn.bizible.com/scripts/bizible.js" async=""></script>
 <script>
 var trackRecruitingLink = function(role, url) {
@@ -157,7 +157,7 @@ HTMLAttrToAdd.setAttribute("lang", "en");
         overflow-y: scroll;
     }
 </style>
-<link href="https://s3-us-west-1.amazonaws.com/cf-ghost-assets-hotfix/css/screen.css" rel="stylesheet">
+<link href="https://blog.cloudflare.com/assets/css/screen.css" rel="stylesheet">
 <link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.8.1/themes/prism.min.css" rel="stylesheet">
 
     <style>

hello/hello.go

@@ -56,7 +56,7 @@ const indexTemplate = `
           <path class="st2" d="M93.6 12.8h-.5c-.1 0-.2.1-.3.2l-.7 2.4c-.3 1-.2 2 .3 2.6.5.6 1.2 1 2.1 1.1l3.7.2c.1 0 .2.1.3.1.1.1.1.2 0 .3-.1.2-.2.3-.4.3l-3.8.2c-2.1.1-4.3 1.8-5.1 3.8l-.2.9c-.1.1 0 .3.2.3h13.2c.2 0 .3-.1.3-.3.2-.8.4-1.7.4-2.6 0-5.2-4.3-9.5-9.5-9.5"/>
           <path class="st3" d="M104.4 30.8c-.5 0-.9-.4-.9-.9s.4-.9.9-.9.9.4.9.9-.4.9-.9.9m0-1.6c-.4 0-.7.3-.7.7 0 .4.3.7.7.7.4 0 .7-.3.7-.7 0-.4-.3-.7-.7-.7m.4 1.2h-.2l-.2-.3h-.2v.3h-.2v-.9h.5c.2 0 .3.1.3.3 0 .1-.1.2-.2.3l.2.3zm-.3-.5c.1 0 .1 0 .1-.1s-.1-.1-.1-.1h-.3v.3h.3zM14.8 29H17v6h3.8v1.9h-6zM23.1 32.9c0-2.3 1.8-4.1 4.3-4.1s4.2 1.8 4.2 4.1-1.8 4.1-4.3 4.1c-2.4 0-4.2-1.8-4.2-4.1m6.3 0c0-1.2-.8-2.2-2-2.2s-2 1-2 2.1.8 2.1 2 2.1c1.2.2 2-.8 2-2M34.3 33.4V29h2.2v4.4c0 1.1.6 1.7 1.5 1.7s1.5-.5 1.5-1.6V29h2.2v4.4c0 2.6-1.5 3.7-3.7 3.7-2.3-.1-3.7-1.2-3.7-3.7M45 29h3.1c2.8 0 4.5 1.6 4.5 3.9s-1.7 4-4.5 4h-3V29zm3.1 5.9c1.3 0 2.2-.7 2.2-2s-.9-2-2.2-2h-.9v4h.9zM55.7 29H62v1.9h-4.1v1.3h3.7V34h-3.7v2.9h-2.2zM65.1 29h2.2v6h3.8v1.9h-6zM76.8 28.9H79l3.4 8H80l-.6-1.4h-3.1l-.6 1.4h-2.3l3.4-8zm2 4.9l-.9-2.2-.9 2.2h1.8zM85.2 29h3.7c1.2 0 2 .3 2.6.9.5.5.7 1.1.7 1.8 0 1.2-.6 2-1.6 2.4l1.9 2.8H90l-1.6-2.4h-1v2.4h-2.2V29zm3.6 3.8c.7 0 1.2-.4 1.2-.9 0-.6-.5-.9-1.2-.9h-1.4v1.9h1.4zM95.3 29h6.4v1.8h-4.2V32h3.8v1.8h-3.8V35h4.3v1.9h-6.5zM10 33.9c-.3.7-1 1.2-1.8 1.2-1.2 0-2-1-2-2.1s.8-2.1 2-2.1c.9 0 1.6.6 1.9 1.3h2.3c-.4-1.9-2-3.3-4.2-3.3-2.4 0-4.3 1.8-4.3 4.1s1.8 4.1 4.2 4.1c2.1 0 3.7-1.4 4.2-3.2H10z"/>
         </svg>
-        <h1 class="f4 f2-ns mt5 fw5">Congrats! You created your first tunnel!</h1>
+        <h1 class="f4 f2-ns mt5 fw5">Congrats! You created a tunnel!</h1>
         <p class="f6 f5-m f4-l measure lh-copy fw3">
           Argo Tunnel exposes locally running applications to the internet by
           running an encrypted, virtual tunnel from your laptop or server to

metrics/metrics.go

@@ -43,7 +43,7 @@ func ServeMetrics(l net.Listener, shutdownC <-chan struct{}, logger *logrus.Logg
 		defer wg.Done()
 		err = server.Serve(l)
 	}()
-	logger.WithField("addr", l.Addr()).Info("Starting metrics server")
+	logger.WithField("addr", fmt.Sprintf("%v/metrics", l.Addr())).Info("Starting metrics server")
 	// server.Serve will hang if server.Shutdown is called before the server is
 	// fully started up. So add artificial delay.
 	time.Sleep(startupTime)

origin/backoffhandler.go

@@ -92,3 +92,8 @@ func (b BackoffHandler) GetBaseTime() time.Duration {
 	}
 	return b.BaseTime
 }
+
+// Retries returns the number of retries consumed so far.
+func (b *BackoffHandler) Retries() int {
+	return int(b.retries)
+}

origin/metrics.go

@@ -58,9 +58,11 @@ type TunnelMetrics struct {
 	// oldServerLocations stores the last server the tunnel was connected to
 	oldServerLocations map[string]string
 
-	regSuccess prometheus.Counter
-	regFail    *prometheus.CounterVec
-	rpcFail    *prometheus.CounterVec
+	regSuccess  *prometheus.CounterVec
+	regFail     *prometheus.CounterVec
+	authSuccess prometheus.Counter
+	authFail    *prometheus.CounterVec
+	rpcFail     *prometheus.CounterVec
 
 	muxerMetrics        *muxerMetrics
 	tunnelsHA           tunnelsForHA
@@ -417,7 +419,7 @@ func NewTunnelMetrics() *TunnelMetrics {
 			Name:      "tunnel_rpc_fail",
 			Help:      "Count of RPC connection errors by type",
 		},
-		[]string{"error"},
+		[]string{"error", "rpcName"},
 	)
 	prometheus.MustRegister(rpcFail)
 
@@ -428,7 +430,7 @@ func NewTunnelMetrics() *TunnelMetrics {
 			Name:      "tunnel_register_fail",
 			Help:      "Count of tunnel registration errors by type",
 		},
-		[]string{"error"},
+		[]string{"error", "rpcName"},
 	)
 	prometheus.MustRegister(registerFail)
 
@@ -443,15 +445,38 @@ func NewTunnelMetrics() *TunnelMetrics {
 	)
 	prometheus.MustRegister(userHostnamesCounts)
 
-	registerSuccess := prometheus.NewCounter(
+	registerSuccess := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Namespace: metricsNamespace,
 			Subsystem: tunnelSubsystem,
 			Name:      "tunnel_register_success",
 			Help:      "Count of successful tunnel registrations",
-		})
+		},
+		[]string{"rpcName"},
+	)
 	prometheus.MustRegister(registerSuccess)
 
+	authSuccess := prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_success",
+			Help:      "Count of successful tunnel authenticate",
+		},
+	)
+	prometheus.MustRegister(authSuccess)
+
+	authFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_fail",
+			Help:      "Count of tunnel authenticate errors by type",
+		},
+		[]string{"error"},
+	)
+	prometheus.MustRegister(authFail)
+
 	return &TunnelMetrics{
 		haConnections:                  haConnections,
 		activeStreams:                  activeStreams,
@@ -472,6 +497,8 @@ func NewTunnelMetrics() *TunnelMetrics {
 		regFail:                        registerFail,
 		rpcFail:                        rpcFail,
 		userHostnamesCounts:            userHostnamesCounts,
+		authSuccess:                    authSuccess,
+		authFail:                       authFail,
 	}
 }
 

origin/supervisor.go

@@ -2,16 +2,21 @@ package origin
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
+	"sync"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
 
+	"github.com/cloudflare/cloudflared/buffer"
 	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/signal"
-
-	"github.com/google/uuid"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
 const (
@@ -21,13 +26,27 @@ const (
 	resolveTTL = time.Hour
 	// Interval between registering new tunnels
 	registrationInterval = time.Second
+
+	subsystemRefreshAuth = "refresh_auth"
+	// Maximum exponent for 'Authenticate' exponential backoff
+	refreshAuthMaxBackoff = 10
+	// Waiting time before retrying a failed 'Authenticate' connection
+	refreshAuthRetryDuration = time.Second * 10
+	// Maximum time to make an Authenticate RPC
+	authTokenTimeout = time.Second * 30
 )
 
+var (
+	errJWTUnset         = errors.New("JWT unset")
+	errEventDigestUnset = errors.New("event digest unset")
+)
+
+// Supervisor manages non-declarative tunnels. Establishes TCP connections with the edge, and
+// reconnects them if they disconnect.
 type Supervisor struct {
-	config  *TunnelConfig
-	edgeIPs []*net.TCPAddr
-	// nextUnusedEdgeIP is the index of the next addr k edgeIPs to try
-	nextUnusedEdgeIP  int
+	cloudflaredUUID   uuid.UUID
+	config            *TunnelConfig
+	edgeIPs           *edgediscovery.Edge
 	lastResolve       time.Time
 	resolverC         chan resolveResult
 	tunnelErrors      chan tunnelError
@@ -38,36 +57,76 @@ type Supervisor struct {
 	nextConnectedSignal chan struct{}
 
 	logger *logrus.Entry
+
+	jwtLock sync.RWMutex
+	jwt     []byte
+
+	eventDigestLock sync.RWMutex
+	eventDigest     []byte
+
+	connDigestLock sync.RWMutex
+	connDigest     map[uint8][]byte
+
+	bufferPool *buffer.Pool
 }
 
 type resolveResult struct {
-	edgeIPs []*net.TCPAddr
-	err     error
+	err error
 }
 
 type tunnelError struct {
 	index int
+	addr  *net.TCPAddr
 	err   error
 }
 
-func NewSupervisor(config *TunnelConfig) *Supervisor {
+func NewSupervisor(config *TunnelConfig, u uuid.UUID) (*Supervisor, error) {
+	var (
+		edgeIPs *edgediscovery.Edge
+		err     error
+	)
+	if len(config.EdgeAddrs) > 0 {
+		edgeIPs, err = edgediscovery.StaticEdge(config.Logger, config.EdgeAddrs)
+	} else {
+		edgeIPs, err = edgediscovery.ResolveEdge(config.Logger)
+	}
+	if err != nil {
+		return nil, err
+	}
 	return &Supervisor{
+		cloudflaredUUID:   u,
 		config:            config,
+		edgeIPs:           edgeIPs,
 		tunnelErrors:      make(chan tunnelError),
 		tunnelsConnecting: map[int]chan struct{}{},
 		logger:            config.Logger.WithField("subsystem", "supervisor"),
-	}
+		connDigest:        make(map[uint8][]byte),
+		bufferPool:        buffer.NewPool(512 * 1024),
+	}, nil
 }
 
-func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, u uuid.UUID) error {
+func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan struct{}) error {
 	logger := s.config.Logger
-	if err := s.initialize(ctx, connectedSignal, u); err != nil {
+	if err := s.initialize(ctx, connectedSignal, reconnectCh); err != nil {
 		return err
 	}
 	var tunnelsWaiting []int
+	tunnelsActive := s.config.HAConnections
+
 	backoff := BackoffHandler{MaxRetries: s.config.Retries, BaseTime: tunnelRetryDuration, RetryForever: true}
 	var backoffTimer <-chan time.Time
-	tunnelsActive := s.config.HAConnections
+
+	refreshAuthBackoff := &BackoffHandler{MaxRetries: refreshAuthMaxBackoff, BaseTime: refreshAuthRetryDuration, RetryForever: true}
+	var refreshAuthBackoffTimer <-chan time.Time
+
+	if s.config.UseReconnectToken {
+		if timer, err := s.refreshAuth(ctx, refreshAuthBackoff, s.authenticate); err == nil {
+			refreshAuthBackoffTimer = timer
+		} else {
+			logger.WithError(err).Errorf("initial refreshAuth failed, retrying in %v", refreshAuthRetryDuration)
+			refreshAuthBackoffTimer = time.After(refreshAuthRetryDuration)
+		}
+	}
 
 	for {
 		select {
@@ -91,23 +150,27 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, u
 					backoffTimer = backoff.BackoffTimer()
 				}
 
-				// If the error is a dial error, the problem is likely to be network related
-				// try another addr before refreshing since we are likely to get back the
-				// same IPs in the same order. Same problem with duplicate connection error.
-				if s.unusedIPs() {
-					s.replaceEdgeIP(tunnelError.index)
-				} else {
-					s.refreshEdgeIPs()
-				}
+				// Previously we'd mark the edge address as bad here, but now we'll just silently use
+				// another.
 			}
 		// Backoff was set and its timer expired
 		case <-backoffTimer:
 			backoffTimer = nil
 			for _, index := range tunnelsWaiting {
-				go s.startTunnel(ctx, index, s.newConnectedTunnelSignal(index), u)
+				go s.startTunnel(ctx, index, s.newConnectedTunnelSignal(index), reconnectCh)
 			}
 			tunnelsActive += len(tunnelsWaiting)
 			tunnelsWaiting = nil
+		// Time to call Authenticate
+		case <-refreshAuthBackoffTimer:
+			newTimer, err := s.refreshAuth(ctx, refreshAuthBackoff, s.authenticate)
+			if err != nil {
+				logger.WithError(err).Error("Authentication failed")
+				// Permanent failure. Leave the `select` without setting the
+				// channel to be non-null, so we'll never hit this case of the `select` again.
+				continue
+			}
+			refreshAuthBackoffTimer = newTimer
 		// Tunnel successfully connected
 		case <-s.nextConnectedSignal:
 			if !s.waitForNextTunnel(s.nextConnectedIndex) && len(tunnelsWaiting) == 0 {
@@ -120,7 +183,6 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, u
 			s.resolverC = nil
 			if result.err == nil {
 				logger.Debug("Service discovery refresh complete")
-				s.edgeIPs = result.edgeIPs
 			} else {
 				logger.WithError(result.err).Error("Service discovery error")
 			}
@@ -128,29 +190,22 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, u
 	}
 }
 
-func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal, u uuid.UUID) error {
+// Returns nil if initialization succeeded, else the initialization error.
+func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan struct{}) error {
 	logger := s.logger
 
-	edgeIPs, err := s.resolveEdgeIPs()
-
-	if err != nil {
-		logger.Infof("ResolveEdgeIPs err")
-		return err
-	}
-	s.edgeIPs = edgeIPs
-	if s.config.HAConnections > len(edgeIPs) {
-		logger.Warnf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, len(edgeIPs))
-		s.config.HAConnections = len(edgeIPs)
-	}
 	s.lastResolve = time.Now()
-	// check entitlement and version too old error before attempting to register more tunnels
-	s.nextUnusedEdgeIP = s.config.HAConnections
-	go s.startFirstTunnel(ctx, connectedSignal, u)
+	availableAddrs := int(s.edgeIPs.AvailableAddrs())
+	if s.config.HAConnections > availableAddrs {
+		logger.Warnf("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
+		s.config.HAConnections = availableAddrs
+	}
+
+	go s.startFirstTunnel(ctx, connectedSignal, reconnectCh)
 	select {
 	case <-ctx.Done():
 		<-s.tunnelErrors
-		// Error can't be nil. A nil error signals that initialization succeed
-		return fmt.Errorf("context was canceled")
+		return ctx.Err()
 	case tunnelError := <-s.tunnelErrors:
 		return tunnelError.err
 	case <-connectedSignal.Wait():
@@ -158,7 +213,7 @@ func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Sig
 	// At least one successful connection, so start the rest
 	for i := 1; i < s.config.HAConnections; i++ {
 		ch := signal.New(make(chan struct{}))
-		go s.startTunnel(ctx, i, ch, u)
+		go s.startTunnel(ctx, i, ch, reconnectCh)
 		time.Sleep(registrationInterval)
 	}
 	return nil
@@ -166,37 +221,64 @@ func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Sig
 
 // startTunnel starts the first tunnel connection. The resulting error will be sent on
 // s.tunnelErrors. It will send a signal via connectedSignal if registration succeed
-func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *signal.Signal, u uuid.UUID) {
-	err := ServeTunnelLoop(ctx, s.config, s.getEdgeIP(0), 0, connectedSignal, u)
+func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan struct{}) {
+	var (
+		addr *net.TCPAddr
+		err  error
+	)
+	const thisConnID = 0
 	defer func() {
-		s.tunnelErrors <- tunnelError{index: 0, err: err}
+		s.tunnelErrors <- tunnelError{index: thisConnID, addr: addr, err: err}
 	}()
 
+	addr, err = s.edgeIPs.GetAddr(thisConnID)
+	if err != nil {
+		return
+	}
+
+	err = ServeTunnelLoop(ctx, s, s.config, addr, thisConnID, connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
+	// If the first tunnel disconnects, keep restarting it.
+	edgeErrors := 0
 	for s.unusedIPs() {
-		select {
-		case <-ctx.Done():
+		if ctx.Err() != nil {
 			return
-		default:
 		}
 		switch err.(type) {
 		case nil:
 			return
 		// try the next address if it was a dialError(network problem) or
 		// dupConnRegisterTunnelError
-		case dialError, dupConnRegisterTunnelError:
-			s.replaceEdgeIP(0)
+		case connection.DialError, dupConnRegisterTunnelError:
+			edgeErrors++
 		default:
 			return
 		}
-		err = ServeTunnelLoop(ctx, s.config, s.getEdgeIP(0), 0, connectedSignal, u)
+		if edgeErrors >= 2 {
+			addr, err = s.edgeIPs.GetDifferentAddr(thisConnID)
+			if err != nil {
+				return
+			}
+		}
+		err = ServeTunnelLoop(ctx, s, s.config, addr, thisConnID, connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
 	}
 }
 
 // startTunnel starts a new tunnel connection. The resulting error will be sent on
 // s.tunnelErrors.
-func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal *signal.Signal, u uuid.UUID) {
-	err := ServeTunnelLoop(ctx, s.config, s.getEdgeIP(index), uint8(index), connectedSignal, u)
-	s.tunnelErrors <- tunnelError{index: index, err: err}
+func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal *signal.Signal, reconnectCh chan struct{}) {
+	var (
+		addr *net.TCPAddr
+		err  error
+	)
+	defer func() {
+		s.tunnelErrors <- tunnelError{index: index, addr: addr, err: err}
+	}()
+
+	addr, err = s.edgeIPs.GetAddr(index)
+	if err != nil {
+		return
+	}
+	err = ServeTunnelLoop(ctx, s, s.config, addr, uint8(index), connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
 }
 
 func (s *Supervisor) newConnectedTunnelSignal(index int) *signal.Signal {
@@ -218,38 +300,140 @@ func (s *Supervisor) waitForNextTunnel(index int) bool {
 	return false
 }
 
-func (s *Supervisor) getEdgeIP(index int) *net.TCPAddr {
-	return s.edgeIPs[index%len(s.edgeIPs)]
-}
-
-func (s *Supervisor) resolveEdgeIPs() ([]*net.TCPAddr, error) {
-	// If --edge is specfied, resolve edge server addresses
-	if len(s.config.EdgeAddrs) > 0 {
-		return connection.ResolveAddrs(s.config.EdgeAddrs)
-	}
-	// Otherwise lookup edge server addresses through service discovery
-	return connection.EdgeDiscovery(s.logger)
-}
-
-func (s *Supervisor) refreshEdgeIPs() {
-	if s.resolverC != nil {
-		return
-	}
-	if time.Since(s.lastResolve) < resolveTTL {
-		return
-	}
-	s.resolverC = make(chan resolveResult)
-	go func() {
-		edgeIPs, err := s.resolveEdgeIPs()
-		s.resolverC <- resolveResult{edgeIPs: edgeIPs, err: err}
-	}()
-}
-
 func (s *Supervisor) unusedIPs() bool {
-	return s.nextUnusedEdgeIP < len(s.edgeIPs)
+	return s.edgeIPs.AvailableAddrs() > s.config.HAConnections
 }
 
-func (s *Supervisor) replaceEdgeIP(badIPIndex int) {
-	s.edgeIPs[badIPIndex] = s.edgeIPs[s.nextUnusedEdgeIP]
-	s.nextUnusedEdgeIP++
+func (s *Supervisor) ReconnectToken() ([]byte, error) {
+	s.jwtLock.RLock()
+	defer s.jwtLock.RUnlock()
+	if s.jwt == nil {
+		return nil, errJWTUnset
+	}
+	return s.jwt, nil
+}
+
+func (s *Supervisor) SetReconnectToken(jwt []byte) {
+	s.jwtLock.Lock()
+	defer s.jwtLock.Unlock()
+	s.jwt = jwt
+}
+
+func (s *Supervisor) EventDigest() ([]byte, error) {
+	s.eventDigestLock.RLock()
+	defer s.eventDigestLock.RUnlock()
+	if s.eventDigest == nil {
+		return nil, errEventDigestUnset
+	}
+	return s.eventDigest, nil
+}
+
+func (s *Supervisor) SetEventDigest(eventDigest []byte) {
+	s.eventDigestLock.Lock()
+	defer s.eventDigestLock.Unlock()
+	s.eventDigest = eventDigest
+}
+
+func (s *Supervisor) ConnDigest(connID uint8) ([]byte, error) {
+	s.connDigestLock.RLock()
+	defer s.connDigestLock.RUnlock()
+	digest, ok := s.connDigest[connID]
+	if !ok {
+		return nil, fmt.Errorf("no connection digest for connection %v", connID)
+	}
+	return digest, nil
+}
+
+func (s *Supervisor) SetConnDigest(connID uint8, connDigest []byte) {
+	s.connDigestLock.Lock()
+	defer s.connDigestLock.Unlock()
+	s.connDigest[connID] = connDigest
+}
+
+func (s *Supervisor) refreshAuth(
+	ctx context.Context,
+	backoff *BackoffHandler,
+	authenticate func(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error),
+) (retryTimer <-chan time.Time, err error) {
+	logger := s.config.Logger.WithField("subsystem", subsystemRefreshAuth)
+	authOutcome, err := authenticate(ctx, backoff.Retries())
+	if err != nil {
+		s.config.Metrics.authFail.WithLabelValues(err.Error()).Inc()
+		if duration, ok := backoff.GetBackoffDuration(ctx); ok {
+			logger.WithError(err).Warnf("Retrying in %v", duration)
+			return backoff.BackoffTimer(), nil
+		}
+		return nil, err
+	}
+	// clear backoff timer
+	backoff.SetGracePeriod()
+
+	switch outcome := authOutcome.(type) {
+	case tunnelpogs.AuthSuccess:
+		s.SetReconnectToken(outcome.JWT())
+		s.config.Metrics.authSuccess.Inc()
+		return timeAfter(outcome.RefreshAfter()), nil
+	case tunnelpogs.AuthUnknown:
+		duration := outcome.RefreshAfter()
+		s.config.Metrics.authFail.WithLabelValues(outcome.Error()).Inc()
+		logger.WithError(outcome).Warnf("Retrying in %v", duration)
+		return timeAfter(duration), nil
+	case tunnelpogs.AuthFail:
+		s.config.Metrics.authFail.WithLabelValues(outcome.Error()).Inc()
+		return nil, outcome
+	default:
+		err := fmt.Errorf("Unexpected outcome type %T", authOutcome)
+		s.config.Metrics.authFail.WithLabelValues(err.Error()).Inc()
+		return nil, err
+	}
+}
+
+func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int) (tunnelpogs.AuthOutcome, error) {
+	arbitraryEdgeIP, err := s.edgeIPs.GetAddrForRPC()
+	if err != nil {
+		return nil, err
+	}
+
+	edgeConn, err := connection.DialEdge(ctx, dialTimeout, s.config.TlsConfig, arbitraryEdgeIP)
+	if err != nil {
+		return nil, err
+	}
+	defer edgeConn.Close()
+
+	handler := h2mux.MuxedStreamFunc(func(*h2mux.MuxedStream) error {
+		// This callback is invoked by h2mux when the edge initiates a stream.
+		return nil // noop
+	})
+	muxerConfig := s.config.muxerConfig(handler)
+	muxerConfig.Logger = muxerConfig.Logger.WithField("subsystem", subsystemRefreshAuth)
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, muxerConfig, s.config.Metrics.activeStreams)
+	if err != nil {
+		return nil, err
+	}
+	go muxer.Serve(ctx)
+	defer func() {
+		// If we don't wait for the muxer shutdown here, edgeConn.Close() runs before the muxer connections are done,
+		// and the user sees log noise: "error writing data", "connection closed unexpectedly"
+		<-muxer.Shutdown()
+	}()
+
+	tunnelServer, err := connection.NewRPCClient(ctx, muxer, s.logger.WithField("subsystem", subsystemRefreshAuth), openStreamTimeout)
+	if err != nil {
+		return nil, err
+	}
+	defer tunnelServer.Close()
+
+	const arbitraryConnectionID = uint8(0)
+	registrationOptions := s.config.RegistrationOptions(arbitraryConnectionID, edgeConn.LocalAddr().String(), s.cloudflaredUUID)
+	registrationOptions.NumPreviousAttempts = uint8(numPreviousAttempts)
+	authResponse, err := tunnelServer.Authenticate(
+		ctx,
+		s.config.OriginCert,
+		s.config.Hostname,
+		registrationOptions,
+	)
+	if err != nil {
+		return nil, err
+	}
+	return authResponse.Outcome(), nil
 }

origin/supervisor_test.go

@@ -0,0 +1,165 @@
+package origin
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+func testConfig(logger *logrus.Logger) *TunnelConfig {
+	metrics := TunnelMetrics{}
+
+	metrics.authSuccess = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_success",
+			Help:      "Count of successful tunnel authenticate",
+		},
+	)
+
+	metrics.authFail = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metricsNamespace,
+			Subsystem: tunnelSubsystem,
+			Name:      "tunnel_authenticate_fail",
+			Help:      "Count of tunnel authenticate errors by type",
+		},
+		[]string{"error"},
+	)
+	return &TunnelConfig{Logger: logger, Metrics: &metrics}
+}
+
+func TestRefreshAuthBackoff(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	var wait time.Duration
+	timeAfter = func(d time.Duration) <-chan time.Time {
+		wait = d
+		return time.After(d)
+	}
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return nil, fmt.Errorf("authentication failure")
+	}
+
+	// authentication failures should consume the backoff
+	for i := uint(0); i < backoff.MaxRetries; i++ {
+		retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+		assert.NoError(t, err)
+		assert.NotNil(t, retryChan)
+		assert.Equal(t, (1<<i)*time.Second, wait)
+	}
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.Error(t, err)
+	assert.Nil(t, retryChan)
+
+	// now we actually make contact with the remote server
+	_, _ = s.refreshAuth(context.Background(), backoff, func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
+	})
+
+	// The backoff timer should have been reset. To confirm this, make timeNow
+	// return a value after the backoff timer's grace period
+	timeNow = func() time.Time {
+		expectedGracePeriod := time.Duration(time.Second * 2 << backoff.MaxRetries)
+		return time.Now().Add(expectedGracePeriod * 2)
+	}
+	_, ok := backoff.GetBackoffDuration(context.Background())
+	assert.True(t, ok)
+}
+
+func TestRefreshAuthSuccess(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	var wait time.Duration
+	timeAfter = func(d time.Duration) <-chan time.Time {
+		wait = d
+		return time.After(d)
+	}
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthSuccess([]byte("jwt"), 19), nil
+	}
+
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.NoError(t, err)
+	assert.NotNil(t, retryChan)
+	assert.Equal(t, 19*time.Hour, wait)
+
+	token, err := s.ReconnectToken()
+	assert.NoError(t, err)
+	assert.Equal(t, []byte("jwt"), token)
+}
+
+func TestRefreshAuthUnknown(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	var wait time.Duration
+	timeAfter = func(d time.Duration) <-chan time.Time {
+		wait = d
+		return time.After(d)
+	}
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthUnknown(errors.New("auth unknown"), 19), nil
+	}
+
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.NoError(t, err)
+	assert.NotNil(t, retryChan)
+	assert.Equal(t, 19*time.Hour, wait)
+
+	token, err := s.ReconnectToken()
+	assert.Equal(t, errJWTUnset, err)
+	assert.Nil(t, token)
+}
+
+func TestRefreshAuthFail(t *testing.T) {
+	logger := logrus.New()
+	logger.Level = logrus.ErrorLevel
+
+	s, err := NewSupervisor(testConfig(logger), uuid.New())
+	if !assert.NoError(t, err) {
+		t.FailNow()
+	}
+	backoff := &BackoffHandler{MaxRetries: 3}
+	auth := func(ctx context.Context, n int) (tunnelpogs.AuthOutcome, error) {
+		return tunnelpogs.NewAuthFail(errors.New("auth fail")), nil
+	}
+
+	retryChan, err := s.refreshAuth(context.Background(), backoff, auth)
+	assert.Error(t, err)
+	assert.Nil(t, retryChan)
+
+	token, err := s.ReconnectToken()
+	assert.Equal(t, errJWTUnset, err)
+	assert.Nil(t, token)
+}

origin/tunnel.go

@@ -14,7 +14,15 @@ import (
 	"sync"
 	"time"
 
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
+
+	"github.com/cloudflare/cloudflared/buffer"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/signal"
 	"github.com/cloudflare/cloudflared/streamhandler"
@@ -22,22 +30,25 @@ import (
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/validation"
 	"github.com/cloudflare/cloudflared/websocket"
-
-	"github.com/google/uuid"
-	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
-	_ "github.com/prometheus/client_golang/prometheus"
-	log "github.com/sirupsen/logrus"
-	"golang.org/x/sync/errgroup"
-	rpc "zombiezen.com/go/capnproto2/rpc"
 )
 
 const (
 	dialTimeout              = 15 * time.Second
 	openStreamTimeout        = 30 * time.Second
+	muxerTimeout             = 5 * time.Second
 	lbProbeUserAgentPrefix   = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
 	TagHeaderNamePrefix      = "Cf-Warp-Tag-"
 	DuplicateConnectionError = "EDUPCONN"
+	FeatureSerializedHeaders = "serialized_headers"
+	FeatureQuickReconnects   = "quick_reconnects"
+)
+
+type registerRPCName string
+
+const (
+	register  registerRPCName = "register"
+	reconnect registerRPCName = "reconnect"
+	unknown   registerRPCName = "unknown"
 )
 
 type TunnelConfig struct {
@@ -73,14 +84,21 @@ type TunnelConfig struct {
 	WSGI                 bool
 	// OriginUrl may not be used if a user specifies a unix socket.
 	OriginUrl string
+
+	// feature-flag to use new edge reconnect tokens
+	UseReconnectToken bool
+	// feature-flag for using ConnectionDigest
+	UseQuickReconnects bool
 }
 
-type dialError struct {
-	cause error
-}
-
-func (e dialError) Error() string {
-	return e.cause.Error()
+// ReconnectTunnelCredentialManager is invoked by functions in this file to
+// get/set parameters for ReconnectTunnel RPC calls.
+type ReconnectTunnelCredentialManager interface {
+	ReconnectToken() ([]byte, error)
+	EventDigest() ([]byte, error)
+	SetEventDigest(eventDigest []byte)
+	ConnDigest(connID uint8) ([]byte, error)
+	SetConnDigest(connID uint8, connDigest []byte)
 }
 
 type dupConnRegisterTunnelError struct{}
@@ -110,8 +128,8 @@ type clientRegisterTunnelError struct {
 	cause error
 }
 
-func newClientRegisterTunnelError(cause error, counter *prometheus.CounterVec) clientRegisterTunnelError {
-	counter.WithLabelValues(cause.Error()).Inc()
+func newClientRegisterTunnelError(cause error, counter *prometheus.CounterVec, name registerRPCName) clientRegisterTunnelError {
+	counter.WithLabelValues(cause.Error(), string(name)).Inc()
 	return clientRegisterTunnelError{cause: cause}
 }
 
@@ -119,6 +137,18 @@ func (e clientRegisterTunnelError) Error() string {
 	return e.cause.Error()
 }
 
+func (c *TunnelConfig) muxerConfig(handler h2mux.MuxedStreamHandler) h2mux.MuxerConfig {
+	return h2mux.MuxerConfig{
+		Timeout:            muxerTimeout,
+		Handler:            handler,
+		IsClient:           true,
+		HeartbeatInterval:  c.HeartbeatInterval,
+		MaxHeartbeats:      c.MaxHeartbeats,
+		Logger:             c.TransportLogger.WithFields(log.Fields{}),
+		CompressionQuality: h2mux.CompressionSetting(c.CompressionQuality),
+	}
+}
+
 func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP string, uuid uuid.UUID) *tunnelpogs.RegistrationOptions {
 	policy := tunnelrpc.ExistingTunnelPolicy_balance
 	if c.HAConnections <= 1 && c.LBPool == "" {
@@ -137,19 +167,35 @@ func (c *TunnelConfig) RegistrationOptions(connectionID uint8, OriginLocalIP str
 		RunFromTerminal:      c.RunFromTerminal,
 		CompressionQuality:   c.CompressionQuality,
 		UUID:                 uuid.String(),
+		Features:             c.SupportedFeatures(),
 	}
 }
 
-func StartTunnelDaemon(ctx context.Context, config *TunnelConfig, connectedSignal *signal.Signal, cloudflaredID uuid.UUID) error {
-	return NewSupervisor(config).Run(ctx, connectedSignal, cloudflaredID)
+func (c *TunnelConfig) SupportedFeatures() []string {
+	basic := []string{FeatureSerializedHeaders}
+	if c.UseQuickReconnects {
+		basic = append(basic, FeatureQuickReconnects)
+	}
+	return basic
+}
+
+func StartTunnelDaemon(ctx context.Context, config *TunnelConfig, connectedSignal *signal.Signal, cloudflaredID uuid.UUID, reconnectCh chan struct{}) error {
+	s, err := NewSupervisor(config, cloudflaredID)
+	if err != nil {
+		return err
+	}
+	return s.Run(ctx, connectedSignal, reconnectCh)
 }
 
 func ServeTunnelLoop(ctx context.Context,
+	credentialManager ReconnectTunnelCredentialManager,
 	config *TunnelConfig,
 	addr *net.TCPAddr,
 	connectionID uint8,
 	connectedSignal *signal.Signal,
 	u uuid.UUID,
+	bufferPool *buffer.Pool,
+	reconnectCh chan struct{},
 ) error {
 	connectionLogger := config.Logger.WithField("connectionID", connectionID)
 	config.Metrics.incrementHaConnections()
@@ -166,12 +212,15 @@ func ServeTunnelLoop(ctx context.Context,
 	for {
 		err, recoverable := ServeTunnel(
 			ctx,
+			credentialManager,
 			config,
 			connectionLogger,
 			addr, connectionID,
 			connectedFuse,
 			&backoff,
 			u,
+			bufferPool,
+			reconnectCh,
 		)
 		if recoverable {
 			if duration, ok := backoff.GetBackoffDuration(ctx); ok {
@@ -186,6 +235,7 @@ func ServeTunnelLoop(ctx context.Context,
 
 func ServeTunnel(
 	ctx context.Context,
+	credentialManager ReconnectTunnelCredentialManager,
 	config *TunnelConfig,
 	logger *log.Entry,
 	addr *net.TCPAddr,
@@ -193,6 +243,8 @@ func ServeTunnel(
 	connectedFuse *h2mux.BooleanFuse,
 	backoff *BackoffHandler,
 	u uuid.UUID,
+	bufferPool *buffer.Pool,
+	reconnectCh chan struct{},
 ) (err error, recoverable bool) {
 	// Treat panics as recoverable errors
 	defer func() {
@@ -213,11 +265,11 @@ func ServeTunnel(
 	tags["ha"] = connectionTag
 
 	// Returns error from parsing the origin URL or handshake errors
-	handler, originLocalIP, err := NewTunnelHandler(ctx, config, addr.String(), connectionID)
+	handler, originLocalIP, err := NewTunnelHandler(ctx, config, addr, connectionID, bufferPool)
 	if err != nil {
 		errLog := logger.WithError(err)
 		switch err.(type) {
-		case dialError:
+		case connection.DialError:
 			errLog.Error("Unable to dial edge")
 		case h2mux.MuxerHandshakeError:
 			errLog.Error("Handshake failed with edge server")
@@ -230,13 +282,38 @@ func ServeTunnel(
 
 	errGroup, serveCtx := errgroup.WithContext(ctx)
 
-	errGroup.Go(func() error {
-		err := RegisterTunnel(serveCtx, handler.muxer, config, logger, connectionID, originLocalIP, u)
-		if err == nil {
-			connectedFuse.Fuse(true)
-			backoff.SetGracePeriod()
+	errGroup.Go(func() (err error) {
+		defer func() {
+			if err == nil {
+				connectedFuse.Fuse(true)
+				backoff.SetGracePeriod()
+			}
+		}()
+
+		if config.UseReconnectToken && connectedFuse.Value() {
+			token, tokenErr := credentialManager.ReconnectToken()
+			eventDigest, eventDigestErr := credentialManager.EventDigest()
+			// if we have both credentials, we can reconnect
+			if tokenErr == nil && eventDigestErr == nil {
+				var connDigest []byte
+
+				// check if we can use Quick Reconnects
+				if config.UseQuickReconnects {
+					if digest, connDigestErr := credentialManager.ConnDigest(connectionID); connDigestErr == nil {
+						connDigest = digest
+					}
+				}
+				return ReconnectTunnel(serveCtx, token, eventDigest, connDigest, handler.muxer, config, logger, connectionID, originLocalIP, u, credentialManager)
+			}
+			// log errors and proceed to RegisterTunnel
+			if tokenErr != nil {
+				logger.WithError(tokenErr).Error("Couldn't get reconnect token")
+			}
+			if eventDigestErr != nil {
+				logger.WithError(eventDigestErr).Error("Couldn't get event digest")
+			}
 		}
-		return err
+		return RegisterTunnel(serveCtx, credentialManager, handler.muxer, config, logger, connectionID, originLocalIP, u)
 	})
 
 	errGroup.Go(func() error {
@@ -254,6 +331,16 @@ func ServeTunnel(
 		}
 	})
 
+	errGroup.Go(func() error {
+		select {
+		case <-reconnectCh:
+			return fmt.Errorf("received disconnect signal")
+		case <-serveCtx.Done():
+			return nil
+		}
+
+	})
+
 	errGroup.Go(func() error {
 		// All routines should stop when muxer finish serving. When muxer is shutdown
 		// gracefully, it doesn't return an error, so we need to return errMuxerShutdown
@@ -267,7 +354,7 @@ func ServeTunnel(
 
 	err = errGroup.Wait()
 	if err != nil {
-		_ = newClientRegisterTunnelError(err, config.Metrics.regFail)
+		_ = newClientRegisterTunnelError(err, config.Metrics.regFail, unknown)
 
 		switch castedErr := err.(type) {
 		case dupConnRegisterTunnelError:
@@ -295,18 +382,9 @@ func ServeTunnel(
 	return nil, true
 }
 
-func IsRPCStreamResponse(headers []h2mux.Header) bool {
-	if len(headers) != 1 {
-		return false
-	}
-	if headers[0].Name != ":status" || headers[0].Value != "200" {
-		return false
-	}
-	return true
-}
-
 func RegisterTunnel(
 	ctx context.Context,
+	credentialManager ReconnectTunnelCredentialManager,
 	muxer *h2mux.Muxer,
 	config *TunnelConfig,
 	logger *log.Entry,
@@ -315,45 +393,82 @@ func RegisterTunnel(
 	uuid uuid.UUID,
 ) error {
 	config.TransportLogger.Debug("initiating RPC stream to register")
-	stream, err := openStream(ctx, muxer)
+	tunnelServer, err := connection.NewRPCClient(ctx, muxer, config.TransportLogger.WithField("subsystem", "rpc-register"), openStreamTimeout)
 	if err != nil {
 		// RPC stream open error
-		return newClientRegisterTunnelError(err, config.Metrics.rpcFail)
+		return newClientRegisterTunnelError(err, config.Metrics.rpcFail, register)
 	}
-	if !IsRPCStreamResponse(stream.Headers) {
-		// stream response error
-		return newClientRegisterTunnelError(err, config.Metrics.rpcFail)
-	}
-	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(config.TransportLogger.WithField("subsystem", "rpc-register"), rpc.StreamTransport(stream)),
-		tunnelrpc.ConnLog(config.TransportLogger.WithField("subsystem", "rpc-transport")),
-	)
-	defer conn.Close()
-	ts := tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx)}
+	defer tunnelServer.Close()
 	// Request server info without blocking tunnel registration; must use capnp library directly.
-	tsClient := tunnelrpc.TunnelServer{Client: ts.Client}
-	serverInfoPromise := tsClient.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+	serverInfoPromise := tunnelrpc.TunnelServer{Client: tunnelServer.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
 		return nil
 	})
-	registration, err := ts.RegisterTunnel(
+	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger)
+	registration := tunnelServer.RegisterTunnel(
 		ctx,
 		config.OriginCert,
 		config.Hostname,
 		config.RegistrationOptions(connectionID, originLocalIP, uuid),
 	)
-	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger)
-	if err != nil {
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
 		// RegisterTunnel RPC failure
-		return newClientRegisterTunnelError(err, config.Metrics.regFail)
+		return processRegisterTunnelError(registrationErr, config.Metrics, register)
 	}
+	credentialManager.SetEventDigest(registration.EventDigest)
+	return processRegistrationSuccess(config, logger, connectionID, registration, register, credentialManager)
+}
+
+func ReconnectTunnel(
+	ctx context.Context,
+	token []byte,
+	eventDigest, connDigest []byte,
+	muxer *h2mux.Muxer,
+	config *TunnelConfig,
+	logger *log.Entry,
+	connectionID uint8,
+	originLocalIP string,
+	uuid uuid.UUID,
+	credentialManager ReconnectTunnelCredentialManager,
+) error {
+	config.TransportLogger.Debug("initiating RPC stream to reconnect")
+	tunnelServer, err := connection.NewRPCClient(ctx, muxer, config.TransportLogger.WithField("subsystem", "rpc-reconnect"), openStreamTimeout)
+	if err != nil {
+		// RPC stream open error
+		return newClientRegisterTunnelError(err, config.Metrics.rpcFail, reconnect)
+	}
+	defer tunnelServer.Close()
+	// Request server info without blocking tunnel registration; must use capnp library directly.
+	serverInfoPromise := tunnelrpc.TunnelServer{Client: tunnelServer.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+		return nil
+	})
+	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger)
+	registration := tunnelServer.ReconnectTunnel(
+		ctx,
+		token,
+		eventDigest,
+		connDigest,
+		config.Hostname,
+		config.RegistrationOptions(connectionID, originLocalIP, uuid),
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// ReconnectTunnel RPC failure
+		return processRegisterTunnelError(registrationErr, config.Metrics, reconnect)
+	}
+	return processRegistrationSuccess(config, logger, connectionID, registration, reconnect, credentialManager)
+}
+
+func processRegistrationSuccess(
+	config *TunnelConfig,
+	logger *log.Entry,
+	connectionID uint8,
+	registration *tunnelpogs.TunnelRegistration,
+	name registerRPCName,
+	credentialManager ReconnectTunnelCredentialManager,
+) error {
 	for _, logLine := range registration.LogLines {
 		logger.Info(logLine)
 	}
 
-	if regErr := processRegisterTunnelError(registration.Err, registration.PermanentFailure, config.Metrics); regErr != nil {
-		return regErr
-	}
-
 	if registration.TunnelID != "" {
 		config.Metrics.tunnelsHA.AddTunnelID(connectionID, registration.TunnelID)
 		logger.Infof("Each HA connection's tunnel IDs: %v", config.Metrics.tunnelsHA.String())
@@ -371,58 +486,36 @@ func RegisterTunnel(
 		}
 	}
 
+	credentialManager.SetConnDigest(connectionID, registration.ConnDigest)
 	config.Metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
 
 	logger.Infof("Route propagating, it may take up to 1 minute for your new route to become functional")
+	config.Metrics.regSuccess.WithLabelValues(string(name)).Inc()
 	return nil
 }
 
-func processRegisterTunnelError(err string, permanentFailure bool, metrics *TunnelMetrics) error {
-	if err == "" {
-		metrics.regSuccess.Inc()
-		return nil
-	}
-
-	metrics.regFail.WithLabelValues(err).Inc()
-	if err == DuplicateConnectionError {
+func processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, metrics *TunnelMetrics, name registerRPCName) error {
+	if err.Error() == DuplicateConnectionError {
+		metrics.regFail.WithLabelValues("dup_edge_conn", string(name)).Inc()
 		return dupConnRegisterTunnelError{}
 	}
+	metrics.regFail.WithLabelValues("server_error", string(name)).Inc()
 	return serverRegisterTunnelError{
-		cause:     fmt.Errorf("Server error: %s", err),
-		permanent: permanentFailure,
+		cause:     fmt.Errorf("Server error: %s", err.Error()),
+		permanent: err.IsPermanent(),
 	}
 }
 
 func UnregisterTunnel(muxer *h2mux.Muxer, gracePeriod time.Duration, logger *log.Logger) error {
 	logger.Debug("initiating RPC stream to unregister")
 	ctx := context.Background()
-	stream, err := openStream(ctx, muxer)
+	tunnelServer, err := connection.NewRPCClient(ctx, muxer, logger.WithField("subsystem", "rpc-unregister"), openStreamTimeout)
 	if err != nil {
 		// RPC stream open error
 		return err
 	}
-	if !IsRPCStreamResponse(stream.Headers) {
-		// stream response error
-		return err
-	}
-	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger.WithField("subsystem", "rpc-unregister"), rpc.StreamTransport(stream)),
-		tunnelrpc.ConnLog(logger.WithField("subsystem", "rpc-transport")),
-	)
-	defer conn.Close()
-	ts := tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx)}
 	// gracePeriod is encoded in int64 using capnproto
-	return ts.UnregisterTunnel(ctx, gracePeriod.Nanoseconds())
-}
-
-func openStream(ctx context.Context, muxer *h2mux.Muxer) (*h2mux.MuxedStream, error) {
-	openStreamCtx, cancel := context.WithTimeout(ctx, openStreamTimeout)
-	defer cancel()
-	return muxer.OpenStream(openStreamCtx, []h2mux.Header{
-		{Name: ":method", Value: "RPC"},
-		{Name: ":scheme", Value: "capnp"},
-		{Name: ":path", Value: "*"},
-	}, nil)
+	return tunnelServer.UnregisterTunnel(ctx, gracePeriod.Nanoseconds())
 }
 
 func LogServerInfo(
@@ -445,16 +538,6 @@ func LogServerInfo(
 	metrics.registerServerLocation(uint8ToString(connectionID), serverInfo.LocationName)
 }
 
-func H1ResponseToH2Response(h1 *http.Response) (h2 []h2mux.Header) {
-	h2 = []h2mux.Header{{Name: ":status", Value: fmt.Sprintf("%d", h1.StatusCode)}}
-	for headerName, headerValues := range h1.Header {
-		for _, headerValue := range headerValues {
-			h2 = append(h2, h2mux.Header{Name: strings.ToLower(headerName), Value: headerValue})
-		}
-	}
-	return
-}
-
 type TunnelHandler struct {
 	originUrl      string
 	httpHostHeader string
@@ -467,15 +550,16 @@ type TunnelHandler struct {
 	connectionID      string
 	logger            *log.Logger
 	noChunkedEncoding bool
-}
 
-var dialer = net.Dialer{DualStack: true}
+	bufferPool *buffer.Pool
+}
 
 // NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
 func NewTunnelHandler(ctx context.Context,
 	config *TunnelConfig,
-	addr string,
+	addr *net.TCPAddr,
 	connectionID uint8,
+	bufferPool *buffer.Pool,
 ) (*TunnelHandler, string, error) {
 	originURL, err := validation.ValidateUrl(config.OriginUrl)
 	if err != nil {
@@ -491,41 +575,23 @@ func NewTunnelHandler(ctx context.Context,
 		connectionID:      uint8ToString(connectionID),
 		logger:            config.Logger,
 		noChunkedEncoding: config.NoChunkedEncoding,
+		bufferPool:        bufferPool,
 	}
 	if h.httpClient == nil {
 		h.httpClient = http.DefaultTransport
 	}
-	// Inherit from parent context so we can cancel (Ctrl-C) while dialing
-	dialCtx, dialCancel := context.WithTimeout(ctx, dialTimeout)
-	// TUN-92: enforce a timeout on dial and handshake (as tls.Dial does not support one)
-	plaintextEdgeConn, err := dialer.DialContext(dialCtx, "tcp", addr)
-	dialCancel()
+
+	edgeConn, err := connection.DialEdge(ctx, dialTimeout, config.TlsConfig, addr)
 	if err != nil {
-		return nil, "", dialError{cause: errors.Wrap(err, "DialContext error")}
+		return nil, "", err
 	}
-	edgeConn := tls.Client(plaintextEdgeConn, config.TlsConfig)
-	edgeConn.SetDeadline(time.Now().Add(dialTimeout))
-	err = edgeConn.Handshake()
-	if err != nil {
-		return nil, "", dialError{cause: errors.Wrap(err, "Handshake with edge error")}
-	}
-	// clear the deadline on the conn; h2mux has its own timeouts
-	edgeConn.SetDeadline(time.Time{})
 	// Establish a muxed connection with the edge
 	// Client mux handshake with agent server
-	h.muxer, err = h2mux.Handshake(edgeConn, edgeConn, h2mux.MuxerConfig{
-		Timeout:            5 * time.Second,
-		Handler:            h,
-		IsClient:           true,
-		HeartbeatInterval:  config.HeartbeatInterval,
-		MaxHeartbeats:      config.MaxHeartbeats,
-		Logger:             config.TransportLogger.WithFields(log.Fields{}),
-		CompressionQuality: h2mux.CompressionSetting(config.CompressionQuality),
-	}, h.metrics.activeStreams)
+	h.muxer, err = h2mux.Handshake(edgeConn, edgeConn, config.muxerConfig(h), h.metrics.activeStreams)
 	if err != nil {
-		return h, "", errors.New("TLS handshake error")
+		return nil, "", errors.Wrap(err, "h2mux handshake with edge error")
 	}
-	return h, edgeConn.LocalAddr().String(), err
+	return h, edgeConn.LocalAddr().String(), nil
 }
 
 func (h *TunnelHandler) AppendTagHeaders(r *http.Request) {
@@ -540,7 +606,7 @@ func (h *TunnelHandler) ServeStream(stream *h2mux.MuxedStream) error {
 
 	req, reqErr := h.createRequest(stream)
 	if reqErr != nil {
-		h.logError(stream, reqErr)
+		h.writeErrorResponse(stream, reqErr)
 		return reqErr
 	}
 
@@ -556,7 +622,7 @@ func (h *TunnelHandler) ServeStream(stream *h2mux.MuxedStream) error {
 		resp, respErr = h.serveHTTP(stream, req)
 	}
 	if respErr != nil {
-		h.logError(stream, respErr)
+		h.writeErrorResponse(stream, respErr)
 		return respErr
 	}
 	h.logResponseOk(resp, cfRay, lbProbe)
@@ -568,7 +634,7 @@ func (h *TunnelHandler) createRequest(stream *h2mux.MuxedStream) (*http.Request,
 	if err != nil {
 		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
 	}
-	err = streamhandler.H2RequestHeadersToH1Request(stream.Headers, req)
+	err = h2mux.H2RequestHeadersToH1Request(stream.Headers, req)
 	if err != nil {
 		return nil, errors.Wrap(err, "invalid request received")
 	}
@@ -587,13 +653,14 @@ func (h *TunnelHandler) serveWebsocket(stream *h2mux.MuxedStream, req *http.Requ
 		return nil, err
 	}
 	defer conn.Close()
-	err = stream.WriteHeaders(H1ResponseToH2Response(response))
+	err = stream.WriteHeaders(h2mux.H1ResponseToH2ResponseHeaders(response))
 	if err != nil {
 		return nil, errors.Wrap(err, "Error writing response header")
 	}
 	// Copy to/from stream to the undelying connection. Use the underlying
 	// connection because cloudflared doesn't operate on the message themselves
 	websocket.Stream(conn.UnderlyingConn(), stream)
+
 	return response, nil
 }
 
@@ -621,7 +688,9 @@ func (h *TunnelHandler) serveHTTP(stream *h2mux.MuxedStream, req *http.Request)
 	}
 	defer response.Body.Close()
 
-	err = stream.WriteHeaders(H1ResponseToH2Response(response))
+	headers := h2mux.H1ResponseToH2ResponseHeaders(response)
+	headers = append(headers, h2mux.CreateResponseMetaHeader(h2mux.ResponseMetaHeaderField, h2mux.ResponseSourceOrigin))
+	err = stream.WriteHeaders(headers)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error writing response header")
 	}
@@ -630,7 +699,9 @@ func (h *TunnelHandler) serveHTTP(stream *h2mux.MuxedStream, req *http.Request)
 	} else {
 		// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
 		// compression generates dictionary on first write
-		io.CopyBuffer(stream, response.Body, make([]byte, 512*1024))
+		buf := h.bufferPool.Get()
+		defer h.bufferPool.Put(buf)
+		io.CopyBuffer(stream, response.Body, buf)
 	}
 	return response, nil
 }
@@ -654,9 +725,12 @@ func (h *TunnelHandler) isEventStream(response *http.Response) bool {
 	return false
 }
 
-func (h *TunnelHandler) logError(stream *h2mux.MuxedStream, err error) {
+func (h *TunnelHandler) writeErrorResponse(stream *h2mux.MuxedStream, err error) {
 	h.logger.WithError(err).Error("HTTP request error")
-	stream.WriteHeaders([]h2mux.Header{{Name: ":status", Value: "502"}})
+	stream.WriteHeaders([]h2mux.Header{
+		{Name: ":status", Value: "502"},
+		h2mux.CreateResponseMetaHeader(h2mux.ResponseMetaHeaderField, h2mux.ResponseSourceCloudflared),
+	})
 	stream.Write([]byte("502 Bad Gateway"))
 	h.metrics.incrementResponses(h.connectionID, "502")
 }

originservice/originservice.go

@@ -12,11 +12,11 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/cloudflare/cloudflared/buffer"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/log"
 	"github.com/cloudflare/cloudflared/websocket"
-
 	"github.com/pkg/errors"
 )
 
@@ -33,6 +33,7 @@ type HTTPService struct {
 	client          http.RoundTripper
 	originURL       *url.URL
 	chunkedEncoding bool
+	bufferPool      *buffer.Pool
 }
 
 func NewHTTPService(transport http.RoundTripper, url *url.URL, chunkedEncoding bool) OriginService {
@@ -40,10 +41,13 @@ func NewHTTPService(transport http.RoundTripper, url *url.URL, chunkedEncoding b
 		client:          transport,
 		originURL:       url,
 		chunkedEncoding: chunkedEncoding,
+		bufferPool:      buffer.NewPool(512 * 1024),
 	}
 }
 
 func (hc *HTTPService) Proxy(stream *h2mux.MuxedStream, req *http.Request) (*http.Response, error) {
+	const responseSourceOrigin = "origin"
+
 	// Support for WSGI Servers by switching transfer encoding from chunked to gzip/deflate
 	if !hc.chunkedEncoding {
 		req.TransferEncoding = []string{"gzip", "deflate"}
@@ -62,7 +66,9 @@ func (hc *HTTPService) Proxy(stream *h2mux.MuxedStream, req *http.Request) (*htt
 	}
 	defer resp.Body.Close()
 
-	err = stream.WriteHeaders(h1ResponseToH2Response(resp))
+	responseHeaders := h1ResponseToH2Response(resp)
+	responseHeaders = append(responseHeaders, h2mux.CreateResponseMetaHeader(h2mux.ResponseMetaHeaderField, responseSourceOrigin))
+	err = stream.WriteHeaders(responseHeaders)
 	if err != nil {
 		return nil, errors.Wrap(err, "error writing response header to HTTP origin")
 	}
@@ -71,7 +77,9 @@ func (hc *HTTPService) Proxy(stream *h2mux.MuxedStream, req *http.Request) (*htt
 	} else {
 		// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
 		// compression generates dictionary on first write
-		io.CopyBuffer(stream, resp.Body, make([]byte, 512*1024))
+		buf := hc.bufferPool.Get()
+		defer hc.bufferPool.Put(buf)
+		io.CopyBuffer(stream, resp.Body, buf)
 	}
 	return resp, nil
 }
@@ -100,7 +108,7 @@ func NewWebSocketService(tlsConfig *tls.Config, url *url.URL) (OriginService, er
 	}
 	shutdownC := make(chan struct{})
 	go func() {
-		websocket.StartProxyServer(log.CreateLogger(), listener, url.String(), shutdownC)
+		websocket.StartProxyServer(log.CreateLogger(), listener, url.String(), shutdownC, websocket.DefaultStreamHandler)
 	}()
 	return &WebsocketService{
 		tlsConfig: tlsConfig,
@@ -142,10 +150,11 @@ func (wsc *WebsocketService) Shutdown() {
 
 // HelloWorldService talks to the hello world example origin
 type HelloWorldService struct {
-	client    http.RoundTripper
-	listener  net.Listener
-	originURL *url.URL
-	shutdownC chan struct{}
+	client     http.RoundTripper
+	listener   net.Listener
+	originURL  *url.URL
+	shutdownC  chan struct{}
+	bufferPool *buffer.Pool
 }
 
 func NewHelloWorldService(transport http.RoundTripper) (OriginService, error) {
@@ -164,7 +173,8 @@ func NewHelloWorldService(transport http.RoundTripper) (OriginService, error) {
 			Scheme: "https",
 			Host:   listener.Addr().String(),
 		},
-		shutdownC: shutdownC,
+		shutdownC:  shutdownC,
+		bufferPool: buffer.NewPool(512 * 1024),
 	}, nil
 }
 
@@ -184,7 +194,9 @@ func (hwc *HelloWorldService) Proxy(stream *h2mux.MuxedStream, req *http.Request
 
 	// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
 	// compression generates dictionary on first write
-	io.CopyBuffer(stream, resp.Body, make([]byte, 512*1024))
+	buf := hwc.bufferPool.Get()
+	defer hwc.bufferPool.Put(buf)
+	io.CopyBuffer(stream, resp.Body, buf)
 
 	return resp, nil
 }

socks/auth_handler.go

@@ -0,0 +1,77 @@
+package socks
+
+import (
+	"fmt"
+	"io"
+)
+
+const (
+	// NoAuth means no authentication is used when connecting
+	NoAuth = uint8(0)
+
+	// UserPassAuth means a user/password is used when connecting
+	UserPassAuth = uint8(2)
+
+	noAcceptable    = uint8(255)
+	userAuthVersion = uint8(1)
+	authSuccess     = uint8(0)
+	authFailure     = uint8(1)
+)
+
+// AuthHandler handles socks authenication requests
+type AuthHandler interface {
+	Handle(io.Reader, io.Writer) error
+	Register(uint8, Authenticator)
+}
+
+// StandardAuthHandler loads the default authenticators
+type StandardAuthHandler struct {
+	authenticators map[uint8]Authenticator
+}
+
+// NewAuthHandler creates a default auth handler
+func NewAuthHandler() AuthHandler {
+	defaults := make(map[uint8]Authenticator)
+	defaults[NoAuth] = NewNoAuthAuthenticator()
+	return &StandardAuthHandler{
+		authenticators: defaults,
+	}
+}
+
+// Register adds/replaces an Authenticator to use when handling Authentication requests
+func (h *StandardAuthHandler) Register(method uint8, a Authenticator) {
+	h.authenticators[method] = a
+}
+
+// Handle gets the methods from the SOCKS5 client and authenicates with the first supported method
+func (h *StandardAuthHandler) Handle(bufConn io.Reader, conn io.Writer) error {
+	methods, err := readMethods(bufConn)
+	if err != nil {
+		return fmt.Errorf("Failed to read auth methods: %v", err)
+	}
+
+	// first supported method is used
+	for _, method := range methods {
+		authenticator := h.authenticators[method]
+		if authenticator != nil {
+			return authenticator.Handle(bufConn, conn)
+		}
+	}
+
+	// failed to authenticate. No supported authentication type found
+	conn.Write([]byte{socks5Version, noAcceptable})
+	return fmt.Errorf("unknown authentication type")
+}
+
+// readMethods is used to read the number and type of methods
+func readMethods(r io.Reader) ([]byte, error) {
+	header := []byte{0}
+	if _, err := r.Read(header); err != nil {
+		return nil, err
+	}
+
+	numMethods := int(header[0])
+	methods := make([]byte, numMethods)
+	_, err := io.ReadAtLeast(r, methods, numMethods)
+	return methods, err
+}

socks/authenticator.go

@@ -0,0 +1,87 @@
+package socks
+
+import (
+	"fmt"
+	"io"
+)
+
+// Authenticator is the connection passed in as a reader/writer to support different authentication types
+type Authenticator interface {
+	Handle(io.Reader, io.Writer) error
+}
+
+// NoAuthAuthenticator is used to handle the No Authentication mode
+type NoAuthAuthenticator struct{}
+
+// NewNoAuthAuthenticator creates a authless Authenticator
+func NewNoAuthAuthenticator() Authenticator {
+	return &NoAuthAuthenticator{}
+}
+
+// Handle writes back the version and NoAuth
+func (a *NoAuthAuthenticator) Handle(reader io.Reader, writer io.Writer) error {
+	_, err := writer.Write([]byte{socks5Version, NoAuth})
+	return err
+}
+
+// UserPassAuthAuthenticator is used to handle the user/password mode
+type UserPassAuthAuthenticator struct {
+	IsValid func(string, string) bool
+}
+
+// NewUserPassAuthAuthenticator creates a new username/password validator Authenticator
+func NewUserPassAuthAuthenticator(isValid func(string, string) bool) Authenticator {
+	return &UserPassAuthAuthenticator{
+		IsValid: isValid,
+	}
+}
+
+// Handle writes back the version and NoAuth
+func (a *UserPassAuthAuthenticator) Handle(reader io.Reader, writer io.Writer) error {
+	if _, err := writer.Write([]byte{socks5Version, UserPassAuth}); err != nil {
+		return err
+	}
+
+	// Get the version and username length
+	header := []byte{0, 0}
+	if _, err := io.ReadAtLeast(reader, header, 2); err != nil {
+		return err
+	}
+
+	// Ensure compatibility. Someone call E-harmony
+	if header[0] != userAuthVersion {
+		return fmt.Errorf("Unsupported auth version: %v", header[0])
+	}
+
+	// Get the user name
+	userLen := int(header[1])
+	user := make([]byte, userLen)
+	if _, err := io.ReadAtLeast(reader, user, userLen); err != nil {
+		return err
+	}
+
+	// Get the password length
+	if _, err := reader.Read(header[:1]); err != nil {
+		return err
+	}
+
+	// Get the password
+	passLen := int(header[0])
+	pass := make([]byte, passLen)
+	if _, err := io.ReadAtLeast(reader, pass, passLen); err != nil {
+		return err
+	}
+
+	// Verify the password
+	if a.IsValid(string(user), string(pass)) {
+		_, err := writer.Write([]byte{userAuthVersion, authSuccess})
+		return err
+	}
+
+	// password failed. Write back failure
+	if _, err := writer.Write([]byte{userAuthVersion, authFailure}); err != nil {
+		return err
+	}
+
+	return fmt.Errorf("User authentication failed")
+}

socks/connection_handler.go

@@ -0,0 +1,57 @@
+package socks
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+)
+
+// ConnectionHandler is the Serve method to handle connections
+// from a local TCP listener of the standard library (net.Listener)
+type ConnectionHandler interface {
+	Serve(io.ReadWriter) error
+}
+
+// StandardConnectionHandler is the base implementation of handling SOCKS5 requests
+type StandardConnectionHandler struct {
+	requestHandler RequestHandler
+	authHandler    AuthHandler
+}
+
+// NewConnectionHandler creates a standard SOCKS5 connection handler
+// This process connections from a generic TCP listener from the standard library
+func NewConnectionHandler(requestHandler RequestHandler) ConnectionHandler {
+	return &StandardConnectionHandler{
+		requestHandler: requestHandler,
+		authHandler:    NewAuthHandler(),
+	}
+}
+
+// Serve process new connection created after calling `Accept()` in the standard library
+func (h *StandardConnectionHandler) Serve(c io.ReadWriter) error {
+	bufConn := bufio.NewReader(c)
+
+	// read the version byte
+	version := []byte{0}
+	if _, err := bufConn.Read(version); err != nil {
+		return err
+	}
+
+	// ensure compatibility
+	if version[0] != socks5Version {
+		return fmt.Errorf("Unsupported SOCKS version: %v", version)
+	}
+
+	// handle auth
+	if err := h.authHandler.Handle(bufConn, c); err != nil {
+		return err
+	}
+
+	// process command/request
+	req, err := NewRequest(bufConn)
+	if err != nil {
+		return err
+	}
+
+	return h.requestHandler.Handle(req, c)
+}

socks/connection_handler_test.go

@@ -0,0 +1,88 @@
+package socks
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/proxy"
+)
+
+type successResponse struct {
+	Status string `json:"status"`
+}
+
+func sendSocksRequest(t *testing.T) []byte {
+	dialer, err := proxy.SOCKS5("tcp", "127.0.0.1:8086", nil, proxy.Direct)
+	assert.NoError(t, err)
+
+	httpTransport := &http.Transport{}
+	httpClient := &http.Client{Transport: httpTransport}
+	// set our socks5 as the dialer
+	httpTransport.Dial = dialer.Dial
+
+	req, err := http.NewRequest("GET", "http://127.0.0.1:8085", nil)
+	assert.NoError(t, err)
+
+	resp, err := httpClient.Do(req)
+	assert.NoError(t, err)
+	defer resp.Body.Close()
+
+	b, err := ioutil.ReadAll(resp.Body)
+	assert.NoError(t, err)
+
+	return b
+}
+
+func startTestServer(t *testing.T, httpHandler func(w http.ResponseWriter, r *http.Request)) {
+	// create a socks server
+	requestHandler := NewRequestHandler(NewNetDialer())
+	socksServer := NewConnectionHandler(requestHandler)
+	listener, err := net.Listen("tcp", ":8086")
+	assert.NoError(t, err)
+
+	go func() {
+		defer listener.Close()
+		for {
+			conn, _ := listener.Accept()
+			go socksServer.Serve(conn)
+		}
+	}()
+
+	// create an http server
+	mux := http.NewServeMux()
+	mux.HandleFunc("/", httpHandler)
+
+	// start the servers
+	go http.ListenAndServe(":8085", mux)
+
+}
+
+func respondWithJSON(w http.ResponseWriter, v interface{}, status int) {
+	data, _ := json.Marshal(v)
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	w.Write(data)
+}
+
+func OkJSONResponseHandler(w http.ResponseWriter, r *http.Request) {
+	resp := successResponse{
+		Status: "ok",
+	}
+	respondWithJSON(w, resp, http.StatusOK)
+}
+
+func TestSocksConnection(t *testing.T) {
+	startTestServer(t, OkJSONResponseHandler)
+	b := sendSocksRequest(t)
+	assert.True(t, len(b) > 0, "no data returned!")
+
+	var resp successResponse
+	json.Unmarshal(b, &resp)
+
+	assert.True(t, resp.Status == "ok", "response didn't return ok")
+}

socks/dialer.go

@@ -0,0 +1,57 @@
+package socks
+
+import (
+	"fmt"
+	"io"
+	"net"
+)
+
+// Dialer is used to provided the transport of the proxy
+type Dialer interface {
+	Dial(string) (io.ReadWriteCloser, *AddrSpec, error)
+}
+
+// NetDialer is a standard TCP dialer
+type NetDialer struct {
+}
+
+// NewNetDialer creates a new dialer
+func NewNetDialer() Dialer {
+	return &NetDialer{}
+}
+
+// Dial is a base TCP dialer
+func (d *NetDialer) Dial(address string) (io.ReadWriteCloser, *AddrSpec, error) {
+	c, err := net.Dial("tcp", address)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	local := c.LocalAddr().(*net.TCPAddr)
+	addr := AddrSpec{IP: local.IP, Port: local.Port}
+
+	return c, &addr, nil
+}
+
+// ConnDialer is like NetDialer but with an existing TCP dialer already created
+type ConnDialer struct {
+	conn net.Conn
+}
+
+// NewConnDialer creates a new dialer with a already created net.conn (TCP expected)
+func NewConnDialer(conn net.Conn) Dialer {
+	return &ConnDialer{
+		conn: conn,
+	}
+}
+
+// Dial is a TCP dialer but already created
+func (d *ConnDialer) Dial(address string) (io.ReadWriteCloser, *AddrSpec, error) {
+	local, ok := d.conn.LocalAddr().(*net.TCPAddr)
+	if !ok {
+		return nil, nil, fmt.Errorf("not a tcp connection")
+	}
+
+	addr := AddrSpec{IP: local.IP, Port: local.Port}
+	return d.conn, &addr, nil
+}

socks/request.go

@@ -0,0 +1,195 @@
+package socks
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"strconv"
+)
+
+const (
+	// version
+	socks5Version = uint8(5)
+
+	// commands https://tools.ietf.org/html/rfc1928#section-4
+	connectCommand   = uint8(1)
+	bindCommand      = uint8(2)
+	associateCommand = uint8(3)
+
+	// address types
+	ipv4Address = uint8(1)
+	fqdnAddress = uint8(3)
+	ipv6Address = uint8(4)
+)
+
+// https://tools.ietf.org/html/rfc1928#section-6
+const (
+	successReply uint8 = iota
+	serverFailure
+	ruleFailure
+	networkUnreachable
+	hostUnreachable
+	connectionRefused
+	ttlExpired
+	commandNotSupported
+	addrTypeNotSupported
+)
+
+// AddrSpec is used to return the target IPv4, IPv6, or a FQDN
+type AddrSpec struct {
+	FQDN string
+	IP   net.IP
+	Port int
+}
+
+// String gives a host version of the Address
+func (a *AddrSpec) String() string {
+	if a.FQDN != "" {
+		return fmt.Sprintf("%s (%s):%d", a.FQDN, a.IP, a.Port)
+	}
+	return fmt.Sprintf("%s:%d", a.IP, a.Port)
+}
+
+// Address returns a string suitable to dial; prefer returning IP-based
+// address, fallback to FQDN
+func (a AddrSpec) Address() string {
+	if len(a.IP) != 0 {
+		return net.JoinHostPort(a.IP.String(), strconv.Itoa(a.Port))
+	}
+	return net.JoinHostPort(a.FQDN, strconv.Itoa(a.Port))
+}
+
+// Request is a SOCKS5 command with supporting field of the connection
+type Request struct {
+	// Protocol version
+	Version uint8
+	// Requested command
+	Command uint8
+	// AddrSpec of the destination
+	DestAddr *AddrSpec
+	// reading from the connection
+	bufConn io.Reader
+}
+
+// NewRequest creates a new request from the connection data stream
+func NewRequest(bufConn io.Reader) (*Request, error) {
+	// Read the version byte
+	header := []byte{0, 0, 0}
+	if _, err := io.ReadAtLeast(bufConn, header, 3); err != nil {
+		return nil, fmt.Errorf("Failed to get command version: %v", err)
+	}
+
+	// ensure compatibility
+	if header[0] != socks5Version {
+		return nil, fmt.Errorf("Unsupported command version: %v", header[0])
+	}
+
+	// Read in the destination address
+	dest, err := readAddrSpec(bufConn)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Request{
+		Version:  socks5Version,
+		Command:  header[1],
+		DestAddr: dest,
+		bufConn:  bufConn,
+	}, nil
+}
+
+func sendReply(w io.Writer, resp uint8, addr *AddrSpec) error {
+	var addrType uint8
+	var addrBody []byte
+	var addrPort uint16
+	switch {
+	case addr == nil:
+		addrType = ipv4Address
+		addrBody = []byte{0, 0, 0, 0}
+		addrPort = 0
+
+	case addr.FQDN != "":
+		addrType = fqdnAddress
+		addrBody = append([]byte{byte(len(addr.FQDN))}, addr.FQDN...)
+		addrPort = uint16(addr.Port)
+
+	case addr.IP.To4() != nil:
+		addrType = ipv4Address
+		addrBody = []byte(addr.IP.To4())
+		addrPort = uint16(addr.Port)
+
+	case addr.IP.To16() != nil:
+		addrType = ipv6Address
+		addrBody = []byte(addr.IP.To16())
+		addrPort = uint16(addr.Port)
+
+	default:
+		return fmt.Errorf("Failed to format address: %v", addr)
+	}
+
+	// Format the message
+	msg := make([]byte, 6+len(addrBody))
+	msg[0] = socks5Version
+	msg[1] = resp
+	msg[2] = 0 // Reserved
+	msg[3] = addrType
+	copy(msg[4:], addrBody)
+	msg[4+len(addrBody)] = byte(addrPort >> 8)
+	msg[4+len(addrBody)+1] = byte(addrPort & 0xff)
+
+	// Send the message
+	_, err := w.Write(msg)
+	return err
+}
+
+// readAddrSpec is used to read AddrSpec.
+// Expects an address type byte, followed by the address and port
+func readAddrSpec(r io.Reader) (*AddrSpec, error) {
+	d := &AddrSpec{}
+
+	// Get the address type
+	addrType := []byte{0}
+	if _, err := r.Read(addrType); err != nil {
+		return nil, err
+	}
+
+	// Handle on a per type basis
+	switch addrType[0] {
+	case ipv4Address:
+		addr := make([]byte, 4)
+		if _, err := io.ReadAtLeast(r, addr, len(addr)); err != nil {
+			return nil, err
+		}
+		d.IP = net.IP(addr)
+
+	case ipv6Address:
+		addr := make([]byte, 16)
+		if _, err := io.ReadAtLeast(r, addr, len(addr)); err != nil {
+			return nil, err
+		}
+		d.IP = net.IP(addr)
+
+	case fqdnAddress:
+		if _, err := r.Read(addrType); err != nil {
+			return nil, err
+		}
+		addrLen := int(addrType[0])
+		fqdn := make([]byte, addrLen)
+		if _, err := io.ReadAtLeast(r, fqdn, addrLen); err != nil {
+			return nil, err
+		}
+		d.FQDN = string(fqdn)
+
+	default:
+		return nil, fmt.Errorf("Unrecognized address type")
+	}
+
+	// Read the port
+	port := []byte{0, 0}
+	if _, err := io.ReadAtLeast(r, port, 2); err != nil {
+		return nil, err
+	}
+	d.Port = (int(port[0]) << 8) | int(port[1])
+
+	return d, nil
+}

socks/request_handler.go

@@ -0,0 +1,106 @@
+package socks
+
+import (
+	"fmt"
+	"io"
+	"strings"
+)
+
+// RequestHandler is the functions needed to handle a SOCKS5 command
+type RequestHandler interface {
+	Handle(*Request, io.ReadWriter) error
+}
+
+// StandardRequestHandler implements the base socks5 command processing
+type StandardRequestHandler struct {
+	dialer Dialer
+}
+
+// NewRequestHandler creates a standard SOCKS5 request handler
+// This handles the SOCKS5 commands and proxies them to their destination
+func NewRequestHandler(dialer Dialer) RequestHandler {
+	return &StandardRequestHandler{
+		dialer: dialer,
+	}
+}
+
+// Handle processes and responds to socks5 commands
+func (h *StandardRequestHandler) Handle(req *Request, conn io.ReadWriter) error {
+	switch req.Command {
+	case connectCommand:
+		return h.handleConnect(conn, req)
+	case bindCommand:
+		return h.handleBind(conn, req)
+	case associateCommand:
+		return h.handleAssociate(conn, req)
+	default:
+		if err := sendReply(conn, commandNotSupported, nil); err != nil {
+			return fmt.Errorf("Failed to send reply: %v", err)
+		}
+		return fmt.Errorf("Unsupported command: %v", req.Command)
+	}
+}
+
+// handleConnect is used to handle a connect command
+func (h *StandardRequestHandler) handleConnect(conn io.ReadWriter, req *Request) error {
+	target, localAddr, err := h.dialer.Dial(req.DestAddr.Address())
+	if err != nil {
+		msg := err.Error()
+		resp := hostUnreachable
+		if strings.Contains(msg, "refused") {
+			resp = connectionRefused
+		} else if strings.Contains(msg, "network is unreachable") {
+			resp = networkUnreachable
+		}
+		if err := sendReply(conn, resp, nil); err != nil {
+			return fmt.Errorf("Failed to send reply: %v", err)
+		}
+		return fmt.Errorf("Connect to %v failed: %v", req.DestAddr, err)
+	}
+	defer target.Close()
+
+	// Send success
+	if err := sendReply(conn, successReply, localAddr); err != nil {
+		return fmt.Errorf("Failed to send reply: %v", err)
+	}
+
+	// Start proxying
+	proxyDone := make(chan error, 2)
+
+	go func() {
+		_, e := io.Copy(target, req.bufConn)
+		proxyDone <- e
+	}()
+
+	go func() {
+		_, e := io.Copy(conn, target)
+		proxyDone <- e
+	}()
+
+	// Wait for both
+	for i := 0; i < 2; i++ {
+		e := <-proxyDone
+		if e != nil {
+			return e
+		}
+	}
+	return nil
+}
+
+// handleBind is used to handle a bind command
+// TODO: Support bind command
+func (h *StandardRequestHandler) handleBind(conn io.ReadWriter, req *Request) error {
+	if err := sendReply(conn, commandNotSupported, nil); err != nil {
+		return fmt.Errorf("Failed to send reply: %v", err)
+	}
+	return nil
+}
+
+// handleAssociate is used to handle a connect command
+// TODO: Support associate command
+func (h *StandardRequestHandler) handleAssociate(conn io.ReadWriter, req *Request) error {
+	if err := sendReply(conn, commandNotSupported, nil); err != nil {
+		return fmt.Errorf("Failed to send reply: %v", err)
+	}
+	return nil
+}

socks/request_handler_test.go

@@ -0,0 +1,28 @@
+package socks
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUnsupportedBind(t *testing.T) {
+	req := createRequest(t, socks5Version, bindCommand, "2001:db8::68", 1337, false)
+	var b bytes.Buffer
+
+	requestHandler := NewRequestHandler(NewNetDialer())
+	err := requestHandler.Handle(req, &b)
+	assert.NoError(t, err)
+	assert.True(t, b.Bytes()[1] == commandNotSupported, "expected a response")
+}
+
+func TestUnsupportedAssociate(t *testing.T) {
+	req := createRequest(t, socks5Version, associateCommand, "127.0.0.1", 1337, false)
+	var b bytes.Buffer
+
+	requestHandler := NewRequestHandler(NewNetDialer())
+	err := requestHandler.Handle(req, &b)
+	assert.NoError(t, err)
+	assert.True(t, b.Bytes()[1] == commandNotSupported, "expected a response")
+}

socks/request_test.go

@@ -0,0 +1,69 @@
+package socks
+
+import (
+	"bytes"
+	"encoding/binary"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func createRequestData(version, command uint8, ip net.IP, port uint16) []byte {
+	// set the command
+	b := []byte{version, command, 0}
+
+	// append the ip
+	if len(ip) == net.IPv4len {
+		b = append(b, 1)
+		b = append(b, ip.To4()...)
+	} else {
+		b = append(b, 4)
+		b = append(b, ip.To16()...)
+	}
+
+	// append the port
+	p := []byte{0, 0}
+	binary.BigEndian.PutUint16(p, port)
+	b = append(b, p...)
+
+	return b
+}
+
+func createRequest(t *testing.T, version, command uint8, ipStr string, port uint16, shouldFail bool) *Request {
+	ip := net.ParseIP(ipStr)
+	data := createRequestData(version, command, ip, port)
+	reader := bytes.NewReader(data)
+	req, err := NewRequest(reader)
+	if shouldFail {
+		assert.Error(t, err)
+		return nil
+	}
+	assert.NoError(t, err)
+	assert.True(t, req.Version == socks5Version, "version doesn't match expectation: %v", req.Version)
+	assert.True(t, req.Command == command, "command doesn't match expectation: %v", req.Command)
+	assert.True(t, req.DestAddr.Port == int(port), "port doesn't match expectation: %v", req.DestAddr.Port)
+	assert.True(t, req.DestAddr.IP.String() == ipStr, "ip doesn't match expectation: %v", req.DestAddr.IP.String())
+
+	return req
+}
+
+func TestValidConnectRequest(t *testing.T) {
+	createRequest(t, socks5Version, connectCommand, "127.0.0.1", 1337, false)
+}
+
+func TestValidBindRequest(t *testing.T) {
+	createRequest(t, socks5Version, bindCommand, "2001:db8::68", 1337, false)
+}
+
+func TestValidAssociateRequest(t *testing.T) {
+	createRequest(t, socks5Version, associateCommand, "127.0.0.1", 1234, false)
+}
+
+func TestInValidVersionRequest(t *testing.T) {
+	createRequest(t, 4, connectCommand, "127.0.0.1", 1337, true)
+}
+
+func TestInValidIPRequest(t *testing.T) {
+	createRequest(t, 4, connectCommand, "127.0.01", 1337, true)
+}

streamhandler/request.go

@@ -1,10 +1,8 @@
 package streamhandler
 
 import (
-	"fmt"
 	"net/http"
 	"net/url"
-	"strconv"
 	"strings"
 
 	"github.com/cloudflare/cloudflared/h2mux"
@@ -28,65 +26,9 @@ func createRequest(stream *h2mux.MuxedStream, url *url.URL) (*http.Request, erro
 	if err != nil {
 		return nil, errors.Wrap(err, "unexpected error from http.NewRequest")
 	}
-	err = H2RequestHeadersToH1Request(stream.Headers, req)
+	err = h2mux.H2RequestHeadersToH1Request(stream.Headers, req)
 	if err != nil {
 		return nil, errors.Wrap(err, "invalid request received")
 	}
 	return req, nil
 }
-
-// H2RequestHeadersToH1Request converts the HTTP/2 headers to an HTTP/1 Request
-// object. This includes conversion of the pseudo-headers into their closest
-// HTTP/1 equivalents. See https://tools.ietf.org/html/rfc7540#section-8.1.2.3
-func H2RequestHeadersToH1Request(h2 []h2mux.Header, h1 *http.Request) error {
-	for _, header := range h2 {
-		switch header.Name {
-		case ":method":
-			h1.Method = header.Value
-		case ":scheme":
-			// noop - use the preexisting scheme from h1.URL
-		case ":authority":
-			// Otherwise the host header will be based on the origin URL
-			h1.Host = header.Value
-		case ":path":
-			// We don't want to be an "opinionated" proxy, so ideally we would use :path as-is.
-			// However, this HTTP/1 Request object belongs to the Go standard library,
-			// whose URL package makes some opinionated decisions about the encoding of
-			// URL characters: see the docs of https://godoc.org/net/url#URL,
-			// in particular the EscapedPath method https://godoc.org/net/url#URL.EscapedPath,
-			// which is always used when computing url.URL.String(), whether we'd like it or not.
-			//
-			// Well, not *always*. We could circumvent this by using url.URL.Opaque. But
-			// that would present unusual difficulties when using an HTTP proxy: url.URL.Opaque
-			// is treated differently when HTTP_PROXY is set!
-			// See https://github.com/golang/go/issues/5684#issuecomment-66080888
-			//
-			// This means we are subject to the behavior of net/url's function `shouldEscape`
-			// (as invoked with mode=encodePath): https://github.com/golang/go/blob/go1.12.7/src/net/url/url.go#L101
-
-			if header.Value == "*" {
-				h1.URL.Path = "*"
-				continue
-			}
-			// Due to the behavior of validation.ValidateUrl, h1.URL may
-			// already have a partial value, with or without a trailing slash.
-			base := h1.URL.String()
-			base = strings.TrimRight(base, "/")
-			// But we know :path begins with '/', because we handled '*' above - see RFC7540
-			url, err := url.Parse(base + header.Value)
-			if err != nil {
-				return errors.Wrap(err, fmt.Sprintf("invalid path '%v'", header.Value))
-			}
-			h1.URL = url
-		case "content-length":
-			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
-			if err != nil {
-				return fmt.Errorf("unparseable content length")
-			}
-			h1.ContentLength = contentLength
-		default:
-			h1.Header.Add(http.CanonicalHeaderKey(header.Name), header.Value)
-		}
-	}
-	return nil
-}

streamhandler/request_test.go

@@ -1,441 +0,0 @@
-package streamhandler
-
-import (
-	"fmt"
-	"math/rand"
-	"net/http"
-	"net/url"
-	"reflect"
-	"regexp"
-	"strings"
-	"testing"
-	"testing/quick"
-
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestH2RequestHeadersToH1Request_RegularHeaders(t *testing.T) {
-	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
-	assert.NoError(t, err)
-
-	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{
-			h2mux.Header{
-				Name:  "Mock header 1",
-				Value: "Mock value 1",
-			},
-			h2mux.Header{
-				Name:  "Mock header 2",
-				Value: "Mock value 2",
-			},
-		},
-		request,
-	)
-
-	assert.Equal(t, http.Header{
-		"Mock header 1": []string{"Mock value 1"},
-		"Mock header 2": []string{"Mock value 2"},
-	}, request.Header)
-
-	assert.NoError(t, headersConversionErr)
-}
-
-func TestH2RequestHeadersToH1Request_NoHeaders(t *testing.T) {
-	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
-	assert.NoError(t, err)
-
-	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{},
-		request,
-	)
-
-	assert.Equal(t, http.Header{}, request.Header)
-
-	assert.NoError(t, headersConversionErr)
-}
-
-func TestH2RequestHeadersToH1Request_InvalidHostPath(t *testing.T) {
-	request, err := http.NewRequest(http.MethodGet, "http://example.com", nil)
-	assert.NoError(t, err)
-
-	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{
-			h2mux.Header{
-				Name:  ":path",
-				Value: "//bad_path/",
-			},
-			h2mux.Header{
-				Name:  "Mock header",
-				Value: "Mock value",
-			},
-		},
-		request,
-	)
-
-	assert.Equal(t, http.Header{
-		"Mock header": []string{"Mock value"},
-	}, request.Header)
-
-	assert.Equal(t, "http://example.com//bad_path/", request.URL.String())
-
-	assert.NoError(t, headersConversionErr)
-}
-
-func TestH2RequestHeadersToH1Request_HostPathWithQuery(t *testing.T) {
-	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
-	assert.NoError(t, err)
-
-	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{
-			h2mux.Header{
-				Name:  ":path",
-				Value: "/?query=mock%20value",
-			},
-			h2mux.Header{
-				Name:  "Mock header",
-				Value: "Mock value",
-			},
-		},
-		request,
-	)
-
-	assert.Equal(t, http.Header{
-		"Mock header": []string{"Mock value"},
-	}, request.Header)
-
-	assert.Equal(t, "http://example.com/?query=mock%20value", request.URL.String())
-
-	assert.NoError(t, headersConversionErr)
-}
-
-func TestH2RequestHeadersToH1Request_HostPathWithURLEncoding(t *testing.T) {
-	request, err := http.NewRequest(http.MethodGet, "http://example.com/", nil)
-	assert.NoError(t, err)
-
-	headersConversionErr := H2RequestHeadersToH1Request(
-		[]h2mux.Header{
-			h2mux.Header{
-				Name:  ":path",
-				Value: "/mock%20path",
-			},
-			h2mux.Header{
-				Name:  "Mock header",
-				Value: "Mock value",
-			},
-		},
-		request,
-	)
-
-	assert.Equal(t, http.Header{
-		"Mock header": []string{"Mock value"},
-	}, request.Header)
-
-	assert.Equal(t, "http://example.com/mock%20path", request.URL.String())
-
-	assert.NoError(t, headersConversionErr)
-}
-
-func TestH2RequestHeadersToH1Request_WeirdURLs(t *testing.T) {
-	type testCase struct {
-		path string
-		want string
-	}
-	testCases := []testCase{
-		{
-			path: "",
-			want: "",
-		},
-		{
-			path: "/",
-			want: "/",
-		},
-		{
-			path: "//",
-			want: "//",
-		},
-		{
-			path: "/test",
-			want: "/test",
-		},
-		{
-			path: "//test",
-			want: "//test",
-		},
-		{
-			// https://github.com/cloudflare/cloudflared/issues/81
-			path: "//test/",
-			want: "//test/",
-		},
-		{
-			path: "/%2Ftest",
-			want: "/%2Ftest",
-		},
-		{
-			path: "//%20test",
-			want: "//%20test",
-		},
-		{
-			// https://github.com/cloudflare/cloudflared/issues/124
-			path: "/test?get=somthing%20a",
-			want: "/test?get=somthing%20a",
-		},
-		{
-			path: "/%20",
-			want: "/%20",
-		},
-		{
-			// stdlib's EscapedPath() will always percent-encode ' '
-			path: "/ ",
-			want: "/%20",
-		},
-		{
-			path: "/ a ",
-			want: "/%20a%20",
-		},
-		{
-			path: "/a%20b",
-			want: "/a%20b",
-		},
-		{
-			path: "/foo/bar;param?query#frag",
-			want: "/foo/bar;param?query#frag",
-		},
-		{
-			// stdlib's EscapedPath() will always percent-encode non-ASCII chars
-			path: "/a␠b",
-			want: "/a%E2%90%A0b",
-		},
-		{
-			path: "/a-umlaut-ä",
-			want: "/a-umlaut-%C3%A4",
-		},
-		{
-			path: "/a-umlaut-%C3%A4",
-			want: "/a-umlaut-%C3%A4",
-		},
-		{
-			path: "/a-umlaut-%c3%a4",
-			want: "/a-umlaut-%c3%a4",
-		},
-		{
-			// here the second '#' is treated as part of the fragment
-			path: "/a#b#c",
-			want: "/a#b%23c",
-		},
-		{
-			path: "/a#b␠c",
-			want: "/a#b%E2%90%A0c",
-		},
-		{
-			path: "/a#b%20c",
-			want: "/a#b%20c",
-		},
-		{
-			path: "/a#b c",
-			want: "/a#b%20c",
-		},
-		{
-			// stdlib's EscapedPath() will always percent-encode '\'
-			path: "/\\",
-			want: "/%5C",
-		},
-		{
-			path: "/a\\",
-			want: "/a%5C",
-		},
-		{
-			path: "/a,b.c.",
-			want: "/a,b.c.",
-		},
-		{
-			path: "/.",
-			want: "/.",
-		},
-		{
-			// stdlib's EscapedPath() will always percent-encode '`'
-			path: "/a`",
-			want: "/a%60",
-		},
-		{
-			path: "/a[0]",
-			want: "/a[0]",
-		},
-		{
-			path: "/?a[0]=5 &b[]=",
-			want: "/?a[0]=5 &b[]=",
-		},
-		{
-			path: "/?a=%22b%20%22",
-			want: "/?a=%22b%20%22",
-		},
-	}
-
-	for index, testCase := range testCases {
-		requestURL := "https://example.com"
-
-		request, err := http.NewRequest(http.MethodGet, requestURL, nil)
-		assert.NoError(t, err)
-		headersConversionErr := H2RequestHeadersToH1Request(
-			[]h2mux.Header{
-				h2mux.Header{
-					Name:  ":path",
-					Value: testCase.path,
-				},
-				h2mux.Header{
-					Name:  "Mock header",
-					Value: "Mock value",
-				},
-			},
-			request,
-		)
-		assert.NoError(t, headersConversionErr)
-
-		assert.Equal(t,
-			http.Header{
-				"Mock header": []string{"Mock value"},
-			},
-			request.Header)
-
-		assert.Equal(t,
-			"https://example.com"+testCase.want,
-			request.URL.String(),
-			"Failed URL index: %v %#v", index, testCase)
-	}
-}
-
-func TestH2RequestHeadersToH1Request_QuickCheck(t *testing.T) {
-	config := &quick.Config{
-		Values: func(args []reflect.Value, rand *rand.Rand) {
-			args[0] = reflect.ValueOf(randomHTTP2Path(t, rand))
-		},
-	}
-
-	type testOrigin struct {
-		url string
-
-		expectedScheme   string
-		expectedBasePath string
-	}
-	testOrigins := []testOrigin{
-		{
-			url:              "http://origin.hostname.example.com:8080",
-			expectedScheme:   "http",
-			expectedBasePath: "http://origin.hostname.example.com:8080",
-		},
-		{
-			url:              "http://origin.hostname.example.com:8080/",
-			expectedScheme:   "http",
-			expectedBasePath: "http://origin.hostname.example.com:8080",
-		},
-		{
-			url:              "http://origin.hostname.example.com:8080/api",
-			expectedScheme:   "http",
-			expectedBasePath: "http://origin.hostname.example.com:8080/api",
-		},
-		{
-			url:              "http://origin.hostname.example.com:8080/api/",
-			expectedScheme:   "http",
-			expectedBasePath: "http://origin.hostname.example.com:8080/api",
-		},
-		{
-			url:              "https://origin.hostname.example.com:8080/api",
-			expectedScheme:   "https",
-			expectedBasePath: "https://origin.hostname.example.com:8080/api",
-		},
-	}
-
-	// use multiple schemes to demonstrate that the URL is based on the
-	// origin's scheme, not the :scheme header
-	for _, testScheme := range []string{"http", "https"} {
-		for _, testOrigin := range testOrigins {
-			assertion := func(testPath string) bool {
-				const expectedMethod = "POST"
-				const expectedHostname = "request.hostname.example.com"
-
-				h2 := []h2mux.Header{
-					h2mux.Header{Name: ":method", Value: expectedMethod},
-					h2mux.Header{Name: ":scheme", Value: testScheme},
-					h2mux.Header{Name: ":authority", Value: expectedHostname},
-					h2mux.Header{Name: ":path", Value: testPath},
-				}
-				h1, err := http.NewRequest("GET", testOrigin.url, nil)
-				require.NoError(t, err)
-
-				err = H2RequestHeadersToH1Request(h2, h1)
-				return assert.NoError(t, err) &&
-					assert.Equal(t, expectedMethod, h1.Method) &&
-					assert.Equal(t, expectedHostname, h1.Host) &&
-					assert.Equal(t, testOrigin.expectedScheme, h1.URL.Scheme) &&
-					assert.Equal(t, testOrigin.expectedBasePath+testPath, h1.URL.String())
-			}
-			err := quick.Check(assertion, config)
-			assert.NoError(t, err)
-		}
-	}
-}
-
-func randomASCIIPrintableChar(rand *rand.Rand) int {
-	// smallest printable ASCII char is 32, largest is 126
-	const startPrintable = 32
-	const endPrintable = 127
-	return startPrintable + rand.Intn(endPrintable-startPrintable)
-}
-
-// randomASCIIText generates an ASCII string, some of whose characters may be
-// percent-encoded. Its "logical length" (ignoring percent-encoding) is
-// between 1 and `maxLength`.
-func randomASCIIText(rand *rand.Rand, minLength int, maxLength int) string {
-	length := minLength + rand.Intn(maxLength)
-	result := ""
-	for i := 0; i < length; i++ {
-		c := randomASCIIPrintableChar(rand)
-
-		// 1/4 chance of using percent encoding when not necessary
-		if c == '%' || rand.Intn(4) == 0 {
-			result += fmt.Sprintf("%%%02X", c)
-		} else {
-			result += string(c)
-		}
-	}
-	return result
-}
-
-// Calls `randomASCIIText` and ensures the result is a valid URL path,
-// i.e. one that can pass unchanged through url.URL.String()
-func randomHTTP1Path(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
-	text := randomASCIIText(rand, minLength, maxLength)
-	regexp, err := regexp.Compile("[^/;,]*")
-	require.NoError(t, err)
-	return "/" + regexp.ReplaceAllStringFunc(text, url.PathEscape)
-}
-
-// Calls `randomASCIIText` and ensures the result is a valid URL query,
-// i.e. one that can pass unchanged through url.URL.String()
-func randomHTTP1Query(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
-	text := randomASCIIText(rand, minLength, maxLength)
-	return "?" + strings.ReplaceAll(text, "#", "%23")
-}
-
-// Calls `randomASCIIText` and ensures the result is a valid URL fragment,
-// i.e. one that can pass unchanged through url.URL.String()
-func randomHTTP1Fragment(t *testing.T, rand *rand.Rand, minLength int, maxLength int) string {
-	text := randomASCIIText(rand, minLength, maxLength)
-	url, err := url.Parse("#" + text)
-	require.NoError(t, err)
-	return url.String()
-}
-
-// Assemble a random :path pseudoheader that is legal by Go stdlib standards
-// (i.e. all characters will satisfy "net/url".shouldEscape for their respective locations)
-func randomHTTP2Path(t *testing.T, rand *rand.Rand) string {
-	result := randomHTTP1Path(t, rand, 1, 64)
-	if rand.Intn(2) == 1 {
-		result += randomHTTP1Query(t, rand, 1, 32)
-	}
-	if rand.Intn(2) == 1 {
-		result += randomHTTP1Fragment(t, rand, 1, 16)
-	}
-	return result
-}

streamhandler/stream_handler.go

@@ -180,11 +180,12 @@ func (s *StreamHandler) requestLogger(req *http.Request, tunnelHostname h2mux.Tu
 }
 
 func (s *StreamHandler) writeErrorStatus(stream *h2mux.MuxedStream, status *httpErrorStatus) {
-	stream.WriteHeaders([]h2mux.Header{
+	_ = stream.WriteHeaders([]h2mux.Header{
 		{
 			Name:  statusPseudoHeader,
 			Value: status.status,
 		},
+		h2mux.CreateResponseMetaHeader(h2mux.ResponseMetaHeaderField, h2mux.ResponseSourceCloudflared),
 	})
-	stream.Write(status.text)
+	_, _ = stream.Write(status.text)
 }

streamhandler/stream_handler_test.go

@@ -31,11 +31,11 @@ var (
 		{Name: ":scheme", Value: "http"},
 		{Name: ":authority", Value: "example.com"},
 		{Name: ":path", Value: "/"},
+
+		// Regular headers must always come after the pseudoheaders
+		{Name: h2mux.RequestUserHeadersField, Value: ""},
 	}
-	tunnelHostnameHeader = h2mux.Header{
-		Name:  h2mux.CloudflaredProxyTunnelHostnameHeader,
-		Value: testTunnelHostname.String(),
-	}
+	tunnelHostnameHeader = h2mux.Header{Name: h2mux.CloudflaredProxyTunnelHostnameHeader, Value: testTunnelHostname.String()}
 )
 
 func TestServeRequest(t *testing.T) {
@@ -69,29 +69,73 @@ func TestServeRequest(t *testing.T) {
 	assertRespBody(t, message, stream)
 }
 
-func TestServeBadRequest(t *testing.T) {
+func createStreamHandler() *StreamHandler {
 	configChan := make(chan *pogs.ClientConfig)
 	useConfigResultChan := make(chan *pogs.UseConfigurationResult)
-	streamHandler := NewStreamHandler(configChan, useConfigResultChan, logrus.New())
 
+	return NewStreamHandler(configChan, useConfigResultChan, logrus.New())
+}
+
+func createRequestMuxPair(t *testing.T, streamHandler *StreamHandler) *DefaultMuxerPair {
 	muxPair := NewDefaultMuxerPair(t, streamHandler)
 	muxPair.Serve(t)
 
+	return muxPair
+}
+
+func TestServeStatusBadRequest(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), testOpenStreamTimeout)
 	defer cancel()
 
 	// No tunnel hostname header, expect to get 400 Bad Request
-	stream, err := muxPair.EdgeMux.OpenStream(ctx, baseHeaders, nil)
+	stream, err := createRequestMuxPair(t, createStreamHandler()).EdgeMux.OpenStream(ctx, baseHeaders, nil)
 	assert.NoError(t, err)
 	assertStatusHeader(t, http.StatusBadRequest, stream.Headers)
 	assertRespBody(t, statusBadRequest.text, stream)
+}
+
+func TestServeInvalidContentLength(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), testOpenStreamTimeout)
+	defer cancel()
+
+	// Invalid content-length, wouldn't be able to create a request
+	// Expect to get 400 Bad Request
+	headers := append(baseHeaders, tunnelHostnameHeader)
+	headers = append(headers, h2mux.Header{
+		Name:  "content-length",
+		Value: "x",
+	})
+	streamHandler := createStreamHandler()
+	streamHandler.UpdateConfig([]*pogs.ReverseProxyConfig{
+		{
+			TunnelHostname: testTunnelHostname,
+			OriginConfig: &pogs.HTTPOriginConfig{
+				URLString: "",
+			},
+		},
+	})
+	mux := createRequestMuxPair(t, streamHandler).EdgeMux
+	stream, err := mux.OpenStream(ctx, headers, nil)
+	assert.NoError(t, err)
+	assertStatusHeader(t, http.StatusBadRequest, stream.Headers)
+	assertRespBody(t, statusBadRequest.text, stream)
+}
+
+func TestServeStatusNotFound(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), testOpenStreamTimeout)
+	defer cancel()
 
 	// No mapping for the tunnel hostname, expect to get 404 Not Found
 	headers := append(baseHeaders, tunnelHostnameHeader)
-	stream, err = muxPair.EdgeMux.OpenStream(ctx, headers, nil)
+	stream, err := createRequestMuxPair(t, createStreamHandler()).EdgeMux.OpenStream(ctx, headers, nil)
 	assert.NoError(t, err)
 	assertStatusHeader(t, http.StatusNotFound, stream.Headers)
 	assertRespBody(t, statusNotFound.text, stream)
+}
+
+func TestServeStatusBadGateway(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), testOpenStreamTimeout)
+	defer cancel()
 
 	// Nothing listening on empty url, so proxy would fail. Expect to get 502 Bad Gateway
 	reverseProxyConfigs := []*pogs.ReverseProxyConfig{
@@ -102,21 +146,13 @@ func TestServeBadRequest(t *testing.T) {
 			},
 		},
 	}
+	streamHandler := createStreamHandler()
 	streamHandler.UpdateConfig(reverseProxyConfigs)
-	stream, err = muxPair.EdgeMux.OpenStream(ctx, headers, nil)
+	headers := append(baseHeaders, tunnelHostnameHeader)
+	stream, err := createRequestMuxPair(t, streamHandler).EdgeMux.OpenStream(ctx, headers, nil)
 	assert.NoError(t, err)
 	assertStatusHeader(t, http.StatusBadGateway, stream.Headers)
 	assertRespBody(t, statusBadGateway.text, stream)
-
-	// Invalid content-length, wouldn't not be able to create a request. Expect to get 400 Bad Request
-	headers = append(headers, h2mux.Header{
-		Name:  "content-length",
-		Value: "x",
-	})
-	stream, err = muxPair.EdgeMux.OpenStream(ctx, headers, nil)
-	assert.NoError(t, err)
-	assertStatusHeader(t, http.StatusBadRequest, stream.Headers)
-	assertRespBody(t, statusBadRequest.text, stream)
 }
 
 func assertStatusHeader(t *testing.T, expectedStatus int, headers []h2mux.Header) {
@@ -218,6 +254,6 @@ type mockHTTPHandler struct {
 	message []byte
 }
 
-func (mth *mockHTTPHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	w.Write(mth.message)
+func (mth *mockHTTPHandler) ServeHTTP(w http.ResponseWriter, _ *http.Request) {
+	_, _ = w.Write(mth.message)
 }

supervisor/supervisor.go

@@ -1,3 +1,4 @@
+// Package supervisor is used by declarative tunnels to get/apply new config from the edge.
 package supervisor
 
 import (
@@ -13,6 +14,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
 	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/streamhandler"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
@@ -56,7 +58,7 @@ func NewSupervisor(
 	defaultClientConfig *pogs.ClientConfig,
 	userCredential []byte,
 	tlsConfig *tls.Config,
-	serviceDiscoverer connection.EdgeServiceDiscoverer,
+	serviceDiscoverer *edgediscovery.Edge,
 	cloudflaredConfig *connection.CloudflaredConfig,
 	autoupdater *updater.AutoUpdater,
 	supportAutoupdate bool,

tlsconfig/cloudflare_ca.go

@@ -26,28 +26,28 @@ mcifak4CQsr+DH4pn5SJD7JxtCG3YGswW8QZsw==
 -----END CERTIFICATE-----
 Issuer: C=US, O=CloudFlare, Inc., OU=CloudFlare Origin SSL Certificate Authority, L=San Francisco, ST=California
 -----BEGIN CERTIFICATE-----
-MIID/DCCAuagAwIBAgIID+rOSdTGfGcwCwYJKoZIhvcNAQELMIGLMQswCQYDVQQG
-EwJVUzEZMBcGA1UEChMQQ2xvdWRGbGFyZSwgSW5jLjE0MDIGA1UECxMrQ2xvdWRG
-bGFyZSBPcmlnaW4gU1NMIENlcnRpZmljYXRlIEF1dGhvcml0eTEWMBQGA1UEBxMN
-U2FuIEZyYW5jaXNjbzETMBEGA1UECBMKQ2FsaWZvcm5pYTAeFw0xNDExMTMyMDM4
-NTBaFw0xOTExMTQwMTQzNTBaMIGLMQswCQYDVQQGEwJVUzEZMBcGA1UEChMQQ2xv
-dWRGbGFyZSwgSW5jLjE0MDIGA1UECxMrQ2xvdWRGbGFyZSBPcmlnaW4gU1NMIENl
-cnRpZmljYXRlIEF1dGhvcml0eTEWMBQGA1UEBxMNU2FuIEZyYW5jaXNjbzETMBEG
-A1UECBMKQ2FsaWZvcm5pYTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
-AMBIlWf1KEKR5hbB75OYrAcUXobpD/AxvSYRXr91mbRu+lqE7YbyyRUShQh15lem
-ef+umeEtPZoLFLhcLyczJxOhI+siLGDQm/a/UDkWvAXYa5DZ+pHU5ct5nZ8pGzqJ
-p8G1Hy5RMVYDXZT9F6EaHjMG0OOffH6Ih25TtgfyyrjXycwDH0u6GXt+G/rywcqz
-/9W4Aki3XNQMUHNQAtBLEEIYHMkyTYJxuL2tXO6ID5cCsoWw8meHufTeZW2DyUpl
-yP3AHt4149RQSyWZMJ6AyntL9d8Xhfpxd9rJkh9Kge2iV9rQTFuE1rRT5s7OSJcK
-xUsklgHcGHYMcNfNMilNHb8CAwEAAaNmMGQwDgYDVR0PAQH/BAQDAgAGMBIGA1Ud
-EwEB/wQIMAYBAf8CAQIwHQYDVR0OBBYEFCToU1ddfDRAh6nrlNu64RZ4/CmkMB8G
-A1UdIwQYMBaAFCToU1ddfDRAh6nrlNu64RZ4/CmkMAsGCSqGSIb3DQEBCwOCAQEA
-cQDBVAoRrhhsGegsSFsv1w8v27zzHKaJNv6ffLGIRvXK8VKKK0gKXh2zQtN9SnaD
-gYNe7Pr4C3I8ooYKRJJWLsmEHdGdnYYmj0OJfGrfQf6MLIc/11bQhLepZTxdhFYh
-QGgDl6gRmb8aDwk7Q92BPvek5nMzaWlP82ixavvYI+okoSY8pwdcVKobx6rWzMWz
-ZEC9M6H3F0dDYE23XcCFIdgNSAmmGyXPBstOe0aAJXwJTxOEPn36VWr0PKIQJy5Y
-4o1wpMpqCOIwWc8J9REV/REzN6Z1LXImdUgXIXOwrz56gKUJzPejtBQyIGj0mveX
-Fu6q54beR89jDc+oABmOgg==
+MIIEADCCAuigAwIBAgIID+rOSdTGfGcwDQYJKoZIhvcNAQELBQAwgYsxCzAJBgNV
+BAYTAlVTMRkwFwYDVQQKExBDbG91ZEZsYXJlLCBJbmMuMTQwMgYDVQQLEytDbG91
+ZEZsYXJlIE9yaWdpbiBTU0wgQ2VydGlmaWNhdGUgQXV0aG9yaXR5MRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMRMwEQYDVQQIEwpDYWxpZm9ybmlhMB4XDTE5MDgyMzIx
+MDgwMFoXDTI5MDgxNTE3MDAwMFowgYsxCzAJBgNVBAYTAlVTMRkwFwYDVQQKExBD
+bG91ZEZsYXJlLCBJbmMuMTQwMgYDVQQLEytDbG91ZEZsYXJlIE9yaWdpbiBTU0wg
+Q2VydGlmaWNhdGUgQXV0aG9yaXR5MRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMRMw
+EQYDVQQIEwpDYWxpZm9ybmlhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEAwEiVZ/UoQpHmFsHvk5isBxRehukP8DG9JhFev3WZtG76WoTthvLJFRKFCHXm
+V6Z5/66Z4S09mgsUuFwvJzMnE6Ej6yIsYNCb9r9QORa8BdhrkNn6kdTly3mdnykb
+OomnwbUfLlExVgNdlP0XoRoeMwbQ4598foiHblO2B/LKuNfJzAMfS7oZe34b+vLB
+yrP/1bgCSLdc1AxQc1AC0EsQQhgcyTJNgnG4va1c7ogPlwKyhbDyZ4e59N5lbYPJ
+SmXI/cAe3jXj1FBLJZkwnoDKe0v13xeF+nF32smSH0qB7aJX2tBMW4TWtFPmzs5I
+lwrFSySWAdwYdgxw180yKU0dvwIDAQABo2YwZDAOBgNVHQ8BAf8EBAMCAQYwEgYD
+VR0TAQH/BAgwBgEB/wIBAjAdBgNVHQ4EFgQUJOhTV118NECHqeuU27rhFnj8KaQw
+HwYDVR0jBBgwFoAUJOhTV118NECHqeuU27rhFnj8KaQwDQYJKoZIhvcNAQELBQAD
+ggEBAHwOf9Ur1l0Ar5vFE6PNrZWrDfQIMyEfdgSKofCdTckbqXNTiXdgbHs+TWoQ
+wAB0pfJDAHJDXOTCWRyTeXOseeOi5Btj5CnEuw3P0oXqdqevM1/+uWp0CM35zgZ8
+VD4aITxity0djzE6Qnx3Syzz+ZkoBgTnNum7d9A66/V636x4vTeqbZFBr9erJzgz
+hhurjcoacvRNhnjtDRM0dPeiCJ50CP3wEYuvUzDHUaowOsnLCjQIkWbR7Ni6KEIk
+MOz2U0OBSif3FTkhCgZWQKOOLo1P42jHC3ssUZAtVNXrCk3fw9/E15k8NPkBazZ6
+0iykLhH1trywrKRMVw67F44IE8Y=
 -----END CERTIFICATE-----
 Issuer: C=US, O=CloudFlare, Inc., OU=Origin Pull, L=San Francisco, ST=California, CN=origin-pull.cloudflare.net
 -----BEGIN CERTIFICATE-----

tunneldns/https_upstream.go

@@ -6,6 +6,7 @@ import (
 	"crypto/tls"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"net/url"
 	"time"
@@ -22,33 +23,18 @@ const (
 
 // UpstreamHTTPS is the upstream implementation for DNS over HTTPS service
 type UpstreamHTTPS struct {
-	client   *http.Client
-	endpoint *url.URL
+	client     *http.Client
+	endpoint   *url.URL
+	bootstraps []string
 }
 
-// NewUpstreamHTTPS creates a new DNS over HTTPS upstream from hostname
-func NewUpstreamHTTPS(endpoint string) (Upstream, error) {
+// NewUpstreamHTTPS creates a new DNS over HTTPS upstream from endpoint
+func NewUpstreamHTTPS(endpoint string, bootstraps []string) (Upstream, error) {
 	u, err := url.Parse(endpoint)
 	if err != nil {
 		return nil, err
 	}
-
-	// Update TLS and HTTP client configuration
-	tls := &tls.Config{ServerName: u.Hostname()}
-	transport := &http.Transport{
-		TLSClientConfig:    tls,
-		DisableCompression: true,
-		MaxIdleConns:       1,
-		Proxy:              http.ProxyFromEnvironment,
-	}
-	http2.ConfigureTransport(transport)
-
-	client := &http.Client{
-		Timeout:   defaultTimeout,
-		Transport: transport,
-	}
-
-	return &UpstreamHTTPS{client: client, endpoint: u}, nil
+	return &UpstreamHTTPS{client: configureClient(u.Hostname()), endpoint: u, bootstraps: bootstraps}, nil
 }
 
 // Exchange provides an implementation for the Upstream interface
@@ -58,34 +44,55 @@ func (u *UpstreamHTTPS) Exchange(ctx context.Context, query *dns.Msg) (*dns.Msg,
 		return nil, errors.Wrap(err, "failed to pack DNS query")
 	}
 
+	if len(query.Question) > 0 && query.Question[0].Name == fmt.Sprintf("%s.", u.endpoint.Hostname()) {
+		for _, bootstrap := range u.bootstraps {
+			endpoint, client, err := configureBootstrap(bootstrap)
+			if err != nil {
+				log.WithError(err).Errorf("failed to configure boostrap upstream %s", bootstrap)
+				continue
+			}
+			msg, err := exchange(queryBuf, query.Id, endpoint, client)
+			if err != nil {
+				log.WithError(err).Errorf("failed to connect to a boostrap upstream %s", bootstrap)
+				continue
+			}
+			return msg, nil
+		}
+		return nil, fmt.Errorf("failed to reach any bootstrap upstream: %v", u.bootstraps)
+	}
+
+	return exchange(queryBuf, query.Id, u.endpoint, u.client)
+}
+
+func exchange(msg []byte, queryID uint16, endpoint *url.URL, client *http.Client) (*dns.Msg, error) {
 	// No content negotiation for now, use DNS wire format
-	buf, backendErr := u.exchangeWireformat(queryBuf)
+	buf, backendErr := exchangeWireformat(msg, endpoint, client)
 	if backendErr == nil {
 		response := &dns.Msg{}
 		if err := response.Unpack(buf); err != nil {
 			return nil, errors.Wrap(err, "failed to unpack DNS response from body")
 		}
 
-		response.Id = query.Id
+		response.Id = queryID
 		return response, nil
 	}
 
-	log.WithError(backendErr).Errorf("failed to connect to an HTTPS backend %q", u.endpoint)
+	log.WithError(backendErr).Errorf("failed to connect to an HTTPS backend %q", endpoint)
 	return nil, backendErr
 }
 
 // Perform message exchange with the default UDP wireformat defined in current draft
 // https://datatracker.ietf.org/doc/draft-ietf-doh-dns-over-https
-func (u *UpstreamHTTPS) exchangeWireformat(msg []byte) ([]byte, error) {
-	req, err := http.NewRequest("POST", u.endpoint.String(), bytes.NewBuffer(msg))
+func exchangeWireformat(msg []byte, endpoint *url.URL, client *http.Client) ([]byte, error) {
+	req, err := http.NewRequest("POST", endpoint.String(), bytes.NewBuffer(msg))
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to create an HTTPS request")
 	}
 
 	req.Header.Add("Content-Type", "application/dns-message")
-	req.Host = u.endpoint.Hostname()
+	req.Host = endpoint.Host
 
-	resp, err := u.client.Do(req)
+	resp, err := client.Do(req)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to perform an HTTPS request")
 	}
@@ -104,3 +111,33 @@ func (u *UpstreamHTTPS) exchangeWireformat(msg []byte) ([]byte, error) {
 
 	return buf, nil
 }
+
+func configureBootstrap(bootstrap string) (*url.URL, *http.Client, error) {
+	b, err := url.Parse(bootstrap)
+	if err != nil {
+		return nil, nil, err
+	}
+	if ip := net.ParseIP(b.Hostname()); ip == nil {
+		return nil, nil, fmt.Errorf("bootstrap address of %s must be an IP address", b.Hostname())
+	}
+
+	return b, configureClient(b.Hostname()), nil
+}
+
+// configureClient will configure a HTTPS client for upstream DoH requests
+func configureClient(hostname string) *http.Client {
+	// Update TLS and HTTP client configuration
+	tls := &tls.Config{ServerName: hostname}
+	transport := &http.Transport{
+		TLSClientConfig:    tls,
+		DisableCompression: true,
+		MaxIdleConns:       1,
+		Proxy:              http.ProxyFromEnvironment,
+	}
+	http2.ConfigureTransport(transport)
+
+	return &http.Client{
+		Timeout:   defaultTimeout,
+		Transport: transport,
+	}
+}

tunneldns/tunnel.go

@@ -35,7 +35,7 @@ func Run(c *cli.Context) error {
 
 	go metrics.ServeMetrics(metricsListener, nil, logger)
 
-	listener, err := CreateListener(c.String("address"), uint16(c.Uint("port")), c.StringSlice("upstream"))
+	listener, err := CreateListener(c.String("address"), uint16(c.Uint("port")), c.StringSlice("upstream"), c.StringSlice("bootstrap"))
 	if err != nil {
 		logger.WithError(err).Errorf("Failed to create the listeners")
 		return err
@@ -117,12 +117,12 @@ func (l *Listener) Stop() error {
 }
 
 // CreateListener configures the server and bound sockets
-func CreateListener(address string, port uint16, upstreams []string) (*Listener, error) {
+func CreateListener(address string, port uint16, upstreams []string, bootstraps []string) (*Listener, error) {
 	// Build the list of upstreams
 	upstreamList := make([]Upstream, 0)
 	for _, url := range upstreams {
 		logger.WithField("url", url).Infof("Adding DNS upstream")
-		upstream, err := NewUpstreamHTTPS(url)
+		upstream, err := NewUpstreamHTTPS(url, bootstraps)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to create HTTPS upstream")
 		}

tunnelrpc/pogs/auth_outcome.go

@@ -1,7 +1,7 @@
 package pogs
 
 import (
-	"fmt"
+	"errors"
 	"time"
 )
 
@@ -18,21 +18,21 @@ type AuthenticateResponse struct {
 
 // Outcome turns the deserialized response of Authenticate into a programmer-friendly sum type.
 func (ar AuthenticateResponse) Outcome() AuthOutcome {
-	// If there was a network error, then cloudflared should retry later,
-	// because origintunneld couldn't prove whether auth was correct or not.
-	if ar.RetryableErr != "" {
-		return &AuthUnknown{Err: fmt.Errorf(ar.RetryableErr), HoursUntilRefresh: ar.HoursUntilRefresh}
-	}
-
 	// If the user's authentication was unsuccessful, the server will return an error explaining why.
 	// cloudflared should fatal with this error.
 	if ar.PermanentErr != "" {
-		return &AuthFail{Err: fmt.Errorf(ar.PermanentErr)}
+		return NewAuthFail(errors.New(ar.PermanentErr))
+	}
+
+	// If there was a network error, then cloudflared should retry later,
+	// because origintunneld couldn't prove whether auth was correct or not.
+	if ar.RetryableErr != "" {
+		return NewAuthUnknown(errors.New(ar.RetryableErr), ar.HoursUntilRefresh)
 	}
 
 	// If auth succeeded, return the token and refresh it when instructed.
-	if ar.PermanentErr == "" && len(ar.Jwt) > 0 {
-		return &AuthSuccess{Jwt: ar.Jwt, HoursUntilRefresh: ar.HoursUntilRefresh}
+	if len(ar.Jwt) > 0 {
+		return NewAuthSuccess(ar.Jwt, ar.HoursUntilRefresh)
 	}
 
 	// Otherwise the state got messed up.
@@ -49,60 +49,84 @@ type AuthOutcome interface {
 
 // AuthSuccess means the backend successfully authenticated this cloudflared.
 type AuthSuccess struct {
-	Jwt               []byte
-	HoursUntilRefresh uint8
+	jwt               []byte
+	hoursUntilRefresh uint8
+}
+
+func NewAuthSuccess(jwt []byte, hoursUntilRefresh uint8) AuthSuccess {
+	return AuthSuccess{jwt: jwt, hoursUntilRefresh: hoursUntilRefresh}
+}
+
+func (ao AuthSuccess) JWT() []byte {
+	return ao.jwt
 }
 
 // RefreshAfter is how long cloudflared should wait before rerunning Authenticate.
-func (ao *AuthSuccess) RefreshAfter() time.Duration {
-	return hoursToTime(ao.HoursUntilRefresh)
+func (ao AuthSuccess) RefreshAfter() time.Duration {
+	return hoursToTime(ao.hoursUntilRefresh)
 }
 
 // Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao *AuthSuccess) Serialize() AuthenticateResponse {
+func (ao AuthSuccess) Serialize() AuthenticateResponse {
 	return AuthenticateResponse{
-		Jwt:               ao.Jwt,
-		HoursUntilRefresh: ao.HoursUntilRefresh,
+		Jwt:               ao.jwt,
+		HoursUntilRefresh: ao.hoursUntilRefresh,
 	}
 }
 
-func (ao *AuthSuccess) isAuthOutcome() {}
+func (ao AuthSuccess) isAuthOutcome() {}
 
 // AuthFail means this cloudflared has the wrong auth and should exit.
 type AuthFail struct {
-	Err error
+	err error
+}
+
+func NewAuthFail(err error) AuthFail {
+	return AuthFail{err: err}
+}
+
+func (ao AuthFail) Error() string {
+	return ao.err.Error()
 }
 
 // Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao *AuthFail) Serialize() AuthenticateResponse {
+func (ao AuthFail) Serialize() AuthenticateResponse {
 	return AuthenticateResponse{
-		PermanentErr: ao.Err.Error(),
+		PermanentErr: ao.err.Error(),
 	}
 }
 
-func (ao *AuthFail) isAuthOutcome() {}
+func (ao AuthFail) isAuthOutcome() {}
 
 // AuthUnknown means the backend couldn't finish checking authentication. Try again later.
 type AuthUnknown struct {
-	Err               error
-	HoursUntilRefresh uint8
+	err               error
+	hoursUntilRefresh uint8
+}
+
+func NewAuthUnknown(err error, hoursUntilRefresh uint8) AuthUnknown {
+	return AuthUnknown{err: err, hoursUntilRefresh: hoursUntilRefresh}
+}
+
+func (ao AuthUnknown) Error() string {
+	return ao.err.Error()
 }
 
 // RefreshAfter is how long cloudflared should wait before rerunning Authenticate.
-func (ao *AuthUnknown) RefreshAfter() time.Duration {
-	return hoursToTime(ao.HoursUntilRefresh)
+func (ao AuthUnknown) RefreshAfter() time.Duration {
+	return hoursToTime(ao.hoursUntilRefresh)
 }
 
 // Serialize into an AuthenticateResponse which can be sent via Capnp
-func (ao *AuthUnknown) Serialize() AuthenticateResponse {
+func (ao AuthUnknown) Serialize() AuthenticateResponse {
 	return AuthenticateResponse{
-		RetryableErr:      ao.Err.Error(),
-		HoursUntilRefresh: ao.HoursUntilRefresh,
+		RetryableErr:      ao.err.Error(),
+		HoursUntilRefresh: ao.hoursUntilRefresh,
 	}
 }
 
-func (ao *AuthUnknown) isAuthOutcome() {}
+func (ao AuthUnknown) isAuthOutcome() {}
 
 func hoursToTime(hours uint8) time.Duration {
 	return time.Duration(hours) * time.Hour
-}
+}

tunnelrpc/pogs/auth_test.go

@@ -31,15 +31,15 @@ func TestAuthenticateResponseOutcome(t *testing.T) {
 	}{
 		{"success",
 			fields{Jwt: []byte("asdf"), HoursUntilRefresh: 6},
-			&AuthSuccess{Jwt: []byte("asdf"), HoursUntilRefresh: 6},
+			AuthSuccess{jwt: []byte("asdf"), hoursUntilRefresh: 6},
 		},
 		{"fail",
 			fields{PermanentErr: "bad creds"},
-			&AuthFail{Err: fmt.Errorf("bad creds")},
+			AuthFail{err: fmt.Errorf("bad creds")},
 		},
 		{"error",
 			fields{RetryableErr: "bad conn", HoursUntilRefresh: 6},
-			&AuthUnknown{Err: fmt.Errorf("bad conn"), HoursUntilRefresh: 6},
+			AuthUnknown{err: fmt.Errorf("bad conn"), hoursUntilRefresh: 6},
 		},
 		{"nil (no fields are set)",
 			fields{},
@@ -69,6 +69,27 @@ func TestAuthenticateResponseOutcome(t *testing.T) {
 	}
 }
 
+func TestAuthSuccess(t *testing.T) {
+	input := NewAuthSuccess([]byte("asdf"), 6)
+	output, ok := input.Serialize().Outcome().(AuthSuccess)
+	assert.True(t, ok)
+	assert.Equal(t, input, output)
+}
+
+func TestAuthUnknown(t *testing.T) {
+	input := NewAuthUnknown(fmt.Errorf("pdx unreachable"), 6)
+	output, ok := input.Serialize().Outcome().(AuthUnknown)
+	assert.True(t, ok)
+	assert.Equal(t, input, output)
+}
+
+func TestAuthFail(t *testing.T) {
+	input := NewAuthFail(fmt.Errorf("wrong creds"))
+	output, ok := input.Serialize().Outcome().(AuthFail)
+	assert.True(t, ok)
+	assert.Equal(t, input, output)
+}
+
 func TestWhenToRefresh(t *testing.T) {
 	expected := 4 * time.Hour
 	actual := hoursToTime(4)

tunnelrpc/pogs/config.go

@@ -197,11 +197,14 @@ func (hc *HTTPOriginConfig) Service() (originservice.OriginService, error) {
 		return nil, err
 	}
 
-	dialContext := (&net.Dialer{
+	dialer := &net.Dialer{
 		Timeout:   hc.ProxyConnectionTimeout,
 		KeepAlive: hc.TCPKeepAlive,
-		DualStack: hc.DialDualStack,
-	}).DialContext
+	}
+	if !hc.DialDualStack {
+		dialer.FallbackDelay = -1
+	}
+	dialContext := dialer.DialContext
 	transport := &http.Transport{
 		Proxy:       http.ProxyFromEnvironment,
 		DialContext: dialContext,
@@ -270,7 +273,6 @@ func (*HelloWorldOriginConfig) Service() (originservice.OriginService, error) {
 		DialContext: (&net.Dialer{
 			Timeout:   30 * time.Second,
 			KeepAlive: 30 * time.Second,
-			DualStack: true,
 		}).DialContext,
 		TLSClientConfig: &tls.Config{
 			RootCAs: rootCAs,

tunnelrpc/pogs/reconnect_tunnel.go

@@ -12,6 +12,14 @@ func (i TunnelServer_PogsImpl) ReconnectTunnel(p tunnelrpc.TunnelServer_reconnec
 	if err != nil {
 		return err
 	}
+	eventDigest, err := p.Params.EventDigest()
+	if err != nil {
+		return err
+	}
+	connDigest, err := p.Params.ConnDigest()
+	if err != nil {
+		return err
+	}
 	hostname, err := p.Params.Hostname()
 	if err != nil {
 		return err
@@ -25,7 +33,7 @@ func (i TunnelServer_PogsImpl) ReconnectTunnel(p tunnelrpc.TunnelServer_reconnec
 		return err
 	}
 	server.Ack(p.Options)
-	registration, err := i.impl.ReconnectTunnel(p.Ctx, jwt, hostname, pogsOptions)
+	registration, err := i.impl.ReconnectTunnel(p.Ctx, jwt, eventDigest, connDigest, hostname, pogsOptions)
 	if err != nil {
 		return err
 	}
@@ -38,16 +46,26 @@ func (i TunnelServer_PogsImpl) ReconnectTunnel(p tunnelrpc.TunnelServer_reconnec
 
 func (c TunnelServer_PogsClient) ReconnectTunnel(
 	ctx context.Context,
-	jwt []byte,
+	jwt,
+	eventDigest []byte,
+	connDigest []byte,
 	hostname string,
 	options *RegistrationOptions,
-) (*TunnelRegistration, error) {
+) *TunnelRegistration {
 	client := tunnelrpc.TunnelServer{Client: c.Client}
 	promise := client.ReconnectTunnel(ctx, func(p tunnelrpc.TunnelServer_reconnectTunnel_Params) error {
 		err := p.SetJwt(jwt)
 		if err != nil {
 			return err
 		}
+		err = p.SetEventDigest(eventDigest)
+		if err != nil {
+			return err
+		}
+		err = p.SetConnDigest(connDigest)
+		if err != nil {
+			return err
+		}
 		err = p.SetHostname(hostname)
 		if err != nil {
 			return err
@@ -64,7 +82,11 @@ func (c TunnelServer_PogsClient) ReconnectTunnel(
 	})
 	retval, err := promise.Result().Struct()
 	if err != nil {
-		return nil, err
+		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
 	}
-	return UnmarshalTunnelRegistration(retval)
+	registration, err := UnmarshalTunnelRegistration(retval)
+	if err != nil {
+		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
+	}
+	return registration
 }

tunnelrpc/pogs/tunnelrpc.go

@@ -9,13 +9,16 @@ import (
 	"github.com/google/uuid"
 	"github.com/pkg/errors"
 
-	log "github.com/sirupsen/logrus"
 	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/pogs"
 	"zombiezen.com/go/capnproto2/rpc"
 	"zombiezen.com/go/capnproto2/server"
 )
 
+const (
+	defaultRetryAfterSeconds = 15
+)
+
 type Authentication struct {
 	Key         string
 	Email       string
@@ -33,14 +36,117 @@ func UnmarshalAuthentication(s tunnelrpc.Authentication) (*Authentication, error
 }
 
 type TunnelRegistration struct {
+	SuccessfulTunnelRegistration
 	Err               string
-	Url               string
-	LogLines          []string
 	PermanentFailure  bool
-	TunnelID          string `capnp:"tunnelID"`
 	RetryAfterSeconds uint16
 }
 
+type SuccessfulTunnelRegistration struct {
+	Url         string
+	LogLines    []string
+	TunnelID    string `capnp:"tunnelID"`
+	EventDigest []byte
+	ConnDigest  []byte
+}
+
+func NewSuccessfulTunnelRegistration(
+	url string,
+	logLines []string,
+	tunnelID string,
+	eventDigest []byte,
+	connDigest []byte,
+) *TunnelRegistration {
+	// Marshal nil will result in an error
+	if logLines == nil {
+		logLines = []string{}
+	}
+	return &TunnelRegistration{
+		SuccessfulTunnelRegistration: SuccessfulTunnelRegistration{
+			Url:         url,
+			LogLines:    logLines,
+			TunnelID:    tunnelID,
+			EventDigest: eventDigest,
+			ConnDigest:  connDigest,
+		},
+	}
+}
+
+// Not calling this function Error() to avoid confusion with implementing error interface
+func (tr TunnelRegistration) DeserializeError() TunnelRegistrationError {
+	if tr.Err != "" {
+		err := fmt.Errorf(tr.Err)
+		if tr.PermanentFailure {
+			return NewPermanentRegistrationError(err)
+		}
+		retryAfterSeconds := tr.RetryAfterSeconds
+		if retryAfterSeconds < defaultRetryAfterSeconds {
+			retryAfterSeconds = defaultRetryAfterSeconds
+		}
+		return NewRetryableRegistrationError(err, retryAfterSeconds)
+	}
+	return nil
+}
+
+type TunnelRegistrationError interface {
+	error
+	Serialize() *TunnelRegistration
+	IsPermanent() bool
+}
+
+type PermanentRegistrationError struct {
+	err string
+}
+
+func NewPermanentRegistrationError(err error) TunnelRegistrationError {
+	return &PermanentRegistrationError{
+		err: err.Error(),
+	}
+}
+
+func (pre *PermanentRegistrationError) Error() string {
+	return pre.err
+}
+
+func (pre *PermanentRegistrationError) Serialize() *TunnelRegistration {
+	return &TunnelRegistration{
+		Err:              pre.err,
+		PermanentFailure: true,
+	}
+}
+
+func (*PermanentRegistrationError) IsPermanent() bool {
+	return true
+}
+
+type RetryableRegistrationError struct {
+	err               string
+	retryAfterSeconds uint16
+}
+
+func NewRetryableRegistrationError(err error, retryAfterSeconds uint16) TunnelRegistrationError {
+	return &RetryableRegistrationError{
+		err:               err.Error(),
+		retryAfterSeconds: retryAfterSeconds,
+	}
+}
+
+func (rre *RetryableRegistrationError) Error() string {
+	return rre.err
+}
+
+func (rre *RetryableRegistrationError) Serialize() *TunnelRegistration {
+	return &TunnelRegistration{
+		Err:               rre.err,
+		PermanentFailure:  false,
+		RetryAfterSeconds: rre.retryAfterSeconds,
+	}
+}
+
+func (*RetryableRegistrationError) IsPermanent() bool {
+	return false
+}
+
 func MarshalTunnelRegistration(s tunnelrpc.TunnelRegistration, p *TunnelRegistration) error {
 	return pogs.Insert(tunnelrpc.TunnelRegistration_TypeID, s.Struct, p)
 }
@@ -65,6 +171,7 @@ type RegistrationOptions struct {
 	CompressionQuality   uint64 `capnp:"compressionQuality"`
 	UUID                 string `capnp:"uuid"`
 	NumPreviousAttempts  uint8
+	Features             []string
 }
 
 func MarshalRegistrationOptions(s tunnelrpc.RegistrationOptions, p *RegistrationOptions) error {
@@ -325,12 +432,12 @@ func UnmarshalConnectParameters(s tunnelrpc.CapnpConnectParameters) (*ConnectPar
 }
 
 type TunnelServer interface {
-	RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error)
+	RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) *TunnelRegistration
 	GetServerInfo(ctx context.Context) (*ServerInfo, error)
 	UnregisterTunnel(ctx context.Context, gracePeriodNanoSec int64) error
 	Connect(ctx context.Context, parameters *ConnectParameters) (ConnectResult, error)
 	Authenticate(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*AuthenticateResponse, error)
-	ReconnectTunnel(ctx context.Context, jwt []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error)
+	ReconnectTunnel(ctx context.Context, jwt, eventDigest, connDigest []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error)
 }
 
 func TunnelServer_ServerToClient(s TunnelServer) tunnelrpc.TunnelServer {
@@ -359,15 +466,12 @@ func (i TunnelServer_PogsImpl) RegisterTunnel(p tunnelrpc.TunnelServer_registerT
 		return err
 	}
 	server.Ack(p.Options)
-	registration, err := i.impl.RegisterTunnel(p.Ctx, originCert, hostname, pogsOptions)
-	if err != nil {
-		return err
-	}
+	registration := i.impl.RegisterTunnel(p.Ctx, originCert, hostname, pogsOptions)
+
 	result, err := p.Results.NewResult()
 	if err != nil {
 		return err
 	}
-	log.Info(registration.TunnelID)
 	return MarshalTunnelRegistration(result, registration)
 }
 
@@ -420,7 +524,7 @@ func (c TunnelServer_PogsClient) Close() error {
 	return c.Conn.Close()
 }
 
-func (c TunnelServer_PogsClient) RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) (*TunnelRegistration, error) {
+func (c TunnelServer_PogsClient) RegisterTunnel(ctx context.Context, originCert []byte, hostname string, options *RegistrationOptions) *TunnelRegistration {
 	client := tunnelrpc.TunnelServer{Client: c.Client}
 	promise := client.RegisterTunnel(ctx, func(p tunnelrpc.TunnelServer_registerTunnel_Params) error {
 		err := p.SetOriginCert(originCert)
@@ -443,9 +547,13 @@ func (c TunnelServer_PogsClient) RegisterTunnel(ctx context.Context, originCert
 	})
 	retval, err := promise.Result().Struct()
 	if err != nil {
-		return nil, err
+		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
 	}
-	return UnmarshalTunnelRegistration(retval)
+	registration, err := UnmarshalTunnelRegistration(retval)
+	if err != nil {
+		return NewRetryableRegistrationError(err, defaultRetryAfterSeconds).Serialize()
+	}
+	return registration
 }
 
 func (c TunnelServer_PogsClient) GetServerInfo(ctx context.Context) (*ServerInfo, error) {

tunnelrpc/pogs/tunnelrpc_test.go

@@ -1,6 +1,7 @@
 package pogs
 
 import (
+	"fmt"
 	"reflect"
 	"testing"
 	"time"
@@ -11,16 +12,31 @@ import (
 	capnp "zombiezen.com/go/capnproto2"
 )
 
+const (
+	testURL               = "tunnel.example.com"
+	testTunnelID          = "asdfghjkl;"
+	testRetryAfterSeconds = 19
+)
+
+var (
+	testErr         = fmt.Errorf("Invalid credential")
+	testLogLines    = []string{"all", "working"}
+	testEventDigest = []byte("asdf")
+	testConnDigest  = []byte("lkjh")
+)
+
+// *PermanentRegistrationError implements TunnelRegistrationError
+var _ TunnelRegistrationError = (*PermanentRegistrationError)(nil)
+
+// *RetryableRegistrationError implements TunnelRegistrationError
+var _ TunnelRegistrationError = (*RetryableRegistrationError)(nil)
+
 func TestTunnelRegistration(t *testing.T) {
 	testCases := []*TunnelRegistration{
-		&TunnelRegistration{
-			Err:               "it broke",
-			Url:               "asdf.cftunnel.com",
-			LogLines:          []string{"it", "was", "broken"},
-			PermanentFailure:  true,
-			TunnelID:          "asdfghjkl;",
-			RetryAfterSeconds: 19,
-		},
+		NewSuccessfulTunnelRegistration(testURL, testLogLines, testTunnelID, testEventDigest, testConnDigest),
+		NewSuccessfulTunnelRegistration(testURL, nil, testTunnelID, testEventDigest, testConnDigest),
+		NewPermanentRegistrationError(testErr).Serialize(),
+		NewRetryableRegistrationError(testErr, testRetryAfterSeconds).Serialize(),
 	}
 	for i, testCase := range testCases {
 		_, seg, err := capnp.NewMessage(capnp.SingleSegment(nil))

tunnelrpc/tunnelrpc.capnp

@@ -21,6 +21,10 @@ struct TunnelRegistration {
     tunnelID @4 :Text;
     # How long should this connection wait to retry in seconds, if the error wasn't permanent
     retryAfterSeconds @5 :UInt16;
+    # A unique ID used to reconnect this tunnel.
+    eventDigest @6 :Data;
+    # A unique ID used to prove this tunnel was previously connected to a given metal.
+    connDigest @7 :Data;
 }
 
 struct RegistrationOptions {
@@ -48,6 +52,8 @@ struct RegistrationOptions {
     uuid @11 :Text;
     # number of previous attempts to send RegisterTunnel/ReconnectTunnel
     numPreviousAttempts @12 :UInt8;
+    # Set of features this cloudflared knows it supports
+    features @13 :List(Text);
 }
 
 struct CapnpConnectParameters {
@@ -291,7 +297,7 @@ interface TunnelServer {
     unregisterTunnel @2 (gracePeriodNanoSec :Int64) -> ();
     connect @3 (parameters :CapnpConnectParameters) -> (result :ConnectResult);
     authenticate @4 (originCert :Data, hostname :Text, options :RegistrationOptions) -> (result :AuthenticateResponse);
-    reconnectTunnel @5 (jwt :Data, hostname :Text, options :RegistrationOptions) -> (result :TunnelRegistration);
+    reconnectTunnel @5 (jwt :Data, eventDigest :Data, connDigest :Data, hostname :Text, options :RegistrationOptions) -> (result :TunnelRegistration);
 }
 
 interface ClientService {

tunnelrpc/tunnelrpc.capnp.go

@@ -125,12 +125,12 @@ type TunnelRegistration struct{ capnp.Struct }
 const TunnelRegistration_TypeID = 0xf41a0f001ad49e46
 
 func NewTunnelRegistration(s *capnp.Segment) (TunnelRegistration, error) {
-	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4})
+	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 6})
 	return TunnelRegistration{st}, err
 }
 
 func NewRootTunnelRegistration(s *capnp.Segment) (TunnelRegistration, error) {
-	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4})
+	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 8, PointerCount: 6})
 	return TunnelRegistration{st}, err
 }
 
@@ -242,12 +242,40 @@ func (s TunnelRegistration) SetRetryAfterSeconds(v uint16) {
 	s.Struct.SetUint16(2, v)
 }
 
+func (s TunnelRegistration) EventDigest() ([]byte, error) {
+	p, err := s.Struct.Ptr(4)
+	return []byte(p.Data()), err
+}
+
+func (s TunnelRegistration) HasEventDigest() bool {
+	p, err := s.Struct.Ptr(4)
+	return p.IsValid() || err != nil
+}
+
+func (s TunnelRegistration) SetEventDigest(v []byte) error {
+	return s.Struct.SetData(4, v)
+}
+
+func (s TunnelRegistration) ConnDigest() ([]byte, error) {
+	p, err := s.Struct.Ptr(5)
+	return []byte(p.Data()), err
+}
+
+func (s TunnelRegistration) HasConnDigest() bool {
+	p, err := s.Struct.Ptr(5)
+	return p.IsValid() || err != nil
+}
+
+func (s TunnelRegistration) SetConnDigest(v []byte) error {
+	return s.Struct.SetData(5, v)
+}
+
 // TunnelRegistration_List is a list of TunnelRegistration.
 type TunnelRegistration_List struct{ capnp.List }
 
 // NewTunnelRegistration creates a new list of TunnelRegistration.
 func NewTunnelRegistration_List(s *capnp.Segment, sz int32) (TunnelRegistration_List, error) {
-	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 4}, sz)
+	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 8, PointerCount: 6}, sz)
 	return TunnelRegistration_List{l}, err
 }
 
@@ -278,12 +306,12 @@ type RegistrationOptions struct{ capnp.Struct }
 const RegistrationOptions_TypeID = 0xc793e50592935b4a
 
 func NewRegistrationOptions(s *capnp.Segment) (RegistrationOptions, error) {
-	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 16, PointerCount: 7})
+	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 16, PointerCount: 8})
 	return RegistrationOptions{st}, err
 }
 
 func NewRootRegistrationOptions(s *capnp.Segment) (RegistrationOptions, error) {
-	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 16, PointerCount: 7})
+	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 16, PointerCount: 8})
 	return RegistrationOptions{st}, err
 }
 
@@ -484,12 +512,37 @@ func (s RegistrationOptions) SetNumPreviousAttempts(v uint8) {
 	s.Struct.SetUint8(4, v)
 }
 
+func (s RegistrationOptions) Features() (capnp.TextList, error) {
+	p, err := s.Struct.Ptr(7)
+	return capnp.TextList{List: p.List()}, err
+}
+
+func (s RegistrationOptions) HasFeatures() bool {
+	p, err := s.Struct.Ptr(7)
+	return p.IsValid() || err != nil
+}
+
+func (s RegistrationOptions) SetFeatures(v capnp.TextList) error {
+	return s.Struct.SetPtr(7, v.List.ToPtr())
+}
+
+// NewFeatures sets the features field to a newly
+// allocated capnp.TextList, preferring placement in s's segment.
+func (s RegistrationOptions) NewFeatures(n int32) (capnp.TextList, error) {
+	l, err := capnp.NewTextList(s.Struct.Segment(), n)
+	if err != nil {
+		return capnp.TextList{}, err
+	}
+	err = s.Struct.SetPtr(7, l.List.ToPtr())
+	return l, err
+}
+
 // RegistrationOptions_List is a list of RegistrationOptions.
 type RegistrationOptions_List struct{ capnp.List }
 
 // NewRegistrationOptions creates a new list of RegistrationOptions.
 func NewRegistrationOptions_List(s *capnp.Segment, sz int32) (RegistrationOptions_List, error) {
-	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 16, PointerCount: 7}, sz)
+	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 16, PointerCount: 8}, sz)
 	return RegistrationOptions_List{l}, err
 }
 
@@ -2876,7 +2929,7 @@ func (c TunnelServer) ReconnectTunnel(ctx context.Context, params func(TunnelSer
 		Options: capnp.NewCallOptions(opts),
 	}
 	if params != nil {
-		call.ParamsSize = capnp.ObjectSize{DataSize: 0, PointerCount: 3}
+		call.ParamsSize = capnp.ObjectSize{DataSize: 0, PointerCount: 5}
 		call.ParamsFunc = func(s capnp.Struct) error { return params(TunnelServer_reconnectTunnel_Params{Struct: s}) }
 	}
 	return TunnelServer_reconnectTunnel_Results_Promise{Pipeline: capnp.NewPipeline(c.Client.Call(call))}
@@ -3874,12 +3927,12 @@ type TunnelServer_reconnectTunnel_Params struct{ capnp.Struct }
 const TunnelServer_reconnectTunnel_Params_TypeID = 0xa353a3556df74984
 
 func NewTunnelServer_reconnectTunnel_Params(s *capnp.Segment) (TunnelServer_reconnectTunnel_Params, error) {
-	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 3})
+	st, err := capnp.NewStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 5})
 	return TunnelServer_reconnectTunnel_Params{st}, err
 }
 
 func NewRootTunnelServer_reconnectTunnel_Params(s *capnp.Segment) (TunnelServer_reconnectTunnel_Params, error) {
-	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 3})
+	st, err := capnp.NewRootStruct(s, capnp.ObjectSize{DataSize: 0, PointerCount: 5})
 	return TunnelServer_reconnectTunnel_Params{st}, err
 }
 
@@ -3907,37 +3960,65 @@ func (s TunnelServer_reconnectTunnel_Params) SetJwt(v []byte) error {
 	return s.Struct.SetData(0, v)
 }
 
-func (s TunnelServer_reconnectTunnel_Params) Hostname() (string, error) {
+func (s TunnelServer_reconnectTunnel_Params) EventDigest() ([]byte, error) {
 	p, err := s.Struct.Ptr(1)
+	return []byte(p.Data()), err
+}
+
+func (s TunnelServer_reconnectTunnel_Params) HasEventDigest() bool {
+	p, err := s.Struct.Ptr(1)
+	return p.IsValid() || err != nil
+}
+
+func (s TunnelServer_reconnectTunnel_Params) SetEventDigest(v []byte) error {
+	return s.Struct.SetData(1, v)
+}
+
+func (s TunnelServer_reconnectTunnel_Params) ConnDigest() ([]byte, error) {
+	p, err := s.Struct.Ptr(2)
+	return []byte(p.Data()), err
+}
+
+func (s TunnelServer_reconnectTunnel_Params) HasConnDigest() bool {
+	p, err := s.Struct.Ptr(2)
+	return p.IsValid() || err != nil
+}
+
+func (s TunnelServer_reconnectTunnel_Params) SetConnDigest(v []byte) error {
+	return s.Struct.SetData(2, v)
+}
+
+func (s TunnelServer_reconnectTunnel_Params) Hostname() (string, error) {
+	p, err := s.Struct.Ptr(3)
 	return p.Text(), err
 }
 
 func (s TunnelServer_reconnectTunnel_Params) HasHostname() bool {
-	p, err := s.Struct.Ptr(1)
+	p, err := s.Struct.Ptr(3)
 	return p.IsValid() || err != nil
 }
 
 func (s TunnelServer_reconnectTunnel_Params) HostnameBytes() ([]byte, error) {
-	p, err := s.Struct.Ptr(1)
+	p, err := s.Struct.Ptr(3)
 	return p.TextBytes(), err
 }
 
 func (s TunnelServer_reconnectTunnel_Params) SetHostname(v string) error {
-	return s.Struct.SetText(1, v)
+	return s.Struct.SetText(3, v)
 }
 
 func (s TunnelServer_reconnectTunnel_Params) Options() (RegistrationOptions, error) {
-	p, err := s.Struct.Ptr(2)
+	p, err := s.Struct.Ptr(4)
 	return RegistrationOptions{Struct: p.Struct()}, err
 }
 
 func (s TunnelServer_reconnectTunnel_Params) HasOptions() bool {
-	p, err := s.Struct.Ptr(2)
+	p, err := s.Struct.Ptr(4)
 	return p.IsValid() || err != nil
 }
 
 func (s TunnelServer_reconnectTunnel_Params) SetOptions(v RegistrationOptions) error {
-	return s.Struct.SetPtr(2, v.Struct.ToPtr())
+	return s.Struct.SetPtr(4, v.Struct.ToPtr())
 }
 
 // NewOptions sets the options field to a newly
@@ -3947,7 +4028,7 @@ func (s TunnelServer_reconnectTunnel_Params) NewOptions() (RegistrationOptions,
 	if err != nil {
 		return RegistrationOptions{}, err
 	}
-	err = s.Struct.SetPtr(2, ss.Struct.ToPtr())
+	err = s.Struct.SetPtr(4, ss.Struct.ToPtr())
 	return ss, err
 }
 
@@ -3956,7 +4037,7 @@ type TunnelServer_reconnectTunnel_Params_List struct{ capnp.List }
 
 // NewTunnelServer_reconnectTunnel_Params creates a new list of TunnelServer_reconnectTunnel_Params.
 func NewTunnelServer_reconnectTunnel_Params_List(s *capnp.Segment, sz int32) (TunnelServer_reconnectTunnel_Params_List, error) {
-	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 3}, sz)
+	l, err := capnp.NewCompositeList(s, capnp.ObjectSize{DataSize: 0, PointerCount: 5}, sz)
 	return TunnelServer_reconnectTunnel_Params_List{l}, err
 }
 
@@ -3982,7 +4063,7 @@ func (p TunnelServer_reconnectTunnel_Params_Promise) Struct() (TunnelServer_reco
 }
 
 func (p TunnelServer_reconnectTunnel_Params_Promise) Options() RegistrationOptions_Promise {
-	return RegistrationOptions_Promise{Pipeline: p.Pipeline.GetPipeline(2)}
+	return RegistrationOptions_Promise{Pipeline: p.Pipeline.GetPipeline(4)}
 }
 
 type TunnelServer_reconnectTunnel_Results struct{ capnp.Struct }
@@ -4302,249 +4383,257 @@ func (p ClientService_useConfiguration_Results_Promise) Result() UseConfiguratio
 	return UseConfigurationResult_Promise{Pipeline: p.Pipeline.GetPipeline(0)}
 }
 
-const schema_db8274f9144abc7e = "x\xda\xc4Z}\x90\x14ez\x7f\x9e\xee\x19\x9a\x85]" +
-	"f\xda^\x8b=\x04\x16(\x8d\xc2\x09Q\xd1D7\xf1" +
-	"f?\x80\xdb\xe5\xf8\x98\xdeYPWL\xd1\xcc\xbc\xbb" +
-	"\xdb\xd0\xd3=t\xf7\x00K\xf0\x10\x0a\xe3\xb9\x91\x13<" +
-	"I\x89\x07W\x80\x1a?\xc2\xe5\xc0\x83\x8a\x1a\xce:\x13" +
-	"\x8d\x98\xbb\x8b\xde\x05R\xc7\xc5\xaa3\"\x95\xd2\x0ae" +
-	"\xd4\xb3,Sj\xa7\x9e\xb7?wv]\xc0\\U\xfe" +
-	"\x81\xa9\xa7\x9f\xf7\xe3\xf9\xfa=\x1f\xef^\xd7;\xa1U" +
-	"\xb8>}u\x06@=\x92\x1e\xe7\xb19\xbf\xda|\xe0" +
-	"\xaa\x7f\xdc\x0e\xea\x14D\xef\xdb'\x167~\xean\xff" +
-	"wH\x8b\x12\xc0\xfcAi3*;%\x09@\xb9_" +
-	"\xfaO@/\xfd\x07\xa7\xdf\xaa\xbc%\xed\x00yJ\x92" +
-	"Y \xe6\xf2\xf8\xc5\xa8l\x1bO\xccw\x8f\xdf\x08\xe8" +
-	"\xfdi\xf9\xf5C\x7f\xb4\xe7g\xc4,\xc4\xcc\x80\xf3\xcf" +
-	"\x8d\xdf\x8c\xca\xa7\x9c\xf3\xe3\xf1\xcb\x01\xbd\x8f\x1en\xfa" +
-	"\x9b\x83\xffr\xf2^\x90\xafF\x08\xcen\xa8\xfb\x0d\x02" +
-	"*3\xeb~\x04\xe8\xfd\xebu[\xdeY\xfd\xd1\xee\xef" +
-	"\x0c?7E|/\xd5\x0d\xa1r\xa6N\x02\xd1\xdb\x7f" +
-	"g\xe3?\xe3\x81Ov\x83|\x0dm\x83\xf4\xf9x\xdd" +
-	"\x04\x01P\xf9E]\x0e\xd0{\xfd\xda\x13/\xec\xfa\xf1" +
-	"}\xdf\x07\xf5jD\xf0\xd7\x9f\xaf\xfb\x1f:\x07'\x10" +
-	"\xc3\x07\x8f\x7f=\xf5\xc3\xd7/\xfb\x01g\xf0\x9e\xf8\xe5" +
-	"m\xcf\xee\xfa\xf1\x8c\xf7`\x85 a\x0a`\xfe\xec\x09" +
-	"6\xf1\xde4\x81t\xf1\xf0\xaf\x7f\xb2\xac\xbc\xfb\xd1C" +
-	"\xfe\xa5\xf9^\x97O\x14\x04Hy;\xba>)\xafx" +
-	"\xac\xf0XR\x9c\xba\x89\x1f\xd2\xd2i\x13I\x9c\x9b~" +
-	"sn\xf9\xd2g\xfb\x9e\x0c\x18\xf8E\x9f\x9f\xf8,1" +
-	"\xbc6\x91\xeeqrc\xf6\x81\xb6?~\xf0\xc9Z\xab" +
-	"\xa4\xf9\x8d'\x0e\xa1\x92\xae\xa7\x9fX\x7f\x1b\x02zC" +
-	"\xb7\xfcd\xe5G\x7f\xe1<\x03\xea\\Ly/\xdf\x7f" +
-	"v\xc3\xec\xa7\xfb^\xe5\xd7\x16\xc9\x8c\x0d\xbf\xa2\xadw" +
-	"6\xd0\xd9\x0d\x7f?g\xd9\x83\xef,9J['\xcc" +
-	"\xe2_b\xee\xa4\x16Tn\x9dD\x96\xb9e\x12q\xbf" +
-	"q\xed\xca\x17_<\xd2\x7f\xb4\xf6\"\xdc\xe2g&-" +
-	"F\xe5<\xe7~\x97s_\xde\x85o\xfe\xf4\xfa\xd4\xdf" +
-	"%\x05\xdf\x9fy\x8f\x0e?\x9a!\x86;?;\xfe\x0f" +
-	"\x0b\xdf?\xf5|\xd2BjV \x0b\xb1,\x09\xde;" +
-	"\x84\xe57[Z_\x04\xf5\x1aDo\xed\x9e-n\xe7" +
-	"#;=X\x81\x12\x0a\x00\xf3wf7\xd3f{\xb3" +
-	"\xe4_\xd3\xce\xb77\x98\xefo\xffi\x8d3\xf2S?" +
-	"\xce.F\xa5N\xa6\xab\xa5\xe5\x1f\x01~\xf2\xcc}\xbb" +
-	"\xba\xce.xU\x9d\x82\xa9Z\xa1\x0f\xca\x9bQ9N" +
-	"\xbc\xf3\x8f\xca\xcd\xa4\xcfH\x835\xec\\\xea\xff\xb8l" +
-	"-*\x1f_F??\xb8\x8c\xb3/\xbe\xf3{\x0f\xa5" +
-	"\xcf}\xef\xd5Z\x95R\xe0\xcc\xff\\\xb1Q\x91\x1b\xb9" +
-	"K7\xfeV\x00\xf4\xa6\x1c\xf9\x93\xbfm/\x9d\xf9\xd9" +
-	"(A\xa4\xdc:\xf9C\xa5k2\xfdZ8\x99d<" +
-	";\xf7\xe8\x9f\xbf\xbb\xf3\x97\xa7\x92\x9erp2\xf7\xd8" +
-	"\xe3\x93Ia\xf7}}p\xf3\xb2\xab\x86N\xd7\x1a\x88" +
-	"s\x9e\x9e<\x84\xcay\xbe\xdd\xbb|;\xe1\x9c\xf6\xb5" +
-	"{\xfe\xed\x1bo&|\xf6\x8e\xa6\xb7\x11R\xde\xb2\x95" +
-	"w\xae\xad\xbb\xfb\xec\xd9\xe4AK\x9b\xb8\xe9\xb4&:" +
-	"\xe8\x98\xfc\x90r\xe2\xe0_\xbfC\x07I\xb5\xea\xde\xd6" +
-	"\xd4\x8b\xca\x9e&\xfa\xb9\xbb\xe9I\x122\x8a\x9d\xd1\x1c" +
-	"g\xe7\x15-\xa8\xec\xbf\x82\xee\xb5\xf7\x0a\xba\xd7M\xab" +
-	"\xdb\xd8\xaa\x9bo\x7f\x0f\xe4)\xe20\xa88C\x9c\xef" +
-	"\x12\xe7\xfcsW\xdc\x87\xca\xf9\xa9\x12\x80\xf7\xdd\xfe\xde" +
-	"\xd7>\xe88\xf8\xdf\xb5\x9bs\x81NOmA\xe5\xdc" +
-	"Tn\xaa\xa9\xdc>\xf3\xaf\xff\xcb\xf3{\x1e\xeb\xf8`" +
-	"\xc4\xees\xa7\xb7\xa3r\xebt\xee\xee\xd3\xbf\xa9\xe8\xd3" +
-	"\xf9\xe6\xdf^\xb0\xfc\x96Y/}\x98\xd4\x84:\x9dG" +
-	"/\x9bN\x9a\xe8\xbb\xf9\xbf\xbey\xd5w\xff\xe9\xc3\x1a" +
-	"\xfbq\xc6{\xa7\xcfAe\x0f\xdfq71\xbf\xbf\xe8" +
-	"\x07\xa7\xa6d\xa6\xfcn\xb4\x8b\x1e\x9f\xbe\x16\x95\xd7\x88" +
-	"w\xfe+\xd3y\x1c\xdf\xfe\xf6\xa3\x1bs\xdf\xff\xdd'" +
-	"$\x97X\x83s\xb7\xcc\xe8Ee\xe9\x0c\xda\xb9k\x06" +
-	"\xc5\xd2\x92\xc3g\xbe1\xb0\xe7\xe4\xa7\xa3\"\xf7\xc73" +
-	"\xb6\xa3R7\x93\xfb\xffLB\xab\xbf\x92\xf6\x9d\xbd\xe7" +
-	"\xb7\x7f\xf6YR\xaa\xf33\xdf\xe6\xd07\x8b\xa4\xda\xf2" +
-	"\xfe\xde\xce\x07W\x1d\xfe\"\xc90s\xd6\x0b\xc4p=" +
-	"g\x88\x82q4OSg\xb5\xa3\xa2\xcd\xa2\xf3\xee\x9a" +
-	"\x95\x83\xb9\x9e[5Mf\xd8\x95T\xf1\x0f\xc3\x9f\xc5" +
-	"yE\xadbVZ\xda\xaa\xee\x003]\xbd\xa8\xb9\xac" +
-	"\x9b\xe5\x9c\x8ae:,\x8f\xa8f\xc5\x14@\x0a\x01d" +
-	"m-\x80\xbaZD\xd5\x10PFl$\xb4\x96u\"" +
-	"\x0e\x88\xa8\xba\x02\xca\x82\xd0H\x88 \xaf\x9f\x05\xa0\x1a" +
-	"\"\xaa\x9b\x04D\xb1\x91\xf0N\xae>\x04\xa0n\x12Q" +
-	"\xdd!\xa0WavY3\x99\x09\x19w\xa1mc=" +
-	"\x08X\x0f\xe8\xd9\xcc\xb5\x07\xb55\x06dX\x82,\xad" +
-	"\xdd\xe8b\x03\x08\xd8\x00\xe8\x0dXU\xdbYa\xba\xa8" +
-	"\x1b\xdd\xac\xcff\x0e\x0e\xe08\x10p\x1c\xe0X\xe2u" +
-	"X\xa6\xc9\x8an\xa1Z,2\xc7\x01 \xc9\xc6G\x92" +
-	"\xcd~\x14@\xbdVD\xf5\xe6\x84d7\x91d7\x8a" +
-	"\xa8\xb6\x0a\xe89\xcc\xde\xc0\xec%\x16\x165W\xb7\xcc" +
-	"e\x9aXf\xd1\xb5\x8b\x86\xceL\xb7\xc3\x82\x8c\xd9\xa7" +
-	"\xf7c6\x0e\x05@\xcc\x8e}\xb1\x85\x9bt\xc7\xd5\xcd" +
-	"\xfe\x1eN\xcf\xe5-C/\x0e\xd2\xed\xea\xb9&\xa7\xb5" +
-	"\xd0\x1e\xf2\xe5\xbd\x00(\xc8r;@N\xef7-\x9b" +
-	"y%\xdd)\x92P \x16\xdd\xadk4C3\x8b," +
-	":h\xdc\xc8\x83\xfc\x03\x0a\\\x8eyZ\xc2\xdaW\xe6" +
-	"5[\x13\xcb\x8eZ\x1f\xe9ca/\x80\xba@D5" +
-	"\x9f\xd0\xc7\xd2\xc5\x00\xea\x12\x11\xd5\xdb\x13\x96^\xd1\x0e" +
-	"\xa0\xe6ETW\x09\xe8Y\xb6\xde\xaf\x9b\x1d\x0cD;" +
-	"i0\xc75\xb52\x03\x80Pa[\xad\x0a)\xd1\xc1" +
-	"l\x8c\xd25\x9aJ\x8f\x14\xa0\x93\x19\x86u\x9be\x1b" +
-	"\xa5\xe5\xfe9\x16i\x9b\x9b2Z&\x8dbyn\x1c" +
-	"\x92[/\xb2yU\x87\xf9\xeb\xaa67\xe4\x95\xdd\xcc" +
-	"\xa9\x1a\xae\x03\xa0\xa6\"\xf1\x1bZ\x00\xd4\xf1\"\xaa\x8d" +
-	"\x02\xe6l\xce\x80\xd9\x18\xd4k\xaez!]WM\x9b" +
-	"\xf5\xeb\x8e\xcbl\x9f|e\x8e\x14^v\x92\x07\x92\xff" +
-	"eET\xa7\x0a\xe8\xf5\xdbZ\x91\xe5\x99\x8d\xbaUZ" +
-	"\xa6\x99VAdEL\x83\x80\xe9\xb1=i\x91\xa6\x1b" +
-	"\xac\xe4K7\xaf\xd8\xcc\xff\xa7\xe8\xad\xf7<?|{" +
-	"\xe3\xf0m\xc0/\xbc ~7\xc7\xf1\xdb |\xee\x8d" +
-	"\x0c\xe0\x06\xf13/\x08a\x8a\x08WD\xf5\x1e\x8a\x88" +
-	"j\x85t\xea\x80h\xd9\x98\x8dQ2\xd0\x0e+\xf5\x93" +
-	"\xa6M\xc8\xb1\")\x1a\xb3a\xb6\xf7\x19\xa4\x925\x80" +
-	"\xd9\xb8\x94\x09\x96\xd9l\x03\xb3\x1d\x96\x87\x8cmm\x1a" +
-	"\xc4l\x9c\xf5k\xb4>\xe9R\xb5\x1e\x1a:Z5\xf6" +
-	"z\x9b\x15}\xc8\x08\x96\xe7\x9b}\xa3%\x82\x84t\xd4" +
-	"*\xa2\xba$\x11$]\x14$\x9d\"\xaa=\x89 Q" +
-	"\xdb\xe3\xc8\xa9\x01\xb3\xdfSl\x0c\xbbzp\xf1\xe8\xca" +
-	"\x09?#\x17\xa8\x17Qm\"\x0c\xa6\xaf\xcc\xa5h\xa5" +
-	"\xd3\xa2\xea\xf6\xc2\xa7u\xd0\xbf\x01\xa2\xe6\x83]\xec\x00" +
-	"T\x9b\xa2\xc3\xf6\xd2a\x8f\x88\xa8>\x9e\xd0\xcfA\x1b" +
-	"@= \xa2zX@\x0c\xd4\xf3\xf4!\x00\xf5\xb0\x88" +
-	"\xeas\x02\xca\xa2\xe0\xfb\xda\xf19\xd4\x0b\x89\xa8\xfe\\" +
-	"@9%6R\xa9/\xbfFq\xf2s\x11\xd5_\x0b" +
-	"(\xa7S\x8d\x98\x06\x90O\xaf\x01PO\x89\xa8\xbe\xf5" +
-	"e\x10T4\xacj\xa9\xcf\xd0\xa0\xd9f\xa5\xae\x05\x11" +
-	"\xdd\xac\x96\xf36\xdb\xa0\xa3Uu\xda\\\x97\x95\xa5\x8a" +
-	"\xeb\x84\xd9$\xe3j\xfd\x0eN\x02\xcc\x8b\x88\xd9\xb8>" +
-	"\x04$b\xb4'\xda\xac\xb4\x92\xd9\x8e.Zf\x94\x10" +
-	"t\xd3e\xa6\xbbD\x03i\x0d3\"\xea\x18\x80\xd1\x1d" +
-	"\xb8=9}\x10\xc1V\x0cr\xd8On7\xd5\xf3\x02" +
-	"%.\x9c\x13{\xde4\xfc\x82\xc8\xa4\xc7\xae\xee\xd8\xf7" +
-	"\xa6\x09\x9f\x13\x994\xa9\xf6\xc6\x10\x9d\x19p\xdd\x0af" +
-	"\xe3\xba10\xf6F\xb6\xc6\xb1\x8a\xeb\x18 !]T" +
-	"\xc4\x04_\x07\x02\xe4\x05\xd1(a6\xee\xfbj<E" +
-	"\xfc\xb2\xb4\x9b\xa3$o\xd9<\xab\xc5\xe1sC,D" +
-	"\x14=\xbd\xc9\xe8i\x0d\xa2gM|\xff\xe6\xa2Vu" +
-	"\xd8\xf0z\xa1\xad\xcf\x05\x91\xd9\x11D:\x03V\xd5(" +
-	"u3\x90\\{\x10\x11\x04\xc4\xb1\x81s\x81\xd5\x99P" +
-	"\xbc\xef\xc6\xa3\xe7\xc2(\x15\xf6&Sa\xa0\xfe\x15\xa4" +
-	"\xfe\x1e\x11\xd5\x8a\x80\x9eA\xd0cvZ :nt" +
-	"]\x9f\x98\xb7\xb8sJ \xa0\x04\xe8U+\x8ek3" +
-	"\xad\x0c\x18y\x1b\xf1O\xba\x84\x0cS\x83ty-S" +
-	"\x0bU\xff\xef\xf9\xfc\xd2\x13\xb3\x9f$\x87\xa5\xe5C\x89" +
-	",Y\x0cV#_\xdea\x99\xd2%\x97^\x01\x82\xf9" +
-	"\x89a^\x90\xe8\xa9*\x0c3\xe6lB\xf8+ET" +
-	"\xafKf\xcc\xb9\xa4\xa2kDTo\x14Pb6%" +
-	"\xbf\xa8}\xf7\x0f\xdd\xea\xf8e&f\xe3\xd9\xcc\x85\xaf" +
-	"\x93\xa8\xc0u\xcb\x1c\xe1\x86\xa3f\x9b\x1b\x12v\x0dM" +
-	"\xb8tM\"\xdb\xacc\x83\xa1\x95\x9aYY\xd3c4" +
-	"\x0a\x8c\xdb\x06\xd2\xb7b\x9e1+\xd5 \xa3\xfb\xf9<" +
-	"\xe7[\x8b.\xd9\x18]\xf2\xee!\x00\xf5\x1e\x11\xd5\x07" +
-	"\x12\x97\xbc\x9f\x0a\xff\x07DT\x1fI\\r\x0f)q" +
-	"\x97\x88\xea>\xc2\xfc\xa0E\xd8K\x06\xde'\xa2\xfa\x94" +
-	"\x80\x98\xf2!\xff\x09\x82\xfc\xa7DT\x8f\x09\x1c\xb0;" +
-	"\xdb:,\x13\x83K8\x00Q\xf1?\xc04\xdb]\xc3" +
-	"4t\xbbL\x97\xd9\x1b44BH\xd8\xea\xeaef" +
-	"U\xdd\x08\"\xca\xda&^-a\xa9\xd3_%i\xae" +
-	"\x83u `\x1dE\xa4\xc3\xec\x0e\x9b\x95\x90\xac\xa1\x19" +
-	"yMt\x07.FA\xc3A<3\x8az\xa8\xd6\xda" +
-	"\"\xa2\xfa\x1d\x82\x12L\x8c\x88\xe4{\xd7\x82\xc0\x91\x84" +
-	"d^\xdf\x1eW_<!\xa6k\xfa'\x9e\x10\xc7\x01" +
-	"\xc8\xdbH;;DTw\x09\xe1\xd5:-\xc8\xf9\x11" +
-	"Zk\xea\xa0?\xd9J\xa8\xa9\xb3X\xde\xa0^\xd0\xd1" +
-	"2{\xb8\xa20\xd6T\xd1*Wlre\xdd2\xd5" +
-	"\xaaf\xe8\xa2;\x18-\x1cS\x17\x04I~(/\xaf" +
-	"4sc\x912n\x0c\x95\xa1\xdc\x85\x8b\x01\x0a\xabP" +
-	"\xc4\xc2\x00\xc6\xee\xa20l\x07(\xac&\xba\x81\xb1\xc7" +
-	"(:N\x01(\x94\x88^\xc1\xa8\xadT\xca\xf8\x0c@" +
-	"\xa1B\xe4-\x18\x97\x0a\xca \xdf~\x13\xd1w`\\" +
-	"-(\xdbp\x0e@a\x0b\xd1\x1f!\xfa8\x81kR" +
-	"\xd9\x83k\x01\x0a\x0f\x13\xfd\x00\xd1\xa5t#\xf5\xd0\xca" +
-	"~\xb4\x01\x0a\xfb\x88\xfe\x14\xd1\xc775\xe2x\x00\xe5" +
-	"\x09N\x7f\x9c\xe8G\x88^\xf7\xb5F\xac\x03P~\x88" +
-	"\xdb\x01\x0a\x87\x89\xfe\x1c\xd1'`#N\x00P\x8e\xe3" +
-	"\xa3\x00\x85\xe7\x88\xfe2\xd1'\x8ek\xc4\x89\x00\xcaK" +
-	"\xfc>'\x88~\x92\xe8\xf5\xa9F\xac\x07P^\xc1C" +
-	"\x00\x85\x93D?\x85\x11\xdeu\x95\x92\xb0K\xee\xa6\xc7" +
-	"e\x87h9\x91\xc9Y\xd0V\xa2\x9f\x13\xf2V\x86\xfa" +
-	"J\xcc\xc4\xf3_@\xcc\x00z\x15\xcb2\x96\x0d\x87\xf3" +
-	"\x0bU>\x81\xbb@\xc62\xbbJQ\xfc\xf9N\xb6\xc4" +
-	"\x82\xe6\xa2ftU\xe2Z\xc8i\xab\xbaV\xb5\x02\xcd" +
-	"%\xcde\xa5(!\xdbUs\x91m\x95{\x90\xd9e" +
-	"\xdd\xd4\x0c\x88\xbe\x8c\xe5s\x99jU/E{\x8fY" +
-	"\xc0E\xee)\xd4\xbags\xa5\xa5G\xeb\xaf\x19\x01\xcc" +
-	"\x89\xb1>\x82\xae\xb97\xc4P\x9fI\x86T\xf3\x06\xcd" +
-	"\xa8\xb2\x8b\xa9\xec\xc6l*\xbas~Sr\xa1\xde3" +
-	"\x1cX]\xb84_Q\x93F\xfd\xe46b\xde\xd1\x1e" +
-	"\x0b\x1b\xc9j\x073\x90N!N`\xa1I\xfa\x82\xde" +
-	"\x12\x9ai\xef\x84sD\x13\xc5\xc09.V\x13\xfd\xcc" +
-	"\xf5\x7fu\x99}\x16\xe5zI+;_qu7s" +
-	"2\x17\xa3\xc5xFx\xe1d\xdc\xd9\xd3\x93\x8f\xc7\x0c" +
-	"\xa2\x8f\xe4\xd7E\xe0\xd5\x86\xdd\x00\x85V\x8a\xce%\x18" +
-	"\xe9P\xe9\xe2 \xd2I\xe4\x1e\x8cKXE\xe5`\x91" +
-	"'\xfa*\x8c\x9b\x1c\xe5\x0e\x1e\xe41\x06\xa6\xda|\xf0" +
-	"b|\xfb\x08\xeb\xe44\xfa\xe0U\xe6\xfb\x1bD\xdf\x94" +
-	"\x04\xaf*\x0e\x0d\x03;I\xf4\xc1k\x1b\x07\x9d\x1dD" +
-	"\xdf\xc5\xc1+\xe5\x83\xd7N|\x16\xa0\xb0\x8b\xe8\xfb8" +
-	"x\xa5}\xf0\xda\x8b/\x0c\x03\xbb\x09\xe3|\xf0z\x82" +
-	"\xf3?E\xf4c\x1c\xbc\xda}\xf0:\xca\xc1\xee\x08\xd1" +
-	"O\x10HUm\xa3\xe0\xda\xba\x09\xd8\x1f\xc7F\xb1\xf2" +
-	"-\xc6*m\x901\xf4\x0d,J,%]3\x16T" +
-	"5\x03\x9a\x0b\xaeV\\\x17\xd7\xe9\x86\xd3\xa9\x99%\x07" +
-	"\x07\xb4u\x8c\xd2\x91\x94L\xdc\xae\xe1\xacd\xb6\xde\x07" +
-	"\x18W\xf6Q!\x93\xc9[Vm}\xc3\x0bDf\xfb" +
-	"\x08\x17}+k\x9b\xbaJ\x06\xeb\xc0\xb0\x9c\x11\xcd8" +
-	"\x1d\xea\xf4\xc52M\xf4k\x8c\x1e\xbdyx\xf1P\x09" +
-	"z\x85\xb0\x08\xe9\xc9\xd5T\x17lS\x85\x15\xdd\x0e\x0b" +
-	"MW7\xabl\xc4\x06\xc5\x81\xaa\xb9\x8e\x95\x16\xa2Y" +
-	"\xb4J\xba\xd9\x0f#\x9a\x14\xf1\xcb\xa6;\x89\xaa\x8bG" +
-	"3&^\xc7\xe4\xd9- p\xe8\xa2\x1aBn\x89[" +
-	"\xfd\\\x91\xaf\xca\xd9Ls\x12]\xea\x18\xa7\x05\xd3H" +
-	"?\xc8\xfc\xb6>\x0d\x10=%a8\x8e\x97\x8fn\x06" +
-	"A~Z\xc2\xf8\x15\x03\xc3G\x0by\xbf\x0d\x82\xbcG" +
-	"B!z\xe2\xc3\xf0yN\xbe\x7f\x08\x04\xf9^\x09\xc5" +
-	"\xe8\xd9\x0d\xc3Y\xb7<\xd8\x0e\x82\\\x960\x15=A" +
-	"b8(\x975\xaa\x93\xee\x900\x1d\xbd\xe7a\xf8\x1a" +
-	"#/\xdd\x0e\x82\xbcP\xf2\xc2v\x08r\xbe\x18\xad\xe8" +
-	"\x85\x80\x01\xcd\x1c2Z\xd1\x0b\xe7C\x18\xb6M\x00\xad" +
-	"\xb85\x80\xe7V\xf4\xc2\x09)d\x8a\x9a\xcbZ\xa9\xd7" +
-	"\xf4?b\x00\xde\xd0\x8a\xc9\xc9\xa3\xf8e\x0d\xce\xe8\x85" +
-	"r{\\\xcc\x85\x00\xbcm(\xae\xe5\xa2\xa6r\xe73" +
-	"\xc9:9\x98\x8d\xec\xdd\x1eLV\x8e%f#G\xa9" +
-	"x>&\xa2\xfa\x86\x10W\x06\xa1O\x87\xc3:\xb4\xec" +
-	"\xb0\xcb\x1dcf\x17x~P\xc3\xd6N\xee\xbc\x925" +
-	"\xc0k\\\xf4\xb7r N\x07\xc9q\xde\xa4\xc48\x0f" +
-	"\xc3\xfeZ\x1a\x96=\x92\xc3\xbdI\x17h\xd6\x92\xdd\"" +
-	"Og)\xee\x92\xe1\xe3%\x86\xef\xcc\xb2L\xae\xd5 " +
-	"yaG\x89a.\x84\x1a\x93]b[\xdd\xcd\x9a\xff" +
-	"/\xc9z\x14\x07\xf1\xcf\xc9\x90G\xfa\x02E\xfb\xaeM" +
-	"\xcc\xe9\x0c+\xe8\x083\xcb\x92E\xfd\x18\xba\xf2/\x1c" +
-	"\x96\xe0\x19Z\\3\x9a\xa3f\xf2a\x11\xd5\x03\x89b" +
-	"g\xff\xac\xc4\xbc.\xec\xd3\x0e.\x0e\xe6u'\xe2\x97" +
-	"\x9c\xe7\xc9QO\x88\xa8\x9eL\xb8\xdf+\xc4\xf8\xb2\xef" +
-	"~a\xbe\x92\x7fA=\xcb\x1b\"\xaao\x06\xadr\xf8" +
-	"\xa2S\xb5c\xa06\xac\xfe%\xba\xc9\x1c\xaa@k\x06" +
-	" \xe13\x11\xba\x04\x7fU\x9b0|8Vv-H" +
-	"\x14\xae\xd1<\x08\x99]\xa0p-\xa1\x13\xcdY\xc6\xd0" +
-	"W!\x88\x0e?8\x82d\x9fh\xbd\x0f%\xa6R\xa1" +
-	"\xb6\xd4\x17\x82i\xcf\xea\x84\xb6\xee\xa2\xd6{\x95\x88\xea" +
-	"\x80\xc0\x11\xc4ZQ)i\xe8\xb2E6[_e\x92" +
-	"Y\x1c\x8c[Pj\xc2\x8a\xce\x0a\xacPY\xbc\xc8f" +
-	"\xb9\xf5U\x96d\x08\x9f\x01@\xd2\xad\xd2\x88\xf9\xff(" +
-	"\xa5\xdfmlM\xc1*\xaec\xee\xb0\xe7\x91\x9a'\xbc" +
-	"\xee\xf8\x0d z\xc1\xebN\xbe\xe0\x05\xb8\xb3\x9e<\xb0" +
-	"\"\xa2\xba%\x81;\x83Cq\xff:z\xae\xff\xfd\xa4" +
-	"\xe7\xaf\xf4\x8aE\x95\xaet1U`\xf4\xc75_q" +
-	"\xac~\xb1E{\xfc6{\x89\xa3(\x88\xc0\x00\x13\x7f" +
-	"{A\x87\x08\xc1\xe6\xff\x1b\x00\x00\xff\xff\x99N\xd0U"
+const schema_db8274f9144abc7e = "x\xda\xccz}\x90\x14uz\xff\xf3t\xcf\xd2\xbb\xb0" +
+	"\xcbL\xdbc\xfdv)\xf67\xc7\x8bQ8!\"G" +
+	"\xa2\x9b\xe4\xf6\x0d\xb8]\x0ea{g\x17u\xe5R\xf6" +
+	"\xce|w\xb6a\xa6{\xe8\xee\x01\x96\xe0\xf1R\x10e" +
+	"\x03'x\x90\x02\x0f\xaf\x00\x8f\xf8\x12\xbd\x13\x0f+\xa7" +
+	"\x11K\x93\xab\xa89\x13\xe5\x82)\xbd\x98\x8a\x11\xa8T" +
+	"\xac\xb3<\xd1\x94eJ\xed\xd4\xf3\xed\xd7\x1d\x96\x05L" +
+	"R\x95\x7f`\xea\xe9\xe7\xfb\xf2\xbc}\x9e\x97\xef\xde\xf4" +
+	"\xe3\xc9m\xc2\x82\x9a\xeb\x93\x00\xeaS5\x93\\6\xf7" +
+	"\x97\x9b\x8e\\\xf7\xd7\xdbA\x9d\x86\xe8~\xf7\xf9e\xe9" +
+	"\xcf\x9c\xed\xff\x045\xa2\x04\xb0pD\xda\x84\xca\x1eI" +
+	"\x02PvI\xff\x06\xe8\xd6\xfc\xd6\x9b\xef\x96\xdf\x95v" +
+	"\x80<-\xce,\x10s\xa9v\x19*\xdbj\x89\xf9\x9e" +
+	"\xda\x0d\x80\xee\xef\x97^?\xf6;\x07~A\xccB\xc4" +
+	"\x0c\xb8\xf0|\xed&T>\xe3\x9c\xffQ\xbb\x12\xd0\xfd" +
+	"x\x7f\xe3\x9f\x1f\xfd\xfbWv\x82|=\x82\x7fvC" +
+	"\xdd\xaf\x10P\x99Q\xf7\x13@\xf7\x1fn\xda|\xee\xee" +
+	"\x8f\xf7\xdd7\xf6\xdc\x04\xf1\xbdT7\x8a\xca\xdbu\x12" +
+	"\x88\xeeCw\xa5\xff\x16\x8f|\xba\x0f\xe4\x1bh\x1b\xa4" +
+	"\xcf\xcf\xd4M\x16\x00\x95\xbf\xabk\x05t_\xbf\xf1\xf9" +
+	"\xe7\xf6\xfe\xf4\xde\x1f\x80z=\"x\xeb?\xa8\xfbO" +
+	":\x07'\x13\xc3G?\xfaz\xe2\xc9\xd7\xaf\xf9!g" +
+	"p\x8f\x9f\xbe\xfd\xe9\xbd?\xfd\xda\xfb\xd0/H\x98\x00" +
+	"X8g\xb2E\xbc\x8b&\x93.\xf6\xbfujEi" +
+	"\xdf\x83\xc7\xbcK\xf3\xbd\xae\x9d\"\x08\x90pwt\x7f" +
+	"Z\xea\x7f8\xfb\xb0/N\x0d}\xaa\x9br\x01\x01\x17" +
+	"6O\xc9 \xa0\xbb\xe8W\xe7W\xde\xf6\xf4\xd0#>" +
+	"\x07\xbf\xe9\xad\xf5O\xd3\xe6\xdd\xf5t\x91W6\xa4v" +
+	"\xb7\xff\xee\xfd\x8fT\x9b\x85\xefU\xaa\x1fEeg=" +
+	"\xfd\xdcV\x7f;\xed7z\xeb\xa9U\x1f\xff\xb1\xfd8" +
+	"\xa8\xf30\xe1\xfe|\xd7\xd9\xf5s\x1e\x1bz\x99\xdf[" +
+	"\x04X\xf8Y\xc3/i\xeb\x86\xa9\xa4\xcb\x86\xbf\x9c\xbb" +
+	"\xe2\xfes\xcbO\xd0\xd61\xbbx\x97xrj\x0b*" +
+	"\xa7\xa6\x92i\x9e\xe5\xdco\xdc\xb8\xea\x85\x17\x9e*\x9c" +
+	"\xa8\xbe\x087\xf9\x9d\xc9e\xa8\x94\x92\xc4\xad'\x89\xfb" +
+	"\xdan|\xe7\xc5\x05\x89\xbf\x88\x1b\xb29\xf5>\x1d\xbe" +
+	" E\x0cw}\xfe\xcc_-\xf9\xf0\xcc\xb3q\x13\x9d" +
+	"N\x09d\xa2\xf3)\x12|`\x14K\xef\xb4\xb4\xbd\x00" +
+	"\xea\x0d\x88\xee\x9a\x03\x9b\x9d\xae\x83{\\\xe8G\x09\x05" +
+	"\xf2\x0ay\x13m\xd6$\x93\x835\x7f\xd0\xd1`|\xb8" +
+	"\xfd\xc5*o\xe4\xa7V\xe4e\xa8\xec\x92\xe9j;\xe5" +
+	"\x9f\x00~\xfa\xf8\xbd{\xbb\xcf.~Y\x9d\x86\x89j" +
+	"\xa1g\\\xb3\x09\x95E\xd7\xd0\xcf\x05\xd7p\xfb\x84\x1a" +
+	"\xacb\xe7Rk\xca\x1aT*\x0a\xfd\\\xa7p\xf6e" +
+	"w}\xff\x81\x9a\xf3\xdf\x7f\xb9Z\xa5\xe4\xe2\x0b\xefI" +
+	"[\xa8\xecK\xd3\xcf=\xe9\xff'\x02\xba\xd3\x9e\xfa\xbd" +
+	"\x1fw\xe4\xdf\xfe\xc58Q\xa444^P\x9a\x1a\xe9" +
+	"\xd7\xb5\x8d$\xe3\xd9y'\xfe\xe8\xdf\xf7\x9c>\x13\xf7" +
+	"\x94u\x8d\xdcew6\x92\xc2\xee\xfd\xfa\xc8\xa6\x15\xd7" +
+	"\x8d\xbeYm \xcey\xbcq\x14\x95S|\xbbg\xf9" +
+	"v\xc2y\xadi\xeb?~\xf3\x9d\x98\xd3\xceiz\x0f" +
+	"!\xe1\xaeXu\xd7\x9a\xba{\xce\x9e\x8d\x1f\xd4\xdc\xe4" +
+	"\x99\xae\x89\x0e:)?\xa0<\x7f\xf4\xcf\xce\xd1AR" +
+	"\xb5\xba\xd5\xa6\x01TX\x13WO\xd3#\x02\xc4\x82g" +
+	"<\xc7\xf9\xce\xf4\x16TJ\xd3\xb9\xe3L\xa7{-\xba" +
+	"\xbb\x9d\xad\xbe\xe5\x8e\xf7A\x9e&\x8e\xc1\x8a\xc7\x88\xf3" +
+	"\xd9\xe9<\x94\xa7\xdf\x8b\xca\xa9f\x09\xc0\xfd^a\xe0" +
+	"\xd5\x8f:\x8f\xfe\xa6zs.\xd0\xf1\xe6\x16T\x9e!" +
+	"\xbe\x85'\x9a\xb9}\x16.\xf8\x93\x0f\x0e<\xdc\xf9\xd1" +
+	"E\xbb\x7f\xf1\xff;Pi\xc8\xd0=\xea2\xdfRn" +
+	"\xcd\xf0\xcd\xbf\xbbx\xe5\xad3_\xba\x10\xd7\xc4\x8c\xcc" +
+	"\x05\x1e\xf9\x19\xd2\xc4\xd0-\xbf\xfe\xd6u\xdf\xfb\x9b\x0b" +
+	"U\xf6\xe3\x8c\xfd\x99\xb9\xa80\xbe\xa3F\xcc\x1f.\xfd" +
+	"\xe1\x99i\xc9i\x9fT]t\x12\xf1\xee\xcc\xacA\xe5" +
+	"\x10\xf1.<\x90y\x99.z\xc7{\x0fnh\xfd\xc1" +
+	"'\x9f\x92\\b\x15\xd0\xed\x9a1\x80\xcaC3h\xe7" +
+	"C3(\x96\x96?\xf1\xf67\x87\x0f\xbc\xf2\xd9\xb8\xd0" +
+	"\xbdd\xe6vT\xee\x9cI\xdc\xfd3\x09\xae\xfeT:" +
+	"|v\xeb\xbf\xfc\xe1\xe7q\xa9\xfe`\xd6{$\x95:" +
+	"\x8b\xa4\xda\xfc\xe1\xa1\xae\xfbW?\xf1\xe5\x18O\x9b\xf5" +
+	"\x1c1l\xe3\x0ca0\x8e\xe7iGgu\xa0rb" +
+	"\x16\x9d\xf7\xe4\xacV\x98\xe7:\x15\xc3`E\xab\x9c\xc8" +
+	"\xfdv\xf037?\xa7\x95\x8drK{\xc5\x19f\x86" +
+	"\xa3\xe74\x87\xf5\xb2V\xbbl\x1a6\xebATSb" +
+	"\x02 \x81\x00\xb2\xb6\x06@\xbd[D\xb5(\xa0\x8c\x98" +
+	"&\xb8\x96u\"\x0e\x8b\xa8:\x02\xca\x82\x90&D\x90" +
+	"\xd7\xcd\x04P\x8b\"\xaa\x1b\x05D1Mx'W\x1e" +
+	"\x00P7\x8a\xa8\xee\x10\xd0-3\xab\xa4\x19\xcc\x80\xa4" +
+	"\xb3\xc4\xb2\xb0\x1e\x04\xac\x07t-\xe6X#\xda`\x11" +
+	"\x92,F\x96\xd6lp\xb0\x01\x04l\x00t\x87\xcd\x8a" +
+	"e\xf7\x1b\x0e\xea\xc5^6d1\x1b\x87q\x12\x088" +
+	"\x09p\"\xf1:M\xc3`9'[\xc9\xe5\x98m\x03" +
+	"\x90d\xb5\xa1ds\x1e\x04Po\x14Q\xbd%&\xd9" +
+	"\"\x92\xec\x1b\"\xaam\x02\xba6\xb3\xd63k\xb9\x89" +
+	"9\xcd\xd1Mc\x85&\x96Xx\xed\\Qg\x86\xd3" +
+	"iB\xd2\x18\xd2\x0b\x98\x8aB\x01\x10S\x13_l\xc9" +
+	"F\xddvt\xa3\xd0\xc7\xe9\xad=fQ\xcf\x8d\xd0\xed" +
+	"\xea\xb9&\x9b[h\x0f\xf9\xda\x01\x00\x14d\xb9\x03\xa0" +
+	"U/\x18\xa6\xc5\xdc\xbcn\xe7H(\x10s\xce\x96A" +
+	"\xad\xa8\x199\x16\x1e4\xe9\xe2\x83\xbc\x03\xb2\\\x8e\xf9" +
+	"Z\xcc\xda\xb3{4K\x13K\xb6Z\x1f\xeac\xc9\x00" +
+	"\x80\xbaXD\xb5'\xa6\x8f\xdb\x96\x01\xa8\xcbET\xef" +
+	"\x88Y\xba\xbf\x03@\xed\x11Q]-\xa0kZzA" +
+	"7:\x19\x88V\xdc`\xb6ch%\x06\x00\x81\xc2\xb6" +
+	"\x98eR\xa2\x8d\xa9\x08\xa5\xab4Us\xb1\x00]\xac" +
+	"X4o7\xadb~\xa5w\x8eI\xda\xe6\xa6\x0c\x97" +
+	"I\xe3X\x9e\x1b\x87\xe4\xd6sl~\xc5f\xde\xba\x8a" +
+	"\xc5\x0d9\xbb\x97\xd9\x95\xa2c\x03\xa8\x89P\xfc\x86\x16" +
+	"\x00\xb5VD5-`\xab\xc5\x190\x15\x81z\xd5U" +
+	"/\xa7\xeb\x8aa\xb1\x82n;\xcc\xf2\xc8\xb3[I\xe1" +
+	"%;~ \xf9_JDu\xba\x80n\xc1\xd2r\xac" +
+	"\x87Y\xa8\x9b\xf9\x15\x9aafE\x96\xc3\x1a\x10\xb0f" +
+	"bOZ\xaa\xe9E\x96\xf7\xa4\x9b\x9f\xcb\xf0\xff)z" +
+	"\xeb]\xd7\x0b\xdf\x81(|\x1b\xf0K\xd7\x8f\xdfMQ" +
+	"\xfc6\x08_\xb8\x17\x07p\x83\xf8\xb9\xeb\x870E\x84" +
+	"#\xa2\xba\x95\"\xa2R&\x9d\xda \x9a\x16\xa6\"\x94" +
+	"\xf4\xb5\xc3\xf2\x05\xd2\xb4\x01\xad,G\x8a\xc6T\x90\xed" +
+	"=\x06)o\x0ec**e\xfce\x16[\xcf,\x9b" +
+	"\xf5@\xd227\x8e`*\xca\xfaUZ\x9fz\xb5Z" +
+	"\x0f\x0c\x1d\xae\x9ax\xbd\xc5r\x1ed\xf8\xcb{2\x9e" +
+	"\xd1\xd2\xa1\xd1\xee\x99\x19\x01Z\x18$\xdb\x06\x01\xd4\xad" +
+	"\"\xaa\xbbcA\xb2\x8b4\x7f\x9f\x88\xea~\x01e\xd1" +
+	"\xc7\xc3}\x14N{ET\x0f\x0b('\x12i*f" +
+	"\xe5C\x14N\xfbET\x8f\x08ca\x8f\xadg\x86\xb3" +
+	"X/\x80\xc4\xec\x88JW\\\xac\x17\x18\x88\xf6\xffB" +
+	"\xc0\x8d\xd1\x87\xaf\x8dP\x0f1\xe7%\xe9\xeaET\x1b" +
+	"\x09\xd8\xe9+s\x08\x02\xe8\xb4\xb0d\xbe\xfci\x9d\xf4" +
+	"\xaf\x0f\xd3=\xfe.\x96\x8f\xd4\x8d\xe1a\x87\xe8\xb0\x83" +
+	"\"\xaa?\x8a)\xfd\xa8\x05\xa0\x1e\x11Q}B@\xf4" +
+	"u\xfe\xd81\x00\xf5\x09\x11\xd5\x9f\x91\xce\x05O\xe7\xcf" +
+	"\xcc\xa5\x0eKD\xf55\xd2\xb9\xe8\xe9\xfcU\x0a\xbe\xd7" +
+	"DT\xdf\x12P\xaeI\xa4\xb1\x06@~\x93\xecxF" +
+	"D\xf5\xddK\xe1Z\xaehV\xf2CE\x0d2\x16\xcb" +
+	"w/\x0e\xe9F\xa5\xd4c\xb1\xf5:\x9a\x15\xbb\xddq" +
+	"XI*;v\x90\xa2\x92\x8eV\xb0q*`\x8f\x88" +
+	"\x98\x8a\x8aN@\"\x86{\xa2\xc5\xf2\xab\x98e\xeb\xa2" +
+	"i\x84YF7\x1cf8\xcb5\x90\x06Y1\xa4N" +
+	"\x80B\xbd~,Q$\xf9\xb0`F\xc8\x89\x05\x02\xfc" +
+	"\xe9\xae\xeb+q\x09\xe9\xa6MDu\xb9\x80\xcd\xf8%" +
+	"\x91I\x8f\xdd\xbd\x00j\x97\x88j\x9f\x80\xcd\xc2\x17D" +
+	"&M\xaa\x03\x11\xee'\x87\x1d\xa7\x8c\xa9\xa8\x18\xf5\x8d" +
+	"\xbd\x81\x0d\xdafn-\x03$\xf8\x0c+#\xff\xeb\xb0" +
+	"\x0f\xe7 \x16\xf3\x98\x8a\xba\xc9*O\x11/\x95\xcb[" +
+	"\xa9r0-\x9e*\xa3\xc4us$D\xe0\x1d\xdd\x03" +
+	"\x91\x04\xb2\xd0\xe6\x89\xa5\x0eF\xf7\xcf\xe4\xb4\x8a\xcd\xc6" +
+	"\x16!\xedC\x0e\x88\xcc\x0aq\xd7\x1e6+\xc5|/" +
+	"\x03\xc9\xb1F\x10A@\x9c\x18\x8d\x17\x9b]1\xc5{" +
+	"n<~\x82\x0d\xf3\xeb@<\xbf\xfa\xea\xef'\xf5\xf7" +
+	"\x89\xa8\x96\x05t\x8b\x84gF\x97\xc9\xc3=\xb8\xaeG" +
+	"\xec1\xb9sJ \xa0\x04\xe8V\xca\xb6c1\xad\x04" +
+	"\x18z\x1b\xf1O\xbd\x8a\xb4U\x05\x9f=Z\x92\xc7\xfd" +
+	"\xff\xa5\"\xe1\xea\xb3\xbd\x97y\xc7\xe4\xfac\xb1\xd4\x9b" +
+	"\xf3W#_\xdei\x1a\xd2U\xd7s>\x82y\xd9f" +
+	"\xbe_=P\xa9\x19\xa4\xe19\x946f\x8b\xa8\xde\x14" +
+	"O\xc3\xf3HE7\x88\xa8~C@\x89Y\x94Q\xc3" +
+	"\x99\x80w\xe8\x16\xdb\xab]1\x15M|.\x7f\x9dX" +
+	"Y\xaf\x9b\xc6En83\x0a\x97\xd0\x84\xdd7\xc7\xec" +
+	"\x1a\x98\xf0\xb6\xc1\xc8\xae\xd2Z6\x12X)\xc3J\x9a" +
+	"\x1e\xa1\x91o\xdcv\x90\xbe\x1d\xf1LX\xfe\xfae\x82" +
+	"W$\xb4z\xd6\xa2K\xc6\xf2\xech,\xa5\x06\x97\xdc" +
+	"E\xdd\xc4n\x11\xd5\x83\xb1K\x1e\xe8\x88\xa5\xd4 \xcf" +
+	"\x1e\"\x03\x1f\x16Q}T@\xf4\xd3\xecq\x82\xfcG" +
+	"ETO\x0a\x1c\xb0\xbb\xda;M\x03\xfdK\xd8\x00a" +
+	"G1\xcc4\xcb\x19d\x1a:\xdd\x86\xc3\xac\xf5\x1a\x16" +
+	"\x03H\xd8\xe2\xe8%fV\x9c\x10\"J\xdaF^\x82" +
+	"a\xbe\xcb[%i\x8e\x8du `\x1dE\xa4\xcd\xac" +
+	"N\x8b\xe5\x91\xac\xa1\x15{4\xd1\x19\xbe\x12\x05\x8d\x05" +
+	"\xf1\xe48\xea\xa1\x02n\xb3\x88\xea}\x04%\x18\x9b;" +
+	"\xc9;\xd7\x80\xc0\x91\x84d^\xd7\x11\x95t<!\xd6" +
+	"T5e<!N\xa2\x1a\x86\xb4\xb3CDu\xaf\x10" +
+	"\\\xad\xcb\x84V/B\xabM\xed7=[\x085u" +
+	"\x16\xc9\xeb\xd7\x0b:\x9aF\x1fW\x14F\x9a\xca\x99\xa5" +
+	"\xb2E\xae\xac\x9b\x86Z\xd1\x8a\xba\xe8\x8c\x84\x0b'\xd4" +
+	"\x05A\x92\x17\xca+\xcb\x19n,R\xc6-\x812\x94" +
+	"\x11\\\x06\x90\xdd\x88\"fw`\xe4.\xca6\xec\x00" +
+	"\xc8n&\xfa}\x18y\x8c\xb2\x13\xa7\x01d\xb7\x12}" +
+	"7\x86\xbd\xaa\xb2\x0b\x1f\x07\xc8\xee&\xf2A\x8cJ\x05" +
+	"\xe5\x00\xdf~?\xd1\x8f`T-(\x0f\xe1\\\x80\xec" +
+	"A\xa2\x9f$\xfa$\x81kR9\x81k\x00\xb2O\x11" +
+	"\xfdy\xa2K5i\xe4s\x1f\xb4\x00\xb2?#\xfa\xcf" +
+	"\x89^\xdb\x98\xc6Z\x00\xe5%N\x7f\x91\xe8\xaf\x11\xbd" +
+	"\xae)\x8du\x00\xca\xab\xb8\x1d \xfb\x0a\xd1\xcf\x10}" +
+	"2\xa6q2\x80r\x1a\x1f\x04\xc8\x9e!\xfa\xbbD\x9f" +
+	"2)\x8dS\x00\x94\x7f\xe6\xf7y\x8b\xe8\xe7\x88^\x9f" +
+	"Hc=\x80\xf2\xafx\x0c {\x8e\xe8\xbf!z\x83" +
+	"\x94\xc6\x06\x00\xe5\x03.\xd7\xaf\x89^+\x848\xd8\x9d" +
+	"\x8f\xc31\xb9\xa1\x1e\x95#\xa2i\x87\xae\xc0\xfc\x1e\x16" +
+	"\xbd\\\xd1c&\xa9\x89\xc5d4m\x06\xc4$\xa0[" +
+	"6\xcd\xe2\x8a\xb10\x7f\xb9\x8a\xc8w#H\x9aFw" +
+	">\x8cK\xcf\xf9\x96\x9b\x90\xc9i\xc5\xeerT#\xd9" +
+	"\xed\x15\xc7\xac\x94!\x93\xd7\x1c\x96\x0f\x13\xb5U1\x96" +
+	"Zf\xa9\x0f\x99U\xd2\x0d\xad\x08\xe1\x97\x89|1Y" +
+	"\xa9\xe8\xf9p\xef\x09\x0b;w\x88iN\xc5b6\x89" +
+	"v\x89\x8c+T{t\xa6\xdc\xd2\xa7\x15\xaaF\x11s" +
+	"\xa3\xf4\x10\xa2\xdd\xbc\x9b\xa3\xec\x90\x8cGaf\xbdV" +
+	"\xac\xb0+)\x06'lnz[\xbd\xe6\xe8r=p" +
+	"08\xbb|5\xdf_\x95y\xbd|x\xd1\xdc\xa5#" +
+	"\x126\x94\xd5\xf2g1]B\x94\xf3\x02k\x0d\xf9=" +
+	".dh\xef\x98\xdf\x84\x93M\xdfo\xaeT\x13\x05\xe6" +
+	"x\xbf\xba\x8d!\x93\xca\x03I+\xd9_qu/\xb3" +
+	"\x93W\xa2\xc5hVy\xf9\xfc\xdd\xd5\xd7\xd7\x13\x8d;" +
+	"D\x0f\xfco\x0a\xf1\xae\x1d{\x01\xb2m\x14\xb8\xcb1" +
+	"\xd4\xa1\xd2\xcdq\xa7\x8b\xc8}\x18U\xbd\x8a\xca\xf1\xa5" +
+	"\x87\xe8\xab1\xea\x8b\x94;9.\xac&\xfa0\xc7\xbb" +
+	"v\x0f\xef\x18\xdf>O\xf42\xc7;\xf4\xf0\xae\xc4\xf7" +
+	"/\x12}c\x1c\xef*8:\x06~%\xd1\xc3\xbbm" +
+	"\x1c\xa7v\x10}/\xc7\xbb\x84\x87w{\xf0i\x80\xec" +
+	"^\xa2\x1f\xe6xW\xe3\xe1\xdd!|\x0e {\x98\xe8" +
+	"\x8fr\xbc\x9b\xe4\xe1\xddq\xce\xffh\x88\xb3S:<" +
+	"\xbc;\xc1\xf11\xc4Y\xb7b\x15\xb3\x8e\xa5\x1b\x80\x85" +
+	"(6r\xe5o3Vn\x87dQ_\xcf\xc2\\\x94" +
+	"\xd7\xb5\xe2\xe2\x8aV\x84L\xd6\xd1rk\xa3\xd2\xbeh" +
+	"wiF\xde\xc6am-\xa3\x0c&\xc5s\xbdS\xb4" +
+	"W1K\x1f\x02\x8c\x9a\x81\xb0\xf6I\xf6\x98fuI" +
+	"\xc4kJfy\xe0\x17~+i\x1b\xbb\xf3E\xd6\x89" +
+	"A\x05$\x1aQ\x06\xd5\xe9\x8bi\x18\xe8\x95%}z" +
+	"fl\xbdQ\xf6\xdb\x8b\xa0n\xe9k\xad*H\xd8\xc6" +
+	"2\xcb9\x9d&\x1a\x8enT\xd8E\x1b\xe4\x86+\xc6" +
+	"Z\x96_\x82F\xce\xcc\xebF\x01.\xeak\xc4KM" +
+	"\x99b\x85\x1a\x8ff\x8c=\xd3\xc9sZ@\xe0\xd0E" +
+	"e\x87\xdc\x12M\x07Zs|U\xab\xc54;\xd6\xd8" +
+	"Np\x9a?\x15\xf5\x82\xcc\x9b\x04\xd4\x00\x84OZ\x18" +
+	"<\x0b\xc8'6\x81 ?&a\xf4\x9a\x82\xc1\xe3\x89" +
+	"\xfc\x90\x05\x82|@B!|k\xc4\xe0\x9dP\xde5" +
+	"\x0a\x82\xbcSB1|\xfe\xc3`\xe6.\x8ft\x80 " +
+	"\x97$L\x84o\xa1\x18\x0c\xece\x8dJ\xab;%\xac" +
+	"\x09\x1f\x161x\x15\x92o\xdb\x0e\x82\xbcDr\x83\x0e" +
+	"\x0aZ=1\xda\xd0\x0d\x00\x032\x1c2\xda\xd0\x0d\xe6" +
+	"T\x18tZ\x00m\xb8\xc5\x87\xe76t\x83I-$" +
+	"s\x9a\xc3\xda\xa8=\xf5>\xa2\x0f\xde\xd0\x86\xf1\x09\xa8" +
+	"x\xa9\x9eh\xfc\xda\xba#\xaa\xff\xc2\x11\xd6hT\xfe" +
+	"\x85}\xe8\x9e\xc7\xe3\xa5\xb5?N9\xb4\xdd\x1f\xc6\x9c" +
+	"\x8c\x8dSNP\xbd}RD\xf5\x0d!*\x1a\x02\x9f" +
+	"\x0e\x86\x86hZAc<\xc1\xec\xd0\xf7|\xbf\xec\xad" +
+	"\x9e \xbays\x98\x97\xc5\xe8meC\x94\x0e\xe2c" +
+	"\xc5\xa9\xb1\xb1\"\x06-\xb94&{\xc4\x87\x8cS/" +
+	"\xd3\xdf\xc5\x1bL\x9e\xce\x12\xdc%\x83GT\x0c\x1e\xbc" +
+	"e\x99\\\xabAr\x83&\x14\x83\\\x08U&\xbb\xca" +
+	"N\xbc\x97e\xfe;\xc9z\x1c\x07\xf1\xceI\x92Gz" +
+	"\x02\x85\xfb\xae\x89\x8d\xf6\x8a\xa6\xdfD&W\xc4\xfb\x80" +
+	"\x09t\xe5]8\xa8\xda\x93\xb4\x98\xf6\xffZ\xb8\xff\xe9" +
+	"\x99\xb1\xd1[\xe0\x7fo\x12\xf1\x0d\x11\xd5wb\xad\xdd" +
+	"\xdb\xcb\x00\xd4\xb7DT?\x89^\x94>\"G\xfdD" +
+	"\xc4\xdeX\x89.\x7fA\x8c\x9fS!\x1bOX5\xf8" +
+	"\x00@\xb6\x96\x12D\x9a'\xac\x84\x97\xb0d\x1c\x04\xc8" +
+	"\xa6\x88>=^\xa07\xe1\x00@\xb6\x91\xe8\xb3\xd1\xef" +
+	"\xc8\x83\xd7\xa8\x8a\x15\x81{\xd1,,\xd7\x8dq\xab\xbe" +
+	"\xe0\x89\x0b\x1d\x82\xcc\x8aE\xb8?\x16_\xbb\x17\xc7\xea" +
+	"\xe0p\xec\x84\xcc\xcaR\x88\xe7\xd1\x0e\xc79W1\xfd" +
+	"\x9d\xc0\x1cY?\xf8\xbc\xd8\xf3k\x89\xd80\xe0Xl" +
+	"N\x16\x18C}\xce\x9f?\xdd\x1d3\xc6w\x06\x01\xd4" +
+	"\xd5\"\xaa\xc3\x02\x07(\xb3\xbf\x9c\xd7\xd0aK-\xb6" +
+	"\xae\xc2$#7\x125\xc5\xd4\x16\xe6\xec~,SA" +
+	"\xbe\xd4b\xad\xeb*,\xce\x10\xbcv\x80\xa4\x9b\xf9\x8b" +
+	"\x9e9\xc6\xa9,og\x83Y3\xb7\x969c^\x81" +
+	"\xaa^*{\xa3\xa7\x8e\xf0\xa1\xb27\xfeP\xe9\xc3\xda" +
+	":r\xf0\xb2\x88\xea\xe6\x18\xac\x8d\x8cF\x1d\xf5\xf8\xa5" +
+	"\xc4\xffL\xf6\xffJ\x8fuTHKWRd\x86\x7f" +
+	"D\xf4\x15\x07\xfdW\xda\x13DO\xd0W9\x1c\x83\x10" +
+	"k0\xf6'&t\x88\xe0o\xfe_\x01\x00\x00\xff\xff" +
+	"\x86\xbe\xf5t"
 
 func init() {
 	schemas.Register(schema_db8274f9144abc7e,

validation/validation.go

@@ -190,10 +190,11 @@ func ValidateHTTPService(originURL string, hostname string, transport http.Round
 		_, secondErr := client.Do(secondRequest)
 		if secondErr == nil { // Worked this time--advise the user to switch protocols
 			return errors.Errorf(
-				"%s doesn't seem to work over %s, but does seem to work over %s. Consider changing the origin URL to %s",
+				"%s doesn't seem to work over %s, but does seem to work over %s. Reason: %v. Consider changing the origin URL to %s",
 				parsedURL.Host,
 				oldScheme,
 				parsedURL.Scheme,
+				initialErr,
 				parsedURL,
 			)
 		}

vendor/github.com/denisenkom/go-mssqldb/README.md

@@ -56,7 +56,7 @@ Other supported formats are listed below.
 * `hostNameInCertificate` - Specifies the Common Name (CN) in the server certificate. Default value is the server host.
 * `ServerSPN` - The kerberos SPN (Service Principal Name) for the server. Default is MSSQLSvc/host:port.
 * `Workstation ID` - The workstation name (default is the host name)
-* `ApplicationIntent` - Can be given the value `ReadOnly` to initiate a read-only connection to an Availability Group listener.
+* `ApplicationIntent` - Can be given the value `ReadOnly` to initiate a read-only connection to an Availability Group listener. The `database` must be specified when connecting with `Application Intent` set to `ReadOnly`. 
 
 ### The connection string can be specified in one of three formats:
 
@@ -187,6 +187,19 @@ _, err := db.ExecContext(ctx, "theproc", &rs)
 log.Printf("status=%d", rs)
 ```
 
+or
+
+```
+var rs mssql.ReturnStatus
+_, err := db.QueryContext(ctx, "theproc", &rs)
+for rows.Next() {
+	err = rows.Scan(&val)
+}
+log.Printf("status=%d", rs)
+```
+
+Limitation: ReturnStatus cannot be retrieved using `QueryRow`.
+
 ## Parameters
 
 The `sqlserver` driver uses normal MS SQL Server syntax and expects parameters in
@@ -207,9 +220,9 @@ are supported:
  * time.Time -> datetimeoffset or datetime (TDS version dependent)
  * mssql.DateTime1 -> datetime
  * mssql.DateTimeOffset -> datetimeoffset
- * "cloud.google.com/go/civil".Date -> date
- * "cloud.google.com/go/civil".DateTime -> datetime2
- * "cloud.google.com/go/civil".Time -> time
+ * "github.com/golang-sql/civil".Date -> date
+ * "github.com/golang-sql/civil".DateTime -> datetime2
+ * "github.com/golang-sql/civil".Time -> time
  * mssql.TVP -> Table Value Parameter (TDS version dependent)
 
 ## Important Notes

vendor/github.com/denisenkom/go-mssqldb/appveyor.yml

@@ -10,7 +10,7 @@ environment:
   SQLUSER: sa
   SQLPASSWORD: Password12!
   DATABASE: test  
-  GOVERSION: 110
+  GOVERSION: 111
   matrix:
     - GOVERSION: 18
       SQLINSTANCE: SQL2016
@@ -18,6 +18,8 @@ environment:
       SQLINSTANCE: SQL2016
     - GOVERSION: 110
       SQLINSTANCE: SQL2016
+    - GOVERSION: 111
+      SQLINSTANCE: SQL2016
     - SQLINSTANCE: SQL2014
     - SQLINSTANCE: SQL2012SP1
     - SQLINSTANCE: SQL2008R2SP2
@@ -27,7 +29,7 @@ install:
   - set PATH=%GOPATH%\bin;%GOROOT%\bin;%PATH%
   - go version
   - go env
-  - go get -u cloud.google.com/go/civil
+  - go get -u github.com/golang-sql/civil
 
 build_script:
   - go build

vendor/github.com/denisenkom/go-mssqldb/buf.go

@@ -221,23 +221,27 @@ func (r *tdsBuffer) uint16() uint16 {
 }
 
 func (r *tdsBuffer) BVarChar() string {
-	l := int(r.byte())
-	return r.readUcs2(l)
+	return readBVarCharOrPanic(r)
 }
 
-func (r *tdsBuffer) UsVarChar() string {
-	l := int(r.uint16())
-	return r.readUcs2(l)
-}
-
-func (r *tdsBuffer) readUcs2(numchars int) string {
-	b := make([]byte, numchars*2)
-	r.ReadFull(b)
-	res, err := ucs22str(b)
+func readBVarCharOrPanic(r io.Reader) string {
+	s, err := readBVarChar(r)
 	if err != nil {
 		badStreamPanic(err)
 	}
-	return res
+	return s
+}
+
+func readUsVarCharOrPanic(r io.Reader) string {
+	s, err := readUsVarChar(r)
+	if err != nil {
+		badStreamPanic(err)
+	}
+	return s
+}
+
+func (r *tdsBuffer) UsVarChar() string {
+	return readUsVarCharOrPanic(r)
 }
 
 func (r *tdsBuffer) Read(buf []byte) (copied int, err error) {

vendor/github.com/denisenkom/go-mssqldb/bulkcopy.go

@@ -7,9 +7,10 @@ import (
 	"fmt"
 	"math"
 	"reflect"
-	"strconv"
 	"strings"
 	"time"
+
+	"github.com/denisenkom/go-mssqldb/internal/decimal"
 )
 
 type Bulk struct {
@@ -42,6 +43,11 @@ type BulkOptions struct {
 
 type DataValue interface{}
 
+const (
+	sqlDateFormat = "2006-01-02"
+	sqlTimeFormat = "2006-01-02 15:04:05.999999999Z07:00"
+)
+
 func (cn *Conn) CreateBulk(table string, columns []string) (_ *Bulk) {
 	b := Bulk{ctx: context.Background(), cn: cn, tablename: table, headerSent: false, columnsName: columns}
 	b.Debug = false
@@ -334,7 +340,7 @@ func (b *Bulk) makeParam(val DataValue, col columnStruct) (res param, err error)
 		case int64:
 			intvalue = val
 		default:
-			err = fmt.Errorf("mssql: invalid type for int column")
+			err = fmt.Errorf("mssql: invalid type for int column: %T", val)
 			return
 		}
 
@@ -361,7 +367,7 @@ func (b *Bulk) makeParam(val DataValue, col columnStruct) (res param, err error)
 		case int64:
 			floatvalue = float64(val)
 		default:
-			err = fmt.Errorf("mssql: invalid type for float column: %s", val)
+			err = fmt.Errorf("mssql: invalid type for float column: %T %s", val, val)
 			return
 		}
 
@@ -380,7 +386,7 @@ func (b *Bulk) makeParam(val DataValue, col columnStruct) (res param, err error)
 		case []byte:
 			res.buffer = val
 		default:
-			err = fmt.Errorf("mssql: invalid type for nvarchar column: %s", val)
+			err = fmt.Errorf("mssql: invalid type for nvarchar column: %T %s", val, val)
 			return
 		}
 		res.ti.Size = len(res.buffer)
@@ -392,14 +398,14 @@ func (b *Bulk) makeParam(val DataValue, col columnStruct) (res param, err error)
 		case []byte:
 			res.buffer = val
 		default:
-			err = fmt.Errorf("mssql: invalid type for varchar column: %s", val)
+			err = fmt.Errorf("mssql: invalid type for varchar column: %T %s", val, val)
 			return
 		}
 		res.ti.Size = len(res.buffer)
 
 	case typeBit, typeBitN:
 		if reflect.TypeOf(val).Kind() != reflect.Bool {
-			err = fmt.Errorf("mssql: invalid type for bit column: %s", val)
+			err = fmt.Errorf("mssql: invalid type for bit column: %T %s", val, val)
 			return
 		}
 		res.ti.TypeId = typeBitN
@@ -413,18 +419,31 @@ func (b *Bulk) makeParam(val DataValue, col columnStruct) (res param, err error)
 		case time.Time:
 			res.buffer = encodeDateTime2(val, int(col.ti.Scale))
 			res.ti.Size = len(res.buffer)
+		case string:
+			var t time.Time
+			if t, err = time.Parse(sqlTimeFormat, val); err != nil {
+				return res, fmt.Errorf("bulk: unable to convert string to date: %v", err)
+			}
+			res.buffer = encodeDateTime2(t, int(col.ti.Scale))
+			res.ti.Size = len(res.buffer)
 		default:
-			err = fmt.Errorf("mssql: invalid type for datetime2 column: %s", val)
+			err = fmt.Errorf("mssql: invalid type for datetime2 column: %T %s", val, val)
 			return
 		}
 	case typeDateTimeOffsetN:
 		switch val := val.(type) {
 		case time.Time:
-			res.buffer = encodeDateTimeOffset(val, int(res.ti.Scale))
+			res.buffer = encodeDateTimeOffset(val, int(col.ti.Scale))
+			res.ti.Size = len(res.buffer)
+		case string:
+			var t time.Time
+			if t, err = time.Parse(sqlTimeFormat, val); err != nil {
+				return res, fmt.Errorf("bulk: unable to convert string to date: %v", err)
+			}
+			res.buffer = encodeDateTimeOffset(t, int(col.ti.Scale))
 			res.ti.Size = len(res.buffer)
-
 		default:
-			err = fmt.Errorf("mssql: invalid type for datetimeoffset column: %s", val)
+			err = fmt.Errorf("mssql: invalid type for datetimeoffset column: %T %s", val, val)
 			return
 		}
 	case typeDateN:
@@ -432,69 +451,79 @@ func (b *Bulk) makeParam(val DataValue, col columnStruct) (res param, err error)
 		case time.Time:
 			res.buffer = encodeDate(val)
 			res.ti.Size = len(res.buffer)
+		case string:
+			var t time.Time
+			if t, err = time.ParseInLocation(sqlDateFormat, val, time.UTC); err != nil {
+				return res, fmt.Errorf("bulk: unable to convert string to date: %v", err)
+			}
+			res.buffer = encodeDate(t)
+			res.ti.Size = len(res.buffer)
 		default:
-			err = fmt.Errorf("mssql: invalid type for date column: %s", val)
+			err = fmt.Errorf("mssql: invalid type for date column: %T %s", val, val)
 			return
 		}
 	case typeDateTime, typeDateTimeN, typeDateTim4:
+		var t time.Time
 		switch val := val.(type) {
 		case time.Time:
-			if col.ti.Size == 4 {
-				res.buffer = encodeDateTim4(val)
-				res.ti.Size = len(res.buffer)
-			} else if col.ti.Size == 8 {
-				res.buffer = encodeDateTime(val)
-				res.ti.Size = len(res.buffer)
-			} else {
-				err = fmt.Errorf("mssql: invalid size of column")
+			t = val
+		case string:
+			if t, err = time.Parse(sqlTimeFormat, val); err != nil {
+				return res, fmt.Errorf("bulk: unable to convert string to date: %v", err)
 			}
-
 		default:
-			err = fmt.Errorf("mssql: invalid type for datetime column: %s", val)
+			err = fmt.Errorf("mssql: invalid type for datetime column: %T %s", val, val)
+			return
+		}
+
+		if col.ti.Size == 4 {
+			res.buffer = encodeDateTim4(t)
+			res.ti.Size = len(res.buffer)
+		} else if col.ti.Size == 8 {
+			res.buffer = encodeDateTime(t)
+			res.ti.Size = len(res.buffer)
+		} else {
+			err = fmt.Errorf("mssql: invalid size of column %d", col.ti.Size)
 		}
 
 	// case typeMoney, typeMoney4, typeMoneyN:
 	case typeDecimal, typeDecimalN, typeNumeric, typeNumericN:
-		var value float64
+		prec := col.ti.Prec
+		scale := col.ti.Scale
+		var dec decimal.Decimal
 		switch v := val.(type) {
 		case int:
-			value = float64(v)
+			dec = decimal.Int64ToDecimalScale(int64(v), 0)
 		case int8:
-			value = float64(v)
+			dec = decimal.Int64ToDecimalScale(int64(v), 0)
 		case int16:
-			value = float64(v)
+			dec = decimal.Int64ToDecimalScale(int64(v), 0)
 		case int32:
-			value = float64(v)
+			dec = decimal.Int64ToDecimalScale(int64(v), 0)
 		case int64:
-			value = float64(v)
+			dec = decimal.Int64ToDecimalScale(int64(v), 0)
 		case float32:
-			value = float64(v)
+			dec, err = decimal.Float64ToDecimalScale(float64(v), scale)
 		case float64:
-			value = v
+			dec, err = decimal.Float64ToDecimalScale(float64(v), scale)
 		case string:
-			if value, err = strconv.ParseFloat(v, 64); err != nil {
-				return res, fmt.Errorf("bulk: unable to convert string to float: %v", err)
-			}
+			dec, err = decimal.StringToDecimalScale(v, scale)
 		default:
-			return res, fmt.Errorf("unknown value for decimal: %#v", v)
+			return res, fmt.Errorf("unknown value for decimal: %T %#v", v, v)
 		}
 
-		perc := col.ti.Prec
-		scale := col.ti.Scale
-		var dec Decimal
-		dec, err = Float64ToDecimalScale(value, scale)
 		if err != nil {
 			return res, err
 		}
-		dec.prec = perc
+		dec.SetPrec(prec)
 
 		var length byte
 		switch {
-		case perc <= 9:
+		case prec <= 9:
 			length = 4
-		case perc <= 19:
+		case prec <= 19:
 			length = 8
-		case perc <= 28:
+		case prec <= 28:
 			length = 12
 		default:
 			length = 16
@@ -504,7 +533,7 @@ func (b *Bulk) makeParam(val DataValue, col columnStruct) (res param, err error)
 		// first byte length written by typeInfo.writer
 		res.ti.Size = int(length) + 1
 		// second byte sign
-		if value < 0 {
+		if !dec.IsPositive() {
 			buf[0] = 0
 		} else {
 			buf[0] = 1
@@ -527,7 +556,7 @@ func (b *Bulk) makeParam(val DataValue, col columnStruct) (res param, err error)
 			res.ti.Size = len(val)
 			res.buffer = val
 		default:
-			err = fmt.Errorf("mssql: invalid type for Binary column: %s", val)
+			err = fmt.Errorf("mssql: invalid type for Binary column: %T %s", val, val)
 			return
 		}
 	case typeGuid:
@@ -536,7 +565,7 @@ func (b *Bulk) makeParam(val DataValue, col columnStruct) (res param, err error)
 			res.ti.Size = len(val)
 			res.buffer = val
 		default:
-			err = fmt.Errorf("mssql: invalid type for Guid column: %s", val)
+			err = fmt.Errorf("mssql: invalid type for Guid column: %T %s", val, val)
 			return
 		}
 

vendor/github.com/denisenkom/go-mssqldb/conn_str.go

@@ -0,0 +1,456 @@
+package mssql
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+	"unicode"
+)
+
+type connectParams struct {
+	logFlags                  uint64
+	port                      uint64
+	host                      string
+	instance                  string
+	database                  string
+	user                      string
+	password                  string
+	dial_timeout              time.Duration
+	conn_timeout              time.Duration
+	keepAlive                 time.Duration
+	encrypt                   bool
+	disableEncryption         bool
+	trustServerCertificate    bool
+	certificate               string
+	hostInCertificate         string
+	hostInCertificateProvided bool
+	serverSPN                 string
+	workstation               string
+	appname                   string
+	typeFlags                 uint8
+	failOverPartner           string
+	failOverPort              uint64
+	packetSize                uint16
+}
+
+func parseConnectParams(dsn string) (connectParams, error) {
+	var p connectParams
+
+	var params map[string]string
+	if strings.HasPrefix(dsn, "odbc:") {
+		parameters, err := splitConnectionStringOdbc(dsn[len("odbc:"):])
+		if err != nil {
+			return p, err
+		}
+		params = parameters
+	} else if strings.HasPrefix(dsn, "sqlserver://") {
+		parameters, err := splitConnectionStringURL(dsn)
+		if err != nil {
+			return p, err
+		}
+		params = parameters
+	} else {
+		params = splitConnectionString(dsn)
+	}
+
+	strlog, ok := params["log"]
+	if ok {
+		var err error
+		p.logFlags, err = strconv.ParseUint(strlog, 10, 64)
+		if err != nil {
+			return p, fmt.Errorf("Invalid log parameter '%s': %s", strlog, err.Error())
+		}
+	}
+	server := params["server"]
+	parts := strings.SplitN(server, `\`, 2)
+	p.host = parts[0]
+	if p.host == "." || strings.ToUpper(p.host) == "(LOCAL)" || p.host == "" {
+		p.host = "localhost"
+	}
+	if len(parts) > 1 {
+		p.instance = parts[1]
+	}
+	p.database = params["database"]
+	p.user = params["user id"]
+	p.password = params["password"]
+
+	p.port = 1433
+	strport, ok := params["port"]
+	if ok {
+		var err error
+		p.port, err = strconv.ParseUint(strport, 10, 16)
+		if err != nil {
+			f := "Invalid tcp port '%v': %v"
+			return p, fmt.Errorf(f, strport, err.Error())
+		}
+	}
+
+	// https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/configure-the-network-packet-size-server-configuration-option
+	// Default packet size remains at 4096 bytes
+	p.packetSize = 4096
+	strpsize, ok := params["packet size"]
+	if ok {
+		var err error
+		psize, err := strconv.ParseUint(strpsize, 0, 16)
+		if err != nil {
+			f := "Invalid packet size '%v': %v"
+			return p, fmt.Errorf(f, strpsize, err.Error())
+		}
+
+		// Ensure packet size falls within the TDS protocol range of 512 to 32767 bytes
+		// NOTE: Encrypted connections have a maximum size of 16383 bytes.  If you request
+		// a higher packet size, the server will respond with an ENVCHANGE request to
+		// alter the packet size to 16383 bytes.
+		p.packetSize = uint16(psize)
+		if p.packetSize < 512 {
+			p.packetSize = 512
+		} else if p.packetSize > 32767 {
+			p.packetSize = 32767
+		}
+	}
+
+	// https://msdn.microsoft.com/en-us/library/dd341108.aspx
+	//
+	// Do not set a connection timeout. Use Context to manage such things.
+	// Default to zero, but still allow it to be set.
+	if strconntimeout, ok := params["connection timeout"]; ok {
+		timeout, err := strconv.ParseUint(strconntimeout, 10, 64)
+		if err != nil {
+			f := "Invalid connection timeout '%v': %v"
+			return p, fmt.Errorf(f, strconntimeout, err.Error())
+		}
+		p.conn_timeout = time.Duration(timeout) * time.Second
+	}
+	p.dial_timeout = 15 * time.Second
+	if strdialtimeout, ok := params["dial timeout"]; ok {
+		timeout, err := strconv.ParseUint(strdialtimeout, 10, 64)
+		if err != nil {
+			f := "Invalid dial timeout '%v': %v"
+			return p, fmt.Errorf(f, strdialtimeout, err.Error())
+		}
+		p.dial_timeout = time.Duration(timeout) * time.Second
+	}
+
+	// default keep alive should be 30 seconds according to spec:
+	// https://msdn.microsoft.com/en-us/library/dd341108.aspx
+	p.keepAlive = 30 * time.Second
+	if keepAlive, ok := params["keepalive"]; ok {
+		timeout, err := strconv.ParseUint(keepAlive, 10, 64)
+		if err != nil {
+			f := "Invalid keepAlive value '%s': %s"
+			return p, fmt.Errorf(f, keepAlive, err.Error())
+		}
+		p.keepAlive = time.Duration(timeout) * time.Second
+	}
+	encrypt, ok := params["encrypt"]
+	if ok {
+		if strings.EqualFold(encrypt, "DISABLE") {
+			p.disableEncryption = true
+		} else {
+			var err error
+			p.encrypt, err = strconv.ParseBool(encrypt)
+			if err != nil {
+				f := "Invalid encrypt '%s': %s"
+				return p, fmt.Errorf(f, encrypt, err.Error())
+			}
+		}
+	} else {
+		p.trustServerCertificate = true
+	}
+	trust, ok := params["trustservercertificate"]
+	if ok {
+		var err error
+		p.trustServerCertificate, err = strconv.ParseBool(trust)
+		if err != nil {
+			f := "Invalid trust server certificate '%s': %s"
+			return p, fmt.Errorf(f, trust, err.Error())
+		}
+	}
+	p.certificate = params["certificate"]
+	p.hostInCertificate, ok = params["hostnameincertificate"]
+	if ok {
+		p.hostInCertificateProvided = true
+	} else {
+		p.hostInCertificate = p.host
+		p.hostInCertificateProvided = false
+	}
+
+	serverSPN, ok := params["serverspn"]
+	if ok {
+		p.serverSPN = serverSPN
+	} else {
+		p.serverSPN = fmt.Sprintf("MSSQLSvc/%s:%d", p.host, p.port)
+	}
+
+	workstation, ok := params["workstation id"]
+	if ok {
+		p.workstation = workstation
+	} else {
+		workstation, err := os.Hostname()
+		if err == nil {
+			p.workstation = workstation
+		}
+	}
+
+	appname, ok := params["app name"]
+	if !ok {
+		appname = "go-mssqldb"
+	}
+	p.appname = appname
+
+	appintent, ok := params["applicationintent"]
+	if ok {
+		if appintent == "ReadOnly" {
+			if p.database == "" {
+				return p, fmt.Errorf("Database must be specified when ApplicationIntent is ReadOnly")
+			}
+			p.typeFlags |= fReadOnlyIntent
+		}
+	}
+
+	failOverPartner, ok := params["failoverpartner"]
+	if ok {
+		p.failOverPartner = failOverPartner
+	}
+
+	failOverPort, ok := params["failoverport"]
+	if ok {
+		var err error
+		p.failOverPort, err = strconv.ParseUint(failOverPort, 0, 16)
+		if err != nil {
+			f := "Invalid tcp port '%v': %v"
+			return p, fmt.Errorf(f, failOverPort, err.Error())
+		}
+	}
+
+	return p, nil
+}
+
+func splitConnectionString(dsn string) (res map[string]string) {
+	res = map[string]string{}
+	parts := strings.Split(dsn, ";")
+	for _, part := range parts {
+		if len(part) == 0 {
+			continue
+		}
+		lst := strings.SplitN(part, "=", 2)
+		name := strings.TrimSpace(strings.ToLower(lst[0]))
+		if len(name) == 0 {
+			continue
+		}
+		var value string = ""
+		if len(lst) > 1 {
+			value = strings.TrimSpace(lst[1])
+		}
+		res[name] = value
+	}
+	return res
+}
+
+// Splits a URL of the form sqlserver://username:password@host/instance?param1=value&param2=value
+func splitConnectionStringURL(dsn string) (map[string]string, error) {
+	res := map[string]string{}
+
+	u, err := url.Parse(dsn)
+	if err != nil {
+		return res, err
+	}
+
+	if u.Scheme != "sqlserver" {
+		return res, fmt.Errorf("scheme %s is not recognized", u.Scheme)
+	}
+
+	if u.User != nil {
+		res["user id"] = u.User.Username()
+		p, exists := u.User.Password()
+		if exists {
+			res["password"] = p
+		}
+	}
+
+	host, port, err := net.SplitHostPort(u.Host)
+	if err != nil {
+		host = u.Host
+	}
+
+	if len(u.Path) > 0 {
+		res["server"] = host + "\\" + u.Path[1:]
+	} else {
+		res["server"] = host
+	}
+
+	if len(port) > 0 {
+		res["port"] = port
+	}
+
+	query := u.Query()
+	for k, v := range query {
+		if len(v) > 1 {
+			return res, fmt.Errorf("key %s provided more than once", k)
+		}
+		res[strings.ToLower(k)] = v[0]
+	}
+
+	return res, nil
+}
+
+// Splits a URL in the ODBC format
+func splitConnectionStringOdbc(dsn string) (map[string]string, error) {
+	res := map[string]string{}
+
+	type parserState int
+	const (
+		// Before the start of a key
+		parserStateBeforeKey parserState = iota
+
+		// Inside a key
+		parserStateKey
+
+		// Beginning of a value. May be bare or braced
+		parserStateBeginValue
+
+		// Inside a bare value
+		parserStateBareValue
+
+		// Inside a braced value
+		parserStateBracedValue
+
+		// A closing brace inside a braced value.
+		// May be the end of the value or an escaped closing brace, depending on the next character
+		parserStateBracedValueClosingBrace
+
+		// After a value. Next character should be a semicolon or whitespace.
+		parserStateEndValue
+	)
+
+	var state = parserStateBeforeKey
+
+	var key string
+	var value string
+
+	for i, c := range dsn {
+		switch state {
+		case parserStateBeforeKey:
+			switch {
+			case c == '=':
+				return res, fmt.Errorf("Unexpected character = at index %d. Expected start of key or semi-colon or whitespace.", i)
+			case !unicode.IsSpace(c) && c != ';':
+				state = parserStateKey
+				key += string(c)
+			}
+
+		case parserStateKey:
+			switch c {
+			case '=':
+				key = normalizeOdbcKey(key)
+				state = parserStateBeginValue
+
+			case ';':
+				// Key without value
+				key = normalizeOdbcKey(key)
+				res[key] = value
+				key = ""
+				value = ""
+				state = parserStateBeforeKey
+
+			default:
+				key += string(c)
+			}
+
+		case parserStateBeginValue:
+			switch {
+			case c == '{':
+				state = parserStateBracedValue
+			case c == ';':
+				// Empty value
+				res[key] = value
+				key = ""
+				state = parserStateBeforeKey
+			case unicode.IsSpace(c):
+				// Ignore whitespace
+			default:
+				state = parserStateBareValue
+				value += string(c)
+			}
+
+		case parserStateBareValue:
+			if c == ';' {
+				res[key] = strings.TrimRightFunc(value, unicode.IsSpace)
+				key = ""
+				value = ""
+				state = parserStateBeforeKey
+			} else {
+				value += string(c)
+			}
+
+		case parserStateBracedValue:
+			if c == '}' {
+				state = parserStateBracedValueClosingBrace
+			} else {
+				value += string(c)
+			}
+
+		case parserStateBracedValueClosingBrace:
+			if c == '}' {
+				// Escaped closing brace
+				value += string(c)
+				state = parserStateBracedValue
+				continue
+			}
+
+			// End of braced value
+			res[key] = value
+			key = ""
+			value = ""
+
+			// This character is the first character past the end,
+			// so it needs to be parsed like the parserStateEndValue state.
+			state = parserStateEndValue
+			switch {
+			case c == ';':
+				state = parserStateBeforeKey
+			case unicode.IsSpace(c):
+				// Ignore whitespace
+			default:
+				return res, fmt.Errorf("Unexpected character %c at index %d. Expected semi-colon or whitespace.", c, i)
+			}
+
+		case parserStateEndValue:
+			switch {
+			case c == ';':
+				state = parserStateBeforeKey
+			case unicode.IsSpace(c):
+				// Ignore whitespace
+			default:
+				return res, fmt.Errorf("Unexpected character %c at index %d. Expected semi-colon or whitespace.", c, i)
+			}
+		}
+	}
+
+	switch state {
+	case parserStateBeforeKey: // Okay
+	case parserStateKey: // Unfinished key. Treat as key without value.
+		key = normalizeOdbcKey(key)
+		res[key] = value
+	case parserStateBeginValue: // Empty value
+		res[key] = value
+	case parserStateBareValue:
+		res[key] = strings.TrimRightFunc(value, unicode.IsSpace)
+	case parserStateBracedValue:
+		return res, fmt.Errorf("Unexpected end of braced value at index %d.", len(dsn))
+	case parserStateBracedValueClosingBrace: // End of braced value
+		res[key] = value
+	case parserStateEndValue: // Okay
+	}
+
+	return res, nil
+}
+
+// Normalizes the given string as an ODBC-format key
+func normalizeOdbcKey(s string) string {
+	return strings.ToLower(strings.TrimRightFunc(s, unicode.IsSpace))
+}

vendor/github.com/denisenkom/go-mssqldb/decimal.go

@@ -1,131 +0,0 @@
-package mssql
-
-import (
-	"encoding/binary"
-	"errors"
-	"math"
-	"math/big"
-)
-
-// http://msdn.microsoft.com/en-us/library/ee780893.aspx
-type Decimal struct {
-	integer  [4]uint32
-	positive bool
-	prec     uint8
-	scale    uint8
-}
-
-var scaletblflt64 [39]float64
-
-func (d Decimal) ToFloat64() float64 {
-	val := float64(0)
-	for i := 3; i >= 0; i-- {
-		val *= 0x100000000
-		val += float64(d.integer[i])
-	}
-	if !d.positive {
-		val = -val
-	}
-	if d.scale != 0 {
-		val /= scaletblflt64[d.scale]
-	}
-	return val
-}
-
-const autoScale = 100
-
-func Float64ToDecimal(f float64) (Decimal, error) {
-	return Float64ToDecimalScale(f, autoScale)
-}
-
-func Float64ToDecimalScale(f float64, scale uint8) (Decimal, error) {
-	var dec Decimal
-	if math.IsNaN(f) {
-		return dec, errors.New("NaN")
-	}
-	if math.IsInf(f, 0) {
-		return dec, errors.New("Infinity can't be converted to decimal")
-	}
-	dec.positive = f >= 0
-	if !dec.positive {
-		f = math.Abs(f)
-	}
-	if f > 3.402823669209385e+38 {
-		return dec, errors.New("Float value is out of range")
-	}
-	dec.prec = 20
-	var integer float64
-	for dec.scale = 0; dec.scale <= scale; dec.scale++ {
-		integer = f * scaletblflt64[dec.scale]
-		_, frac := math.Modf(integer)
-		if frac == 0 && scale == autoScale {
-			break
-		}
-	}
-	for i := 0; i < 4; i++ {
-		mod := math.Mod(integer, 0x100000000)
-		integer -= mod
-		integer /= 0x100000000
-		dec.integer[i] = uint32(mod)
-	}
-	return dec, nil
-}
-
-func init() {
-	var acc float64 = 1
-	for i := 0; i <= 38; i++ {
-		scaletblflt64[i] = acc
-		acc *= 10
-	}
-}
-
-func (d Decimal) BigInt() big.Int {
-	bytes := make([]byte, 16)
-	binary.BigEndian.PutUint32(bytes[0:4], d.integer[3])
-	binary.BigEndian.PutUint32(bytes[4:8], d.integer[2])
-	binary.BigEndian.PutUint32(bytes[8:12], d.integer[1])
-	binary.BigEndian.PutUint32(bytes[12:16], d.integer[0])
-	var x big.Int
-	x.SetBytes(bytes)
-	if !d.positive {
-		x.Neg(&x)
-	}
-	return x
-}
-
-func (d Decimal) Bytes() []byte {
-	x := d.BigInt()
-	return scaleBytes(x.String(), d.scale)
-}
-
-func (d Decimal) UnscaledBytes() []byte {
-	x := d.BigInt()
-	return x.Bytes()
-}
-
-func scaleBytes(s string, scale uint8) []byte {
-	z := make([]byte, 0, len(s)+1)
-	if s[0] == '-' || s[0] == '+' {
-		z = append(z, byte(s[0]))
-		s = s[1:]
-	}
-	pos := len(s) - int(scale)
-	if pos <= 0 {
-		z = append(z, byte('0'))
-	} else if pos > 0 {
-		z = append(z, []byte(s[:pos])...)
-	}
-	if scale > 0 {
-		z = append(z, byte('.'))
-		for pos < 0 {
-			z = append(z, byte('0'))
-			pos++
-		}
-		z = append(z, []byte(s[pos:])...)
-	}
-	return z
-}
-
-func (d Decimal) String() string {
-	return string(d.Bytes())
-}

vendor/github.com/denisenkom/go-mssqldb/go.mod

@@ -3,8 +3,6 @@ module github.com/denisenkom/go-mssqldb
 go 1.11
 
 require (
-	cloud.google.com/go v0.37.4
+	github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe
 	golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	gopkg.in/yaml.v2 v2.2.2 // indirect
 )

vendor/github.com/denisenkom/go-mssqldb/go.sum

@@ -1,168 +1,5 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.31.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.37.2 h1:4y4L7BdHenTfZL0HervofNTHh9Ad6mNX72cQvl+5eH0=
-cloud.google.com/go v0.37.2/go.mod h1:H8IAquKe2L30IxoupDgqTaQvKSwF/c8prYHynGIWQbA=
-git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
-git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
-github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
-github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs=
-github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
-github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I=
-github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
-github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
-github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
-github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
-github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
-github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8=
-github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
-github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
-go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
-go.opencensus.io v0.19.1/go.mod h1:gug0GbSHa8Pafr0d2urOSgoXHZ6x/RUlaiT0d9pqb4A=
-go.opencensus.io v0.19.2/go.mod h1:NO/8qkisMZLZ1FCsKNqtJPwc8/TaclWyY0B6wcYNg9M=
-go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
-golang.org/x/build v0.0.0-20190314133821-5284462c4bec/go.mod h1:atTaCNAy0f16Ah5aV1gMSwgiKVHwu/JncqDpuRr7lS4=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181029044818-c44066c5c816/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181029174526-d69651ed3497/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
-google.golang.org/api v0.0.0-20181030000543-1d582fd0359e/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
-google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
-google.golang.org/api v0.2.0/go.mod h1:IfRCZScioGtypHNTlz3gFk67J8uePVW7uDTBzXuIkhU=
-google.golang.org/api v0.3.0/go.mod h1:IuvZyQh8jgscv8qWfQ4ABd8m7hEudgBFM/EdhA3BnXw=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
-google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
-google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
-gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o=
-honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

vendor/github.com/denisenkom/go-mssqldb/internal/decimal/decimal.go

@@ -0,0 +1,252 @@
+package decimal
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"math"
+	"math/big"
+	"strings"
+)
+
+// Decimal represents decimal type in the Microsoft Open Specifications: http://msdn.microsoft.com/en-us/library/ee780893.aspx
+type Decimal struct {
+	integer  [4]uint32 // Little-endian
+	positive bool
+	prec     uint8
+	scale    uint8
+}
+
+var (
+	scaletblflt64 [39]float64
+	int10         big.Int
+	int1e5        big.Int
+)
+
+func init() {
+	var acc float64 = 1
+	for i := 0; i <= 38; i++ {
+		scaletblflt64[i] = acc
+		acc *= 10
+	}
+
+	int10.SetInt64(10)
+	int1e5.SetInt64(1e5)
+}
+
+const autoScale = 100
+
+// SetInteger sets the ind'th element in the integer array
+func (d *Decimal) SetInteger(integer uint32, ind uint8) {
+	d.integer[ind] = integer
+}
+
+// SetPositive sets the positive member
+func (d *Decimal) SetPositive(positive bool) {
+	d.positive = positive
+}
+
+// SetPrec sets the prec member
+func (d *Decimal) SetPrec(prec uint8) {
+	d.prec = prec
+}
+
+// SetScale sets the scale member
+func (d *Decimal) SetScale(scale uint8) {
+	d.scale = scale
+}
+
+// IsPositive returns true if the Decimal is positive
+func (d *Decimal) IsPositive() bool {
+	return d.positive
+}
+
+// ToFloat64 converts decimal to a float64
+func (d Decimal) ToFloat64() float64 {
+	val := float64(0)
+	for i := 3; i >= 0; i-- {
+		val *= 0x100000000
+		val += float64(d.integer[i])
+	}
+	if !d.positive {
+		val = -val
+	}
+	if d.scale != 0 {
+		val /= scaletblflt64[d.scale]
+	}
+	return val
+}
+
+// BigInt converts decimal to a bigint
+func (d Decimal) BigInt() big.Int {
+	bytes := make([]byte, 16)
+	binary.BigEndian.PutUint32(bytes[0:4], d.integer[3])
+	binary.BigEndian.PutUint32(bytes[4:8], d.integer[2])
+	binary.BigEndian.PutUint32(bytes[8:12], d.integer[1])
+	binary.BigEndian.PutUint32(bytes[12:16], d.integer[0])
+	var x big.Int
+	x.SetBytes(bytes)
+	if !d.positive {
+		x.Neg(&x)
+	}
+	return x
+}
+
+// Bytes converts decimal to a scaled byte slice
+func (d Decimal) Bytes() []byte {
+	x := d.BigInt()
+	return ScaleBytes(x.String(), d.scale)
+}
+
+// UnscaledBytes converts decimal to a unscaled byte slice
+func (d Decimal) UnscaledBytes() []byte {
+	x := d.BigInt()
+	return x.Bytes()
+}
+
+// String converts decimal to a string
+func (d Decimal) String() string {
+	return string(d.Bytes())
+}
+
+// Float64ToDecimal converts float64 to decimal
+func Float64ToDecimal(f float64) (Decimal, error) {
+	return Float64ToDecimalScale(f, autoScale)
+}
+
+// Float64ToDecimalScale converts float64 to decimal; user can specify the scale
+func Float64ToDecimalScale(f float64, scale uint8) (Decimal, error) {
+	var dec Decimal
+	if math.IsNaN(f) {
+		return dec, errors.New("NaN")
+	}
+	if math.IsInf(f, 0) {
+		return dec, errors.New("Infinity can't be converted to decimal")
+	}
+	dec.positive = f >= 0
+	if !dec.positive {
+		f = math.Abs(f)
+	}
+	if f > 3.402823669209385e+38 {
+		return dec, errors.New("Float value is out of range")
+	}
+	dec.prec = 20
+	var integer float64
+	for dec.scale = 0; dec.scale <= scale; dec.scale++ {
+		integer = f * scaletblflt64[dec.scale]
+		_, frac := math.Modf(integer)
+		if frac == 0 && scale == autoScale {
+			break
+		}
+	}
+	for i := 0; i < 4; i++ {
+		mod := math.Mod(integer, 0x100000000)
+		integer -= mod
+		integer /= 0x100000000
+		dec.integer[i] = uint32(mod)
+		if mod-math.Trunc(mod) >= 0.5 {
+			dec.integer[i] = uint32(mod) + 1
+		}
+	}
+	return dec, nil
+}
+
+// Int64ToDecimalScale converts float64 to decimal; user can specify the scale
+func Int64ToDecimalScale(v int64, scale uint8) Decimal {
+	positive := v >= 0
+	if !positive {
+		if v == math.MinInt64 {
+			// Special case - can't negate
+			return Decimal{
+				integer:  [4]uint32{0, 0x80000000, 0, 0},
+				positive: false,
+				prec:     20,
+				scale:    0,
+			}
+		}
+		v = -v
+	}
+	return Decimal{
+		integer:  [4]uint32{uint32(v), uint32(v >> 32), 0, 0},
+		positive: positive,
+		prec:     20,
+		scale:    scale,
+	}
+}
+
+// StringToDecimalScale converts string to decimal
+func StringToDecimalScale(v string, outScale uint8) (Decimal, error) {
+	var r big.Int
+	var unscaled string
+	var inScale int
+
+	point := strings.LastIndexByte(v, '.')
+	if point == -1 {
+		inScale = 0
+		unscaled = v
+	} else {
+		inScale = len(v) - point - 1
+		unscaled = v[:point] + v[point+1:]
+	}
+	if inScale > math.MaxUint8 {
+		return Decimal{}, fmt.Errorf("can't parse %q as a decimal number: scale too large", v)
+	}
+
+	_, ok := r.SetString(unscaled, 10)
+	if !ok {
+		return Decimal{}, fmt.Errorf("can't parse %q as a decimal number", v)
+	}
+
+	if inScale > int(outScale) {
+		return Decimal{}, fmt.Errorf("can't parse %q as a decimal number: scale %d is larger than the scale %d of the target column", v, inScale, outScale)
+	}
+	for inScale < int(outScale) {
+		if int(outScale)-inScale >= 5 {
+			r.Mul(&r, &int1e5)
+			inScale += 5
+		} else {
+			r.Mul(&r, &int10)
+			inScale++
+		}
+	}
+
+	bytes := r.Bytes()
+	if len(bytes) > 16 {
+		return Decimal{}, fmt.Errorf("can't parse %q as a decimal number: precision too large", v)
+	}
+	var out [4]uint32
+	for i, b := range bytes {
+		pos := len(bytes) - i - 1
+		out[pos/4] += uint32(b) << uint(pos%4*8)
+	}
+	return Decimal{
+		integer:  out,
+		positive: r.Sign() >= 0,
+		prec:     20,
+		scale:    uint8(inScale),
+	}, nil
+}
+
+// ScaleBytes converts a stringified decimal to a scaled byte slice
+func ScaleBytes(s string, scale uint8) []byte {
+	z := make([]byte, 0, len(s)+1)
+	if s[0] == '-' || s[0] == '+' {
+		z = append(z, byte(s[0]))
+		s = s[1:]
+	}
+	pos := len(s) - int(scale)
+	if pos <= 0 {
+		z = append(z, byte('0'))
+	} else if pos > 0 {
+		z = append(z, []byte(s[:pos])...)
+	}
+	if scale > 0 {
+		z = append(z, byte('.'))
+		for pos < 0 {
+			z = append(z, byte('0'))
+			pos++
+		}
+		z = append(z, []byte(s[pos:])...)
+	}
+	return z
+}

vendor/github.com/denisenkom/go-mssqldb/mssql.go

@@ -716,6 +716,8 @@ func (rc *Rows) Next(dest []driver.Value) error {
 			if tokdata.isError() {
 				return rc.stmt.c.checkBadConn(tokdata.getError())
 			}
+		case ReturnStatus:
+			rc.stmt.c.setReturnStatus(tokdata)
 		case error:
 			return rc.stmt.c.checkBadConn(tokdata)
 		}
@@ -874,29 +876,6 @@ func (r *Result) RowsAffected() (int64, error) {
 	return r.rowsAffected, nil
 }
 
-func (r *Result) LastInsertId() (int64, error) {
-	s, err := r.c.Prepare("select cast(@@identity as bigint)")
-	if err != nil {
-		return 0, err
-	}
-	defer s.Close()
-	rows, err := s.Query(nil)
-	if err != nil {
-		return 0, err
-	}
-	defer rows.Close()
-	dest := make([]driver.Value, 1)
-	err = rows.Next(dest)
-	if err != nil {
-		return 0, err
-	}
-	if dest[0] == nil {
-		return -1, errors.New("There is no generated identity value")
-	}
-	lastInsertId := dest[0].(int64)
-	return lastInsertId, nil
-}
-
 var _ driver.Pinger = &Conn{}
 
 // Ping is used to check if the remote server is available and satisfies the Pinger interface.

vendor/github.com/denisenkom/go-mssqldb/mssql_go110.go

@@ -5,6 +5,7 @@ package mssql
 import (
 	"context"
 	"database/sql/driver"
+	"errors"
 )
 
 var _ driver.Connector = &Connector{}
@@ -45,3 +46,7 @@ func (c *Connector) Connect(ctx context.Context) (driver.Conn, error) {
 func (c *Connector) Driver() driver.Driver {
 	return c.driver
 }
+
+func (r *Result) LastInsertId() (int64, error) {
+	return -1, errors.New("LastInsertId is not supported. Please use the OUTPUT clause or add `select ID = convert(bigint, SCOPE_IDENTITY())` to the end of your query.")
+}

vendor/github.com/denisenkom/go-mssqldb/mssql_go110pre.go

@@ -0,0 +1,31 @@
+// +build !go1.10
+
+package mssql
+
+import (
+	"database/sql/driver"
+	"errors"
+)
+
+func (r *Result) LastInsertId() (int64, error) {
+	s, err := r.c.Prepare("select cast(@@identity as bigint)")
+	if err != nil {
+		return 0, err
+	}
+	defer s.Close()
+	rows, err := s.Query(nil)
+	if err != nil {
+		return 0, err
+	}
+	defer rows.Close()
+	dest := make([]driver.Value, 1)
+	err = rows.Next(dest)
+	if err != nil {
+		return 0, err
+	}
+	if dest[0] == nil {
+		return -1, errors.New("There is no generated identity value")
+	}
+	lastInsertId := dest[0].(int64)
+	return lastInsertId, nil
+}

vendor/github.com/denisenkom/go-mssqldb/mssql_go19.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	// "github.com/cockroachdb/apd"
-	"cloud.google.com/go/civil"
+	"github.com/golang-sql/civil"
 )
 
 // Type alias provided for compatibility.

vendor/github.com/denisenkom/go-mssqldb/net.go

@@ -9,9 +9,6 @@ import (
 type timeoutConn struct {
 	c             net.Conn
 	timeout       time.Duration
-	buf           *tdsBuffer
-	packetPending bool
-	continueRead  bool
 }
 
 func newTimeoutConn(conn net.Conn, timeout time.Duration) *timeoutConn {
@@ -22,32 +19,6 @@ func newTimeoutConn(conn net.Conn, timeout time.Duration) *timeoutConn {
 }
 
 func (c *timeoutConn) Read(b []byte) (n int, err error) {
-	if c.buf != nil {
-		if c.packetPending {
-			c.packetPending = false
-			err = c.buf.FinishPacket()
-			if err != nil {
-				err = fmt.Errorf("Cannot send handshake packet: %s", err.Error())
-				return
-			}
-			c.continueRead = false
-		}
-		if !c.continueRead {
-			var packet packetType
-			packet, err = c.buf.BeginRead()
-			if err != nil {
-				err = fmt.Errorf("Cannot read handshake packet: %s", err.Error())
-				return
-			}
-			if packet != packPrelogin {
-				err = fmt.Errorf("unexpected packet %d, expecting prelogin", packet)
-				return
-			}
-			c.continueRead = true
-		}
-		n, err = c.buf.Read(b)
-		return
-	}
 	if c.timeout > 0 {
 		err = c.c.SetDeadline(time.Now().Add(c.timeout))
 		if err != nil {
@@ -58,17 +29,6 @@ func (c *timeoutConn) Read(b []byte) (n int, err error) {
 }
 
 func (c *timeoutConn) Write(b []byte) (n int, err error) {
-	if c.buf != nil {
-		if !c.packetPending {
-			c.buf.BeginPacket(packPrelogin, false)
-			c.packetPending = true
-		}
-		n, err = c.buf.Write(b)
-		if err != nil {
-			return
-		}
-		return
-	}
 	if c.timeout > 0 {
 		err = c.c.SetDeadline(time.Now().Add(c.timeout))
 		if err != nil {
@@ -101,3 +61,108 @@ func (c timeoutConn) SetReadDeadline(t time.Time) error {
 func (c timeoutConn) SetWriteDeadline(t time.Time) error {
 	panic("Not implemented")
 }
+
+// this connection is used during TLS Handshake
+// TDS protocol requires TLS handshake messages to be sent inside TDS packets
+type tlsHandshakeConn struct {
+	buf *tdsBuffer
+	packetPending bool
+	continueRead  bool
+}
+
+func (c *tlsHandshakeConn) Read(b []byte) (n int, err error) {
+	if c.packetPending {
+		c.packetPending = false
+		err = c.buf.FinishPacket()
+		if err != nil {
+			err = fmt.Errorf("Cannot send handshake packet: %s", err.Error())
+			return
+		}
+		c.continueRead = false
+	}
+	if !c.continueRead {
+		var packet packetType
+		packet, err = c.buf.BeginRead()
+		if err != nil {
+			err = fmt.Errorf("Cannot read handshake packet: %s", err.Error())
+			return
+		}
+		if packet != packPrelogin {
+			err = fmt.Errorf("unexpected packet %d, expecting prelogin", packet)
+			return
+		}
+		c.continueRead = true
+	}
+	return c.buf.Read(b)
+}
+
+func (c *tlsHandshakeConn) Write(b []byte) (n int, err error) {
+	if !c.packetPending {
+		c.buf.BeginPacket(packPrelogin, false)
+		c.packetPending = true
+	}
+	return c.buf.Write(b)
+}
+
+func (c *tlsHandshakeConn) Close() error {
+	panic("Not implemented")
+}
+
+func (c *tlsHandshakeConn) LocalAddr() net.Addr {
+	panic("Not implemented")
+}
+
+func (c *tlsHandshakeConn) RemoteAddr() net.Addr {
+	panic("Not implemented")
+}
+
+func (c *tlsHandshakeConn) SetDeadline(t time.Time) error {
+	panic("Not implemented")
+}
+
+func (c *tlsHandshakeConn) SetReadDeadline(t time.Time) error {
+	panic("Not implemented")
+}
+
+func (c *tlsHandshakeConn) SetWriteDeadline(t time.Time) error {
+	panic("Not implemented")
+}
+
+// this connection just delegates all methods to it's wrapped connection
+// it also allows switching underlying connection on the fly
+// it is needed because tls.Conn does not allow switching underlying connection
+type passthroughConn struct {
+	c net.Conn
+}
+
+func (c passthroughConn) Read(b []byte) (n int, err error) {
+	return c.c.Read(b)
+}
+
+func (c passthroughConn) Write(b []byte) (n int, err error) {
+	return c.c.Write(b)
+}
+
+func (c passthroughConn) Close() error {
+	return c.c.Close()
+}
+
+func (c passthroughConn) LocalAddr() net.Addr {
+	panic("Not implemented")
+}
+
+func (c passthroughConn) RemoteAddr() net.Addr {
+	panic("Not implemented")
+}
+
+func (c passthroughConn) SetDeadline(t time.Time) error {
+	panic("Not implemented")
+}
+
+func (c passthroughConn) SetReadDeadline(t time.Time) error {
+	panic("Not implemented")
+}
+
+func (c passthroughConn) SetWriteDeadline(t time.Time) error {
+	panic("Not implemented")
+}

vendor/github.com/denisenkom/go-mssqldb/tds.go

@@ -10,13 +10,9 @@ import (
 	"io"
 	"io/ioutil"
 	"net"
-	"net/url"
-	"os"
 	"sort"
 	"strconv"
 	"strings"
-	"time"
-	"unicode"
 	"unicode/utf16"
 	"unicode/utf8"
 )
@@ -51,15 +47,13 @@ func parseInstances(msg []byte) map[string]map[string]string {
 }
 
 func getInstances(ctx context.Context, d Dialer, address string) (map[string]map[string]string, error) {
-	maxTime := 5 * time.Second
-	ctx, cancel := context.WithTimeout(ctx, maxTime)
-	defer cancel()
 	conn, err := d.DialContext(ctx, "udp", address+":1434")
 	if err != nil {
 		return nil, err
 	}
 	defer conn.Close()
-	conn.SetDeadline(time.Now().Add(maxTime))
+	deadline, _ := ctx.Deadline()
+	conn.SetDeadline(deadline)
 	_, err = conn.Write([]byte{3})
 	if err != nil {
 		return nil, err
@@ -474,10 +468,9 @@ func readUcs2(r io.Reader, numchars int) (res string, err error) {
 }
 
 func readUsVarChar(r io.Reader) (res string, err error) {
-	var numchars uint16
-	err = binary.Read(r, binary.LittleEndian, &numchars)
+	numchars, err := readUshort(r)
 	if err != nil {
-		return "", err
+		return
 	}
 	return readUcs2(r, int(numchars))
 }
@@ -497,8 +490,7 @@ func writeUsVarChar(w io.Writer, s string) (err error) {
 }
 
 func readBVarChar(r io.Reader) (res string, err error) {
-	var numchars uint8
-	err = binary.Read(r, binary.LittleEndian, &numchars)
+	numchars, err := readByte(r)
 	if err != nil {
 		return "", err
 	}
@@ -525,8 +517,7 @@ func writeBVarChar(w io.Writer, s string) (err error) {
 }
 
 func readBVarByte(r io.Reader) (res []byte, err error) {
-	var length uint8
-	err = binary.Read(r, binary.LittleEndian, &length)
+	length, err := readByte(r)
 	if err != nil {
 		return
 	}
@@ -654,458 +645,6 @@ func sendAttention(buf *tdsBuffer) error {
 	return buf.FinishPacket()
 }
 
-type connectParams struct {
-	logFlags                  uint64
-	port                      uint64
-	host                      string
-	instance                  string
-	database                  string
-	user                      string
-	password                  string
-	dial_timeout              time.Duration
-	conn_timeout              time.Duration
-	keepAlive                 time.Duration
-	encrypt                   bool
-	disableEncryption         bool
-	trustServerCertificate    bool
-	certificate               string
-	hostInCertificate         string
-	hostInCertificateProvided bool
-	serverSPN                 string
-	workstation               string
-	appname                   string
-	typeFlags                 uint8
-	failOverPartner           string
-	failOverPort              uint64
-	packetSize                uint16
-}
-
-func splitConnectionString(dsn string) (res map[string]string) {
-	res = map[string]string{}
-	parts := strings.Split(dsn, ";")
-	for _, part := range parts {
-		if len(part) == 0 {
-			continue
-		}
-		lst := strings.SplitN(part, "=", 2)
-		name := strings.TrimSpace(strings.ToLower(lst[0]))
-		if len(name) == 0 {
-			continue
-		}
-		var value string = ""
-		if len(lst) > 1 {
-			value = strings.TrimSpace(lst[1])
-		}
-		res[name] = value
-	}
-	return res
-}
-
-// Splits a URL in the ODBC format
-func splitConnectionStringOdbc(dsn string) (map[string]string, error) {
-	res := map[string]string{}
-
-	type parserState int
-	const (
-		// Before the start of a key
-		parserStateBeforeKey parserState = iota
-
-		// Inside a key
-		parserStateKey
-
-		// Beginning of a value. May be bare or braced
-		parserStateBeginValue
-
-		// Inside a bare value
-		parserStateBareValue
-
-		// Inside a braced value
-		parserStateBracedValue
-
-		// A closing brace inside a braced value.
-		// May be the end of the value or an escaped closing brace, depending on the next character
-		parserStateBracedValueClosingBrace
-
-		// After a value. Next character should be a semicolon or whitespace.
-		parserStateEndValue
-	)
-
-	var state = parserStateBeforeKey
-
-	var key string
-	var value string
-
-	for i, c := range dsn {
-		switch state {
-		case parserStateBeforeKey:
-			switch {
-			case c == '=':
-				return res, fmt.Errorf("Unexpected character = at index %d. Expected start of key or semi-colon or whitespace.", i)
-			case !unicode.IsSpace(c) && c != ';':
-				state = parserStateKey
-				key += string(c)
-			}
-
-		case parserStateKey:
-			switch c {
-			case '=':
-				key = normalizeOdbcKey(key)
-				if len(key) == 0 {
-					return res, fmt.Errorf("Unexpected end of key at index %d.", i)
-				}
-
-				state = parserStateBeginValue
-
-			case ';':
-				// Key without value
-				key = normalizeOdbcKey(key)
-				if len(key) == 0 {
-					return res, fmt.Errorf("Unexpected end of key at index %d.", i)
-				}
-
-				res[key] = value
-				key = ""
-				value = ""
-				state = parserStateBeforeKey
-
-			default:
-				key += string(c)
-			}
-
-		case parserStateBeginValue:
-			switch {
-			case c == '{':
-				state = parserStateBracedValue
-			case c == ';':
-				// Empty value
-				res[key] = value
-				key = ""
-				state = parserStateBeforeKey
-			case unicode.IsSpace(c):
-				// Ignore whitespace
-			default:
-				state = parserStateBareValue
-				value += string(c)
-			}
-
-		case parserStateBareValue:
-			if c == ';' {
-				res[key] = strings.TrimRightFunc(value, unicode.IsSpace)
-				key = ""
-				value = ""
-				state = parserStateBeforeKey
-			} else {
-				value += string(c)
-			}
-
-		case parserStateBracedValue:
-			if c == '}' {
-				state = parserStateBracedValueClosingBrace
-			} else {
-				value += string(c)
-			}
-
-		case parserStateBracedValueClosingBrace:
-			if c == '}' {
-				// Escaped closing brace
-				value += string(c)
-				state = parserStateBracedValue
-				continue
-			}
-
-			// End of braced value
-			res[key] = value
-			key = ""
-			value = ""
-
-			// This character is the first character past the end,
-			// so it needs to be parsed like the parserStateEndValue state.
-			state = parserStateEndValue
-			switch {
-			case c == ';':
-				state = parserStateBeforeKey
-			case unicode.IsSpace(c):
-				// Ignore whitespace
-			default:
-				return res, fmt.Errorf("Unexpected character %c at index %d. Expected semi-colon or whitespace.", c, i)
-			}
-
-		case parserStateEndValue:
-			switch {
-			case c == ';':
-				state = parserStateBeforeKey
-			case unicode.IsSpace(c):
-				// Ignore whitespace
-			default:
-				return res, fmt.Errorf("Unexpected character %c at index %d. Expected semi-colon or whitespace.", c, i)
-			}
-		}
-	}
-
-	switch state {
-	case parserStateBeforeKey: // Okay
-	case parserStateKey: // Unfinished key. Treat as key without value.
-		key = normalizeOdbcKey(key)
-		if len(key) == 0 {
-			return res, fmt.Errorf("Unexpected end of key at index %d.", len(dsn))
-		}
-		res[key] = value
-	case parserStateBeginValue: // Empty value
-		res[key] = value
-	case parserStateBareValue:
-		res[key] = strings.TrimRightFunc(value, unicode.IsSpace)
-	case parserStateBracedValue:
-		return res, fmt.Errorf("Unexpected end of braced value at index %d.", len(dsn))
-	case parserStateBracedValueClosingBrace: // End of braced value
-		res[key] = value
-	case parserStateEndValue: // Okay
-	}
-
-	return res, nil
-}
-
-// Normalizes the given string as an ODBC-format key
-func normalizeOdbcKey(s string) string {
-	return strings.ToLower(strings.TrimRightFunc(s, unicode.IsSpace))
-}
-
-// Splits a URL of the form sqlserver://username:password@host/instance?param1=value&param2=value
-func splitConnectionStringURL(dsn string) (map[string]string, error) {
-	res := map[string]string{}
-
-	u, err := url.Parse(dsn)
-	if err != nil {
-		return res, err
-	}
-
-	if u.Scheme != "sqlserver" {
-		return res, fmt.Errorf("scheme %s is not recognized", u.Scheme)
-	}
-
-	if u.User != nil {
-		res["user id"] = u.User.Username()
-		p, exists := u.User.Password()
-		if exists {
-			res["password"] = p
-		}
-	}
-
-	host, port, err := net.SplitHostPort(u.Host)
-	if err != nil {
-		host = u.Host
-	}
-
-	if len(u.Path) > 0 {
-		res["server"] = host + "\\" + u.Path[1:]
-	} else {
-		res["server"] = host
-	}
-
-	if len(port) > 0 {
-		res["port"] = port
-	}
-
-	query := u.Query()
-	for k, v := range query {
-		if len(v) > 1 {
-			return res, fmt.Errorf("key %s provided more than once", k)
-		}
-		res[strings.ToLower(k)] = v[0]
-	}
-
-	return res, nil
-}
-
-func parseConnectParams(dsn string) (connectParams, error) {
-	var p connectParams
-
-	var params map[string]string
-	if strings.HasPrefix(dsn, "odbc:") {
-		parameters, err := splitConnectionStringOdbc(dsn[len("odbc:"):])
-		if err != nil {
-			return p, err
-		}
-		params = parameters
-	} else if strings.HasPrefix(dsn, "sqlserver://") {
-		parameters, err := splitConnectionStringURL(dsn)
-		if err != nil {
-			return p, err
-		}
-		params = parameters
-	} else {
-		params = splitConnectionString(dsn)
-	}
-
-	strlog, ok := params["log"]
-	if ok {
-		var err error
-		p.logFlags, err = strconv.ParseUint(strlog, 10, 64)
-		if err != nil {
-			return p, fmt.Errorf("Invalid log parameter '%s': %s", strlog, err.Error())
-		}
-	}
-	server := params["server"]
-	parts := strings.SplitN(server, `\`, 2)
-	p.host = parts[0]
-	if p.host == "." || strings.ToUpper(p.host) == "(LOCAL)" || p.host == "" {
-		p.host = "localhost"
-	}
-	if len(parts) > 1 {
-		p.instance = parts[1]
-	}
-	p.database = params["database"]
-	p.user = params["user id"]
-	p.password = params["password"]
-
-	p.port = 1433
-	strport, ok := params["port"]
-	if ok {
-		var err error
-		p.port, err = strconv.ParseUint(strport, 10, 16)
-		if err != nil {
-			f := "Invalid tcp port '%v': %v"
-			return p, fmt.Errorf(f, strport, err.Error())
-		}
-	}
-
-	// https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/configure-the-network-packet-size-server-configuration-option
-	// Default packet size remains at 4096 bytes
-	p.packetSize = 4096
-	strpsize, ok := params["packet size"]
-	if ok {
-		var err error
-		psize, err := strconv.ParseUint(strpsize, 0, 16)
-		if err != nil {
-			f := "Invalid packet size '%v': %v"
-			return p, fmt.Errorf(f, strpsize, err.Error())
-		}
-
-		// Ensure packet size falls within the TDS protocol range of 512 to 32767 bytes
-		// NOTE: Encrypted connections have a maximum size of 16383 bytes.  If you request
-		// a higher packet size, the server will respond with an ENVCHANGE request to
-		// alter the packet size to 16383 bytes.
-		p.packetSize = uint16(psize)
-		if p.packetSize < 512 {
-			p.packetSize = 512
-		} else if p.packetSize > 32767 {
-			p.packetSize = 32767
-		}
-	}
-
-	// https://msdn.microsoft.com/en-us/library/dd341108.aspx
-	//
-	// Do not set a connection timeout. Use Context to manage such things.
-	// Default to zero, but still allow it to be set.
-	if strconntimeout, ok := params["connection timeout"]; ok {
-		timeout, err := strconv.ParseUint(strconntimeout, 10, 64)
-		if err != nil {
-			f := "Invalid connection timeout '%v': %v"
-			return p, fmt.Errorf(f, strconntimeout, err.Error())
-		}
-		p.conn_timeout = time.Duration(timeout) * time.Second
-	}
-	p.dial_timeout = 15 * time.Second
-	if strdialtimeout, ok := params["dial timeout"]; ok {
-		timeout, err := strconv.ParseUint(strdialtimeout, 10, 64)
-		if err != nil {
-			f := "Invalid dial timeout '%v': %v"
-			return p, fmt.Errorf(f, strdialtimeout, err.Error())
-		}
-		p.dial_timeout = time.Duration(timeout) * time.Second
-	}
-
-	// default keep alive should be 30 seconds according to spec:
-	// https://msdn.microsoft.com/en-us/library/dd341108.aspx
-	p.keepAlive = 30 * time.Second
-	if keepAlive, ok := params["keepalive"]; ok {
-		timeout, err := strconv.ParseUint(keepAlive, 10, 64)
-		if err != nil {
-			f := "Invalid keepAlive value '%s': %s"
-			return p, fmt.Errorf(f, keepAlive, err.Error())
-		}
-		p.keepAlive = time.Duration(timeout) * time.Second
-	}
-	encrypt, ok := params["encrypt"]
-	if ok {
-		if strings.EqualFold(encrypt, "DISABLE") {
-			p.disableEncryption = true
-		} else {
-			var err error
-			p.encrypt, err = strconv.ParseBool(encrypt)
-			if err != nil {
-				f := "Invalid encrypt '%s': %s"
-				return p, fmt.Errorf(f, encrypt, err.Error())
-			}
-		}
-	} else {
-		p.trustServerCertificate = true
-	}
-	trust, ok := params["trustservercertificate"]
-	if ok {
-		var err error
-		p.trustServerCertificate, err = strconv.ParseBool(trust)
-		if err != nil {
-			f := "Invalid trust server certificate '%s': %s"
-			return p, fmt.Errorf(f, trust, err.Error())
-		}
-	}
-	p.certificate = params["certificate"]
-	p.hostInCertificate, ok = params["hostnameincertificate"]
-	if ok {
-		p.hostInCertificateProvided = true
-	} else {
-		p.hostInCertificate = p.host
-		p.hostInCertificateProvided = false
-	}
-
-	serverSPN, ok := params["serverspn"]
-	if ok {
-		p.serverSPN = serverSPN
-	} else {
-		p.serverSPN = fmt.Sprintf("MSSQLSvc/%s:%d", p.host, p.port)
-	}
-
-	workstation, ok := params["workstation id"]
-	if ok {
-		p.workstation = workstation
-	} else {
-		workstation, err := os.Hostname()
-		if err == nil {
-			p.workstation = workstation
-		}
-	}
-
-	appname, ok := params["app name"]
-	if !ok {
-		appname = "go-mssqldb"
-	}
-	p.appname = appname
-
-	appintent, ok := params["applicationintent"]
-	if ok {
-		if appintent == "ReadOnly" {
-			p.typeFlags |= fReadOnlyIntent
-		}
-	}
-
-	failOverPartner, ok := params["failoverpartner"]
-	if ok {
-		p.failOverPartner = failOverPartner
-	}
-
-	failOverPort, ok := params["failoverport"]
-	if ok {
-		var err error
-		p.failOverPort, err = strconv.ParseUint(failOverPort, 0, 16)
-		if err != nil {
-			f := "Invalid tcp port '%v': %v"
-			return p, fmt.Errorf(f, failOverPort, err.Error())
-		}
-	}
-
-	return p, nil
-}
-
 type auth interface {
 	InitialBytes() ([]byte, error)
 	NextBytes([]byte) ([]byte, error)
@@ -1277,12 +816,12 @@ initiate_connection:
 		// while SQL Server seems to expect one TCP segment per encrypted TDS package.
 		// Setting DynamicRecordSizingDisabled to true disables that algorithm and uses 16384 bytes per TLS package
 		config.DynamicRecordSizingDisabled = true
-		outbuf.transport = conn
-		toconn.buf = outbuf
-		tlsConn := tls.Client(toconn, &config)
+		// setting up connection handler which will allow wrapping of TLS handshake packets inside TDS stream
+		handshakeConn := tlsHandshakeConn{buf: outbuf}
+		passthrough := passthroughConn{c: &handshakeConn}
+		tlsConn := tls.Client(&passthrough, &config)
 		err = tlsConn.Handshake()
-
-		toconn.buf = nil
+		passthrough.c = toconn
 		outbuf.transport = tlsConn
 		if err != nil {
 			return nil, fmt.Errorf("TLS Handshake failed: %v", err)

vendor/github.com/denisenkom/go-mssqldb/types.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"github.com/denisenkom/go-mssqldb/internal/cp"
+	"github.com/denisenkom/go-mssqldb/internal/decimal"
 )
 
 // fixed-length data types
@@ -818,12 +819,12 @@ func decodeMoney(buf []byte) []byte {
 		uint64(buf[1])<<40 |
 		uint64(buf[2])<<48 |
 		uint64(buf[3])<<56)
-	return scaleBytes(strconv.FormatInt(money, 10), 4)
+	return decimal.ScaleBytes(strconv.FormatInt(money, 10), 4)
 }
 
 func decodeMoney4(buf []byte) []byte {
 	money := int32(binary.LittleEndian.Uint32(buf[0:4]))
-	return scaleBytes(strconv.FormatInt(int64(money), 10), 4)
+	return decimal.ScaleBytes(strconv.FormatInt(int64(money), 10), 4)
 }
 
 func decodeGuid(buf []byte) []byte {
@@ -835,15 +836,14 @@ func decodeGuid(buf []byte) []byte {
 func decodeDecimal(prec uint8, scale uint8, buf []byte) []byte {
 	var sign uint8
 	sign = buf[0]
-	dec := Decimal{
-		positive: sign != 0,
-		prec:     prec,
-		scale:    scale,
-	}
+	var dec decimal.Decimal
+	dec.SetPositive(sign != 0)
+	dec.SetPrec(prec)
+	dec.SetScale(scale)
 	buf = buf[1:]
 	l := len(buf) / 4
 	for i := 0; i < l; i++ {
-		dec.integer[i] = binary.LittleEndian.Uint32(buf[0:4])
+		dec.SetInteger(binary.LittleEndian.Uint32(buf[0:4]), uint8(i))
 		buf = buf[4:]
 	}
 	return dec.Bytes()
@@ -1186,7 +1186,7 @@ func makeDecl(ti typeInfo) string {
 	case typeBigChar, typeChar:
 		return fmt.Sprintf("char(%d)", ti.Size)
 	case typeBigVarChar, typeVarChar:
-		if ti.Size > 4000 || ti.Size == 0 {
+		if ti.Size > 8000 || ti.Size == 0 {
 			return fmt.Sprintf("varchar(max)")
 		} else {
 			return fmt.Sprintf("varchar(%d)", ti.Size)

vendor/github.com/denisenkom/go-mssqldb/uniqueidentifier.go

@@ -72,3 +72,9 @@ func (u UniqueIdentifier) Value() (driver.Value, error) {
 func (u UniqueIdentifier) String() string {
 	return fmt.Sprintf("%X-%X-%X-%X-%X", u[0:4], u[4:6], u[6:8], u[8:10], u[10:])
 }
+
+// MarshalText converts Uniqueidentifier to bytes corresponding to the stringified hexadecimal representation of the Uniqueidentifier
+// e.g., "AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA" -> [65 65 65 65 65 65 65 65 45 65 65 65 65 45 65 65 65 65 45 65 65 65 65 65 65 65 65 65 65 65 65]
+func (u UniqueIdentifier) MarshalText() []byte {
+	return []byte(u.String())
+}

vendor/github.com/felixge/httpsnoop/.travis.yml

@@ -1,6 +0,0 @@
-language: go
-
-go:
-  - 1.6
-  - 1.7
-  - 1.8