Release 2020.11.4

RELEASE_NOTES

@@ -1,3 +1,20 @@
+2020.11.4
+- 2020-11-11 TUN-3534: Specific error message when credentials file is a .pem not .json
+- 2020-11-02 TUN-3500: Integrate replace h2mux by http2 work with multiple origin support
+- 2020-11-09 TUN-3514: Transport logger write to UI when UI is enabled
+- 2020-10-30 TUN-3490: Make sure OriginClient implementation doesn't write after Proxy return
+- 2020-10-20 TUN-3403: Unit test for origin/proxy to test serving HTTP and Websocket
+- 2020-10-23 TUN-3480: Support SSE with http2 connection, and add SSE handler to hello-world server
+- 2020-10-27 TUN-3489: Add unit tests to cover proxy logic in connection package of cloudflared
+- 2020-10-16 TUN-3467: Serialize cf-cloudflared-response-meta during package initialization using jsoniter
+- 2020-10-14 TUN-3456: New protocol option auto to automatically select between http2 and h2mux
+- 2020-10-14 TUN-3458: Upgrade to http2 when available, fallback to h2mux when we reach max retries
+- 2020-10-08 TUN-3449: Use flag to select transport protocol implementation
+- 2020-10-08 TUN-3462: Refactor cloudflared to separate origin from connection
+- 2020-09-21 TUN-3406: Proxy websocket requests over Go http2
+- 2020-09-25 TUN-3420: Establish control plane and send RPC over control plane
+- 2020-09-11 TUN-3400: Use Go HTTP2 library as transport to connect with the edge
+
 2020.11.3
 - 2020-11-11 TUN-3533: Set config for single origin ingress
 

cmd/cloudflared/access/carrier.go

@@ -2,7 +2,6 @@ package access
 
 import (
 	"net/http"
-	"net/url"
 	"strings"
 
 	"github.com/cloudflare/cloudflared/carrier"
@@ -17,16 +16,11 @@ import (
 
 // StartForwarder starts a client side websocket forward
 func StartForwarder(forwarder config.Forwarder, shutdown <-chan struct{}, logger logger.Service) error {
-	validURLString, err := validation.ValidateUrl(forwarder.Listener)
+	validURL, err := validation.ValidateUrl(forwarder.Listener)
 	if err != nil {
 		return errors.Wrap(err, "error validating origin URL")
 	}
 
-	validURL, err := url.Parse(validURLString)
-	if err != nil {
-		return errors.Wrap(err, "error parsing origin URL")
-	}
-
 	// get the headers from the config file and add to the request
 	headers := make(http.Header)
 	if forwarder.TokenClientID != "" {
@@ -106,12 +100,7 @@ func ssh(c *cli.Context) error {
 	wsConn := carrier.NewWSConnection(logger, false)
 
 	if c.NArg() > 0 || c.IsSet(sshURLFlag) {
-		localForwarder, err := config.ValidateUrl(c, true)
-		if err != nil {
-			logger.Errorf("Error validating origin URL: %s", err)
-			return errors.Wrap(err, "error validating origin URL")
-		}
-		forwarder, err := url.Parse(localForwarder)
+		forwarder, err := config.ValidateUrl(c, true)
 		if err != nil {
 			logger.Errorf("Error validating origin URL: %s", err)
 			return errors.Wrap(err, "error validating origin URL")

cmd/cloudflared/config/configuration.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"fmt"
+	"net/url"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -189,11 +190,11 @@ func ValidateUnixSocket(c *cli.Context) (string, error) {
 
 // ValidateUrl will validate url flag correctness. It can be either from --url or argument
 // Notice ValidateUnixSocket, it will enforce --unix-socket is not used with --url or argument
-func ValidateUrl(c *cli.Context, allowFromArgs bool) (string, error) {
+func ValidateUrl(c *cli.Context, allowURLFromArgs bool) (*url.URL, error) {
 	var url = c.String("url")
-	if allowFromArgs && c.NArg() > 0 {
+	if allowURLFromArgs && c.NArg() > 0 {
 		if c.IsSet("url") {
-			return "", errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
+			return nil, errors.New("Specified origin urls using both --url and argument. Decide which one you want, I can only support one.")
 		}
 		url = c.Args().Get(0)
 	}

cmd/cloudflared/tunnel/cmd.go

@@ -18,6 +18,7 @@ import (
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/updater"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/dbconnect"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
@@ -247,8 +248,8 @@ func StartServer(
 	version string,
 	shutdownC,
 	graceShutdownC chan struct{},
-	namedTunnel *origin.NamedTunnelConfig,
-	log logger.Service,
+	namedTunnel *connection.NamedTunnelConfig,
+	generalLogger logger.Service,
 	isUIEnabled bool,
 ) error {
 	_ = raven.SetDSN(sentryDSN)
@@ -259,45 +260,45 @@ func StartServer(
 	dnsReadySignal := make(chan struct{})
 
 	if config.GetConfiguration().Source() == "" {
-		log.Infof(config.ErrNoConfigFile.Error())
+		generalLogger.Infof(config.ErrNoConfigFile.Error())
 	}
 
 	if c.IsSet("trace-output") {
 		tmpTraceFile, err := ioutil.TempFile("", "trace")
 		if err != nil {
-			log.Errorf("Failed to create new temporary file to save trace output: %s", err)
+			generalLogger.Errorf("Failed to create new temporary file to save trace output: %s", err)
 		}
 
 		defer func() {
 			if err := tmpTraceFile.Close(); err != nil {
-				log.Errorf("Failed to close trace output file %s with error: %s", tmpTraceFile.Name(), err)
+				generalLogger.Errorf("Failed to close trace output file %s with error: %s", tmpTraceFile.Name(), err)
 			}
 			if err := os.Rename(tmpTraceFile.Name(), c.String("trace-output")); err != nil {
-				log.Errorf("Failed to rename temporary trace output file %s to %s with error: %s", tmpTraceFile.Name(), c.String("trace-output"), err)
+				generalLogger.Errorf("Failed to rename temporary trace output file %s to %s with error: %s", tmpTraceFile.Name(), c.String("trace-output"), err)
 			} else {
 				err := os.Remove(tmpTraceFile.Name())
 				if err != nil {
-					log.Errorf("Failed to remove the temporary trace file %s with error: %s", tmpTraceFile.Name(), err)
+					generalLogger.Errorf("Failed to remove the temporary trace file %s with error: %s", tmpTraceFile.Name(), err)
 				}
 			}
 		}()
 
 		if err := trace.Start(tmpTraceFile); err != nil {
-			log.Errorf("Failed to start trace: %s", err)
+			generalLogger.Errorf("Failed to start trace: %s", err)
 			return errors.Wrap(err, "Error starting tracing")
 		}
 		defer trace.Stop()
 	}
 
 	buildInfo := buildinfo.GetBuildInfo(version)
-	buildInfo.Log(log)
-	logClientOptions(c, log)
+	buildInfo.Log(generalLogger)
+	logClientOptions(c, generalLogger)
 
 	if c.IsSet("proxy-dns") {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			errC <- runDNSProxyServer(c, dnsReadySignal, shutdownC, log)
+			errC <- runDNSProxyServer(c, dnsReadySignal, shutdownC, generalLogger)
 		}()
 	} else {
 		close(dnsReadySignal)
@@ -308,24 +309,24 @@ func StartServer(
 
 	metricsListener, err := listeners.Listen("tcp", c.String("metrics"))
 	if err != nil {
-		log.Errorf("Error opening metrics server listener: %s", err)
+		generalLogger.Errorf("Error opening metrics server listener: %s", err)
 		return errors.Wrap(err, "Error opening metrics server listener")
 	}
 	defer metricsListener.Close()
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		errC <- metrics.ServeMetrics(metricsListener, shutdownC, log)
+		errC <- metrics.ServeMetrics(metricsListener, shutdownC, generalLogger)
 	}()
 
 	go notifySystemd(connectedSignal)
 	if c.IsSet("pidfile") {
-		go writePidFile(connectedSignal, c.String("pidfile"), log)
+		go writePidFile(connectedSignal, c.String("pidfile"), generalLogger)
 	}
 
 	cloudflaredID, err := uuid.NewRandom()
 	if err != nil {
-		log.Errorf("Cannot generate cloudflared ID: %s", err)
+		generalLogger.Errorf("Cannot generate cloudflared ID: %s", err)
 		return err
 	}
 
@@ -336,12 +337,12 @@ func StartServer(
 	}()
 
 	// update needs to be after DNS proxy is up to resolve equinox server address
-	if updater.IsAutoupdateEnabled(c, log) {
-		log.Infof("Autoupdate frequency is set to %v", c.Duration("autoupdate-freq"))
+	if updater.IsAutoupdateEnabled(c, generalLogger) {
+		generalLogger.Infof("Autoupdate frequency is set to %v", c.Duration("autoupdate-freq"))
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			autoupdater := updater.NewAutoUpdater(c.Duration("autoupdate-freq"), &listeners, log)
+			autoupdater := updater.NewAutoUpdater(c.Duration("autoupdate-freq"), &listeners, generalLogger)
 			errC <- autoupdater.Run(ctx)
 		}()
 	}
@@ -350,33 +351,33 @@ func StartServer(
 	if dnsProxyStandAlone(c) {
 		connectedSignal.Notify()
 		// no grace period, handle SIGINT/SIGTERM immediately
-		return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, 0, log)
+		return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, 0, generalLogger)
 	}
 
 	url := c.String("url")
 	hostname := c.String("hostname")
 	if url == hostname && url != "" && hostname != "" {
 		errText := "hostname and url shouldn't match. See --help for more information"
-		log.Error(errText)
+		generalLogger.Error(errText)
 		return fmt.Errorf(errText)
 	}
 
-	transportLogger, err := createLogger(c, true, false)
+	transportLogger, err := createLogger(c, true, isUIEnabled)
 	if err != nil {
 		return errors.Wrap(err, "error setting up transport logger")
 	}
 
-	tunnelConfig, err := prepareTunnelConfig(c, buildInfo, version, log, transportLogger, namedTunnel)
+	tunnelConfig, ingressRules, err := prepareTunnelConfig(c, buildInfo, version, generalLogger, transportLogger, namedTunnel, isUIEnabled)
 	if err != nil {
 		return err
 	}
 
-	tunnelConfig.IngressRules.StartOrigins(&wg, log, shutdownC, errC)
+	ingressRules.StartOrigins(&wg, generalLogger, shutdownC, errC)
 
 	reconnectCh := make(chan origin.ReconnectSignal, 1)
 	if c.IsSet("stdin-control") {
-		log.Info("Enabling control through stdin")
-		go stdinControl(reconnectCh, log)
+		generalLogger.Info("Enabling control through stdin")
+		go stdinControl(reconnectCh, generalLogger)
 	}
 
 	wg.Add(1)
@@ -386,26 +387,21 @@ func StartServer(
 	}()
 
 	if isUIEnabled {
-		const tunnelEventChanBufferSize = 16
-		tunnelEventChan := make(chan ui.TunnelEvent, tunnelEventChanBufferSize)
-		tunnelConfig.TunnelEventChan = tunnelEventChan
-
 		tunnelInfo := ui.NewUIModel(
 			version,
 			hostname,
 			metricsListener.Addr().String(),
-			// TODO (TUN-3461): Update UI to show multiple origin URLs
-			&tunnelConfig.IngressRules,
+			&ingressRules,
 			tunnelConfig.HAConnections,
 		)
 		logLevels, err := logger.ParseLevelString(c.String("loglevel"))
 		if err != nil {
 			return err
 		}
-		tunnelInfo.LaunchUI(ctx, log, logLevels, tunnelEventChan)
+		tunnelInfo.LaunchUI(ctx, generalLogger, transportLogger, logLevels, tunnelConfig.TunnelEventChan)
 	}
 
-	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"), log)
+	return waitToShutdown(&wg, errC, shutdownC, graceShutdownC, c.Duration("grace-period"), generalLogger)
 }
 
 // forceSetFlag attempts to set the given flag value in the closest context that has it defined
@@ -719,6 +715,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			Value:  false,
 			Hidden: shouldHide,
 		}),
+		selectProtocolFlag,
 	}...)
 
 	return flags
@@ -985,7 +982,7 @@ func configureLoggingFlags(shouldHide bool) []cli.Flag {
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    "transport-loglevel",
 			Aliases: []string{"proto-loglevel"}, // This flag used to be called proto-loglevel
-			Value:   "fatal",
+			Value:   "info",
 			Usage:   "Transport logging level(previously called protocol logging level) {fatal, error, info, debug}",
 			EnvVars: []string{"TUNNEL_PROTO_LOGLEVEL", "TUNNEL_TRANSPORT_LOGLEVEL"},
 			Hidden:  shouldHide,

cmd/cloudflared/tunnel/configuration.go

@@ -1,6 +1,7 @@
 package tunnel
 
 import (
+	"crypto/tls"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -9,6 +10,10 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/origin"
@@ -154,28 +159,29 @@ func prepareTunnelConfig(
 	version string,
 	logger logger.Service,
 	transportLogger logger.Service,
-	namedTunnel *origin.NamedTunnelConfig,
-) (*origin.TunnelConfig, error) {
-	compatibilityMode := namedTunnel == nil
+	namedTunnel *connection.NamedTunnelConfig,
+	uiIsEnabled bool,
+) (*origin.TunnelConfig, ingress.Ingress, error) {
+	isNamedTunnel := namedTunnel != nil
 
 	hostname, err := validation.ValidateHostname(c.String("hostname"))
 	if err != nil {
 		logger.Errorf("Invalid hostname: %s", err)
-		return nil, errors.Wrap(err, "Invalid hostname")
+		return nil, ingress.Ingress{}, errors.Wrap(err, "Invalid hostname")
 	}
 	isFreeTunnel := hostname == ""
 	clientID := c.String("id")
 	if !c.IsSet("id") {
 		clientID, err = generateRandomClientID(logger)
 		if err != nil {
-			return nil, err
+			return nil, ingress.Ingress{}, err
 		}
 	}
 
 	tags, err := NewTagSliceFromCLI(c.StringSlice("tag"))
 	if err != nil {
 		logger.Errorf("Tag parse failure: %s", err)
-		return nil, errors.Wrap(err, "Tag parse failure")
+		return nil, ingress.Ingress{}, errors.Wrap(err, "Tag parse failure")
 	}
 
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
@@ -184,17 +190,18 @@ func prepareTunnelConfig(
 	if !isFreeTunnel {
 		originCert, err = getOriginCert(c, logger)
 		if err != nil {
-			return nil, errors.Wrap(err, "Error getting origin cert")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "Error getting origin cert")
 		}
 	}
 
-	tunnelMetrics := origin.NewTunnelMetrics()
-
-	var ingressRules ingress.Ingress
-	if namedTunnel != nil {
+	var (
+		ingressRules  ingress.Ingress
+		classicTunnel *connection.ClassicTunnelConfig
+	)
+	if isNamedTunnel {
 		clientUUID, err := uuid.NewRandom()
 		if err != nil {
-			return nil, errors.Wrap(err, "can't generate clientUUID")
+			return nil, ingress.Ingress{}, errors.Wrap(err, "can't generate clientUUID")
 		}
 		namedTunnel.Client = tunnelpogs.ClientInfo{
 			ClientID: clientUUID[:],
@@ -204,56 +211,84 @@ func prepareTunnelConfig(
 		}
 		ingressRules, err = ingress.ParseIngress(config.GetConfiguration())
 		if err != nil && err != ingress.ErrNoIngressRules {
-			return nil, err
+			return nil, ingress.Ingress{}, err
 		}
 		if !ingressRules.IsEmpty() && c.IsSet("url") {
-			return nil, ingress.ErrURLIncompatibleWithIngress
+			return nil, ingress.Ingress{}, ingress.ErrURLIncompatibleWithIngress
+		}
+	} else {
+		classicTunnel = &connection.ClassicTunnelConfig{
+			Hostname:   hostname,
+			OriginCert: originCert,
+			// turn off use of reconnect token and auth refresh when using named tunnels
+			UseReconnectToken: !isNamedTunnel && c.Bool("use-reconnect-token"),
 		}
 	}
 
 	// Convert single-origin configuration into multi-origin configuration.
 	if ingressRules.IsEmpty() {
-		ingressRules, err = ingress.NewSingleOrigin(c, compatibilityMode, logger)
+		ingressRules, err = ingress.NewSingleOrigin(c, !isNamedTunnel, logger)
 		if err != nil {
-			return nil, err
+			return nil, ingress.Ingress{}, err
 		}
 	}
 
-	toEdgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c)
+	protocolSelector, err := connection.NewProtocolSelector(c.String("protocol"), namedTunnel, edgediscovery.HTTP2Percentage, origin.ResolveTTL, logger)
 	if err != nil {
-		logger.Errorf("unable to create TLS config to connect with edge: %s", err)
-		return nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
+		return nil, ingress.Ingress{}, err
 	}
-	return &origin.TunnelConfig{
-		BuildInfo:          buildInfo,
-		ClientID:           clientID,
-		CompressionQuality: c.Uint64("compression-quality"),
-		EdgeAddrs:          c.StringSlice("edge"),
-		GracePeriod:        c.Duration("grace-period"),
-		HAConnections:      c.Int("ha-connections"),
+	logger.Infof("Initial protocol %s", protocolSelector.Current())
+
+	edgeTLSConfigs := make(map[connection.Protocol]*tls.Config, len(connection.ProtocolList))
+	for _, p := range connection.ProtocolList {
+		edgeTLSConfig, err := tlsconfig.CreateTunnelConfig(c, p.ServerName())
+		if err != nil {
+			return nil, ingress.Ingress{}, errors.Wrap(err, "unable to create TLS config to connect with edge")
+		}
+		edgeTLSConfigs[p] = edgeTLSConfig
+	}
+
+	originClient := origin.NewClient(ingressRules, tags, logger)
+	connectionConfig := &connection.Config{
+		OriginClient:    originClient,
+		GracePeriod:     c.Duration("grace-period"),
+		ReplaceExisting: c.Bool("force"),
+	}
+	muxerConfig := &connection.MuxerConfig{
 		HeartbeatInterval:  c.Duration("heartbeat-interval"),
-		Hostname:           hostname,
-		IncidentLookup:     origin.NewIncidentLookup(),
-		IsAutoupdated:      c.Bool("is-autoupdated"),
-		IsFreeTunnel:       isFreeTunnel,
-		LBPool:             c.String("lb-pool"),
-		Logger:             logger,
-		TransportLogger:    transportLogger,
 		MaxHeartbeats:      c.Uint64("heartbeat-count"),
-		Metrics:            tunnelMetrics,
+		CompressionSetting: h2mux.CompressionSetting(c.Uint64("compression-quality")),
 		MetricsUpdateFreq:  c.Duration("metrics-update-freq"),
-		OriginCert:         originCert,
-		ReportedVersion:    version,
-		Retries:            c.Uint("retries"),
-		RunFromTerminal:    isRunningFromTerminal(),
-		Tags:               tags,
-		TlsConfig:          toEdgeTLSConfig,
-		NamedTunnel:        namedTunnel,
-		ReplaceExisting:    c.Bool("force"),
-		IngressRules:       ingressRules,
-		// turn off use of reconnect token and auth refresh when using named tunnels
-		UseReconnectToken: compatibilityMode && c.Bool("use-reconnect-token"),
-	}, nil
+	}
+
+	var tunnelEventChan chan ui.TunnelEvent
+	if uiIsEnabled {
+		tunnelEventChan = make(chan ui.TunnelEvent, 16)
+	}
+
+	return &origin.TunnelConfig{
+		ConnectionConfig: connectionConfig,
+		BuildInfo:        buildInfo,
+		ClientID:         clientID,
+		EdgeAddrs:        c.StringSlice("edge"),
+		HAConnections:    c.Int("ha-connections"),
+		IncidentLookup:   origin.NewIncidentLookup(),
+		IsAutoupdated:    c.Bool("is-autoupdated"),
+		IsFreeTunnel:     isFreeTunnel,
+		LBPool:           c.String("lb-pool"),
+		Tags:             tags,
+		Logger:           logger,
+		Observer:         connection.NewObserver(transportLogger, tunnelEventChan),
+		ReportedVersion:  version,
+		Retries:          c.Uint("retries"),
+		RunFromTerminal:  isRunningFromTerminal(),
+		NamedTunnel:      namedTunnel,
+		ClassicTunnel:    classicTunnel,
+		MuxerConfig:      muxerConfig,
+		TunnelEventChan:  tunnelEventChan,
+		ProtocolSelector: protocolSelector,
+		EdgeTLSConfigs:   edgeTLSConfigs,
+	}, ingressRules, nil
 }
 
 func isRunningFromTerminal() bool {

cmd/cloudflared/tunnel/subcommand_context.go

@@ -14,8 +14,8 @@ import (
 
 	"github.com/cloudflare/cloudflared/certutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/tunnelstore"
 )
@@ -120,6 +120,12 @@ func (sc *subcommandContext) readTunnelCredentials(tunnelID uuid.UUID) (*pogs.Tu
 
 	var auth pogs.TunnelAuth
 	if err = json.Unmarshal(body, &auth); err != nil {
+		if strings.HasSuffix(filePath, ".pem") {
+			return nil, fmt.Errorf("The tunnel credentials file should be .json but you gave a .pem. "+
+				"The tunnel credentials file was originally created by `cloudflared tunnel create` and named %s.json."+
+				"You may have accidentally used the filepath to cert.pem, which is generated by `cloudflared tunnel "+
+				"login`.", tunnelID)
+		}
 		return nil, errInvalidJSONCredential{path: filePath, err: err}
 	}
 	return &auth, nil
@@ -265,7 +271,7 @@ func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 		version,
 		shutdownC,
 		graceShutdownC,
-		&origin.NamedTunnelConfig{Auth: *credentials, ID: tunnelID},
+		&connection.NamedTunnelConfig{Auth: *credentials, ID: tunnelID},
 		sc.logger,
 		sc.isUIEnabled,
 	)

cmd/cloudflared/tunnel/subcommands.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/tunnelstore"
@@ -77,21 +78,31 @@ var (
 		Name:    "credentials-file",
 		Aliases: []string{credFileFlagAlias},
 		Usage:   "File path of tunnel credentials",
+		EnvVars: []string{"TUNNEL_CRED_FILE"},
 	})
 	forceDeleteFlag = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
 		Usage:   "Allows you to delete a tunnel, even if it has active connections.",
+		EnvVars: []string{"TUNNEL_RUN_FORCE_OVERWRITE"},
 	}
+	selectProtocolFlag = altsrc.NewStringFlag(&cli.StringFlag{
+		Name:    "protocol",
+		Value:   "h2mux",
+		Aliases: []string{"p"},
+		Usage:   fmt.Sprintf("Protocol implementation to connect with Cloudflare's edge network. %s", connection.AvailableProtocolFlagMessage),
+		EnvVars: []string{"TUNNEL_TRANSPORT_PROTOCOL"},
+		Hidden:  true,
+	})
 )
 
 func buildCreateCommand() *cli.Command {
 	return &cli.Command{
-		Name:               "create",
-		Action:             cliutil.ErrorHandler(createCommand),
-		Usage:              "Create a new tunnel with given name",
-		UsageText:          "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
-		Description:        `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
+		Name:      "create",
+		Action:    cliutil.ErrorHandler(createCommand),
+		Usage:     "Create a new tunnel with given name",
+		UsageText: "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
+		Description: `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
   Use "cloudflared tunnel route" subcommand to map a DNS name to this tunnel and "cloudflared tunnel run" to start the connection. 
 
   For example, to create a tunnel named 'my-tunnel' run:
@@ -309,6 +320,7 @@ func buildRunCommand() *cli.Command {
 	flags := []cli.Flag{
 		forceFlag,
 		credentialsFileFlag,
+		selectProtocolFlag,
 	}
 	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{

cmd/cloudflared/ui/launch_ui.go

@@ -67,7 +67,7 @@ func NewUIModel(version, hostname, metricsURL string, ing *ingress.Ingress, haCo
 
 func (data *uiModel) LaunchUI(
 	ctx context.Context,
-	log logger.Service,
+	generalLogger, transportLogger logger.Service,
 	logLevels []logger.Level,
 	tunnelEventChan <-chan TunnelEvent,
 ) {
@@ -75,7 +75,8 @@ func (data *uiModel) LaunchUI(
 
 	// Add TextView as a group to write output to
 	logTextView := NewDynamicColorTextView()
-	log.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+	generalLogger.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+	transportLogger.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
 
 	// Construct the UI
 	palette := palette{
@@ -140,7 +141,7 @@ func (data *uiModel) LaunchUI(
 				case Connected:
 					data.setConnTableCell(event, connTable, palette)
 				case Disconnected, Reconnecting:
-					data.changeConnStatus(event, connTable, log, palette)
+					data.changeConnStatus(event, connTable, generalLogger, palette)
 				case SetUrl:
 					tunnelHostText.SetText(event.Url)
 					data.edgeURL = event.Url
@@ -156,7 +157,7 @@ func (data *uiModel) LaunchUI(
 
 	go func() {
 		if err := app.SetRoot(frame, true).Run(); err != nil {
-			log.Errorf("Error launching UI: %s", err)
+			generalLogger.Errorf("Error launching UI: %s", err)
 		}
 	}()
 }

connection/connection.go

@@ -0,0 +1,62 @@
+package connection
+
+import (
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/google/uuid"
+)
+
+const (
+	lbProbeUserAgentPrefix = "Mozilla/5.0 (compatible; Cloudflare-Traffic-Manager/1.0; +https://www.cloudflare.com/traffic-manager/;"
+)
+
+type Config struct {
+	OriginClient    OriginClient
+	GracePeriod     time.Duration
+	ReplaceExisting bool
+}
+
+type NamedTunnelConfig struct {
+	Auth   pogs.TunnelAuth
+	ID     uuid.UUID
+	Client pogs.ClientInfo
+}
+
+type ClassicTunnelConfig struct {
+	Hostname   string
+	OriginCert []byte
+	// feature-flag to use new edge reconnect tokens
+	UseReconnectToken bool
+}
+
+func (c *ClassicTunnelConfig) IsTrialZone() bool {
+	return c.Hostname == ""
+}
+
+type OriginClient interface {
+	Proxy(w ResponseWriter, req *http.Request, isWebsocket bool) error
+}
+
+type ResponseWriter interface {
+	WriteRespHeaders(*http.Response) error
+	WriteErrorResponse()
+	io.ReadWriter
+}
+
+type ConnectedFuse interface {
+	Connected()
+	IsConnected() bool
+}
+
+func uint8ToString(input uint8) string {
+	return strconv.FormatUint(uint64(input), 10)
+}
+
+func isServerSentEvent(headers http.Header) bool {
+	return strings.ToLower(headers.Get("content-type")) == "text/event-stream"
+}

connection/connection_test.go

@@ -0,0 +1,113 @@
+package connection
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/gobwas/ws/wsutil"
+)
+
+const (
+	largeFileSize = 2 * 1024 * 1024
+)
+
+var (
+	testConfig = &Config{
+		OriginClient: &mockOriginClient{},
+		GracePeriod:  time.Millisecond * 100,
+	}
+	testLogger, _ = logger.New()
+	testOriginURL = &url.URL{
+		Scheme: "https",
+		Host:   "connectiontest.argotunnel.com",
+	}
+	testTunnelEventChan = make(chan ui.TunnelEvent)
+	testObserver        = &Observer{
+		testLogger,
+		m,
+		testTunnelEventChan,
+	}
+	testLargeResp = make([]byte, largeFileSize)
+)
+
+type testRequest struct {
+	name           string
+	endpoint       string
+	expectedStatus int
+	expectedBody   []byte
+	isProxyError   bool
+}
+
+type mockOriginClient struct {
+}
+
+func (moc *mockOriginClient) Proxy(w ResponseWriter, r *http.Request, isWebsocket bool) error {
+	if isWebsocket {
+		return wsEndpoint(w, r)
+	}
+	switch r.URL.Path {
+	case "/ok":
+		originRespEndpoint(w, http.StatusOK, []byte(http.StatusText(http.StatusOK)))
+	case "/large_file":
+		originRespEndpoint(w, http.StatusOK, testLargeResp)
+	case "/400":
+		originRespEndpoint(w, http.StatusBadRequest, []byte(http.StatusText(http.StatusBadRequest)))
+	case "/500":
+		originRespEndpoint(w, http.StatusInternalServerError, []byte(http.StatusText(http.StatusInternalServerError)))
+	case "/error":
+		return fmt.Errorf("Failed to proxy to origin")
+	default:
+		originRespEndpoint(w, http.StatusNotFound, []byte("page not found"))
+	}
+	return nil
+}
+
+type nowriter struct {
+	io.Reader
+}
+
+func (nowriter) Write(p []byte) (int, error) {
+	return 0, fmt.Errorf("Writer not implemented")
+}
+
+func wsEndpoint(w ResponseWriter, r *http.Request) error {
+	resp := &http.Response{
+		StatusCode: http.StatusSwitchingProtocols,
+	}
+	w.WriteRespHeaders(resp)
+	clientReader := nowriter{r.Body}
+	go func() {
+		for {
+			data, err := wsutil.ReadClientText(clientReader)
+			if err != nil {
+				return
+			}
+			if err := wsutil.WriteServerText(w, data); err != nil {
+				return
+			}
+		}
+	}()
+	<-r.Context().Done()
+	return nil
+}
+
+func originRespEndpoint(w ResponseWriter, status int, data []byte) {
+	resp := &http.Response{
+		StatusCode: status,
+	}
+	w.WriteRespHeaders(resp)
+	w.Write(data)
+}
+
+type mockConnectedFuse struct{}
+
+func (mcf mockConnectedFuse) Connected() {}
+
+func (mcf mockConnectedFuse) IsConnected() bool {
+	return true
+}

connection/errors.go

@@ -0,0 +1,76 @@
+package connection
+
+import (
+	"github.com/cloudflare/cloudflared/edgediscovery"
+	"github.com/cloudflare/cloudflared/h2mux"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	DuplicateConnectionError = "EDUPCONN"
+)
+
+// RegisterTunnel error from client
+type clientRegisterTunnelError struct {
+	cause error
+}
+
+func newRPCError(cause error, counter *prometheus.CounterVec, name rpcName) clientRegisterTunnelError {
+	counter.WithLabelValues(cause.Error(), string(name)).Inc()
+	return clientRegisterTunnelError{cause: cause}
+}
+
+func (e clientRegisterTunnelError) Error() string {
+	return e.cause.Error()
+}
+
+type DupConnRegisterTunnelError struct{}
+
+var errDuplicationConnection = &DupConnRegisterTunnelError{}
+
+func (e DupConnRegisterTunnelError) Error() string {
+	return "already connected to this server, trying another address"
+}
+
+// RegisterTunnel error from server
+type serverRegisterTunnelError struct {
+	cause     error
+	permanent bool
+}
+
+func (e serverRegisterTunnelError) Error() string {
+	return e.cause.Error()
+}
+
+func serverRegistrationErrorFromRPC(err error) *serverRegisterTunnelError {
+	if retryable, ok := err.(*tunnelpogs.RetryableError); ok {
+		return &serverRegisterTunnelError{
+			cause:     retryable.Unwrap(),
+			permanent: false,
+		}
+	}
+	return &serverRegisterTunnelError{
+		cause:     err,
+		permanent: true,
+	}
+}
+
+type muxerShutdownError struct{}
+
+func (e muxerShutdownError) Error() string {
+	return "muxer shutdown"
+}
+
+func isHandshakeErrRecoverable(err error, connIndex uint8, observer *Observer) bool {
+	switch err.(type) {
+	case edgediscovery.DialError:
+		observer.Errorf("Connection %d unable to dial edge: %s", connIndex, err)
+	case h2mux.MuxerHandshakeError:
+		observer.Errorf("Connection %d handshake with edge server failed: %s", connIndex, err)
+	default:
+		observer.Errorf("Connection %d failed: %s", connIndex, err)
+		return false
+	}
+	return true
+}

connection/h2mux.go

@@ -0,0 +1,220 @@
+package connection
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	muxerTimeout      = 5 * time.Second
+	openStreamTimeout = 30 * time.Second
+)
+
+type h2muxConnection struct {
+	config      *Config
+	muxerConfig *MuxerConfig
+	muxer       *h2mux.Muxer
+	// connectionID is only used by metrics, and prometheus requires labels to be string
+	connIndexStr string
+	connIndex    uint8
+
+	observer *Observer
+}
+
+type MuxerConfig struct {
+	HeartbeatInterval  time.Duration
+	MaxHeartbeats      uint64
+	CompressionSetting h2mux.CompressionSetting
+	MetricsUpdateFreq  time.Duration
+}
+
+func (mc *MuxerConfig) H2MuxerConfig(h h2mux.MuxedStreamHandler, logger logger.Service) *h2mux.MuxerConfig {
+	return &h2mux.MuxerConfig{
+		Timeout:            muxerTimeout,
+		Handler:            h,
+		IsClient:           true,
+		HeartbeatInterval:  mc.HeartbeatInterval,
+		MaxHeartbeats:      mc.MaxHeartbeats,
+		Logger:             logger,
+		CompressionQuality: mc.CompressionSetting,
+	}
+}
+
+// NewTunnelHandler returns a TunnelHandler, origin LAN IP and error
+func NewH2muxConnection(ctx context.Context,
+	config *Config,
+	muxerConfig *MuxerConfig,
+	edgeConn net.Conn,
+	connIndex uint8,
+	observer *Observer,
+) (*h2muxConnection, error, bool) {
+	h := &h2muxConnection{
+		config:       config,
+		muxerConfig:  muxerConfig,
+		connIndexStr: uint8ToString(connIndex),
+		connIndex:    connIndex,
+		observer:     observer,
+	}
+
+	// Establish a muxed connection with the edge
+	// Client mux handshake with agent server
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig.H2MuxerConfig(h, observer), h2mux.ActiveStreams)
+	if err != nil {
+		recoverable := isHandshakeErrRecoverable(err, connIndex, observer)
+		return nil, err, recoverable
+	}
+	h.muxer = muxer
+	return h, nil, false
+}
+
+func (h *h2muxConnection) ServeNamedTunnel(ctx context.Context, namedTunnel *NamedTunnelConfig, credentialManager CredentialManager, connOptions *tunnelpogs.ConnectionOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() error {
+		stream, err := h.newRPCStream(serveCtx, register)
+		if err != nil {
+			return err
+		}
+		rpcClient := newRegistrationRPCClient(ctx, stream, h.observer)
+		defer rpcClient.Close()
+
+		if err = rpcClient.RegisterConnection(serveCtx, namedTunnel, connOptions, h.connIndex, h.observer); err != nil {
+			return err
+		}
+		connectedFuse.Connected()
+		return nil
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, true)
+		return nil
+	})
+	return errGroup.Wait()
+}
+
+func (h *h2muxConnection) ServeClassicTunnel(ctx context.Context, classicTunnel *ClassicTunnelConfig, credentialManager CredentialManager, registrationOptions *tunnelpogs.RegistrationOptions, connectedFuse ConnectedFuse) error {
+	errGroup, serveCtx := errgroup.WithContext(ctx)
+	errGroup.Go(func() error {
+		return h.serveMuxer(serveCtx)
+	})
+
+	errGroup.Go(func() (err error) {
+		defer func() {
+			if err == nil {
+				connectedFuse.Connected()
+			}
+		}()
+		if classicTunnel.UseReconnectToken && connectedFuse.IsConnected() {
+			err := h.reconnectTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
+			if err == nil {
+				return nil
+			}
+			// log errors and proceed to RegisterTunnel
+			h.observer.Errorf("Couldn't reconnect connection %d. Reregistering it instead. Error was: %v", h.connIndex, err)
+		}
+		return h.registerTunnel(ctx, credentialManager, classicTunnel, registrationOptions)
+	})
+
+	errGroup.Go(func() error {
+		h.controlLoop(serveCtx, connectedFuse, false)
+		return nil
+	})
+	return errGroup.Wait()
+}
+
+func (h *h2muxConnection) serveMuxer(ctx context.Context) error {
+	// All routines should stop when muxer finish serving. When muxer is shutdown
+	// gracefully, it doesn't return an error, so we need to return errMuxerShutdown
+	// here to notify other routines to stop
+	err := h.muxer.Serve(ctx)
+	if err == nil {
+		return muxerShutdownError{}
+	}
+	return err
+}
+
+func (h *h2muxConnection) controlLoop(ctx context.Context, connectedFuse ConnectedFuse, isNamedTunnel bool) {
+	updateMetricsTickC := time.Tick(h.muxerConfig.MetricsUpdateFreq)
+	for {
+		select {
+		case <-ctx.Done():
+			// UnregisterTunnel blocks until the RPC call returns
+			if connectedFuse.IsConnected() {
+				h.unregister(isNamedTunnel)
+			}
+			h.muxer.Shutdown()
+			return
+		case <-updateMetricsTickC:
+			h.observer.metrics.updateMuxerMetrics(h.connIndexStr, h.muxer.Metrics())
+		}
+	}
+}
+
+func (h *h2muxConnection) newRPCStream(ctx context.Context, rpcName rpcName) (*h2mux.MuxedStream, error) {
+	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer openStreamCancel()
+	stream, err := h.muxer.OpenRPCStream(openStreamCtx)
+	if err != nil {
+		return nil, err
+	}
+	return stream, nil
+}
+
+func (h *h2muxConnection) ServeStream(stream *h2mux.MuxedStream) error {
+	respWriter := &h2muxRespWriter{stream}
+
+	req, reqErr := h.newRequest(stream)
+	if reqErr != nil {
+		respWriter.WriteErrorResponse()
+		return reqErr
+	}
+
+	err := h.config.OriginClient.Proxy(respWriter, req, websocket.IsWebSocketUpgrade(req))
+	if err != nil {
+		respWriter.WriteErrorResponse()
+		return err
+	}
+	return nil
+}
+
+func (h *h2muxConnection) newRequest(stream *h2mux.MuxedStream) (*http.Request, error) {
+	req, err := http.NewRequest("GET", "http://localhost:8080", h2mux.MuxedStreamReader{MuxedStream: stream})
+	if err != nil {
+		return nil, errors.Wrap(err, "Unexpected error from http.NewRequest")
+	}
+	err = h2mux.H2RequestHeadersToH1Request(stream.Headers, req)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid request received")
+	}
+	return req, nil
+}
+
+type h2muxRespWriter struct {
+	*h2mux.MuxedStream
+}
+
+func (rp *h2muxRespWriter) WriteRespHeaders(resp *http.Response) error {
+	headers := h2mux.H1ResponseToH2ResponseHeaders(resp)
+	headers = append(headers, h2mux.Header{Name: responseMetaHeaderField, Value: responseMetaHeaderOrigin})
+	return rp.WriteHeaders(headers)
+}
+
+func (rp *h2muxRespWriter) WriteErrorResponse() {
+	rp.WriteHeaders([]h2mux.Header{
+		{Name: ":status", Value: "502"},
+		{Name: responseMetaHeaderField, Value: responseMetaHeaderCfd},
+	})
+	rp.Write([]byte("502 Bad Gateway"))
+}

connection/h2mux_test.go

@@ -0,0 +1,242 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testMuxerConfig = &MuxerConfig{
+		HeartbeatInterval:  time.Second * 5,
+		MaxHeartbeats:      5,
+		CompressionSetting: 0,
+		MetricsUpdateFreq:  time.Second * 5,
+	}
+)
+
+func newH2MuxConnection(ctx context.Context, t require.TestingT) (*h2muxConnection, *h2mux.Muxer) {
+	edgeConn, originConn := net.Pipe()
+	edgeMuxChan := make(chan *h2mux.Muxer)
+	go func() {
+		edgeMuxConfig := h2mux.MuxerConfig{
+			Logger: testObserver,
+		}
+		edgeMux, err := h2mux.Handshake(edgeConn, edgeConn, edgeMuxConfig, h2mux.ActiveStreams)
+		require.NoError(t, err)
+		edgeMuxChan <- edgeMux
+	}()
+	var connIndex = uint8(0)
+	h2muxConn, err, _ := NewH2muxConnection(ctx, testConfig, testMuxerConfig, originConn, connIndex, testObserver)
+	require.NoError(t, err)
+	return h2muxConn, <-edgeMuxChan
+}
+
+func TestServeStreamHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "/ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "/large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "/400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "/500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "/error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	for _, test := range tests {
+		headers := []h2mux.Header{
+			{
+				Name:  ":path",
+				Value: test.endpoint,
+			},
+		}
+		stream, err := edgeMux.OpenStream(ctx, headers, nil)
+		require.NoError(t, err)
+		require.True(t, hasHeader(stream, ":status", strconv.Itoa(test.expectedStatus)))
+
+		if test.isProxyError {
+			assert.True(t, hasHeader(stream, responseMetaHeaderField, responseMetaHeaderCfd))
+		} else {
+			assert.True(t, hasHeader(stream, responseMetaHeaderField, responseMetaHeaderOrigin))
+			body := make([]byte, len(test.expectedBody))
+			_, err = stream.Read(body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, body)
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+func TestServeStreamWS(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, t)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(t, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: "/ws",
+		},
+		{
+			Name:  "connection",
+			Value: "upgrade",
+		},
+		{
+			Name:  "upgrade",
+			Value: "websocket",
+		},
+	}
+
+	readPipe, writePipe := io.Pipe()
+	stream, err := edgeMux.OpenStream(ctx, headers, readPipe)
+	require.NoError(t, err)
+
+	require.True(t, hasHeader(stream, ":status", strconv.Itoa(http.StatusSwitchingProtocols)))
+	assert.True(t, hasHeader(stream, responseMetaHeaderField, responseMetaHeaderOrigin))
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientText(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerText(stream)
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	wg.Wait()
+}
+
+func hasHeader(stream *h2mux.MuxedStream, name, val string) bool {
+	for _, header := range stream.Headers {
+		if header.Name == name && header.Value == val {
+			return true
+		}
+	}
+	return false
+}
+
+func benchmarkServeStreamHTTPSimple(b *testing.B, test testRequest) {
+	ctx, cancel := context.WithCancel(context.Background())
+	h2muxConn, edgeMux := newH2MuxConnection(ctx, b)
+
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		edgeMux.Serve(ctx)
+	}()
+	go func() {
+		defer wg.Done()
+		err := h2muxConn.serveMuxer(ctx)
+		require.Error(b, err)
+	}()
+
+	headers := []h2mux.Header{
+		{
+			Name:  ":path",
+			Value: test.endpoint,
+		},
+	}
+
+	body := make([]byte, len(test.expectedBody))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		stream, openstreamErr := edgeMux.OpenStream(ctx, headers, nil)
+		_, readBodyErr := stream.Read(body)
+		b.StopTimer()
+
+		require.NoError(b, openstreamErr)
+		assert.True(b, hasHeader(stream, responseMetaHeaderField, responseMetaHeaderOrigin))
+		require.True(b, hasHeader(stream, ":status", strconv.Itoa(http.StatusOK)))
+		require.NoError(b, readBodyErr)
+		require.Equal(b, test.expectedBody, body)
+	}
+
+	cancel()
+	wg.Wait()
+}
+
+func BenchmarkServeStreamHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "/ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}
+
+func BenchmarkServeStreamHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "/large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeStreamHTTPSimple(b, test)
+}

connection/header.go

@@ -0,0 +1,31 @@
+package connection
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+)
+
+const (
+	responseMetaHeaderField = "cf-cloudflared-response-meta"
+)
+
+var (
+	canonicalResponseUserHeadersField = http.CanonicalHeaderKey(h2mux.ResponseUserHeadersField)
+	canonicalResponseMetaHeaderField  = http.CanonicalHeaderKey(responseMetaHeaderField)
+	responseMetaHeaderCfd             = mustInitRespMetaHeader("cloudflared")
+	responseMetaHeaderOrigin          = mustInitRespMetaHeader("origin")
+)
+
+type responseMetaHeader struct {
+	Source string `json:"src"`
+}
+
+func mustInitRespMetaHeader(src string) string {
+	header, err := json.Marshal(responseMetaHeader{Source: src})
+	if err != nil {
+		panic(fmt.Sprintf("Failed to serialize response meta header = %s, err: %v", src, err))
+	}
+	return string(header)
+}

connection/http2.go

@@ -0,0 +1,221 @@
+package connection
+
+import (
+	"context"
+	"errors"
+	"io"
+	"math"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+
+	"golang.org/x/net/http2"
+)
+
+const (
+	internalUpgradeHeader = "Cf-Cloudflared-Proxy-Connection-Upgrade"
+	websocketUpgrade      = "websocket"
+	controlStreamUpgrade  = "control-stream"
+)
+
+var (
+	errNotFlusher = errors.New("ResponseWriter doesn't implement http.Flusher")
+)
+
+type http2Connection struct {
+	conn         net.Conn
+	server       *http2.Server
+	config       *Config
+	namedTunnel  *NamedTunnelConfig
+	connOptions  *tunnelpogs.ConnectionOptions
+	observer     *Observer
+	connIndexStr string
+	connIndex    uint8
+	wg           *sync.WaitGroup
+	// newRPCClientFunc allows us to mock RPCs during testing
+	newRPCClientFunc func(context.Context, io.ReadWriteCloser, logger.Service) NamedTunnelRPCClient
+	connectedFuse    ConnectedFuse
+}
+
+func NewHTTP2Connection(
+	conn net.Conn,
+	config *Config,
+	namedTunnelConfig *NamedTunnelConfig,
+	connOptions *tunnelpogs.ConnectionOptions,
+	observer *Observer,
+	connIndex uint8,
+	connectedFuse ConnectedFuse,
+) *http2Connection {
+	return &http2Connection{
+		conn: conn,
+		server: &http2.Server{
+			MaxConcurrentStreams: math.MaxUint32,
+		},
+		config:           config,
+		namedTunnel:      namedTunnelConfig,
+		connOptions:      connOptions,
+		observer:         observer,
+		connIndexStr:     uint8ToString(connIndex),
+		connIndex:        connIndex,
+		wg:               &sync.WaitGroup{},
+		newRPCClientFunc: newRegistrationRPCClient,
+		connectedFuse:    connectedFuse,
+	}
+}
+
+func (c *http2Connection) Serve(ctx context.Context) {
+	go func() {
+		<-ctx.Done()
+		c.close()
+	}()
+	c.server.ServeConn(c.conn, &http2.ServeConnOpts{
+		Context: ctx,
+		Handler: c,
+	})
+}
+
+func (c *http2Connection) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	c.wg.Add(1)
+	defer c.wg.Done()
+
+	respWriter := &http2RespWriter{
+		r: r.Body,
+		w: w,
+	}
+	flusher, isFlusher := w.(http.Flusher)
+	if !isFlusher {
+		c.observer.Errorf("%T doesn't implement http.Flusher", w)
+		respWriter.WriteErrorResponse()
+		return
+	}
+	respWriter.flusher = flusher
+	var err error
+	if isControlStreamUpgrade(r) {
+		respWriter.shouldFlush = true
+		err = c.serveControlStream(r.Context(), respWriter)
+	} else if isWebsocketUpgrade(r) {
+		respWriter.shouldFlush = true
+		stripWebsocketUpgradeHeader(r)
+		err = c.config.OriginClient.Proxy(respWriter, r, true)
+	} else {
+		err = c.config.OriginClient.Proxy(respWriter, r, false)
+	}
+
+	if err != nil {
+		respWriter.WriteErrorResponse()
+	}
+}
+
+func (c *http2Connection) serveControlStream(ctx context.Context, respWriter *http2RespWriter) error {
+	rpcClient := c.newRPCClientFunc(ctx, respWriter, c.observer)
+	defer rpcClient.Close()
+
+	if err := rpcClient.RegisterConnection(ctx, c.namedTunnel, c.connOptions, c.connIndex, c.observer); err != nil {
+		return err
+	}
+	c.connectedFuse.Connected()
+
+	<-ctx.Done()
+	rpcClient.GracefulShutdown(ctx, c.config.GracePeriod)
+	return nil
+}
+
+func (c *http2Connection) close() {
+	// Wait for all serve HTTP handlers to return
+	c.wg.Wait()
+	c.conn.Close()
+}
+
+type http2RespWriter struct {
+	r           io.Reader
+	w           http.ResponseWriter
+	flusher     http.Flusher
+	shouldFlush bool
+}
+
+func (rp *http2RespWriter) WriteRespHeaders(resp *http.Response) error {
+	dest := rp.w.Header()
+	userHeaders := make(http.Header, len(resp.Header))
+	for header, values := range resp.Header {
+		// Since these are http2 headers, they're required to be lowercase
+		h2name := strings.ToLower(header)
+		for _, v := range values {
+			if h2name == "content-length" {
+				// This header has meaning in HTTP/2 and will be used by the edge,
+				// so it should be sent as an HTTP/2 response header.
+				dest.Add(h2name, v)
+				// Since these are http2 headers, they're required to be lowercase
+			} else if !h2mux.IsControlHeader(h2name) || h2mux.IsWebsocketClientHeader(h2name) {
+				// User headers, on the other hand, must all be serialized so that
+				// HTTP/2 header validation won't be applied to HTTP/1 header values
+				userHeaders.Add(h2name, v)
+			}
+		}
+	}
+
+	// Perform user header serialization and set them in the single header
+	dest.Set(canonicalResponseUserHeadersField, h2mux.SerializeHeaders(userHeaders))
+	rp.setResponseMetaHeader(responseMetaHeaderOrigin)
+	status := resp.StatusCode
+	// HTTP2 removes support for 101 Switching Protocols https://tools.ietf.org/html/rfc7540#section-8.1.1
+	if status == http.StatusSwitchingProtocols {
+		status = http.StatusOK
+	}
+	rp.w.WriteHeader(status)
+	if isServerSentEvent(resp.Header) {
+		rp.shouldFlush = true
+	}
+	if rp.shouldFlush {
+		rp.flusher.Flush()
+	}
+	return nil
+}
+
+func (rp *http2RespWriter) WriteErrorResponse() {
+	rp.setResponseMetaHeader(responseMetaHeaderCfd)
+	rp.w.WriteHeader(http.StatusBadGateway)
+}
+
+func (rp *http2RespWriter) setResponseMetaHeader(value string) {
+	rp.w.Header().Set(canonicalResponseMetaHeaderField, value)
+}
+
+func (rp *http2RespWriter) Read(p []byte) (n int, err error) {
+	return rp.r.Read(p)
+}
+
+func (rp *http2RespWriter) Write(p []byte) (n int, err error) {
+	defer func() {
+		// Implementer of OriginClient should make sure it doesn't write to the connection after Proxy returns
+		// Register a recover routine just in case.
+		if r := recover(); r != nil {
+			println("Recover from http2 response writer panic, error", r)
+		}
+	}()
+	n, err = rp.w.Write(p)
+	if err == nil && rp.shouldFlush {
+		rp.flusher.Flush()
+	}
+	return n, err
+}
+
+func (rp *http2RespWriter) Close() error {
+	return nil
+}
+
+func isControlStreamUpgrade(r *http.Request) bool {
+	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == controlStreamUpgrade
+}
+
+func isWebsocketUpgrade(r *http.Request) bool {
+	return strings.ToLower(r.Header.Get(internalUpgradeHeader)) == websocketUpgrade
+}
+
+func stripWebsocketUpgradeHeader(r *http.Request) {
+	r.Header.Del(internalUpgradeHeader)
+}

connection/http2_test.go

@@ -0,0 +1,303 @@
+package connection
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/net/http2"
+)
+
+var (
+	testTransport = http2.Transport{}
+)
+
+func newTestHTTP2Connection() (*http2Connection, net.Conn) {
+	edgeConn, originConn := net.Pipe()
+	var connIndex = uint8(0)
+	return NewHTTP2Connection(
+		originConn,
+		testConfig,
+		&NamedTunnelConfig{},
+		&pogs.ConnectionOptions{},
+		testObserver,
+		connIndex,
+		mockConnectedFuse{},
+	), edgeConn
+}
+
+func TestServeHTTP(t *testing.T) {
+	tests := []testRequest{
+		{
+			name:           "ok",
+			endpoint:       "ok",
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte(http.StatusText(http.StatusOK)),
+		},
+		{
+			name:           "large_file",
+			endpoint:       "large_file",
+			expectedStatus: http.StatusOK,
+			expectedBody:   testLargeResp,
+		},
+		{
+			name:           "Bad request",
+			endpoint:       "400",
+			expectedStatus: http.StatusBadRequest,
+			expectedBody:   []byte(http.StatusText(http.StatusBadRequest)),
+		},
+		{
+			name:           "Internal server error",
+			endpoint:       "500",
+			expectedStatus: http.StatusInternalServerError,
+			expectedBody:   []byte(http.StatusText(http.StatusInternalServerError)),
+		},
+		{
+			name:           "Proxy error",
+			endpoint:       "error",
+			expectedStatus: http.StatusBadGateway,
+			expectedBody:   nil,
+			isProxyError:   true,
+		},
+	}
+
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	for _, test := range tests {
+		endpoint := fmt.Sprintf("http://localhost:8080/%s", test.endpoint)
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+		require.NoError(t, err)
+
+		resp, err := edgeHTTP2Conn.RoundTrip(req)
+		require.NoError(t, err)
+		require.Equal(t, test.expectedStatus, resp.StatusCode)
+		if test.expectedBody != nil {
+			respBody, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+			require.Equal(t, test.expectedBody, respBody)
+		}
+		if test.isProxyError {
+			require.Equal(t, responseMetaHeaderCfd, resp.Header.Get(responseMetaHeaderField))
+		} else {
+			require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(responseMetaHeaderField))
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+type mockNamedTunnelRPCClient struct {
+	registered   chan struct{}
+	unregistered chan struct{}
+}
+
+func (mc mockNamedTunnelRPCClient) RegisterConnection(
+	c context.Context,
+	config *NamedTunnelConfig,
+	options *tunnelpogs.ConnectionOptions,
+	connIndex uint8,
+	observer *Observer,
+) error {
+	close(mc.registered)
+	return nil
+}
+
+func (mc mockNamedTunnelRPCClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+	close(mc.unregistered)
+}
+
+func (mockNamedTunnelRPCClient) Close() {}
+
+type mockRPCClientFactory struct {
+	registered   chan struct{}
+	unregistered chan struct{}
+}
+
+func (mf *mockRPCClientFactory) newMockRPCClient(context.Context, io.ReadWriteCloser, logger.Service) NamedTunnelRPCClient {
+	return mockNamedTunnelRPCClient{
+		registered:   mf.registered,
+		unregistered: mf.unregistered,
+	}
+}
+
+type wsRespWriter struct {
+	*httptest.ResponseRecorder
+	readPipe  *io.PipeReader
+	writePipe *io.PipeWriter
+}
+
+func newWSRespWriter() *wsRespWriter {
+	readPipe, writePipe := io.Pipe()
+	return &wsRespWriter{
+		httptest.NewRecorder(),
+		readPipe,
+		writePipe,
+	}
+}
+
+func (w *wsRespWriter) RespBody() io.ReadWriter {
+	return nowriter{w.readPipe}
+}
+
+func (w *wsRespWriter) Write(data []byte) (n int, err error) {
+	return w.writePipe.Write(data)
+}
+
+func TestServeWS(t *testing.T) {
+	http2Conn, _ := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	respWriter := newWSRespWriter()
+	readPipe, writePipe := io.Pipe()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/ws", readPipe)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, websocketUpgrade)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.ServeHTTP(respWriter, req)
+	}()
+
+	data := []byte("test websocket")
+	err = wsutil.WriteClientText(writePipe, data)
+	require.NoError(t, err)
+
+	respBody, err := wsutil.ReadServerText(respWriter.RespBody())
+	require.NoError(t, err)
+	require.Equal(t, data, respBody, fmt.Sprintf("Expect %s, got %s", string(data), string(respBody)))
+
+	cancel()
+	resp := respWriter.Result()
+	// http2RespWriter should rewrite status 101 to 200
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+	require.Equal(t, responseMetaHeaderOrigin, resp.Header.Get(responseMetaHeaderField))
+
+	wg.Wait()
+}
+
+func TestServeControlStream(t *testing.T) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	rpcClientFactory := mockRPCClientFactory{
+		registered:   make(chan struct{}),
+		unregistered: make(chan struct{}),
+	}
+	http2Conn.newRPCClientFunc = rpcClientFactory.newMockRPCClient
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/", nil)
+	require.NoError(t, err)
+	req.Header.Set(internalUpgradeHeader, controlStreamUpgrade)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(t, err)
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		edgeHTTP2Conn.RoundTrip(req)
+	}()
+
+	<-rpcClientFactory.registered
+	cancel()
+	<-rpcClientFactory.unregistered
+
+	wg.Wait()
+}
+
+func benchmarkServeHTTP(b *testing.B, test testRequest) {
+	http2Conn, edgeConn := newTestHTTP2Connection()
+
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		http2Conn.Serve(ctx)
+	}()
+
+	endpoint := fmt.Sprintf("http://localhost:8080/%s", test.endpoint)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)
+	require.NoError(b, err)
+
+	edgeHTTP2Conn, err := testTransport.NewClientConn(edgeConn)
+	require.NoError(b, err)
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		b.StartTimer()
+		resp, err := edgeHTTP2Conn.RoundTrip(req)
+		b.StopTimer()
+		require.NoError(b, err)
+		require.Equal(b, test.expectedStatus, resp.StatusCode)
+		if test.expectedBody != nil {
+			respBody, err := ioutil.ReadAll(resp.Body)
+			require.NoError(b, err)
+			require.Equal(b, test.expectedBody, respBody)
+		}
+		resp.Body.Close()
+	}
+
+	cancel()
+	wg.Wait()
+}
+func BenchmarkServeHTTPSimple(b *testing.B) {
+	test := testRequest{
+		name:           "ok",
+		endpoint:       "ok",
+		expectedStatus: http.StatusOK,
+		expectedBody:   []byte(http.StatusText(http.StatusOK)),
+	}
+
+	benchmarkServeHTTP(b, test)
+}
+
+func BenchmarkServeHTTPLargeFile(b *testing.B) {
+	test := testRequest{
+		name:           "large_file",
+		endpoint:       "large_file",
+		expectedStatus: http.StatusOK,
+		expectedBody:   testLargeResp,
+	}
+
+	benchmarkServeHTTP(b, test)
+}

connection/json.go

@@ -0,0 +1,7 @@
+package connection
+
+import (
+	jsoniter "github.com/json-iterator/go"
+)
+
+var json = jsoniter.ConfigFastest

connection/metrics.go

@@ -0,0 +1,405 @@
+package connection
+
+import (
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	MetricsNamespace = "cloudflared"
+	TunnelSubsystem  = "tunnel"
+	muxerSubsystem   = "muxer"
+)
+
+type muxerMetrics struct {
+	rtt              *prometheus.GaugeVec
+	rttMin           *prometheus.GaugeVec
+	rttMax           *prometheus.GaugeVec
+	receiveWindowAve *prometheus.GaugeVec
+	sendWindowAve    *prometheus.GaugeVec
+	receiveWindowMin *prometheus.GaugeVec
+	receiveWindowMax *prometheus.GaugeVec
+	sendWindowMin    *prometheus.GaugeVec
+	sendWindowMax    *prometheus.GaugeVec
+	inBoundRateCurr  *prometheus.GaugeVec
+	inBoundRateMin   *prometheus.GaugeVec
+	inBoundRateMax   *prometheus.GaugeVec
+	outBoundRateCurr *prometheus.GaugeVec
+	outBoundRateMin  *prometheus.GaugeVec
+	outBoundRateMax  *prometheus.GaugeVec
+	compBytesBefore  *prometheus.GaugeVec
+	compBytesAfter   *prometheus.GaugeVec
+	compRateAve      *prometheus.GaugeVec
+}
+
+type tunnelMetrics struct {
+	timerRetries    prometheus.Gauge
+	serverLocations *prometheus.GaugeVec
+	// locationLock is a mutex for oldServerLocations
+	locationLock sync.Mutex
+	// oldServerLocations stores the last server the tunnel was connected to
+	oldServerLocations map[string]string
+
+	regSuccess *prometheus.CounterVec
+	regFail    *prometheus.CounterVec
+	rpcFail    *prometheus.CounterVec
+
+	muxerMetrics        *muxerMetrics
+	tunnelsHA           tunnelsForHA
+	userHostnamesCounts *prometheus.CounterVec
+}
+
+func newMuxerMetrics() *muxerMetrics {
+	rtt := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt",
+			Help:      "Round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rtt)
+
+	rttMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_min",
+			Help:      "Shortest round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rttMin)
+
+	rttMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "rtt_max",
+			Help:      "Longest round-trip time in millisecond",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(rttMax)
+
+	receiveWindowAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_ave",
+			Help:      "Average receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowAve)
+
+	sendWindowAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_ave",
+			Help:      "Average send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowAve)
+
+	receiveWindowMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_min",
+			Help:      "Smallest receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowMin)
+
+	receiveWindowMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "receive_window_max",
+			Help:      "Largest receive window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(receiveWindowMax)
+
+	sendWindowMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_min",
+			Help:      "Smallest send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowMin)
+
+	sendWindowMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "send_window_max",
+			Help:      "Largest send window size in bytes",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(sendWindowMax)
+
+	inBoundRateCurr := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_curr",
+			Help:      "Current inbounding bytes per second, 0 if there is no incoming connection",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateCurr)
+
+	inBoundRateMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero inbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateMin)
+
+	inBoundRateMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "inbound_bytes_per_sec_max",
+			Help:      "Maximum inbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(inBoundRateMax)
+
+	outBoundRateCurr := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_curr",
+			Help:      "Current outbounding bytes per second, 0 if there is no outgoing traffic",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateCurr)
+
+	outBoundRateMin := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_min",
+			Help:      "Minimum non-zero outbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateMin)
+
+	outBoundRateMax := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "outbound_bytes_per_sec_max",
+			Help:      "Maximum outbounding bytes per second",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(outBoundRateMax)
+
+	compBytesBefore := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_before",
+			Help:      "Bytes sent via cross-stream compression, pre compression",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compBytesBefore)
+
+	compBytesAfter := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_bytes_after",
+			Help:      "Bytes sent via cross-stream compression, post compression",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compBytesAfter)
+
+	compRateAve := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: muxerSubsystem,
+			Name:      "comp_rate_ave",
+			Help:      "Average outbound cross-stream compression ratio",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(compRateAve)
+
+	return &muxerMetrics{
+		rtt:              rtt,
+		rttMin:           rttMin,
+		rttMax:           rttMax,
+		receiveWindowAve: receiveWindowAve,
+		sendWindowAve:    sendWindowAve,
+		receiveWindowMin: receiveWindowMin,
+		receiveWindowMax: receiveWindowMax,
+		sendWindowMin:    sendWindowMin,
+		sendWindowMax:    sendWindowMax,
+		inBoundRateCurr:  inBoundRateCurr,
+		inBoundRateMin:   inBoundRateMin,
+		inBoundRateMax:   inBoundRateMax,
+		outBoundRateCurr: outBoundRateCurr,
+		outBoundRateMin:  outBoundRateMin,
+		outBoundRateMax:  outBoundRateMax,
+		compBytesBefore:  compBytesBefore,
+		compBytesAfter:   compBytesAfter,
+		compRateAve:      compRateAve,
+	}
+}
+
+func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
+	m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
+	m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
+	m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
+	m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
+	m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
+	m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
+	m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
+	m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
+	m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
+	m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
+	m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
+	m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
+	m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
+	m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
+	m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
+	m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
+	m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
+	m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
+}
+
+func convertRTTMilliSec(t time.Duration) float64 {
+	return float64(t / time.Millisecond)
+}
+
+// Metrics that can be collected without asking the edge
+func newTunnelMetrics() *tunnelMetrics {
+	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "max_concurrent_requests_per_tunnel",
+			Help:      "Largest number of concurrent requests proxied through each tunnel so far",
+		},
+		[]string{"connection_id"},
+	)
+	prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
+
+	timerRetries := prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "timer_retries",
+			Help:      "Unacknowledged heart beats count",
+		})
+	prometheus.MustRegister(timerRetries)
+
+	serverLocations := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "server_locations",
+			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
+		},
+		[]string{"connection_id", "location"},
+	)
+	prometheus.MustRegister(serverLocations)
+
+	rpcFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_rpc_fail",
+			Help:      "Count of RPC connection errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(rpcFail)
+
+	registerFail := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_register_fail",
+			Help:      "Count of tunnel registration errors by type",
+		},
+		[]string{"error", "rpcName"},
+	)
+	prometheus.MustRegister(registerFail)
+
+	userHostnamesCounts := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "user_hostnames_counts",
+			Help:      "Which user hostnames cloudflared is serving",
+		},
+		[]string{"userHostname"},
+	)
+	prometheus.MustRegister(userHostnamesCounts)
+
+	registerSuccess := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: MetricsNamespace,
+			Subsystem: TunnelSubsystem,
+			Name:      "tunnel_register_success",
+			Help:      "Count of successful tunnel registrations",
+		},
+		[]string{"rpcName"},
+	)
+	prometheus.MustRegister(registerSuccess)
+
+	return &tunnelMetrics{
+		timerRetries:        timerRetries,
+		serverLocations:     serverLocations,
+		oldServerLocations:  make(map[string]string),
+		muxerMetrics:        newMuxerMetrics(),
+		tunnelsHA:           NewTunnelsForHA(),
+		regSuccess:          registerSuccess,
+		regFail:             registerFail,
+		rpcFail:             rpcFail,
+		userHostnamesCounts: userHostnamesCounts,
+	}
+}
+
+func (t *tunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
+	t.muxerMetrics.update(connectionID, metrics)
+}
+
+func (t *tunnelMetrics) registerServerLocation(connectionID, loc string) {
+	t.locationLock.Lock()
+	defer t.locationLock.Unlock()
+	if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
+		return
+	} else if ok {
+		t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
+	}
+	t.serverLocations.WithLabelValues(connectionID, loc).Inc()
+	t.oldServerLocations[connectionID] = loc
+}

connection/observer.go

@@ -0,0 +1,99 @@
+package connection
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+)
+
+type Observer struct {
+	logger.Service
+	metrics         *tunnelMetrics
+	tunnelEventChan chan<- ui.TunnelEvent
+}
+
+func NewObserver(logger logger.Service, tunnelEventChan chan<- ui.TunnelEvent) *Observer {
+	return &Observer{
+		logger,
+		newTunnelMetrics(),
+		tunnelEventChan,
+	}
+}
+
+func (o *Observer) logServerInfo(connIndex uint8, location, msg string) {
+	// If launch-ui flag is set, send connect msg
+	if o.tunnelEventChan != nil {
+		o.tunnelEventChan <- ui.TunnelEvent{Index: connIndex, EventType: ui.Connected, Location: location}
+	}
+	o.Infof(msg)
+	o.metrics.registerServerLocation(uint8ToString(connIndex), location)
+}
+
+func (o *Observer) logTrialHostname(registration *tunnelpogs.TunnelRegistration) error {
+	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
+	if o.tunnelEventChan == nil {
+		if registrationURL, err := url.Parse(registration.Url); err == nil {
+			for _, line := range asciiBox(trialZoneMsg(registrationURL.String()), 2) {
+				o.Info(line)
+			}
+		} else {
+			o.Error("Failed to connect tunnel, please try again.")
+			return fmt.Errorf("empty URL in response from Cloudflare edge")
+		}
+	}
+	return nil
+}
+
+// Print out the given lines in a nice ASCII box.
+func asciiBox(lines []string, padding int) (box []string) {
+	maxLen := maxLen(lines)
+	spacer := strings.Repeat(" ", padding)
+
+	border := "+" + strings.Repeat("-", maxLen+(padding*2)) + "+"
+
+	box = append(box, border)
+	for _, line := range lines {
+		box = append(box, "|"+spacer+line+strings.Repeat(" ", maxLen-len(line))+spacer+"|")
+	}
+	box = append(box, border)
+	return
+}
+
+func maxLen(lines []string) int {
+	max := 0
+	for _, line := range lines {
+		if len(line) > max {
+			max = len(line)
+		}
+	}
+	return max
+}
+
+func trialZoneMsg(url string) []string {
+	return []string{
+		"Your free tunnel has started! Visit it:",
+		"  " + url,
+	}
+}
+
+func (o *Observer) sendRegisteringEvent() {
+	if o.tunnelEventChan != nil {
+		o.tunnelEventChan <- ui.TunnelEvent{EventType: ui.RegisteringTunnel}
+	}
+}
+
+func (o *Observer) sendConnectedEvent(connIndex uint8, location string) {
+	if o.tunnelEventChan != nil {
+		o.tunnelEventChan <- ui.TunnelEvent{Index: connIndex, EventType: ui.Connected, Location: location}
+	}
+}
+
+func (o *Observer) sendURL(url string) {
+	if o.tunnelEventChan != nil {
+		o.tunnelEventChan <- ui.TunnelEvent{EventType: ui.SetUrl, Url: url}
+	}
+}

connection/observer_test.go

@@ -0,0 +1,45 @@
+package connection
+
+import (
+	"strconv"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// can only be called once
+var m = newTunnelMetrics()
+
+func TestRegisterServerLocation(t *testing.T) {
+	tunnels := 20
+	var wg sync.WaitGroup
+	wg.Add(tunnels)
+	for i := 0; i < tunnels; i++ {
+		go func(i int) {
+			id := strconv.Itoa(i)
+			m.registerServerLocation(id, "LHR")
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	for i := 0; i < tunnels; i++ {
+		id := strconv.Itoa(i)
+		assert.Equal(t, "LHR", m.oldServerLocations[id])
+	}
+
+	wg.Add(tunnels)
+	for i := 0; i < tunnels; i++ {
+		go func(i int) {
+			id := strconv.Itoa(i)
+			m.registerServerLocation(id, "AUS")
+			wg.Done()
+		}(i)
+	}
+	wg.Wait()
+	for i := 0; i < tunnels; i++ {
+		id := strconv.Itoa(i)
+		assert.Equal(t, "AUS", m.oldServerLocations[id])
+	}
+
+}

connection/protocol.go

@@ -0,0 +1,179 @@
+package connection
+
+import (
+	"fmt"
+	"hash/fnv"
+	"sync"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+)
+
+const (
+	AvailableProtocolFlagMessage = "Available protocols: http2 - Go's implementation, h2mux - Cloudflare's implementation of HTTP/2, and auto - automatically select between http2 and h2mux"
+	// edgeH2muxTLSServerName is the server name to establish h2mux connection with edge
+	edgeH2muxTLSServerName = "cftunnel.com"
+	// edgeH2TLSServerName is the server name to establish http2 connection with edge
+	edgeH2TLSServerName = "h2.cftunnel.com"
+	// threshold to switch back to h2mux when the user intentionally pick --protocol http2
+	explicitHTTP2FallbackThreshold = -1
+	autoSelectFlag                 = "auto"
+)
+
+var (
+	ProtocolList = []Protocol{H2mux, HTTP2}
+)
+
+type Protocol int64
+
+const (
+	H2mux Protocol = iota
+	HTTP2
+)
+
+func (p Protocol) ServerName() string {
+	switch p {
+	case H2mux:
+		return edgeH2muxTLSServerName
+	case HTTP2:
+		return edgeH2TLSServerName
+	default:
+		return ""
+	}
+}
+
+// Fallback returns the fallback protocol and whether the protocol has a fallback
+func (p Protocol) fallback() (Protocol, bool) {
+	switch p {
+	case H2mux:
+		return 0, false
+	case HTTP2:
+		return H2mux, true
+	default:
+		return 0, false
+	}
+}
+
+func (p Protocol) String() string {
+	switch p {
+	case H2mux:
+		return "h2mux"
+	case HTTP2:
+		return "http2"
+	default:
+		return fmt.Sprintf("unknown protocol")
+	}
+}
+
+type ProtocolSelector interface {
+	Current() Protocol
+	Fallback() (Protocol, bool)
+}
+
+type staticProtocolSelector struct {
+	current Protocol
+}
+
+func (s *staticProtocolSelector) Current() Protocol {
+	return s.current
+}
+
+func (s *staticProtocolSelector) Fallback() (Protocol, bool) {
+	return 0, false
+}
+
+type autoProtocolSelector struct {
+	lock           sync.RWMutex
+	current        Protocol
+	switchThrehold int32
+	fetchFunc      PercentageFetcher
+	refreshAfter   time.Time
+	ttl            time.Duration
+	logger         logger.Service
+}
+
+func newAutoProtocolSelector(
+	current Protocol,
+	switchThrehold int32,
+	fetchFunc PercentageFetcher,
+	ttl time.Duration,
+	logger logger.Service,
+) *autoProtocolSelector {
+	return &autoProtocolSelector{
+		current:        current,
+		switchThrehold: switchThrehold,
+		fetchFunc:      fetchFunc,
+		refreshAfter:   time.Now().Add(ttl),
+		ttl:            ttl,
+		logger:         logger,
+	}
+}
+
+func (s *autoProtocolSelector) Current() Protocol {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	if time.Now().Before(s.refreshAfter) {
+		return s.current
+	}
+
+	percentage, err := s.fetchFunc()
+	if err != nil {
+		s.logger.Errorf("Failed to refresh protocol, err: %v", err)
+		return s.current
+	}
+
+	if s.switchThrehold < percentage {
+		s.current = HTTP2
+	} else {
+		s.current = H2mux
+	}
+	s.refreshAfter = time.Now().Add(s.ttl)
+	return s.current
+}
+
+func (s *autoProtocolSelector) Fallback() (Protocol, bool) {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	return s.current.fallback()
+}
+
+type PercentageFetcher func() (int32, error)
+
+func NewProtocolSelector(protocolFlag string, namedTunnel *NamedTunnelConfig, fetchFunc PercentageFetcher, ttl time.Duration, logger logger.Service) (ProtocolSelector, error) {
+	if namedTunnel == nil {
+		return &staticProtocolSelector{
+			current: H2mux,
+		}, nil
+	}
+	if protocolFlag == H2mux.String() {
+		return &staticProtocolSelector{
+			current: H2mux,
+		}, nil
+	}
+
+	http2Percentage, err := fetchFunc()
+	if err != nil {
+		return nil, err
+	}
+	if protocolFlag == HTTP2.String() {
+		if http2Percentage < 0 {
+			return newAutoProtocolSelector(H2mux, explicitHTTP2FallbackThreshold, fetchFunc, ttl, logger), nil
+		}
+		return newAutoProtocolSelector(HTTP2, explicitHTTP2FallbackThreshold, fetchFunc, ttl, logger), nil
+	}
+
+	if protocolFlag != autoSelectFlag {
+		return nil, fmt.Errorf("Unknown protocol %s, %s", protocolFlag, AvailableProtocolFlagMessage)
+	}
+	threshold := switchThreshold(namedTunnel.Auth.AccountTag)
+	if threshold < http2Percentage {
+		return newAutoProtocolSelector(HTTP2, threshold, fetchFunc, ttl, logger), nil
+	}
+	return newAutoProtocolSelector(H2mux, threshold, fetchFunc, ttl, logger), nil
+}
+
+func switchThreshold(accountTag string) int32 {
+	h := fnv.New32a()
+	h.Write([]byte(accountTag))
+	return int32(h.Sum32() % 100)
+}

connection/protocol_test.go

@@ -0,0 +1,220 @@
+package connection
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testNoTTL = 0
+)
+
+var (
+	testNamedTunnelConfig = &NamedTunnelConfig{
+		Auth: pogs.TunnelAuth{
+			AccountTag: "testAccountTag",
+		},
+	}
+)
+
+func mockFetcher(percentage int32) PercentageFetcher {
+	return func() (int32, error) {
+		return percentage, nil
+	}
+}
+
+func mockFetcherWithError() PercentageFetcher {
+	return func() (int32, error) {
+		return 0, fmt.Errorf("failed to fetch precentage")
+	}
+}
+
+type dynamicMockFetcher struct {
+	percentage int32
+	err        error
+}
+
+func (dmf *dynamicMockFetcher) fetch() PercentageFetcher {
+	return func() (int32, error) {
+		if dmf.err != nil {
+			return 0, dmf.err
+		}
+		return dmf.percentage, nil
+	}
+}
+
+func TestNewProtocolSelector(t *testing.T) {
+	tests := []struct {
+		name              string
+		protocol          string
+		expectedProtocol  Protocol
+		hasFallback       bool
+		expectedFallback  Protocol
+		namedTunnelConfig *NamedTunnelConfig
+		fetchFunc         PercentageFetcher
+		wantErr           bool
+	}{
+		{
+			name:              "classic tunnel",
+			protocol:          "h2mux",
+			expectedProtocol:  H2mux,
+			namedTunnelConfig: nil,
+		},
+		{
+			name:              "named tunnel over h2mux",
+			protocol:          "h2mux",
+			expectedProtocol:  H2mux,
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel over http2",
+			protocol:          "http2",
+			expectedProtocol:  HTTP2,
+			hasFallback:       true,
+			expectedFallback:  H2mux,
+			fetchFunc:         mockFetcher(0),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel http2 disabled",
+			protocol:          "http2",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(-1),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto all http2 disabled",
+			protocol:          "auto",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(-1),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto to h2mux",
+			protocol:          "auto",
+			expectedProtocol:  H2mux,
+			fetchFunc:         mockFetcher(0),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			name:              "named tunnel auto to http2",
+			protocol:          "auto",
+			expectedProtocol:  HTTP2,
+			hasFallback:       true,
+			expectedFallback:  H2mux,
+			fetchFunc:         mockFetcher(100),
+			namedTunnelConfig: testNamedTunnelConfig,
+		},
+		{
+			// None named tunnel can only use h2mux, so specifying an unknown protocol is not an error
+			name:             "classic tunnel unknown protocol",
+			protocol:         "unknown",
+			expectedProtocol: H2mux,
+		},
+		{
+			name:              "named tunnel unknown protocol",
+			protocol:          "unknown",
+			fetchFunc:         mockFetcher(100),
+			namedTunnelConfig: testNamedTunnelConfig,
+			wantErr:           true,
+		},
+		{
+			name:              "named tunnel fetch error",
+			protocol:          "unknown",
+			fetchFunc:         mockFetcherWithError(),
+			namedTunnelConfig: testNamedTunnelConfig,
+			wantErr:           true,
+		},
+	}
+	logger, _ := logger.New()
+	for _, test := range tests {
+		selector, err := NewProtocolSelector(test.protocol, test.namedTunnelConfig, test.fetchFunc, testNoTTL, logger)
+		if test.wantErr {
+			assert.Error(t, err, fmt.Sprintf("test %s failed", test.name))
+		} else {
+			assert.NoError(t, err, fmt.Sprintf("test %s failed", test.name))
+			assert.Equal(t, test.expectedProtocol, selector.Current(), fmt.Sprintf("test %s failed", test.name))
+			fallback, ok := selector.Fallback()
+			assert.Equal(t, test.hasFallback, ok, fmt.Sprintf("test %s failed", test.name))
+			if test.hasFallback {
+				assert.Equal(t, test.expectedFallback, fallback, fmt.Sprintf("test %s failed", test.name))
+			}
+		}
+	}
+}
+
+func TestAutoProtocolSelectorRefresh(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{}
+	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.err = fmt.Errorf("failed to fetch")
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	fetcher.err = nil
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+}
+
+func TestHTTP2ProtocolSelectorRefresh(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{}
+	selector, err := NewProtocolSelector("http2", testNamedTunnelConfig, fetcher.fetch(), testNoTTL, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.err = fmt.Errorf("failed to fetch")
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	fetcher.err = nil
+	assert.Equal(t, H2mux, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 100
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = -1
+	assert.Equal(t, H2mux, selector.Current())
+}
+
+func TestProtocolSelectorRefreshTTL(t *testing.T) {
+	logger, _ := logger.New()
+	fetcher := dynamicMockFetcher{percentage: 100}
+	selector, err := NewProtocolSelector("auto", testNamedTunnelConfig, fetcher.fetch(), time.Hour, logger)
+	assert.NoError(t, err)
+	assert.Equal(t, HTTP2, selector.Current())
+
+	fetcher.percentage = 0
+	assert.Equal(t, HTTP2, selector.Current())
+}

connection/rpc.go

@@ -2,40 +2,294 @@ package connection
 
 import (
 	"context"
+	"fmt"
 	"io"
-
-	rpc "zombiezen.com/go/capnproto2/rpc"
+	"time"
 
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"zombiezen.com/go/capnproto2/rpc"
 )
 
-// NewTunnelRPCClient creates and returns a new RPC client, which will communicate
-// using a stream on the given muxer
-func NewTunnelRPCClient(
+type tunnelServerClient struct {
+	client    tunnelpogs.TunnelServer_PogsClient
+	transport rpc.Transport
+}
+
+// NewTunnelRPCClient creates and returns a new RPC client, which will communicate using a stream on the given muxer.
+// This method is exported for supervisor to call Authenticate RPC
+func NewTunnelServerClient(
 	ctx context.Context,
 	stream io.ReadWriteCloser,
 	logger logger.Service,
-) (client tunnelpogs.TunnelServer_PogsClient, err error) {
+) *tunnelServerClient {
+	transport := tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream))
 	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
+		transport,
 		tunnelrpc.ConnLog(logger),
 	)
 	registrationClient := tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	client = tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn}
-	return client, nil
+	return &tunnelServerClient{
+		client:    tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn},
+		transport: transport,
+	}
 }
 
-func NewRegistrationRPCClient(
+func (tsc *tunnelServerClient) Authenticate(ctx context.Context, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) (tunnelpogs.AuthOutcome, error) {
+	authResp, err := tsc.client.Authenticate(ctx, classicTunnel.OriginCert, classicTunnel.Hostname, registrationOptions)
+	if err != nil {
+		return nil, err
+	}
+	return authResp.Outcome(), nil
+}
+
+func (tsc *tunnelServerClient) Close() {
+	// Closing the client will also close the connection
+	tsc.client.Close()
+	tsc.transport.Close()
+}
+
+type NamedTunnelRPCClient interface {
+	RegisterConnection(
+		c context.Context,
+		config *NamedTunnelConfig,
+		options *tunnelpogs.ConnectionOptions,
+		connIndex uint8,
+		observer *Observer,
+	) error
+	GracefulShutdown(ctx context.Context, gracePeriod time.Duration)
+	Close()
+}
+
+type registrationServerClient struct {
+	client    tunnelpogs.RegistrationServer_PogsClient
+	transport rpc.Transport
+}
+
+func newRegistrationRPCClient(
 	ctx context.Context,
 	stream io.ReadWriteCloser,
 	logger logger.Service,
-) (client tunnelpogs.RegistrationServer_PogsClient, err error) {
+) NamedTunnelRPCClient {
+	transport := tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream))
 	conn := rpc.NewConn(
-		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
+		transport,
 		tunnelrpc.ConnLog(logger),
 	)
-	client = tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
-	return client, nil
+	return &registrationServerClient{
+		client:    tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn},
+		transport: transport,
+	}
+}
+
+func (rsc *registrationServerClient) RegisterConnection(
+	ctx context.Context,
+	config *NamedTunnelConfig,
+	options *tunnelpogs.ConnectionOptions,
+	connIndex uint8,
+	observer *Observer,
+) error {
+	conn, err := rsc.client.RegisterConnection(
+		ctx,
+		config.Auth,
+		config.ID,
+		connIndex,
+		options,
+	)
+	if err != nil {
+		if err.Error() == DuplicateConnectionError {
+			observer.metrics.regFail.WithLabelValues("dup_edge_conn", "registerConnection").Inc()
+			return errDuplicationConnection
+		}
+		observer.metrics.regFail.WithLabelValues("server_error", "registerConnection").Inc()
+		return serverRegistrationErrorFromRPC(err)
+	}
+
+	observer.metrics.regSuccess.WithLabelValues("registerConnection").Inc()
+
+	observer.logServerInfo(connIndex, conn.Location, fmt.Sprintf("Connection %d registered with %s using ID %s", connIndex, conn.Location, conn.UUID))
+	observer.sendConnectedEvent(connIndex, conn.Location)
+
+	return nil
+}
+
+func (rsc *registrationServerClient) GracefulShutdown(ctx context.Context, gracePeriod time.Duration) {
+	ctx, cancel := context.WithTimeout(ctx, gracePeriod)
+	defer cancel()
+	rsc.client.UnregisterConnection(ctx)
+}
+
+func (rsc *registrationServerClient) Close() {
+	// Closing the client will also close the connection
+	rsc.client.Close()
+	// Closing the transport also closes the stream
+	rsc.transport.Close()
+}
+
+type rpcName string
+
+const (
+	register     rpcName = "register"
+	reconnect    rpcName = "reconnect"
+	unregister   rpcName = "unregister"
+	authenticate rpcName = " authenticate"
+)
+
+func (h *h2muxConnection) registerTunnel(ctx context.Context, credentialSetter CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+	h.observer.sendRegisteringEvent()
+
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := NewTunnelServerClient(ctx, stream, h.observer)
+	defer rpcClient.Close()
+
+	h.logServerInfo(ctx, rpcClient)
+	registration := rpcClient.client.RegisterTunnel(
+		ctx,
+		classicTunnel.OriginCert,
+		classicTunnel.Hostname,
+		registrationOptions,
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// RegisterTunnel RPC failure
+		return h.processRegisterTunnelError(registrationErr, register)
+	}
+
+	// Send free tunnel URL to UI
+	h.observer.sendURL(registration.Url)
+	credentialSetter.SetEventDigest(h.connIndex, registration.EventDigest)
+	return h.processRegistrationSuccess(registration, register, credentialSetter, classicTunnel)
+}
+
+type CredentialManager interface {
+	ReconnectToken() ([]byte, error)
+	EventDigest(connID uint8) ([]byte, error)
+	SetEventDigest(connID uint8, digest []byte)
+	ConnDigest(connID uint8) ([]byte, error)
+	SetConnDigest(connID uint8, digest []byte)
+}
+
+func (h *h2muxConnection) processRegistrationSuccess(
+	registration *tunnelpogs.TunnelRegistration,
+	name rpcName,
+	credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig,
+) error {
+	for _, logLine := range registration.LogLines {
+		h.observer.Info(logLine)
+	}
+
+	if registration.TunnelID != "" {
+		h.observer.metrics.tunnelsHA.AddTunnelID(h.connIndex, registration.TunnelID)
+		h.observer.Infof("Each HA connection's tunnel IDs: %v", h.observer.metrics.tunnelsHA.String())
+	}
+
+	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
+	if classicTunnel.IsTrialZone() {
+		err := h.observer.logTrialHostname(registration)
+		if err != nil {
+			return err
+		}
+	}
+
+	credentialManager.SetConnDigest(h.connIndex, registration.ConnDigest)
+	h.observer.metrics.userHostnamesCounts.WithLabelValues(registration.Url).Inc()
+
+	h.observer.Infof("Route propagating, it may take up to 1 minute for your new route to become functional")
+	h.observer.metrics.regSuccess.WithLabelValues(string(name)).Inc()
+	return nil
+}
+
+func (h *h2muxConnection) processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, name rpcName) error {
+	if err.Error() == DuplicateConnectionError {
+		h.observer.metrics.regFail.WithLabelValues("dup_edge_conn", string(name)).Inc()
+		return errDuplicationConnection
+	}
+	h.observer.metrics.regFail.WithLabelValues("server_error", string(name)).Inc()
+	return serverRegisterTunnelError{
+		cause:     err,
+		permanent: err.IsPermanent(),
+	}
+}
+
+func (h *h2muxConnection) reconnectTunnel(ctx context.Context, credentialManager CredentialManager, classicTunnel *ClassicTunnelConfig, registrationOptions *tunnelpogs.RegistrationOptions) error {
+	token, err := credentialManager.ReconnectToken()
+	if err != nil {
+		return err
+	}
+	eventDigest, err := credentialManager.EventDigest(h.connIndex)
+	if err != nil {
+		return err
+	}
+	connDigest, err := credentialManager.ConnDigest(h.connIndex)
+	if err != nil {
+		return err
+	}
+
+	h.observer.Debug("initiating RPC stream to reconnect")
+	stream, err := h.newRPCStream(ctx, register)
+	if err != nil {
+		return err
+	}
+	rpcClient := NewTunnelServerClient(ctx, stream, h.observer)
+	defer rpcClient.Close()
+
+	h.logServerInfo(ctx, rpcClient)
+	registration := rpcClient.client.ReconnectTunnel(
+		ctx,
+		token,
+		eventDigest,
+		connDigest,
+		classicTunnel.Hostname,
+		registrationOptions,
+	)
+	if registrationErr := registration.DeserializeError(); registrationErr != nil {
+		// ReconnectTunnel RPC failure
+		return h.processRegisterTunnelError(registrationErr, reconnect)
+	}
+	return h.processRegistrationSuccess(registration, reconnect, credentialManager, classicTunnel)
+}
+
+func (h *h2muxConnection) logServerInfo(ctx context.Context, rpcClient *tunnelServerClient) error {
+	// Request server info without blocking tunnel registration; must use capnp library directly.
+	serverInfoPromise := tunnelrpc.TunnelServer{Client: rpcClient.client.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+		return nil
+	})
+	serverInfoMessage, err := serverInfoPromise.Result().Struct()
+	if err != nil {
+		h.observer.Errorf("Failed to retrieve server information: %s", err)
+		return err
+	}
+	serverInfo, err := tunnelpogs.UnmarshalServerInfo(serverInfoMessage)
+	if err != nil {
+		h.observer.Errorf("Failed to retrieve server information: %s", err)
+		return err
+	}
+	h.observer.logServerInfo(h.connIndex, serverInfo.LocationName, fmt.Sprintf("Connnection %d connected to %s", h.connIndex, serverInfo.LocationName))
+	return nil
+}
+
+func (h *h2muxConnection) unregister(isNamedTunnel bool) {
+	unregisterCtx, cancel := context.WithTimeout(context.Background(), h.config.GracePeriod)
+	defer cancel()
+
+	stream, err := h.newRPCStream(unregisterCtx, register)
+	if err != nil {
+		return
+	}
+
+	if isNamedTunnel {
+		rpcClient := newRegistrationRPCClient(unregisterCtx, stream, h.observer)
+		defer rpcClient.Close()
+
+		rpcClient.GracefulShutdown(unregisterCtx, h.config.GracePeriod)
+	} else {
+		rpcClient := NewTunnelServerClient(unregisterCtx, stream, h.observer)
+		defer rpcClient.Close()
+
+		// gracePeriod is encoded in int64 using capnproto
+		rpcClient.client.UnregisterTunnel(unregisterCtx, h.config.GracePeriod.Nanoseconds())
+	}
 }

connection/tunnelsforha.go

@@ -0,0 +1,50 @@
+package connection
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// tunnelsForHA maps this cloudflared instance's HA connections to the tunnel IDs they serve.
+type tunnelsForHA struct {
+	sync.Mutex
+	metrics *prometheus.GaugeVec
+	entries map[uint8]string
+}
+
+// NewTunnelsForHA initializes the Prometheus metrics etc for a tunnelsForHA.
+func NewTunnelsForHA() tunnelsForHA {
+	metrics := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "tunnel_ids",
+			Help: "The ID of all tunnels (and their corresponding HA connection ID) running in this instance of cloudflared.",
+		},
+		[]string{"tunnel_id", "ha_conn_id"},
+	)
+	prometheus.MustRegister(metrics)
+
+	return tunnelsForHA{
+		metrics: metrics,
+		entries: make(map[uint8]string),
+	}
+}
+
+// Track a new tunnel ID, removing the disconnected tunnel (if any) and update metrics.
+func (t *tunnelsForHA) AddTunnelID(haConn uint8, tunnelID string) {
+	t.Lock()
+	defer t.Unlock()
+	haStr := fmt.Sprintf("%v", haConn)
+	if oldTunnelID, ok := t.entries[haConn]; ok {
+		t.metrics.WithLabelValues(oldTunnelID, haStr).Dec()
+	}
+	t.entries[haConn] = tunnelID
+	t.metrics.WithLabelValues(tunnelID, haStr).Inc()
+}
+
+func (t *tunnelsForHA) String() string {
+	t.Lock()
+	defer t.Unlock()
+	return fmt.Sprintf("%v", t.entries)
+}

edgediscovery/dial.go

@@ -1,4 +1,4 @@
-package connection
+package edgediscovery
 
 import (
 	"context"
@@ -9,7 +9,7 @@ import (
 	"github.com/pkg/errors"
 )
 
-// DialEdge makes a TLS connection to a Cloudflare edge node
+// DialEdgeWithH2Mux makes a TLS connection to a Cloudflare edge node
 func DialEdge(
 	ctx context.Context,
 	timeout time.Duration,
@@ -25,6 +25,7 @@ func DialEdge(
 	if err != nil {
 		return nil, newDialError(err, "DialContext error")
 	}
+
 	tlsEdgeConn := tls.Client(edgeConn, tlsConfig)
 	tlsEdgeConn.SetDeadline(time.Now().Add(timeout))
 

edgediscovery/mocks_for_test.go

@@ -1,4 +1,4 @@
-package connection
+package edgediscovery
 
 import (
 	"fmt"

edgediscovery/protocol.go

@@ -0,0 +1,45 @@
+package edgediscovery
+
+import (
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+)
+
+const (
+	protocolRecord = "protocol.argotunnel.com"
+)
+
+var (
+	errNoProtocolRecord = fmt.Errorf("No TXT record found for %s to determine connection protocol", protocolRecord)
+)
+
+func HTTP2Percentage() (int32, error) {
+	records, err := net.LookupTXT(protocolRecord)
+	if err != nil {
+		return 0, err
+	}
+	if len(records) == 0 {
+		return 0, errNoProtocolRecord
+	}
+	return parseHTTP2Precentage(records[0])
+}
+
+// The record looks like http2=percentage
+func parseHTTP2Precentage(record string) (int32, error) {
+	const key = "http2"
+	slices := strings.Split(record, "=")
+	if len(slices) != 2 {
+		return 0, fmt.Errorf("Malformed TXT record %s, expect http2=percentage", record)
+	}
+	if slices[0] != key {
+		return 0, fmt.Errorf("Incorrect key %s, expect %s", slices[0], key)
+	}
+	percentage, err := strconv.ParseInt(slices[1], 10, 32)
+	if err != nil {
+		return 0, err
+	}
+	return int32(percentage), nil
+
+}

edgediscovery/protocol_test.go

@@ -0,0 +1,80 @@
+package edgediscovery
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHTTP2Percentage(t *testing.T) {
+	_, err := HTTP2Percentage()
+	assert.NoError(t, err)
+}
+
+func TestParseHTTP2Precentage(t *testing.T) {
+	tests := []struct {
+		record     string
+		percentage int32
+		wantErr    bool
+	}{
+		{
+			record:     "http2=-1",
+			percentage: -1,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=0",
+			percentage: 0,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=50",
+			percentage: 50,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=100",
+			percentage: 100,
+			wantErr:    false,
+		},
+		{
+			record:     "http2=1000",
+			percentage: 1000,
+			wantErr:    false,
+		},
+		{
+			record:  "http2=10.5",
+			wantErr: true,
+		},
+		{
+			record:  "http2=10 h2mux=90",
+			wantErr: true,
+		},
+		{
+			record:  "http2=ten",
+			wantErr: true,
+		},
+
+		{
+			record:  "h2mux=100",
+			wantErr: true,
+		},
+		{
+			record:  "http2",
+			wantErr: true,
+		},
+		{
+			record:  "http2=",
+			wantErr: true,
+		},
+	}
+
+	for _, test := range tests {
+		p, err := parseHTTP2Precentage(test.record)
+		if test.wantErr {
+			assert.Error(t, err)
+		} else {
+			assert.Equal(t, test.percentage, p)
+		}
+	}
+}

go.mod

@@ -15,7 +15,7 @@ require (
 	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0
-	github.com/equinox-io/equinox v1.2.0
+	github.com/equinox-io/equinox v1.2.0 // indirect
 	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
 	github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 // indirect
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
@@ -27,12 +27,16 @@ require (
 	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
 	github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0
 	github.com/go-sql-driver/mysql v1.5.0
+	github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 // indirect
+	github.com/gobwas/pool v0.2.1 // indirect
+	github.com/gobwas/ws v1.0.4
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/google/go-cmp v0.5.2 // indirect
 	github.com/google/uuid v1.1.2
 	github.com/gorilla/mux v1.7.3
 	github.com/gorilla/websocket v1.4.2
 	github.com/jmoiron/sqlx v1.2.0
+	github.com/json-iterator/go v1.1.10
 	github.com/kr/text v0.2.0 // indirect
 	github.com/kshvakov/clickhouse v1.3.11
 	github.com/kylelemons/godebug v1.1.0 // indirect

go.sum

@@ -233,6 +233,12 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58 h1:YyrUZvJaU8Q0QsoVo+xLFBgWDTam29PKea6GYmwvSiQ=
+github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
+github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
+github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
+github.com/gobwas/ws v1.0.4 h1:5eXU1CZhpQdq5kXbKb+sECH5Ia5KiO6CYzIzdlVx6Bs=
+github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
@@ -369,6 +375,7 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@@ -441,8 +448,10 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=

h2mux/activestreammap.go

@@ -7,6 +7,19 @@ import (
 	"golang.org/x/net/http2"
 )
 
+var (
+	ActiveStreams = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: "cloudflared",
+		Subsystem: "tunnel",
+		Name:      "active_streams",
+		Help:      "Number of active streams created by all muxers.",
+	})
+)
+
+func init() {
+	prometheus.MustRegister(ActiveStreams)
+}
+
 // activeStreamMap is used to moderate access to active streams between the read and write
 // threads, and deny access to new peer streams while shutting down.
 type activeStreamMap struct {

h2mux/activestreammap_test.go

@@ -9,7 +9,7 @@ import (
 
 func TestShutdown(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	// Add all the streams
 	{
@@ -62,7 +62,7 @@ func TestShutdown(t *testing.T) {
 
 func TestEmptyBeforeShutdown(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	// Add all the streams
 	{
@@ -138,7 +138,7 @@ func (_ *noopReadyList) Signal(streamID uint32) {}
 
 func TestAbort(t *testing.T) {
 	const numStreams = 1000
-	m := newActiveStreamMap(true, NewActiveStreamsMetrics("test", t.Name()))
+	m := newActiveStreamMap(true, ActiveStreams)
 
 	var openedStreams sync.Map
 

h2mux/h2mux_test.go

@@ -113,11 +113,11 @@ func (p *DefaultMuxerPair) Handshake(testName string) error {
 	defer cancel()
 	errGroup, _ := errgroup.WithContext(ctx)
 	errGroup.Go(func() (err error) {
-		p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig, NewActiveStreamsMetrics(testName, "edge"))
+		p.EdgeMux, err = Handshake(p.EdgeConn, p.EdgeConn, p.EdgeMuxConfig, ActiveStreams)
 		return errors.Wrap(err, "edge handshake failure")
 	})
 	errGroup.Go(func() (err error) {
-		p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig, NewActiveStreamsMetrics(testName, "origin"))
+		p.OriginMux, err = Handshake(p.OriginConn, p.OriginConn, p.OriginMuxConfig, ActiveStreams)
 		return errors.Wrap(err, "origin handshake failure")
 	})
 

h2mux/header.go

@@ -2,7 +2,6 @@ package h2mux
 
 import (
 	"encoding/base64"
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -22,10 +21,6 @@ const (
 	RequestUserHeadersField  = "cf-cloudflared-request-headers"
 	ResponseUserHeadersField = "cf-cloudflared-response-headers"
 
-	ResponseMetaHeaderField   = "cf-cloudflared-response-meta"
-	ResponseSourceCloudflared = "cloudflared"
-	ResponseSourceOrigin      = "origin"
-
 	CFAccessTokenHeader        = "cf-access-token"
 	CFJumpDestinationHeader    = "CF-Access-Jump-Destination"
 	CFAccessClientIDHeader     = "CF-Access-Client-Id"
@@ -124,7 +119,7 @@ func IsControlHeader(headerName string) bool {
 }
 
 // isWebsocketClientHeader returns true if the header name is required by the client to upgrade properly
-func isWebsocketClientHeader(headerName string) bool {
+func IsWebsocketClientHeader(headerName string) bool {
 	return headerName == "sec-websocket-accept" ||
 		headerName == "connection" ||
 		headerName == "upgrade"
@@ -143,7 +138,7 @@ func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
 
 			// Since these are http2 headers, they're required to be lowercase
 			h2 = append(h2, Header{Name: "content-length", Value: values[0]})
-		} else if !IsControlHeader(h2name) || isWebsocketClientHeader(h2name) {
+		} else if !IsControlHeader(h2name) || IsWebsocketClientHeader(h2name) {
 			// User headers, on the other hand, must all be serialized so that
 			// HTTP/2 header validation won't be applied to HTTP/1 header values
 			userHeaders[header] = values
@@ -152,7 +147,6 @@ func H1ResponseToH2ResponseHeaders(h1 *http.Response) (h2 []Header) {
 
 	// Perform user header serialization and set them in the single header
 	h2 = append(h2, Header{ResponseUserHeadersField, SerializeHeaders(userHeaders)})
-
 	return h2
 }
 
@@ -238,19 +232,3 @@ func DeserializeHeaders(serializedHeaders string) ([]Header, error) {
 
 	return deserialized, nil
 }
-
-type ResponseMetaHeader struct {
-	Source string `json:"src"`
-}
-
-func CreateResponseMetaHeader(headerName, source string) Header {
-	jsonResponseMetaHeader, err := json.Marshal(ResponseMetaHeader{Source: source})
-	if err != nil {
-		panic(err)
-	}
-
-	return Header{
-		Name:  headerName,
-		Value: string(jsonResponseMetaHeader),
-	}
-}

h2mux/muxmetrics.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/golang-collections/collections/queue"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 // data points used to compute average receive window and send window size
@@ -295,14 +294,3 @@ func (r *rate) get() (curr, min, max uint64) {
 	defer r.lock.RUnlock()
 	return r.curr, r.min, r.max
 }
-
-func NewActiveStreamsMetrics(namespace, subsystem string) prometheus.Gauge {
-	activeStreams := prometheus.NewGauge(prometheus.GaugeOpts{
-		Namespace: namespace,
-		Subsystem: subsystem,
-		Name:      "active_streams",
-		Help:      "Number of active streams created by all muxers.",
-	})
-	prometheus.MustRegister(activeStreams)
-	return activeStreams
-}

hello/hello.go

@@ -18,6 +18,14 @@ import (
 	"github.com/cloudflare/cloudflared/tlsconfig"
 )
 
+const (
+	UptimeRoute    = "/uptime"
+	WSRoute        = "/ws"
+	SSERoute       = "/sse"
+	HealthRoute    = "/_health"
+	defaultSSEFreq = time.Second * 10
+)
+
 type templateData struct {
 	ServerName string
 	Request    *http.Request
@@ -104,8 +112,10 @@ func StartHelloWorldServer(logger logger.Service, listener net.Listener, shutdow
 	}
 
 	muxer := http.NewServeMux()
-	muxer.HandleFunc("/uptime", uptimeHandler(time.Now()))
-	muxer.HandleFunc("/ws", websocketHandler(logger, upgrader))
+	muxer.HandleFunc(UptimeRoute, uptimeHandler(time.Now()))
+	muxer.HandleFunc(WSRoute, websocketHandler(logger, upgrader))
+	muxer.HandleFunc(SSERoute, sseHandler(logger))
+	muxer.HandleFunc(HealthRoute, healthHandler())
 	muxer.HandleFunc("/", rootHandler(serverName))
 	httpServer := &http.Server{Addr: listener.Addr().String(), Handler: muxer}
 	go func() {
@@ -177,6 +187,48 @@ func websocketHandler(logger logger.Service, upgrader websocket.Upgrader) http.H
 	}
 }
 
+func sseHandler(logger logger.Service) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/event-stream")
+		flusher, ok := w.(http.Flusher)
+		if !ok {
+			w.WriteHeader(http.StatusInternalServerError)
+			logger.Errorf("Can't support SSE. ResponseWriter %T doesn't implement http.Flusher interface", w)
+			return
+		}
+
+		freq := defaultSSEFreq
+		if requestedFreq := r.URL.Query()["freq"]; len(requestedFreq) > 0 {
+			parsedFreq, err := time.ParseDuration(requestedFreq[0])
+			if err == nil {
+				freq = parsedFreq
+			}
+		}
+		logger.Infof("Server Sent Events every %s", freq)
+		ticker := time.NewTicker(freq)
+		counter := 0
+		for {
+			select {
+			case <-r.Context().Done():
+				return
+			case <-ticker.C:
+			}
+			_, err := fmt.Fprintf(w, "%d\n\n", counter)
+			if err != nil {
+				return
+			}
+			flusher.Flush()
+			counter++
+		}
+	}
+}
+
+func healthHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		w.Write([]byte("ok"))
+	}
+}
+
 func rootHandler(serverName string) http.HandlerFunc {
 	responseTemplate := template.Must(template.New("index").Parse(indexTemplate))
 	return func(w http.ResponseWriter, r *http.Request) {

ingress/ingress.go

@@ -63,9 +63,9 @@ type Ingress struct {
 
 // NewSingleOrigin constructs an Ingress set with only one rule, constructed from
 // legacy CLI parameters like --url or --no-chunked-encoding.
-func NewSingleOrigin(c *cli.Context, compatibilityMode bool, logger logger.Service) (Ingress, error) {
+func NewSingleOrigin(c *cli.Context, allowURLFromArgs bool, logger logger.Service) (Ingress, error) {
 
-	service, err := parseSingleOriginService(c, compatibilityMode)
+	service, err := parseSingleOriginService(c, allowURLFromArgs)
 	if err != nil {
 		return Ingress{}, err
 	}
@@ -85,19 +85,15 @@ func NewSingleOrigin(c *cli.Context, compatibilityMode bool, logger logger.Servi
 }
 
 // Get a single origin service from the CLI/config.
-func parseSingleOriginService(c *cli.Context, compatibilityMode bool) (OriginService, error) {
+func parseSingleOriginService(c *cli.Context, allowURLFromArgs bool) (OriginService, error) {
 	if c.IsSet("hello-world") {
 		return new(helloWorld), nil
 	}
 	if c.IsSet("url") {
-		originURLStr, err := config.ValidateUrl(c, compatibilityMode)
+		originURL, err := config.ValidateUrl(c, allowURLFromArgs)
 		if err != nil {
 			return nil, errors.Wrap(err, "Error validating origin URL")
 		}
-		originURL, err := url.Parse(originURLStr)
-		if err != nil {
-			return nil, errors.Wrap(err, "couldn't parse origin URL")
-		}
 		return &localService{URL: originURL, RootURL: originURL}, nil
 	}
 	if c.IsSet("unix-socket") {

ingress/origin_service.go

@@ -245,7 +245,7 @@ type statusCode struct {
 func newStatusCode(status int) statusCode {
 	resp := &http.Response{
 		StatusCode: status,
-		Status:     http.StatusText(status),
+		Status:     fmt.Sprintf("%d %s", status, http.StatusText(status)),
 		Body:       new(NopReadCloser),
 	}
 	return statusCode{resp: resp}
@@ -318,3 +318,20 @@ func newHTTPTransport(service OriginService, cfg OriginRequestConfig) (*http.Tra
 
 	return &httpTransport, nil
 }
+
+// MockOriginService should only be used by other packages to mock OriginService. Set Transport to configure desired RoundTripper behavior.
+type MockOriginService struct {
+	Transport http.RoundTripper
+}
+
+func (mos MockOriginService) RoundTrip(req *http.Request) (*http.Response, error) {
+	return mos.Transport.RoundTrip(req)
+}
+
+func (mos MockOriginService) String() string {
+	return "MockOriginService"
+}
+
+func (mos MockOriginService) start(wg *sync.WaitGroup, log logger.Service, shutdownC <-chan struct{}, errC chan error, cfg OriginRequestConfig) error {
+	return nil
+}

origin/backoffhandler.go

@@ -97,3 +97,11 @@ func (b BackoffHandler) GetBaseTime() time.Duration {
 func (b *BackoffHandler) Retries() int {
 	return int(b.retries)
 }
+
+func (b *BackoffHandler) ReachedMaxRetries() bool {
+	return b.retries == b.MaxRetries
+}
+
+func (b *BackoffHandler) resetNow() {
+	b.resetDeadline = time.Now()
+}

origin/metrics.go

@@ -1,540 +1,72 @@
 package origin
 
 import (
-	"sync"
-	"time"
-
-	"github.com/cloudflare/cloudflared/h2mux"
-
+	"github.com/cloudflare/cloudflared/connection"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-const (
-	metricsNamespace = "cloudflared"
-	tunnelSubsystem  = "tunnel"
-	muxerSubsystem   = "muxer"
-)
+// Metrics uses connection.MetricsNamespace(aka cloudflared) as namespace and connection.TunnelSubsystem
+// (tunnel) as subsystem to keep them consistent with the previous qualifier.
 
-type muxerMetrics struct {
-	rtt              *prometheus.GaugeVec
-	rttMin           *prometheus.GaugeVec
-	rttMax           *prometheus.GaugeVec
-	receiveWindowAve *prometheus.GaugeVec
-	sendWindowAve    *prometheus.GaugeVec
-	receiveWindowMin *prometheus.GaugeVec
-	receiveWindowMax *prometheus.GaugeVec
-	sendWindowMin    *prometheus.GaugeVec
-	sendWindowMax    *prometheus.GaugeVec
-	inBoundRateCurr  *prometheus.GaugeVec
-	inBoundRateMin   *prometheus.GaugeVec
-	inBoundRateMax   *prometheus.GaugeVec
-	outBoundRateCurr *prometheus.GaugeVec
-	outBoundRateMin  *prometheus.GaugeVec
-	outBoundRateMax  *prometheus.GaugeVec
-	compBytesBefore  *prometheus.GaugeVec
-	compBytesAfter   *prometheus.GaugeVec
-	compRateAve      *prometheus.GaugeVec
-}
-
-type TunnelMetrics struct {
-	haConnections     prometheus.Gauge
-	activeStreams     prometheus.Gauge
-	totalRequests     prometheus.Counter
-	requestsPerTunnel *prometheus.CounterVec
-	// concurrentRequestsLock is a mutex for concurrentRequests and maxConcurrentRequests
-	concurrentRequestsLock      sync.Mutex
-	concurrentRequestsPerTunnel *prometheus.GaugeVec
-	// concurrentRequests records count of concurrent requests for each tunnel
-	concurrentRequests             map[string]uint64
-	maxConcurrentRequestsPerTunnel *prometheus.GaugeVec
-	// concurrentRequests records max count of concurrent requests for each tunnel
-	maxConcurrentRequests map[string]uint64
-	timerRetries          prometheus.Gauge
-	responseByCode        *prometheus.CounterVec
-	responseCodePerTunnel *prometheus.CounterVec
-	serverLocations       *prometheus.GaugeVec
-	// locationLock is a mutex for oldServerLocations
-	locationLock sync.Mutex
-	// oldServerLocations stores the last server the tunnel was connected to
-	oldServerLocations map[string]string
-
-	regSuccess *prometheus.CounterVec
-	regFail    *prometheus.CounterVec
-	rpcFail    *prometheus.CounterVec
-
-	muxerMetrics        *muxerMetrics
-	tunnelsHA           tunnelsForHA
-	userHostnamesCounts *prometheus.CounterVec
-}
-
-func newMuxerMetrics() *muxerMetrics {
-	rtt := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "rtt",
-			Help:      "Round-trip time in millisecond",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(rtt)
-
-	rttMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "rtt_min",
-			Help:      "Shortest round-trip time in millisecond",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(rttMin)
-
-	rttMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "rtt_max",
-			Help:      "Longest round-trip time in millisecond",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(rttMax)
-
-	receiveWindowAve := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "receive_window_ave",
-			Help:      "Average receive window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(receiveWindowAve)
-
-	sendWindowAve := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "send_window_ave",
-			Help:      "Average send window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(sendWindowAve)
-
-	receiveWindowMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "receive_window_min",
-			Help:      "Smallest receive window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(receiveWindowMin)
-
-	receiveWindowMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "receive_window_max",
-			Help:      "Largest receive window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(receiveWindowMax)
-
-	sendWindowMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "send_window_min",
-			Help:      "Smallest send window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(sendWindowMin)
-
-	sendWindowMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "send_window_max",
-			Help:      "Largest send window size in bytes",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(sendWindowMax)
-
-	inBoundRateCurr := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "inbound_bytes_per_sec_curr",
-			Help:      "Current inbounding bytes per second, 0 if there is no incoming connection",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(inBoundRateCurr)
-
-	inBoundRateMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "inbound_bytes_per_sec_min",
-			Help:      "Minimum non-zero inbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(inBoundRateMin)
-
-	inBoundRateMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "inbound_bytes_per_sec_max",
-			Help:      "Maximum inbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(inBoundRateMax)
-
-	outBoundRateCurr := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "outbound_bytes_per_sec_curr",
-			Help:      "Current outbounding bytes per second, 0 if there is no outgoing traffic",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(outBoundRateCurr)
-
-	outBoundRateMin := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "outbound_bytes_per_sec_min",
-			Help:      "Minimum non-zero outbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(outBoundRateMin)
-
-	outBoundRateMax := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "outbound_bytes_per_sec_max",
-			Help:      "Maximum outbounding bytes per second",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(outBoundRateMax)
-
-	compBytesBefore := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "comp_bytes_before",
-			Help:      "Bytes sent via cross-stream compression, pre compression",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(compBytesBefore)
-
-	compBytesAfter := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "comp_bytes_after",
-			Help:      "Bytes sent via cross-stream compression, post compression",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(compBytesAfter)
-
-	compRateAve := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: muxerSubsystem,
-			Name:      "comp_rate_ave",
-			Help:      "Average outbound cross-stream compression ratio",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(compRateAve)
-
-	return &muxerMetrics{
-		rtt:              rtt,
-		rttMin:           rttMin,
-		rttMax:           rttMax,
-		receiveWindowAve: receiveWindowAve,
-		sendWindowAve:    sendWindowAve,
-		receiveWindowMin: receiveWindowMin,
-		receiveWindowMax: receiveWindowMax,
-		sendWindowMin:    sendWindowMin,
-		sendWindowMax:    sendWindowMax,
-		inBoundRateCurr:  inBoundRateCurr,
-		inBoundRateMin:   inBoundRateMin,
-		inBoundRateMax:   inBoundRateMax,
-		outBoundRateCurr: outBoundRateCurr,
-		outBoundRateMin:  outBoundRateMin,
-		outBoundRateMax:  outBoundRateMax,
-		compBytesBefore:  compBytesBefore,
-		compBytesAfter:   compBytesAfter,
-		compRateAve:      compRateAve,
-	}
-}
-
-func (m *muxerMetrics) update(connectionID string, metrics *h2mux.MuxerMetrics) {
-	m.rtt.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTT))
-	m.rttMin.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMin))
-	m.rttMax.WithLabelValues(connectionID).Set(convertRTTMilliSec(metrics.RTTMax))
-	m.receiveWindowAve.WithLabelValues(connectionID).Set(metrics.ReceiveWindowAve)
-	m.sendWindowAve.WithLabelValues(connectionID).Set(metrics.SendWindowAve)
-	m.receiveWindowMin.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMin))
-	m.receiveWindowMax.WithLabelValues(connectionID).Set(float64(metrics.ReceiveWindowMax))
-	m.sendWindowMin.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMin))
-	m.sendWindowMax.WithLabelValues(connectionID).Set(float64(metrics.SendWindowMax))
-	m.inBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateCurr))
-	m.inBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMin))
-	m.inBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.InBoundRateMax))
-	m.outBoundRateCurr.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateCurr))
-	m.outBoundRateMin.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMin))
-	m.outBoundRateMax.WithLabelValues(connectionID).Set(float64(metrics.OutBoundRateMax))
-	m.compBytesBefore.WithLabelValues(connectionID).Set(float64(metrics.CompBytesBefore.Value()))
-	m.compBytesAfter.WithLabelValues(connectionID).Set(float64(metrics.CompBytesAfter.Value()))
-	m.compRateAve.WithLabelValues(connectionID).Set(float64(metrics.CompRateAve()))
-}
-
-func convertRTTMilliSec(t time.Duration) float64 {
-	return float64(t / time.Millisecond)
-}
-
-// Metrics that can be collected without asking the edge
-func NewTunnelMetrics() *TunnelMetrics {
-	haConnections := prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "ha_connections",
-			Help:      "Number of active ha connections",
-		})
-	prometheus.MustRegister(haConnections)
-
-	activeStreams := h2mux.NewActiveStreamsMetrics(metricsNamespace, tunnelSubsystem)
-
-	totalRequests := prometheus.NewCounter(
+var (
+	totalRequests = prometheus.NewCounter(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
 			Name:      "total_requests",
 			Help:      "Amount of requests proxied through all the tunnels",
-		})
-	prometheus.MustRegister(totalRequests)
-
-	requestsPerTunnel := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "requests_per_tunnel",
-			Help:      "Amount of requests proxied through each tunnel",
 		},
-		[]string{"connection_id"},
 	)
-	prometheus.MustRegister(requestsPerTunnel)
-
-	concurrentRequestsPerTunnel := prometheus.NewGaugeVec(
+	concurrentRequests = prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
 			Name:      "concurrent_requests_per_tunnel",
 			Help:      "Concurrent requests proxied through each tunnel",
 		},
-		[]string{"connection_id"},
 	)
-	prometheus.MustRegister(concurrentRequestsPerTunnel)
-
-	maxConcurrentRequestsPerTunnel := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "max_concurrent_requests_per_tunnel",
-			Help:      "Largest number of concurrent requests proxied through each tunnel so far",
-		},
-		[]string{"connection_id"},
-	)
-	prometheus.MustRegister(maxConcurrentRequestsPerTunnel)
-
-	timerRetries := prometheus.NewGauge(
-		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "timer_retries",
-			Help:      "Unacknowledged heart beats count",
-		})
-	prometheus.MustRegister(timerRetries)
-
-	responseByCode := prometheus.NewCounterVec(
+	responseByCode = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
 			Name:      "response_by_code",
 			Help:      "Count of responses by HTTP status code",
 		},
 		[]string{"status_code"},
 	)
-	prometheus.MustRegister(responseByCode)
-
-	responseCodePerTunnel := prometheus.NewCounterVec(
+	requestErrors = prometheus.NewCounter(
 		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "response_code_per_tunnel",
-			Help:      "Count of responses by HTTP status code fore each tunnel",
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
+			Name:      "request_errors",
+			Help:      "Count of error proxying to origin",
 		},
-		[]string{"connection_id", "status_code"},
 	)
-	prometheus.MustRegister(responseCodePerTunnel)
-
-	serverLocations := prometheus.NewGaugeVec(
+	haConnections = prometheus.NewGauge(
 		prometheus.GaugeOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "server_locations",
-			Help:      "Where each tunnel is connected to. 1 means current location, 0 means previous locations.",
+			Namespace: connection.MetricsNamespace,
+			Subsystem: connection.TunnelSubsystem,
+			Name:      "ha_connections",
+			Help:      "Number of active ha connections",
 		},
-		[]string{"connection_id", "location"},
 	)
-	prometheus.MustRegister(serverLocations)
+)
 
-	rpcFail := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "tunnel_rpc_fail",
-			Help:      "Count of RPC connection errors by type",
-		},
-		[]string{"error", "rpcName"},
+func init() {
+	prometheus.MustRegister(
+		totalRequests,
+		concurrentRequests,
+		responseByCode,
+		requestErrors,
+		haConnections,
 	)
-	prometheus.MustRegister(rpcFail)
-
-	registerFail := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "tunnel_register_fail",
-			Help:      "Count of tunnel registration errors by type",
-		},
-		[]string{"error", "rpcName"},
-	)
-	prometheus.MustRegister(registerFail)
-
-	userHostnamesCounts := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "user_hostnames_counts",
-			Help:      "Which user hostnames cloudflared is serving",
-		},
-		[]string{"userHostname"},
-	)
-	prometheus.MustRegister(userHostnamesCounts)
-
-	registerSuccess := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: metricsNamespace,
-			Subsystem: tunnelSubsystem,
-			Name:      "tunnel_register_success",
-			Help:      "Count of successful tunnel registrations",
-		},
-		[]string{"rpcName"},
-	)
-	prometheus.MustRegister(registerSuccess)
-
-	return &TunnelMetrics{
-		haConnections:                  haConnections,
-		activeStreams:                  activeStreams,
-		totalRequests:                  totalRequests,
-		requestsPerTunnel:              requestsPerTunnel,
-		concurrentRequestsPerTunnel:    concurrentRequestsPerTunnel,
-		concurrentRequests:             make(map[string]uint64),
-		maxConcurrentRequestsPerTunnel: maxConcurrentRequestsPerTunnel,
-		maxConcurrentRequests:          make(map[string]uint64),
-		timerRetries:                   timerRetries,
-		responseByCode:                 responseByCode,
-		responseCodePerTunnel:          responseCodePerTunnel,
-		serverLocations:                serverLocations,
-		oldServerLocations:             make(map[string]string),
-		muxerMetrics:                   newMuxerMetrics(),
-		tunnelsHA:                      NewTunnelsForHA(),
-		regSuccess:                     registerSuccess,
-		regFail:                        registerFail,
-		rpcFail:                        rpcFail,
-		userHostnamesCounts:            userHostnamesCounts,
-	}
 }
 
-func (t *TunnelMetrics) incrementHaConnections() {
-	t.haConnections.Inc()
+func incrementRequests() {
+	totalRequests.Inc()
+	concurrentRequests.Inc()
 }
 
-func (t *TunnelMetrics) decrementHaConnections() {
-	t.haConnections.Dec()
-}
-
-func (t *TunnelMetrics) updateMuxerMetrics(connectionID string, metrics *h2mux.MuxerMetrics) {
-	t.muxerMetrics.update(connectionID, metrics)
-}
-
-func (t *TunnelMetrics) incrementRequests(connectionID string) {
-	t.concurrentRequestsLock.Lock()
-	var concurrentRequests uint64
-	var ok bool
-	if concurrentRequests, ok = t.concurrentRequests[connectionID]; ok {
-		t.concurrentRequests[connectionID]++
-		concurrentRequests++
-	} else {
-		t.concurrentRequests[connectionID] = 1
-		concurrentRequests = 1
-	}
-	if maxConcurrentRequests, ok := t.maxConcurrentRequests[connectionID]; (ok && maxConcurrentRequests < concurrentRequests) || !ok {
-		t.maxConcurrentRequests[connectionID] = concurrentRequests
-		t.maxConcurrentRequestsPerTunnel.WithLabelValues(connectionID).Set(float64(concurrentRequests))
-	}
-	t.concurrentRequestsLock.Unlock()
-
-	t.totalRequests.Inc()
-	t.requestsPerTunnel.WithLabelValues(connectionID).Inc()
-	t.concurrentRequestsPerTunnel.WithLabelValues(connectionID).Inc()
-}
-
-func (t *TunnelMetrics) decrementConcurrentRequests(connectionID string) {
-	t.concurrentRequestsLock.Lock()
-	if _, ok := t.concurrentRequests[connectionID]; ok {
-		t.concurrentRequests[connectionID]--
-	}
-	t.concurrentRequestsLock.Unlock()
-
-	t.concurrentRequestsPerTunnel.WithLabelValues(connectionID).Dec()
-}
-
-func (t *TunnelMetrics) incrementResponses(connectionID, code string) {
-	t.responseByCode.WithLabelValues(code).Inc()
-	t.responseCodePerTunnel.WithLabelValues(connectionID, code).Inc()
-
-}
-
-func (t *TunnelMetrics) registerServerLocation(connectionID, loc string) {
-	t.locationLock.Lock()
-	defer t.locationLock.Unlock()
-	if oldLoc, ok := t.oldServerLocations[connectionID]; ok && oldLoc == loc {
-		return
-	} else if ok {
-		t.serverLocations.WithLabelValues(connectionID, oldLoc).Dec()
-	}
-	t.serverLocations.WithLabelValues(connectionID, loc).Inc()
-	t.oldServerLocations[connectionID] = loc
+func decrementConcurrentRequests() {
+	concurrentRequests.Dec()
 }

origin/metrics_test.go

@@ -1,121 +0,0 @@
-package origin
-
-import (
-	"strconv"
-	"sync"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-// can only be called once
-var m = NewTunnelMetrics()
-
-func TestConcurrentRequestsSingleTunnel(t *testing.T) {
-	routines := 20
-	var wg sync.WaitGroup
-	wg.Add(routines)
-	for i := 0; i < routines; i++ {
-		go func() {
-			m.incrementRequests("0")
-			wg.Done()
-		}()
-	}
-	wg.Wait()
-	assert.Len(t, m.concurrentRequests, 1)
-	assert.Equal(t, uint64(routines), m.concurrentRequests["0"])
-	assert.Len(t, m.maxConcurrentRequests, 1)
-	assert.Equal(t, uint64(routines), m.maxConcurrentRequests["0"])
-
-	wg.Add(routines / 2)
-	for i := 0; i < routines/2; i++ {
-		go func() {
-			m.decrementConcurrentRequests("0")
-			wg.Done()
-		}()
-	}
-	wg.Wait()
-	assert.Equal(t, uint64(routines-routines/2), m.concurrentRequests["0"])
-	assert.Equal(t, uint64(routines), m.maxConcurrentRequests["0"])
-}
-
-func TestConcurrentRequestsMultiTunnel(t *testing.T) {
-	m.concurrentRequests = make(map[string]uint64)
-	m.maxConcurrentRequests = make(map[string]uint64)
-	tunnels := 20
-	var wg sync.WaitGroup
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			// if we have j < i, then tunnel 0 won't have a chance to call incrementRequests
-			for j := 0; j < i+1; j++ {
-				id := strconv.Itoa(i)
-				m.incrementRequests(id)
-			}
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-
-	assert.Len(t, m.concurrentRequests, tunnels)
-	assert.Len(t, m.maxConcurrentRequests, tunnels)
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, uint64(i+1), m.concurrentRequests[id])
-		assert.Equal(t, uint64(i+1), m.maxConcurrentRequests[id])
-	}
-
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			for j := 0; j < i+1; j++ {
-				id := strconv.Itoa(i)
-				m.decrementConcurrentRequests(id)
-			}
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-
-	assert.Len(t, m.concurrentRequests, tunnels)
-	assert.Len(t, m.maxConcurrentRequests, tunnels)
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, uint64(0), m.concurrentRequests[id])
-		assert.Equal(t, uint64(i+1), m.maxConcurrentRequests[id])
-	}
-
-}
-
-func TestRegisterServerLocation(t *testing.T) {
-	tunnels := 20
-	var wg sync.WaitGroup
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			id := strconv.Itoa(i)
-			m.registerServerLocation(id, "LHR")
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, "LHR", m.oldServerLocations[id])
-	}
-
-	wg.Add(tunnels)
-	for i := 0; i < tunnels; i++ {
-		go func(i int) {
-			id := strconv.Itoa(i)
-			m.registerServerLocation(id, "AUS")
-			wg.Done()
-		}(i)
-	}
-	wg.Wait()
-	for i := 0; i < tunnels; i++ {
-		id := strconv.Itoa(i)
-		assert.Equal(t, "AUS", m.oldServerLocations[id])
-	}
-
-}

origin/proxy.go

@@ -0,0 +1,235 @@
+package origin
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/cloudflare/cloudflared/buffer"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/cloudflare/cloudflared/websocket"
+	"github.com/pkg/errors"
+)
+
+const (
+	TagHeaderNamePrefix = "Cf-Warp-Tag-"
+)
+
+type client struct {
+	ingressRules ingress.Ingress
+	tags         []tunnelpogs.Tag
+	logger       logger.Service
+	bufferPool   *buffer.Pool
+}
+
+func NewClient(ingressRules ingress.Ingress, tags []tunnelpogs.Tag, logger logger.Service) connection.OriginClient {
+	return &client{
+		ingressRules: ingressRules,
+		tags:         tags,
+		logger:       logger,
+		bufferPool:   buffer.NewPool(512 * 1024),
+	}
+}
+
+func (c *client) Proxy(w connection.ResponseWriter, req *http.Request, isWebsocket bool) error {
+	incrementRequests()
+	defer decrementConcurrentRequests()
+
+	cfRay := findCfRayHeader(req)
+	lbProbe := isLBProbeRequest(req)
+
+	c.appendTagHeaders(req)
+	rule, ruleNum := c.ingressRules.FindMatchingRule(req.Host, req.URL.Path)
+	c.logRequest(req, cfRay, lbProbe, ruleNum)
+
+	var (
+		resp *http.Response
+		err  error
+	)
+	if isWebsocket {
+		resp, err = c.proxyWebsocket(w, req, rule)
+	} else {
+		resp, err = c.proxyHTTP(w, req, rule)
+	}
+	if err != nil {
+		c.logRequestError(err, cfRay, ruleNum)
+		w.WriteErrorResponse()
+		return err
+	}
+	c.logOriginResponse(resp, cfRay, lbProbe, ruleNum)
+	return nil
+}
+
+func (c *client) proxyHTTP(w connection.ResponseWriter, req *http.Request, rule *ingress.Rule) (*http.Response, error) {
+	// Support for WSGI Servers by switching transfer encoding from chunked to gzip/deflate
+	if rule.Config.DisableChunkedEncoding {
+		req.TransferEncoding = []string{"gzip", "deflate"}
+		cLength, err := strconv.Atoi(req.Header.Get("Content-Length"))
+		if err == nil {
+			req.ContentLength = int64(cLength)
+		}
+	}
+
+	// Request origin to keep connection alive to improve performance
+	req.Header.Set("Connection", "keep-alive")
+
+	if hostHeader := rule.Config.HTTPHostHeader; hostHeader != "" {
+		req.Header.Set("Host", hostHeader)
+		req.Host = hostHeader
+	}
+
+	resp, err := rule.Service.RoundTrip(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error proxying request to origin")
+	}
+	defer resp.Body.Close()
+
+	err = w.WriteRespHeaders(resp)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error writing response header")
+	}
+	if isEventStream(resp) {
+		c.logger.Debug("Detected Server-Side Events from Origin")
+		c.writeEventStream(w, resp.Body)
+	} else {
+		// Use CopyBuffer, because Copy only allocates a 32KiB buffer, and cross-stream
+		// compression generates dictionary on first write
+		buf := c.bufferPool.Get()
+		defer c.bufferPool.Put(buf)
+		io.CopyBuffer(w, resp.Body, buf)
+	}
+	return resp, nil
+}
+
+func (c *client) proxyWebsocket(w connection.ResponseWriter, req *http.Request, rule *ingress.Rule) (*http.Response, error) {
+	if hostHeader := rule.Config.HTTPHostHeader; hostHeader != "" {
+		req.Header.Set("Host", hostHeader)
+		req.Host = hostHeader
+	}
+
+	dialler, ok := rule.Service.(websocket.Dialler)
+	if !ok {
+		return nil, fmt.Errorf("Websockets aren't supported by the origin service '%s'", rule.Service)
+	}
+	conn, resp, err := websocket.ClientConnect(req, dialler)
+	if err != nil {
+		return nil, err
+	}
+
+	serveCtx, cancel := context.WithCancel(req.Context())
+	connClosedChan := make(chan struct{})
+	go func() {
+		// serveCtx is done if req is cancelled, or streamWebsocket returns
+		<-serveCtx.Done()
+		conn.Close()
+		close(connClosedChan)
+	}()
+
+	// Copy to/from stream to the undelying connection. Use the underlying
+	// connection because cloudflared doesn't operate on the message themselves
+	err = c.streamWebsocket(w, conn.UnderlyingConn(), resp)
+	cancel()
+
+	// We need to make sure conn is closed before returning, otherwise we might write to conn after Proxy returns
+	<-connClosedChan
+	return resp, err
+}
+
+func (c *client) streamWebsocket(w connection.ResponseWriter, conn net.Conn, resp *http.Response) error {
+	err := w.WriteRespHeaders(resp)
+	if err != nil {
+		return errors.Wrap(err, "Error writing websocket response header")
+	}
+	websocket.Stream(conn, w)
+	return nil
+}
+
+func (c *client) writeEventStream(w connection.ResponseWriter, respBody io.ReadCloser) {
+	reader := bufio.NewReader(respBody)
+	for {
+		line, err := reader.ReadBytes('\n')
+		if err != nil {
+			break
+		}
+		w.Write(line)
+	}
+}
+
+func (c *client) appendTagHeaders(r *http.Request) {
+	for _, tag := range c.tags {
+		r.Header.Add(TagHeaderNamePrefix+tag.Name, tag.Value)
+	}
+}
+
+func (c *client) logRequest(r *http.Request, cfRay string, lbProbe bool, ruleNum int) {
+	if cfRay != "" {
+		c.logger.Debugf("CF-RAY: %s %s %s %s", cfRay, r.Method, r.URL, r.Proto)
+	} else if lbProbe {
+		c.logger.Debugf("CF-RAY: %s Load Balancer health check %s %s %s", cfRay, r.Method, r.URL, r.Proto)
+	} else {
+		c.logger.Debugf("All requests should have a CF-RAY header. Please open a support ticket with Cloudflare. %s %s %s ", r.Method, r.URL, r.Proto)
+	}
+	c.logger.Debugf("CF-RAY: %s Request Headers %+v", cfRay, r.Header)
+	c.logger.Debugf("CF-RAY: %s Serving with ingress rule %d", cfRay, ruleNum)
+
+	if contentLen := r.ContentLength; contentLen == -1 {
+		c.logger.Debugf("CF-RAY: %s Request Content length unknown", cfRay)
+	} else {
+		c.logger.Debugf("CF-RAY: %s Request content length %d", cfRay, contentLen)
+	}
+}
+
+func (c *client) logOriginResponse(r *http.Response, cfRay string, lbProbe bool, ruleNum int) {
+	responseByCode.WithLabelValues(strconv.Itoa(r.StatusCode)).Inc()
+	if cfRay != "" {
+		c.logger.Infof("CF-RAY: %s Status: %s served by ingress %d", cfRay, r.Status, ruleNum)
+	} else if lbProbe {
+		c.logger.Debugf("Response to Load Balancer health check %s", r.Status)
+	} else {
+		c.logger.Debugf("Status: %s served by ingress %d", r.Status, ruleNum)
+	}
+	c.logger.Debugf("CF-RAY: %s Response Headers %+v", cfRay, r.Header)
+
+	if contentLen := r.ContentLength; contentLen == -1 {
+		c.logger.Debugf("CF-RAY: %s Response content length unknown", cfRay)
+	} else {
+		c.logger.Debugf("CF-RAY: %s Response content length %d", cfRay, contentLen)
+	}
+}
+
+func (c *client) logRequestError(err error, cfRay string, ruleNum int) {
+	requestErrors.Inc()
+	if cfRay != "" {
+		c.logger.Errorf("CF-RAY: %s Proxying to ingress %d error: %v", cfRay, ruleNum, err)
+	} else {
+		c.logger.Errorf("Proxying to ingress %d error: %v", ruleNum, err)
+	}
+
+}
+
+func findCfRayHeader(req *http.Request) string {
+	return req.Header.Get("Cf-Ray")
+}
+
+func isLBProbeRequest(req *http.Request) bool {
+	return strings.HasPrefix(req.UserAgent(), lbProbeUserAgentPrefix)
+}
+
+func uint8ToString(input uint8) string {
+	return strconv.FormatUint(uint64(input), 10)
+}
+
+func isEventStream(response *http.Response) bool {
+	if response.Header.Get("content-type") == "text/event-stream" {
+		return true
+	}
+	return false
+}

origin/proxy_test.go

@@ -0,0 +1,352 @@
+package origin
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/hello"
+	"github.com/cloudflare/cloudflared/ingress"
+	"github.com/cloudflare/cloudflared/logger"
+	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/urfave/cli/v2"
+
+	"github.com/gobwas/ws/wsutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	testTags = []tunnelpogs.Tag(nil)
+)
+
+type mockHTTPRespWriter struct {
+	*httptest.ResponseRecorder
+}
+
+func newMockHTTPRespWriter() *mockHTTPRespWriter {
+	return &mockHTTPRespWriter{
+		httptest.NewRecorder(),
+	}
+}
+
+func (w *mockHTTPRespWriter) WriteRespHeaders(resp *http.Response) error {
+	w.WriteHeader(resp.StatusCode)
+	for header, val := range resp.Header {
+		w.Header()[header] = val
+	}
+	return nil
+}
+
+func (w *mockHTTPRespWriter) WriteErrorResponse() {
+	w.WriteHeader(http.StatusBadGateway)
+	w.Write([]byte("http response error"))
+}
+
+func (w *mockHTTPRespWriter) Read(data []byte) (int, error) {
+	return 0, fmt.Errorf("mockHTTPRespWriter doesn't implement io.Reader")
+}
+
+type mockWSRespWriter struct {
+	*mockHTTPRespWriter
+	writeNotification chan []byte
+	reader            io.Reader
+}
+
+func newMockWSRespWriter(reader io.Reader) *mockWSRespWriter {
+	return &mockWSRespWriter{
+		newMockHTTPRespWriter(),
+		make(chan []byte),
+		reader,
+	}
+}
+
+func (w *mockWSRespWriter) Write(data []byte) (int, error) {
+	w.writeNotification <- data
+	return len(data), nil
+}
+
+func (w *mockWSRespWriter) respBody() io.ReadWriter {
+	data := <-w.writeNotification
+	return bytes.NewBuffer(data)
+}
+
+func (w *mockWSRespWriter) Read(data []byte) (int, error) {
+	return w.reader.Read(data)
+}
+
+type mockSSERespWriter struct {
+	*mockHTTPRespWriter
+	writeNotification chan []byte
+}
+
+func newMockSSERespWriter() *mockSSERespWriter {
+	return &mockSSERespWriter{
+		newMockHTTPRespWriter(),
+		make(chan []byte),
+	}
+}
+
+func (w *mockSSERespWriter) Write(data []byte) (int, error) {
+	w.writeNotification <- data
+	return len(data), nil
+}
+
+func (w *mockSSERespWriter) ReadBytes() []byte {
+	return <-w.writeNotification
+}
+
+func TestProxySingleOrigin(t *testing.T) {
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	flagSet := flag.NewFlagSet(t.Name(), flag.PanicOnError)
+	flagSet.Bool("hello-world", true, "")
+
+	cliCtx := cli.NewContext(cli.NewApp(), flagSet, nil)
+	err = cliCtx.Set("hello-world", "true")
+	require.NoError(t, err)
+
+	allowURLFromArgs := false
+	ingressRule, err := ingress.NewSingleOrigin(cliCtx, allowURLFromArgs, logger)
+	require.NoError(t, err)
+
+	var wg sync.WaitGroup
+	errC := make(chan error)
+	ingressRule.StartOrigins(&wg, logger, ctx.Done(), errC)
+
+	client := NewClient(ingressRule, testTags, logger)
+	t.Run("testProxyHTTP", testProxyHTTP(t, client))
+	t.Run("testProxyWebsocket", testProxyWebsocket(t, client))
+	t.Run("testProxySSE", testProxySSE(t, client))
+	cancel()
+	wg.Wait()
+}
+
+func testProxyHTTP(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		respWriter := newMockHTTPRespWriter()
+		req, err := http.NewRequest(http.MethodGet, "http://localhost:8080", nil)
+		require.NoError(t, err)
+
+		err = client.Proxy(respWriter, req, false)
+		require.NoError(t, err)
+
+		assert.Equal(t, http.StatusOK, respWriter.Code)
+	}
+}
+
+func testProxyWebsocket(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		// WSRoute is a websocket echo handler
+		ctx, cancel := context.WithCancel(context.Background())
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://localhost:8080%s", hello.WSRoute), nil)
+
+		readPipe, writePipe := io.Pipe()
+		respWriter := newMockWSRespWriter(readPipe)
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err = client.Proxy(respWriter, req, true)
+			require.NoError(t, err)
+
+			require.Equal(t, http.StatusSwitchingProtocols, respWriter.Code)
+		}()
+
+		msg := []byte("test websocket")
+		err = wsutil.WriteClientText(writePipe, msg)
+		require.NoError(t, err)
+
+		// ReadServerText reads next data message from rw, considering that caller represents client side.
+		returnedMsg, err := wsutil.ReadServerText(respWriter.respBody())
+		require.NoError(t, err)
+		require.Equal(t, msg, returnedMsg)
+
+		err = wsutil.WriteClientBinary(writePipe, msg)
+		require.NoError(t, err)
+
+		returnedMsg, err = wsutil.ReadServerBinary(respWriter.respBody())
+		require.NoError(t, err)
+		require.Equal(t, msg, returnedMsg)
+
+		cancel()
+		wg.Wait()
+	}
+}
+
+func testProxySSE(t *testing.T, client connection.OriginClient) func(t *testing.T) {
+	return func(t *testing.T) {
+		var (
+			pushCount = 50
+			pushFreq  = time.Duration(time.Millisecond * 10)
+		)
+		respWriter := newMockSSERespWriter()
+		ctx, cancel := context.WithCancel(context.Background())
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://localhost:8080%s?freq=%s", hello.SSERoute, pushFreq), nil)
+		require.NoError(t, err)
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err = client.Proxy(respWriter, req, false)
+			require.NoError(t, err)
+
+			require.Equal(t, http.StatusOK, respWriter.Code)
+		}()
+
+		for i := 0; i < pushCount; i++ {
+			line := respWriter.ReadBytes()
+			expect := fmt.Sprintf("%d\n", i)
+			require.Equal(t, []byte(expect), line, fmt.Sprintf("Expect to read %v, got %v", expect, line))
+
+			line = respWriter.ReadBytes()
+			require.Equal(t, []byte("\n"), line, fmt.Sprintf("Expect to read '\n', got %v", line))
+		}
+
+		cancel()
+		wg.Wait()
+	}
+}
+
+func TestProxyMultipleOrigins(t *testing.T) {
+	api := httptest.NewServer(mockAPI{})
+	defer api.Close()
+
+	unvalidatedIngress := []config.UnvalidatedIngressRule{
+		{
+			Hostname: "api.example.com",
+			Service:  api.URL,
+		},
+		{
+			Hostname: "hello.example.com",
+			Service:  "hello-world",
+		},
+		{
+			Hostname: "health.example.com",
+			Path:     "/health",
+			Service:  "http_status:200",
+		},
+		{
+			Hostname: "*",
+			Service:  "http_status:404",
+		},
+	}
+
+	ingress, err := ingress.ParseIngress(&config.Configuration{
+		TunnelID: t.Name(),
+		Ingress:  unvalidatedIngress,
+	})
+	require.NoError(t, err)
+
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	errC := make(chan error)
+	var wg sync.WaitGroup
+	ingress.StartOrigins(&wg, logger, ctx.Done(), errC)
+
+	client := NewClient(ingress, testTags, logger)
+
+	tests := []struct {
+		url            string
+		expectedStatus int
+		expectedBody   []byte
+	}{
+		{
+			url:            "http://api.example.com",
+			expectedStatus: http.StatusCreated,
+			expectedBody:   []byte("Created"),
+		},
+		{
+			url:            fmt.Sprintf("http://hello.example.com%s", hello.HealthRoute),
+			expectedStatus: http.StatusOK,
+			expectedBody:   []byte("ok"),
+		},
+		{
+			url:            "http://health.example.com/health",
+			expectedStatus: http.StatusOK,
+		},
+		{
+			url:            "http://health.example.com/",
+			expectedStatus: http.StatusNotFound,
+		},
+		{
+			url:            "http://not-found.example.com",
+			expectedStatus: http.StatusNotFound,
+		},
+	}
+
+	for _, test := range tests {
+		respWriter := newMockHTTPRespWriter()
+		req, err := http.NewRequest(http.MethodGet, test.url, nil)
+		require.NoError(t, err)
+
+		err = client.Proxy(respWriter, req, false)
+		require.NoError(t, err)
+
+		assert.Equal(t, test.expectedStatus, respWriter.Code)
+		if test.expectedBody != nil {
+			assert.Equal(t, test.expectedBody, respWriter.Body.Bytes())
+		} else {
+			assert.Equal(t, 0, respWriter.Body.Len())
+		}
+	}
+	cancel()
+	wg.Wait()
+}
+
+type mockAPI struct{}
+
+func (ma mockAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	w.WriteHeader(http.StatusCreated)
+	w.Write([]byte("Created"))
+}
+
+type errorOriginTransport struct{}
+
+func (errorOriginTransport) RoundTrip(*http.Request) (*http.Response, error) {
+	return nil, fmt.Errorf("Proxy error")
+}
+
+func TestProxyError(t *testing.T) {
+	ingress := ingress.Ingress{
+		Rules: []ingress.Rule{
+			{
+				Hostname: "*",
+				Path:     nil,
+				Service: ingress.MockOriginService{
+					Transport: errorOriginTransport{},
+				},
+			},
+		},
+	}
+
+	logger, err := logger.New()
+	require.NoError(t, err)
+
+	client := NewClient(ingress, testTags, logger)
+
+	respWriter := newMockHTTPRespWriter()
+	req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
+	assert.NoError(t, err)
+
+	err = client.Proxy(respWriter, req, false)
+	assert.Error(t, err)
+	assert.Equal(t, http.StatusBadGateway, respWriter.Code)
+	assert.Equal(t, "http response error", respWriter.Body.String())
+}

origin/reconnect.go

@@ -7,11 +7,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/h2mux"
-	"github.com/cloudflare/cloudflared/logger"
-	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
-	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -138,52 +134,3 @@ func (cm *reconnectCredentialManager) RefreshAuth(
 		return nil, err
 	}
 }
-
-func ReconnectTunnel(
-	ctx context.Context,
-	muxer *h2mux.Muxer,
-	config *TunnelConfig,
-	logger logger.Service,
-	connectionID uint8,
-	originLocalAddr string,
-	uuid uuid.UUID,
-	credentialManager *reconnectCredentialManager,
-) error {
-	token, err := credentialManager.ReconnectToken()
-	if err != nil {
-		return err
-	}
-	eventDigest, err := credentialManager.EventDigest(connectionID)
-	if err != nil {
-		return err
-	}
-	connDigest, err := credentialManager.ConnDigest(connectionID)
-	if err != nil {
-		return err
-	}
-
-	config.TransportLogger.Debug("initiating RPC stream to reconnect")
-	rpcClient, err := newTunnelRPCClient(ctx, muxer, config, reconnect)
-	if err != nil {
-		return err
-	}
-	defer rpcClient.Close()
-	// Request server info without blocking tunnel registration; must use capnp library directly.
-	serverInfoPromise := tunnelrpc.TunnelServer{Client: rpcClient.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
-		return nil
-	})
-	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger, config.TunnelEventChan)
-	registration := rpcClient.ReconnectTunnel(
-		ctx,
-		token,
-		eventDigest,
-		connDigest,
-		config.Hostname,
-		config.RegistrationOptions(connectionID, originLocalAddr, uuid),
-	)
-	if registrationErr := registration.DeserializeError(); registrationErr != nil {
-		// ReconnectTunnel RPC failure
-		return processRegisterTunnelError(registrationErr, config.Metrics, reconnect)
-	}
-	return processRegistrationSuccess(config, logger, connectionID, registration, reconnect, credentialManager)
-}

origin/supervisor.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/google/uuid"
 
-	"github.com/cloudflare/cloudflared/buffer"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/edgediscovery"
 	"github.com/cloudflare/cloudflared/h2mux"
@@ -18,10 +17,10 @@ import (
 )
 
 const (
+	// SRV and TXT record resolution TTL
+	ResolveTTL = time.Hour
 	// Waiting time before retrying a failed tunnel connection
 	tunnelRetryDuration = time.Second * 10
-	// SRV record resolution TTL
-	resolveTTL = time.Hour
 	// Interval between registering new tunnels
 	registrationInterval = time.Second
 
@@ -44,8 +43,6 @@ type Supervisor struct {
 	cloudflaredUUID   uuid.UUID
 	config            *TunnelConfig
 	edgeIPs           *edgediscovery.Edge
-	lastResolve       time.Time
-	resolverC         chan resolveResult
 	tunnelErrors      chan tunnelError
 	tunnelsConnecting map[int]chan struct{}
 	// nextConnectedIndex and nextConnectedSignal are used to wait for all
@@ -56,12 +53,7 @@ type Supervisor struct {
 	logger logger.Service
 
 	reconnectCredentialManager *reconnectCredentialManager
-
-	bufferPool *buffer.Pool
-}
-
-type resolveResult struct {
-	err error
+	useReconnectToken          bool
 }
 
 type tunnelError struct {
@@ -84,6 +76,11 @@ func NewSupervisor(config *TunnelConfig, cloudflaredUUID uuid.UUID) (*Supervisor
 		return nil, err
 	}
 
+	useReconnectToken := false
+	if config.ClassicTunnel != nil {
+		useReconnectToken = config.ClassicTunnel.UseReconnectToken
+	}
+
 	return &Supervisor{
 		cloudflaredUUID:            cloudflaredUUID,
 		config:                     config,
@@ -91,13 +88,12 @@ func NewSupervisor(config *TunnelConfig, cloudflaredUUID uuid.UUID) (*Supervisor
 		tunnelErrors:               make(chan tunnelError),
 		tunnelsConnecting:          map[int]chan struct{}{},
 		logger:                     config.Logger,
-		reconnectCredentialManager: newReconnectCredentialManager(metricsNamespace, tunnelSubsystem, config.HAConnections),
-		bufferPool:                 buffer.NewPool(512 * 1024),
+		reconnectCredentialManager: newReconnectCredentialManager(connection.MetricsNamespace, connection.TunnelSubsystem, config.HAConnections),
+		useReconnectToken:          useReconnectToken,
 	}, nil
 }
 
 func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) error {
-	logger := s.config.Logger
 	if err := s.initialize(ctx, connectedSignal, reconnectCh); err != nil {
 		return err
 	}
@@ -110,11 +106,11 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 	refreshAuthBackoff := &BackoffHandler{MaxRetries: refreshAuthMaxBackoff, BaseTime: refreshAuthRetryDuration, RetryForever: true}
 	var refreshAuthBackoffTimer <-chan time.Time
 
-	if s.config.UseReconnectToken {
+	if s.useReconnectToken {
 		if timer, err := s.reconnectCredentialManager.RefreshAuth(ctx, refreshAuthBackoff, s.authenticate); err == nil {
 			refreshAuthBackoffTimer = timer
 		} else {
-			logger.Errorf("supervisor: initial refreshAuth failed, retrying in %v: %s", refreshAuthRetryDuration, err)
+			s.logger.Errorf("supervisor: initial refreshAuth failed, retrying in %v: %s", refreshAuthRetryDuration, err)
 			refreshAuthBackoffTimer = time.After(refreshAuthRetryDuration)
 		}
 	}
@@ -133,7 +129,7 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 		case tunnelError := <-s.tunnelErrors:
 			tunnelsActive--
 			if tunnelError.err != nil {
-				logger.Infof("supervisor: Tunnel disconnected due to error: %s", tunnelError.err)
+				s.logger.Infof("supervisor: Tunnel disconnected due to error: %s", tunnelError.err)
 				tunnelsWaiting = append(tunnelsWaiting, tunnelError.index)
 				s.waitForNextTunnel(tunnelError.index)
 
@@ -156,7 +152,7 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 		case <-refreshAuthBackoffTimer:
 			newTimer, err := s.reconnectCredentialManager.RefreshAuth(ctx, refreshAuthBackoff, s.authenticate)
 			if err != nil {
-				logger.Errorf("supervisor: Authentication failed: %s", err)
+				s.logger.Errorf("supervisor: Authentication failed: %s", err)
 				// Permanent failure. Leave the `select` without setting the
 				// channel to be non-null, so we'll never hit this case of the `select` again.
 				continue
@@ -168,27 +164,15 @@ func (s *Supervisor) Run(ctx context.Context, connectedSignal *signal.Signal, re
 				// No more tunnels outstanding, clear backoff timer
 				backoff.SetGracePeriod()
 			}
-		// DNS resolution returned
-		case result := <-s.resolverC:
-			s.lastResolve = time.Now()
-			s.resolverC = nil
-			if result.err == nil {
-				logger.Debug("supervisor: Service discovery refresh complete")
-			} else {
-				logger.Errorf("supervisor: Service discovery error: %s", result.err)
-			}
 		}
 	}
 }
 
 // Returns nil if initialization succeeded, else the initialization error.
 func (s *Supervisor) initialize(ctx context.Context, connectedSignal *signal.Signal, reconnectCh chan ReconnectSignal) error {
-	logger := s.logger
-
-	s.lastResolve = time.Now()
 	availableAddrs := int(s.edgeIPs.AvailableAddrs())
 	if s.config.HAConnections > availableAddrs {
-		logger.Infof("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
+		s.logger.Infof("You requested %d HA connections but I can give you at most %d.", s.config.HAConnections, availableAddrs)
 		s.config.HAConnections = availableAddrs
 	}
 
@@ -227,7 +211,7 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 		return
 	}
 
-	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
+	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, reconnectCh)
 	// If the first tunnel disconnects, keep restarting it.
 	edgeErrors := 0
 	for s.unusedIPs() {
@@ -239,7 +223,7 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 			return
 		// try the next address if it was a dialError(network problem) or
 		// dupConnRegisterTunnelError
-		case connection.DialError, dupConnRegisterTunnelError:
+		case edgediscovery.DialError, connection.DupConnRegisterTunnelError:
 			edgeErrors++
 		default:
 			return
@@ -250,7 +234,7 @@ func (s *Supervisor) startFirstTunnel(ctx context.Context, connectedSignal *sign
 				return
 			}
 		}
-		err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
+		err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, firstConnIndex, connectedSignal, s.cloudflaredUUID, reconnectCh)
 	}
 }
 
@@ -269,7 +253,7 @@ func (s *Supervisor) startTunnel(ctx context.Context, index int, connectedSignal
 	if err != nil {
 		return
 	}
-	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, uint8(index), connectedSignal, s.cloudflaredUUID, s.bufferPool, reconnectCh)
+	err = ServeTunnelLoop(ctx, s.reconnectCredentialManager, s.config, addr, uint8(index), connectedSignal, s.cloudflaredUUID, reconnectCh)
 }
 
 func (s *Supervisor) newConnectedTunnelSignal(index int) *signal.Signal {
@@ -301,7 +285,7 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		return nil, err
 	}
 
-	edgeConn, err := connection.DialEdge(ctx, dialTimeout, s.config.TlsConfig, arbitraryEdgeIP)
+	edgeConn, err := edgediscovery.DialEdge(ctx, dialTimeout, s.config.EdgeTLSConfigs[connection.H2mux], arbitraryEdgeIP)
 	if err != nil {
 		return nil, err
 	}
@@ -311,8 +295,8 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		// This callback is invoked by h2mux when the edge initiates a stream.
 		return nil // noop
 	})
-	muxerConfig := s.config.muxerConfig(handler)
-	muxer, err := h2mux.Handshake(edgeConn, edgeConn, muxerConfig, s.config.Metrics.activeStreams)
+	muxerConfig := s.config.MuxerConfig.H2MuxerConfig(handler, s.logger)
+	muxer, err := h2mux.Handshake(edgeConn, edgeConn, *muxerConfig, h2mux.ActiveStreams)
 	if err != nil {
 		return nil, err
 	}
@@ -323,23 +307,15 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		<-muxer.Shutdown()
 	}()
 
-	rpcClient, err := newTunnelRPCClient(ctx, muxer, s.config, authenticate)
+	stream, err := muxer.OpenRPCStream(ctx)
 	if err != nil {
 		return nil, err
 	}
+	rpcClient := connection.NewTunnelServerClient(ctx, stream, s.logger)
 	defer rpcClient.Close()
 
 	const arbitraryConnectionID = uint8(0)
 	registrationOptions := s.config.RegistrationOptions(arbitraryConnectionID, edgeConn.LocalAddr().String(), s.cloudflaredUUID)
 	registrationOptions.NumPreviousAttempts = uint8(numPreviousAttempts)
-	authResponse, err := rpcClient.Authenticate(
-		ctx,
-		s.config.OriginCert,
-		s.config.Hostname,
-		registrationOptions,
-	)
-	if err != nil {
-		return nil, err
-	}
-	return authResponse.Outcome(), nil
+	return rpcClient.Authenticate(ctx, s.config.ClassicTunnel, registrationOptions)
 }

origin/tunnel_test.go

@@ -0,0 +1,90 @@
+package origin
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/cloudflare/cloudflared/connection"
+	"github.com/cloudflare/cloudflared/logger"
+	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
+	"github.com/stretchr/testify/assert"
+)
+
+type dynamicMockFetcher struct {
+	percentage int32
+	err        error
+}
+
+func (dmf *dynamicMockFetcher) fetch() connection.PercentageFetcher {
+	return func() (int32, error) {
+		if dmf.err != nil {
+			return 0, dmf.err
+		}
+		return dmf.percentage, nil
+	}
+}
+func TestWaitForBackoffFallback(t *testing.T) {
+	maxRetries := uint(3)
+	backoff := BackoffHandler{
+		MaxRetries: maxRetries,
+		BaseTime:   time.Millisecond * 10,
+	}
+	ctx := context.Background()
+	logger, err := logger.New()
+	assert.NoError(t, err)
+	resolveTTL := time.Duration(0)
+	namedTunnel := &connection.NamedTunnelConfig{
+		Auth: pogs.TunnelAuth{
+			AccountTag: "test-account",
+		},
+	}
+	mockFetcher := dynamicMockFetcher{
+		percentage: 0,
+	}
+	protocolSelector, err := connection.NewProtocolSelector(connection.HTTP2.String(), namedTunnel, mockFetcher.fetch(), resolveTTL, logger)
+	assert.NoError(t, err)
+	config := &TunnelConfig{
+		Logger:           logger,
+		ProtocolSelector: protocolSelector,
+	}
+	connIndex := uint8(1)
+
+	initProtocol := protocolSelector.Current()
+	assert.Equal(t, connection.HTTP2, initProtocol)
+
+	protocallFallback := &protocallFallback{
+		backoff,
+		initProtocol,
+		false,
+	}
+
+	// Retry #0 and #1. At retry #2, we switch protocol, so the fallback loop has one more retry than this
+	for i := 0; i < int(maxRetries-1); i++ {
+		err := waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("Some error"))
+		assert.NoError(t, err)
+		assert.Equal(t, initProtocol, protocallFallback.protocol)
+	}
+
+	// Retry fallback protocol
+	for i := 0; i < int(maxRetries); i++ {
+		err := waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("Some error"))
+		assert.NoError(t, err)
+		fallback, ok := protocolSelector.Fallback()
+		assert.True(t, ok)
+		assert.Equal(t, fallback, protocallFallback.protocol)
+	}
+
+	currentGlobalProtocol := protocolSelector.Current()
+	assert.Equal(t, initProtocol, currentGlobalProtocol)
+
+	// No protocol to fallback, return error
+	err = waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("Some error"))
+	assert.Error(t, err)
+
+	protocallFallback.reset()
+	err = waitForBackoff(ctx, protocallFallback, config, connIndex, fmt.Errorf("New error"))
+	assert.NoError(t, err)
+	assert.Equal(t, initProtocol, protocallFallback.protocol)
+}

tlsconfig/certreloader.go

@@ -17,8 +17,6 @@ import (
 const (
 	OriginCAPoolFlag = "origin-ca-pool"
 	CaCertFlag       = "cacert"
-
-	edgeTLSServerName = "cftunnel.com"
 )
 
 // CertReloader can load and reload a TLS certificate from a particular filepath.
@@ -120,13 +118,13 @@ func LoadCustomOriginCA(originCAFilename string) (*x509.CertPool, error) {
 	return certPool, nil
 }
 
-func CreateTunnelConfig(c *cli.Context) (*tls.Config, error) {
+func CreateTunnelConfig(c *cli.Context, serverName string) (*tls.Config, error) {
 	var rootCAs []string
 	if c.String(CaCertFlag) != "" {
 		rootCAs = append(rootCAs, c.String(CaCertFlag))
 	}
 
-	userConfig := &TLSParameters{RootCAs: rootCAs, ServerName: edgeTLSServerName}
+	userConfig := &TLSParameters{RootCAs: rootCAs, ServerName: serverName}
 	tlsConfig, err := GetConfig(userConfig)
 	if err != nil {
 		return nil, err

tunnelrpc/pogs/connectionrpc.go

@@ -93,6 +93,11 @@ type RegistrationServer_PogsClient struct {
 	Conn   *rpc.Conn
 }
 
+func (c RegistrationServer_PogsClient) Close() error {
+	c.Client.Close()
+	return c.Conn.Close()
+}
+
 func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error) {
 	client := tunnelrpc.TunnelServer{Client: c.Client}
 	promise := client.RegisterConnection(ctx, func(p tunnelrpc.RegistrationServer_registerConnection_Params) error {

validation/validation.go

@@ -66,7 +66,15 @@ func ValidateHostname(hostname string) (string, error) {
 // but when it does not, the path is preserved:
 //   ValidateUrl("localhost:8080/api/") => "http://localhost:8080/api/"
 // This is arguably a bug, but changing it might break some cloudflared users.
-func ValidateUrl(originUrl string) (string, error) {
+func ValidateUrl(originUrl string) (*url.URL, error) {
+	urlStr, err := validateUrlString(originUrl)
+	if err != nil {
+		return nil, err
+	}
+	return url.Parse(urlStr)
+}
+
+func validateUrlString(originUrl string) (string, error) {
 	if originUrl == "" {
 		return "", fmt.Errorf("URL should not be empty")
 	}
@@ -157,6 +165,7 @@ func validateIP(scheme, host, port string) (string, error) {
 	return fmt.Sprintf("%s://%s", scheme, host), nil
 }
 
+// originURL shouldn't be a pointer, because this function might change the scheme
 func ValidateHTTPService(originURL string, hostname string, transport http.RoundTripper) error {
 	parsedURL, err := url.Parse(originURL)
 	if err != nil {
@@ -176,28 +185,32 @@ func ValidateHTTPService(originURL string, hostname string, transport http.Round
 		return err
 	}
 	initialRequest.Host = hostname
-	_, initialErr := client.Do(initialRequest)
-	if initialErr != nil {
-		// Attempt the same endpoint via the other protocol (http/https); maybe we have better luck?
-		oldScheme := parsedURL.Scheme
-		parsedURL.Scheme = toggleProtocol(parsedURL.Scheme)
+	resp, initialErr := client.Do(initialRequest)
+	if initialErr == nil {
+		resp.Body.Close()
+		return nil
+	}
 
-		secondRequest, err := http.NewRequest("GET", parsedURL.String(), nil)
-		if err != nil {
-			return err
-		}
-		secondRequest.Host = hostname
-		_, secondErr := client.Do(secondRequest)
-		if secondErr == nil { // Worked this time--advise the user to switch protocols
-			return errors.Errorf(
-				"%s doesn't seem to work over %s, but does seem to work over %s. Reason: %v. Consider changing the origin URL to %s",
-				parsedURL.Host,
-				oldScheme,
-				parsedURL.Scheme,
-				initialErr,
-				parsedURL,
-			)
-		}
+	// Attempt the same endpoint via the other protocol (http/https); maybe we have better luck?
+	oldScheme := parsedURL.Scheme
+	parsedURL.Scheme = toggleProtocol(oldScheme)
+
+	secondRequest, err := http.NewRequest("GET", parsedURL.String(), nil)
+	if err != nil {
+		return err
+	}
+	secondRequest.Host = hostname
+	resp, secondErr := client.Do(secondRequest)
+	if secondErr == nil { // Worked this time--advise the user to switch protocols
+		resp.Body.Close()
+		return errors.Errorf(
+			"%s doesn't seem to work over %s, but does seem to work over %s. Reason: %v. Consider changing the origin URL to %v",
+			parsedURL.Host,
+			oldScheme,
+			parsedURL.Scheme,
+			initialErr,
+			originURL,
+		)
 	}
 
 	return initialErr
@@ -220,12 +233,12 @@ type Access struct {
 }
 
 func NewAccessValidator(ctx context.Context, domain, issuer, applicationAUD string) (*Access, error) {
-	domainURL, err := ValidateUrl(domain)
+	domainURL, err := validateUrlString(domain)
 	if err != nil {
 		return nil, err
 	}
 
-	issuerURL, err := ValidateUrl(issuer)
+	issuerURL, err := validateUrlString(issuer)
 	if err != nil {
 		return nil, err
 	}

validation/validation_test.go

@@ -101,7 +101,7 @@ func TestValidateUrl(t *testing.T) {
 	for i, testCase := range testCases {
 		validUrl, err := ValidateUrl(testCase.input)
 		assert.NoError(t, err, "test case %v", i)
-		assert.Equal(t, testCase.expectedOutput, validUrl, "test case %v", i)
+		assert.Equal(t, testCase.expectedOutput, validUrl.String(), "test case %v", i)
 	}
 
 	validUrl, err := ValidateUrl("")
@@ -151,7 +151,7 @@ func TestValidateHTTPService_HTTP2HTTP(t *testing.T) {
 
 // Happy path 2: originURL is HTTPS, and HTTPS connections work
 func TestValidateHTTPService_HTTPS2HTTPS(t *testing.T) {
-	originURL := "https://127.0.0.1/"
+	originURL := "https://127.0.0.1:1234/"
 	hostname := "example.com"
 
 	assert.Nil(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {

vendor/github.com/equinox-io/equinox/.travis.yml

@@ -1,13 +0,0 @@
-language: go
-go:
-  - "1.4"
-  - "1.5"
-  - "1.7"
-  - "1.9"
-  - "1.10"
-  - "1.11"
-  - "tip"
-matrix:
-  allow_failures:
-    - go: tip
-script: go test -v github.com/equinox-io/equinox github.com/equinox-io/equinox/proto

vendor/github.com/equinox-io/equinox/README.md

@@ -1,99 +0,0 @@
-# equinox client SDK [![godoc reference](https://godoc.org/github.com/equinox-io/equinox?status.png)](https://godoc.org/github.com/equinox-io/equinox)
-
-Package equinox allows applications to remotely update themselves with the [equinox.io](https://equinox.io) service.
-
-## Minimal Working Example
-
-```go
-import "github.com/equinox-io/equinox"
-
-const appID = "<YOUR EQUINOX APP ID>"
-
-var publicKey = []byte(`
------BEGIN PUBLIC KEY-----
-MFYwEAYHKoZIzj0CAQYFK4EEAAoDQgAEtrVmBxQvheRArXjg2vG1xIprWGuCyESx
-MMY8pjmjepSy2kuz+nl9aFLqmr+rDNdYvEBqQaZrYMc6k29gjvoQnQ==
------END PUBLIC KEY-----
-`)
-
-func update(channel string) error {
-    opts := equinox.Options{Channel: channel}
-    if err := opts.SetPublicKeyPEM(publicKey); err != nil {
-        return err
-    }
-
-    // check for the update
-    resp, err := equinox.Check(appID, opts)
-    switch {
-    case err == equinox.NotAvailableErr:
-        fmt.Println("No update available, already at the latest version!")
-        return nil
-    case err != nil:
-        return err
-    }
-
-    // fetch the update and apply it
-    err = resp.Apply()
-    if err != nil {
-        return err
-    }
-
-    fmt.Printf("Updated to new version: %s!\n", resp.ReleaseVersion)
-    return nil
-}
-```
-
-
-## Update To Specific Version
-
-When you specify a channel in the update options, equinox will try to update the application
-to the latest release of your application published to that channel. Instead, you may wish to
-update the application to a specific (possibly older) version. You can do this by explicitly setting
-Version in the Options struct:
-
-```go
-opts := equinox.Options{Version: "0.1.2"}
-```
-
-## Prompt For Update
-
-You may wish to ask the user for approval before updating to a new version. This is as simple
-as calling the Check function and only calling Apply on the returned result if the user approves.
-Example:
-
-```go
-// check for the update
-resp, err := equinox.Check(appID, opts)
-switch {
-case err == equinox.NotAvailableErr:
-    fmt.Println("No update available, already at the latest version!")
-    return nil
-case err != nil:
-    return err
-}
-
-fmt.Println("New version available!")
-fmt.Println("Version:", resp.ReleaseVersion)
-fmt.Println("Name:", resp.ReleaseTitle)
-fmt.Println("Details:", resp.ReleaseDescription)
-
-ok := prompt("Would you like to update?")
-
-if !ok {
-    return
-}
-
-err = resp.Apply()
-// ...
-```
-
-## Generating Keys
-
-All equinox releases must be signed with a private ECDSA key, and all updates verified with the
-public key portion. To do that, you'll need to generate a key pair. The equinox release tool can
-generate an ecdsa key pair for you easily:
-
-```shell
-equinox genkey
-```
-

vendor/github.com/equinox-io/equinox/doc.go

@@ -1,92 +0,0 @@
-/*
-Package equinox allows applications to remotely update themselves with the equinox.io service.
-
-Minimal Working Example
-
-	import "github.com/equinox-io/equinox"
-
-	const appID = "<YOUR EQUINOX APP ID>"
-
-	var publicKey = []byte(`
-	-----BEGIN PUBLIC KEY-----
-	MFYwEAYHKoZIzj0CAQYFK4EEAAoDQgAEtrVmBxQvheRArXjg2vG1xIprWGuCyESx
-	MMY8pjmjepSy2kuz+nl9aFLqmr+rDNdYvEBqQaZrYMc6k29gjvoQnQ==
-	-----END PUBLIC KEY-----
-	`)
-
-	func update(channel string) error {
-		opts := equinox.Options{Channel: channel}
-		if err := opts.SetPublicKeyPEM(publicKey); err != nil {
-			return err
-		}
-
-		// check for the update
-		resp, err := equinox.Check(appID, opts)
-		switch {
-		case err == equinox.NotAvailableErr:
-			fmt.Println("No update available, already at the latest version!")
-			return nil
-		case err != nil:
-			return err
-		}
-
-		// fetch the update and apply it
-		err = resp.Apply()
-		if err != nil {
-			return err
-		}
-
-		fmt.Printf("Updated to new version: %s!\n", resp.ReleaseVersion)
-		return nil
-	}
-
-
-Update To Specific Version
-
-When you specify a channel in the update options, equinox will try to update the application
-to the latest release of your application published to that channel. Instead, you may wish to
-update the application to a specific (possibly older) version. You can do this by explicitly setting
-Version in the Options struct:
-
-		opts := equinox.Options{Version: "0.1.2"}
-
-Prompt For Update
-
-You may wish to ask the user for approval before updating to a new version. This is as simple
-as calling the Check function and only calling Apply on the returned result if the user approves.
-Example:
-
-	// check for the update
-	resp, err := equinox.Check(appID, opts)
-	switch {
-	case err == equinox.NotAvailableErr:
-		fmt.Println("No update available, already at the latest version!")
-		return nil
-	case err != nil:
-		return err
-	}
-
-	fmt.Println("New version available!")
-	fmt.Println("Version:", resp.ReleaseVersion)
-	fmt.Println("Name:", resp.ReleaseTitle)
-	fmt.Println("Details:", resp.ReleaseDescription)
-
-	ok := prompt("Would you like to update?")
-
-	if !ok {
-		return
-	}
-
-	err = resp.Apply()
-	// ...
-
-Generating Keys
-
-All equinox releases must be signed with a private ECDSA key, and all updates verified with the
-public key portion. To do that, you'll need to generate a key pair. The equinox release tool can
-generate an ecdsa key pair for you easily:
-
-	equinox genkey
-
-*/
-package equinox

vendor/github.com/equinox-io/equinox/go.mod

@@ -1 +0,0 @@
-module github.com/equinox-io/equinox

vendor/github.com/equinox-io/equinox/internal/go-update/LICENSE

@@ -1,13 +0,0 @@
-Copyright 2015 Alan Shreve
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.

vendor/github.com/equinox-io/equinox/internal/go-update/README.md

@@ -1,65 +0,0 @@
-# go-update: Build self-updating Go programs [![godoc reference](https://godoc.org/github.com/inconshreveable/go-update?status.png)](https://godoc.org/github.com/inconshreveable/go-update)
-
-Package update provides functionality to implement secure, self-updating Go programs (or other single-file targets)
-A program can update itself by replacing its executable file with a new version.
-
-It provides the flexibility to implement different updating user experiences
-like auto-updating, or manual user-initiated updates. It also boasts
-advanced features like binary patching and code signing verification.
-
-Example of updating from a URL:
-
-```go
-import (
-    "fmt"
-    "net/http"
-
-    "github.com/inconshreveable/go-update"
-)
-
-func doUpdate(url string) error {
-    resp, err := http.Get(url)
-    if err != nil {
-        return err
-    }
-    defer resp.Body.Close()
-    err := update.Apply(resp.Body, update.Options{})
-    if err != nil {
-        // error handling
-    }
-    return err
-}
-```
-
-## Features
-
-- Cross platform support (Windows too!)
-- Binary patch application
-- Checksum verification
-- Code signing verification
-- Support for updating arbitrary files
-
-## [equinox.io](https://equinox.io)
-[equinox.io](https://equinox.io) is a complete ready-to-go updating solution built on top of go-update that provides:
-
-- Hosted updates
-- Update channels (stable, beta, nightly, ...)
-- Dynamically computed binary diffs
-- Automatic key generation and code
-- Release tooling with proper code signing
-- Update/download metrics
-
-## API Compatibility Promises
-The master branch of `go-update` is *not* guaranteed to have a stable API over time. For any production application, you should vendor
-your dependency on `go-update` with a tool like git submodules, [gb](http://getgb.io/) or [govendor](https://github.com/kardianos/govendor).
-
-The `go-update` package makes the following promises about API compatibility:
-1. A list of all API-breaking changes will be documented in this README.
-1. `go-update` will strive for as few API-breaking changes as possible.
-
-## API Breaking Changes
-- **Sept 3, 2015**: The `Options` struct passed to `Apply` was changed to be passed by value instead of passed by pointer. Old API at `28de026`.
-- **Aug 9, 2015**: 2.0 API. Old API at `221d034` or `gopkg.in/inconshreveable/go-update.v0`.
-
-## License
-Apache

vendor/github.com/equinox-io/equinox/internal/go-update/apply.go

@@ -1,322 +0,0 @@
-package update
-
-import (
-	"bytes"
-	"crypto"
-	"crypto/x509"
-	"encoding/pem"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path/filepath"
-
-	"github.com/equinox-io/equinox/internal/go-update/internal/osext"
-)
-
-var (
-	openFile = os.OpenFile
-)
-
-// Apply performs an update of the current executable (or opts.TargetFile, if set) with the contents of the given io.Reader.
-//
-// Apply performs the following actions to ensure a safe cross-platform update:
-//
-// 1. If configured, applies the contents of the update io.Reader as a binary patch.
-//
-// 2. If configured, computes the checksum of the new executable and verifies it matches.
-//
-// 3. If configured, verifies the signature with a public key.
-//
-// 4. Creates a new file, /path/to/.target.new with the TargetMode with the contents of the updated file
-//
-// 5. Renames /path/to/target to /path/to/.target.old
-//
-// 6. Renames /path/to/.target.new to /path/to/target
-//
-// 7. If the final rename is successful, deletes /path/to/.target.old, returns no error. On Windows,
-// the removal of /path/to/target.old always fails, so instead Apply hides the old file instead.
-//
-// 8. If the final rename fails, attempts to roll back by renaming /path/to/.target.old
-// back to /path/to/target.
-//
-// If the roll back operation fails, the file system is left in an inconsistent state (betweet steps 5 and 6) where
-// there is no new executable file and the old executable file could not be be moved to its original location. In this
-// case you should notify the user of the bad news and ask them to recover manually. Applications can determine whether
-// the rollback failed by calling RollbackError, see the documentation on that function for additional detail.
-func Apply(update io.Reader, opts Options) error {
-	// validate
-	verify := false
-	switch {
-	case opts.Signature != nil && opts.PublicKey != nil:
-		// okay
-		verify = true
-	case opts.Signature != nil:
-		return errors.New("no public key to verify signature with")
-	case opts.PublicKey != nil:
-		return errors.New("No signature to verify with")
-	}
-
-	// set defaults
-	if opts.Hash == 0 {
-		opts.Hash = crypto.SHA256
-	}
-	if opts.Verifier == nil {
-		opts.Verifier = NewECDSAVerifier()
-	}
-	if opts.TargetMode == 0 {
-		opts.TargetMode = 0755
-	}
-
-	// get target path
-	var err error
-	opts.TargetPath, err = opts.getPath()
-	if err != nil {
-		return err
-	}
-
-	var newBytes []byte
-	if opts.Patcher != nil {
-		if newBytes, err = opts.applyPatch(update); err != nil {
-			return err
-		}
-	} else {
-		// no patch to apply, go on through
-		if newBytes, err = ioutil.ReadAll(update); err != nil {
-			return err
-		}
-	}
-
-	// verify checksum if requested
-	if opts.Checksum != nil {
-		if err = opts.verifyChecksum(newBytes); err != nil {
-			return err
-		}
-	}
-
-	if verify {
-		if err = opts.verifySignature(newBytes); err != nil {
-			return err
-		}
-	}
-
-	// get the directory the executable exists in
-	updateDir := filepath.Dir(opts.TargetPath)
-	filename := filepath.Base(opts.TargetPath)
-
-	// Copy the contents of newbinary to a new executable file
-	newPath := filepath.Join(updateDir, fmt.Sprintf(".%s.new", filename))
-	fp, err := openFile(newPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, opts.TargetMode)
-	if err != nil {
-		return err
-	}
-	defer fp.Close()
-
-	_, err = io.Copy(fp, bytes.NewReader(newBytes))
-	if err != nil {
-		return err
-	}
-
-	// if we don't call fp.Close(), windows won't let us move the new executable
-	// because the file will still be "in use"
-	fp.Close()
-
-	// this is where we'll move the executable to so that we can swap in the updated replacement
-	oldPath := opts.OldSavePath
-	removeOld := opts.OldSavePath == ""
-	if removeOld {
-		oldPath = filepath.Join(updateDir, fmt.Sprintf(".%s.old", filename))
-	}
-
-	// delete any existing old exec file - this is necessary on Windows for two reasons:
-	// 1. after a successful update, Windows can't remove the .old file because the process is still running
-	// 2. windows rename operations fail if the destination file already exists
-	_ = os.Remove(oldPath)
-
-	// move the existing executable to a new file in the same directory
-	err = os.Rename(opts.TargetPath, oldPath)
-	if err != nil {
-		return err
-	}
-
-	// move the new exectuable in to become the new program
-	err = os.Rename(newPath, opts.TargetPath)
-
-	if err != nil {
-		// move unsuccessful
-		//
-		// The filesystem is now in a bad state. We have successfully
-		// moved the existing binary to a new location, but we couldn't move the new
-		// binary to take its place. That means there is no file where the current executable binary
-		// used to be!
-		// Try to rollback by restoring the old binary to its original path.
-		rerr := os.Rename(oldPath, opts.TargetPath)
-		if rerr != nil {
-			return &rollbackErr{err, rerr}
-		}
-
-		return err
-	}
-
-	// move successful, remove the old binary if needed
-	if removeOld {
-		errRemove := os.Remove(oldPath)
-
-		// windows has trouble with removing old binaries, so hide it instead
-		if errRemove != nil {
-			_ = hideFile(oldPath)
-		}
-	}
-
-	return nil
-}
-
-// RollbackError takes an error value returned by Apply and returns the error, if any,
-// that occurred when attempting to roll back from a failed update. Applications should
-// always call this function on any non-nil errors returned by Apply.
-//
-// If no rollback was needed or if the rollback was successful, RollbackError returns nil,
-// otherwise it returns the error encountered when trying to roll back.
-func RollbackError(err error) error {
-	if err == nil {
-		return nil
-	}
-	if rerr, ok := err.(*rollbackErr); ok {
-		return rerr.rollbackErr
-	}
-	return nil
-}
-
-type rollbackErr struct {
-	error             // original error
-	rollbackErr error // error encountered while rolling back
-}
-
-type Options struct {
-	// TargetPath defines the path to the file to update.
-	// The emptry string means 'the executable file of the running program'.
-	TargetPath string
-
-	// Create TargetPath replacement with this file mode. If zero, defaults to 0755.
-	TargetMode os.FileMode
-
-	// Checksum of the new binary to verify against. If nil, no checksum or signature verification is done.
-	Checksum []byte
-
-	// Public key to use for signature verification. If nil, no signature verification is done.
-	PublicKey crypto.PublicKey
-
-	// Signature to verify the updated file. If nil, no signature verification is done.
-	Signature []byte
-
-	// Pluggable signature verification algorithm. If nil, ECDSA is used.
-	Verifier Verifier
-
-	// Use this hash function to generate the checksum. If not set, SHA256 is used.
-	Hash crypto.Hash
-
-	// If nil, treat the update as a complete replacement for the contents of the file at TargetPath.
-	// If non-nil, treat the update contents as a patch and use this object to apply the patch.
-	Patcher Patcher
-
-	// Store the old executable file at this path after a successful update.
-	// The empty string means the old executable file will be removed after the update.
-	OldSavePath string
-}
-
-// CheckPermissions determines whether the process has the correct permissions to
-// perform the requested update. If the update can proceed, it returns nil, otherwise
-// it returns the error that would occur if an update were attempted.
-func (o *Options) CheckPermissions() error {
-	// get the directory the file exists in
-	path, err := o.getPath()
-	if err != nil {
-		return err
-	}
-
-	fileDir := filepath.Dir(path)
-	fileName := filepath.Base(path)
-
-	// attempt to open a file in the file's directory
-	newPath := filepath.Join(fileDir, fmt.Sprintf(".%s.new", fileName))
-	fp, err := openFile(newPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, o.TargetMode)
-	if err != nil {
-		return err
-	}
-	fp.Close()
-
-	_ = os.Remove(newPath)
-	return nil
-}
-
-// SetPublicKeyPEM is a convenience method to set the PublicKey property
-// used for checking a completed update's signature by parsing a
-// Public Key formatted as PEM data.
-func (o *Options) SetPublicKeyPEM(pembytes []byte) error {
-	block, _ := pem.Decode(pembytes)
-	if block == nil {
-		return errors.New("couldn't parse PEM data")
-	}
-
-	pub, err := x509.ParsePKIXPublicKey(block.Bytes)
-	if err != nil {
-		return err
-	}
-	o.PublicKey = pub
-	return nil
-}
-
-func (o *Options) getPath() (string, error) {
-	if o.TargetPath == "" {
-		return osext.Executable()
-	} else {
-		return o.TargetPath, nil
-	}
-}
-
-func (o *Options) applyPatch(patch io.Reader) ([]byte, error) {
-	// open the file to patch
-	old, err := os.Open(o.TargetPath)
-	if err != nil {
-		return nil, err
-	}
-	defer old.Close()
-
-	// apply the patch
-	var applied bytes.Buffer
-	if err = o.Patcher.Patch(old, &applied, patch); err != nil {
-		return nil, err
-	}
-
-	return applied.Bytes(), nil
-}
-
-func (o *Options) verifyChecksum(updated []byte) error {
-	checksum, err := checksumFor(o.Hash, updated)
-	if err != nil {
-		return err
-	}
-
-	if !bytes.Equal(o.Checksum, checksum) {
-		return fmt.Errorf("Updated file has wrong checksum. Expected: %x, got: %x", o.Checksum, checksum)
-	}
-	return nil
-}
-
-func (o *Options) verifySignature(updated []byte) error {
-	checksum, err := checksumFor(o.Hash, updated)
-	if err != nil {
-		return err
-	}
-	return o.Verifier.VerifySignature(checksum, o.Signature, o.Hash, o.PublicKey)
-}
-
-func checksumFor(h crypto.Hash, payload []byte) ([]byte, error) {
-	if !h.Available() {
-		return nil, errors.New("requested hash function not available")
-	}
-	hash := h.New()
-	hash.Write(payload) // guaranteed not to error
-	return hash.Sum([]byte{}), nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/doc.go

@@ -1,172 +0,0 @@
-/*
-Package update provides functionality to implement secure, self-updating Go programs (or other single-file targets).
-
-For complete updating solutions please see Equinox (https://equinox.io) and go-tuf (https://github.com/flynn/go-tuf).
-
-Basic Example
-
-This example shows how to update a program remotely from a URL.
-
-	import (
-		"fmt"
-		"net/http"
-
-		"github.com/inconshreveable/go-update"
-	)
-
-	func doUpdate(url string) error {
-		// request the new file
-		resp, err := http.Get(url)
-		if err != nil {
-			return err
-		}
-		defer resp.Body.Close()
-		err := update.Apply(resp.Body, update.Options{})
-		if err != nil {
-			if rerr := update.RollbackError(err); rerr != nil {
-				fmt.Println("Failed to rollback from bad update: %v", rerr)
-			}
-		}
-		return err
-	}
-
-
-Binary Patching
-
-Go binaries can often be large. It can be advantageous to only ship a binary patch to a client
-instead of the complete program text of a new version.
-
-This example shows how to update a program with a bsdiff binary patch. Other patch formats
-may be applied by implementing the Patcher interface.
-
-	import (
-		"encoding/hex"
-		"io"
-
-		"github.com/inconshreveable/go-update"
-	)
-
-	func updateWithPatch(patch io.Reader) error {
-		err := update.Apply(patch, update.Options{
-			Patcher: update.NewBSDiffPatcher()
-		})
-		if err != nil {
-			// error handling
-		}
-		return err
-	}
-
-Checksum Verification
-
-Updating executable code on a computer can be a dangerous operation unless you
-take the appropriate steps to guarantee the authenticity of the new code. While
-checksum verification is important, it should always be combined with signature
-verification (next section) to guarantee that the code came from a trusted party.
-
-go-update validates SHA256 checksums by default, but this is pluggable via the Hash
-property on the Options struct.
-
-This example shows how to guarantee that the newly-updated binary is verified to
-have an appropriate checksum (that was otherwise retrived via a secure channel)
-specified as a hex string.
-
-	import (
-		"crypto"
-		_ "crypto/sha256"
-		"encoding/hex"
-		"io"
-
-		"github.com/inconshreveable/go-update"
-	)
-
-	func updateWithChecksum(binary io.Reader, hexChecksum string) error {
-		checksum, err := hex.DecodeString(hexChecksum)
-		if err != nil {
-			return err
-		}
-		err = update.Apply(binary, update.Options{
-			Hash: crypto.SHA256, 	// this is the default, you don't need to specify it
-			Checksum: checksum,
-		})
-		if err != nil {
-			// error handling
-		}
-		return err
-	}
-
-Cryptographic Signature Verification
-
-Cryptographic verification of new code from an update is an extremely important way to guarantee the
-security and integrity of your updates.
-
-Verification is performed by validating the signature of a hash of the new file. This
-means nothing changes if you apply your update with a patch.
-
-This example shows how to add signature verification to your updates. To make all of this work
-an application distributor must first create a public/private key pair and embed the public key
-into their application. When they issue a new release, the issuer must sign the new executable file
-with the private key and distribute the signature along with the update.
-
-	import (
-		"crypto"
-		_ "crypto/sha256"
-		"encoding/hex"
-		"io"
-
-		"github.com/inconshreveable/go-update"
-	)
-
-	var publicKey = []byte(`
-	-----BEGIN PUBLIC KEY-----
-	MFYwEAYHKoZIzj0CAQYFK4EEAAoDQgAEtrVmBxQvheRArXjg2vG1xIprWGuCyESx
-	MMY8pjmjepSy2kuz+nl9aFLqmr+rDNdYvEBqQaZrYMc6k29gjvoQnQ==
-	-----END PUBLIC KEY-----
-	`)
-
-	func verifiedUpdate(binary io.Reader, hexChecksum, hexSignature string) {
-		checksum, err := hex.DecodeString(hexChecksum)
-		if err != nil {
-			return err
-		}
-		signature, err := hex.DecodeString(hexSignature)
-		if err != nil {
-			return err
-		}
-		opts := update.Options{
-			Checksum: checksum,
-			Signature: signature,
-			Hash: crypto.SHA256, 	                 // this is the default, you don't need to specify it
-			Verifier: update.NewECDSAVerifier(),   // this is the default, you don't need to specify it
-		}
-		err = opts.SetPublicKeyPEM(publicKey)
-		if err != nil {
-			return err
-		}
-		err = update.Apply(binary, opts)
-		if err != nil {
-			// error handling
-		}
-		return err
-	}
-
-
-Building Single-File Go Binaries
-
-In order to update a Go application with go-update, you must distributed it as a single executable.
-This is often easy, but some applications require static assets (like HTML and CSS asset files or TLS certificates).
-In order to update applications like these, you'll want to make sure to embed those asset files into
-the distributed binary with a tool like go-bindata (my favorite): https://github.com/jteeuwen/go-bindata
-
-Non-Goals
-
-Mechanisms and protocols for determining whether an update should be applied and, if so, which one are
-out of scope for this package. Please consult go-tuf (https://github.com/flynn/go-tuf) or Equinox (https://equinox.io)
-for more complete solutions.
-
-go-update only works for self-updating applications that are distributed as a single binary, i.e.
-applications that do not have additional assets or dependency files.
-Updating application that are distributed as mutliple on-disk files is out of scope, although this
-may change in future versions of this library.
-
-*/
-package update

vendor/github.com/equinox-io/equinox/internal/go-update/hide_noop.go

@@ -1,7 +0,0 @@
-// +build !windows
-
-package update
-
-func hideFile(path string) error {
-	return nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/hide_windows.go

@@ -1,19 +0,0 @@
-package update
-
-import (
-	"syscall"
-	"unsafe"
-)
-
-func hideFile(path string) error {
-	kernel32 := syscall.NewLazyDLL("kernel32.dll")
-	setFileAttributes := kernel32.NewProc("SetFileAttributesW")
-
-	r1, _, err := setFileAttributes.Call(uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))), 2)
-
-	if r1 == 0 {
-		return err
-	} else {
-		return nil
-	}
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/License

@@ -1,22 +0,0 @@
-Copyright 2012 Keith Rarick
-
-Permission is hereby granted, free of charge, to any person
-obtaining a copy of this software and associated documentation
-files (the "Software"), to deal in the Software without
-restriction, including without limitation the rights to use,
-copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the
-Software is furnished to do so, subject to the following
-conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/Readme.md

@@ -1,7 +0,0 @@
-# binarydist
-
-Package binarydist implements binary diff and patch as described on
-<http://www.daemonology.net/bsdiff/>. It reads and writes files
-compatible with the tools there.
-
-Documentation at <http://go.pkgdoc.org/github.com/kr/binarydist>.

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/bzip2.go

@@ -1,40 +0,0 @@
-package binarydist
-
-import (
-	"io"
-	"os/exec"
-)
-
-type bzip2Writer struct {
-	c *exec.Cmd
-	w io.WriteCloser
-}
-
-func (w bzip2Writer) Write(b []byte) (int, error) {
-	return w.w.Write(b)
-}
-
-func (w bzip2Writer) Close() error {
-	if err := w.w.Close(); err != nil {
-		return err
-	}
-	return w.c.Wait()
-}
-
-// Package compress/bzip2 implements only decompression,
-// so we'll fake it by running bzip2 in another process.
-func newBzip2Writer(w io.Writer) (wc io.WriteCloser, err error) {
-	var bw bzip2Writer
-	bw.c = exec.Command("bzip2", "-c")
-	bw.c.Stdout = w
-
-	if bw.w, err = bw.c.StdinPipe(); err != nil {
-		return nil, err
-	}
-
-	if err = bw.c.Start(); err != nil {
-		return nil, err
-	}
-
-	return bw, nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/diff.go

@@ -1,408 +0,0 @@
-package binarydist
-
-import (
-	"bytes"
-	"encoding/binary"
-	"io"
-	"io/ioutil"
-)
-
-func swap(a []int, i, j int) { a[i], a[j] = a[j], a[i] }
-
-func split(I, V []int, start, length, h int) {
-	var i, j, k, x, jj, kk int
-
-	if length < 16 {
-		for k = start; k < start+length; k += j {
-			j = 1
-			x = V[I[k]+h]
-			for i = 1; k+i < start+length; i++ {
-				if V[I[k+i]+h] < x {
-					x = V[I[k+i]+h]
-					j = 0
-				}
-				if V[I[k+i]+h] == x {
-					swap(I, k+i, k+j)
-					j++
-				}
-			}
-			for i = 0; i < j; i++ {
-				V[I[k+i]] = k + j - 1
-			}
-			if j == 1 {
-				I[k] = -1
-			}
-		}
-		return
-	}
-
-	x = V[I[start+length/2]+h]
-	jj = 0
-	kk = 0
-	for i = start; i < start+length; i++ {
-		if V[I[i]+h] < x {
-			jj++
-		}
-		if V[I[i]+h] == x {
-			kk++
-		}
-	}
-	jj += start
-	kk += jj
-
-	i = start
-	j = 0
-	k = 0
-	for i < jj {
-		if V[I[i]+h] < x {
-			i++
-		} else if V[I[i]+h] == x {
-			swap(I, i, jj+j)
-			j++
-		} else {
-			swap(I, i, kk+k)
-			k++
-		}
-	}
-
-	for jj+j < kk {
-		if V[I[jj+j]+h] == x {
-			j++
-		} else {
-			swap(I, jj+j, kk+k)
-			k++
-		}
-	}
-
-	if jj > start {
-		split(I, V, start, jj-start, h)
-	}
-
-	for i = 0; i < kk-jj; i++ {
-		V[I[jj+i]] = kk - 1
-	}
-	if jj == kk-1 {
-		I[jj] = -1
-	}
-
-	if start+length > kk {
-		split(I, V, kk, start+length-kk, h)
-	}
-}
-
-func qsufsort(obuf []byte) []int {
-	var buckets [256]int
-	var i, h int
-	I := make([]int, len(obuf)+1)
-	V := make([]int, len(obuf)+1)
-
-	for _, c := range obuf {
-		buckets[c]++
-	}
-	for i = 1; i < 256; i++ {
-		buckets[i] += buckets[i-1]
-	}
-	copy(buckets[1:], buckets[:])
-	buckets[0] = 0
-
-	for i, c := range obuf {
-		buckets[c]++
-		I[buckets[c]] = i
-	}
-
-	I[0] = len(obuf)
-	for i, c := range obuf {
-		V[i] = buckets[c]
-	}
-
-	V[len(obuf)] = 0
-	for i = 1; i < 256; i++ {
-		if buckets[i] == buckets[i-1]+1 {
-			I[buckets[i]] = -1
-		}
-	}
-	I[0] = -1
-
-	for h = 1; I[0] != -(len(obuf) + 1); h += h {
-		var n int
-		for i = 0; i < len(obuf)+1; {
-			if I[i] < 0 {
-				n -= I[i]
-				i -= I[i]
-			} else {
-				if n != 0 {
-					I[i-n] = -n
-				}
-				n = V[I[i]] + 1 - i
-				split(I, V, i, n, h)
-				i += n
-				n = 0
-			}
-		}
-		if n != 0 {
-			I[i-n] = -n
-		}
-	}
-
-	for i = 0; i < len(obuf)+1; i++ {
-		I[V[i]] = i
-	}
-	return I
-}
-
-func matchlen(a, b []byte) (i int) {
-	for i < len(a) && i < len(b) && a[i] == b[i] {
-		i++
-	}
-	return i
-}
-
-func search(I []int, obuf, nbuf []byte, st, en int) (pos, n int) {
-	if en-st < 2 {
-		x := matchlen(obuf[I[st]:], nbuf)
-		y := matchlen(obuf[I[en]:], nbuf)
-
-		if x > y {
-			return I[st], x
-		} else {
-			return I[en], y
-		}
-	}
-
-	x := st + (en-st)/2
-	if bytes.Compare(obuf[I[x]:], nbuf) < 0 {
-		return search(I, obuf, nbuf, x, en)
-	} else {
-		return search(I, obuf, nbuf, st, x)
-	}
-	panic("unreached")
-}
-
-// Diff computes the difference between old and new, according to the bsdiff
-// algorithm, and writes the result to patch.
-func Diff(old, new io.Reader, patch io.Writer) error {
-	obuf, err := ioutil.ReadAll(old)
-	if err != nil {
-		return err
-	}
-
-	nbuf, err := ioutil.ReadAll(new)
-	if err != nil {
-		return err
-	}
-
-	pbuf, err := diffBytes(obuf, nbuf)
-	if err != nil {
-		return err
-	}
-
-	_, err = patch.Write(pbuf)
-	return err
-}
-
-func diffBytes(obuf, nbuf []byte) ([]byte, error) {
-	var patch seekBuffer
-	err := diff(obuf, nbuf, &patch)
-	if err != nil {
-		return nil, err
-	}
-	return patch.buf, nil
-}
-
-func diff(obuf, nbuf []byte, patch io.WriteSeeker) error {
-	var lenf int
-	I := qsufsort(obuf)
-	db := make([]byte, len(nbuf))
-	eb := make([]byte, len(nbuf))
-	var dblen, eblen int
-
-	var hdr header
-	hdr.Magic = magic
-	hdr.NewSize = int64(len(nbuf))
-	err := binary.Write(patch, signMagLittleEndian{}, &hdr)
-	if err != nil {
-		return err
-	}
-
-	// Compute the differences, writing ctrl as we go
-	pfbz2, err := newBzip2Writer(patch)
-	if err != nil {
-		return err
-	}
-	var scan, pos, length int
-	var lastscan, lastpos, lastoffset int
-	for scan < len(nbuf) {
-		var oldscore int
-		scan += length
-		for scsc := scan; scan < len(nbuf); scan++ {
-			pos, length = search(I, obuf, nbuf[scan:], 0, len(obuf))
-
-			for ; scsc < scan+length; scsc++ {
-				if scsc+lastoffset < len(obuf) &&
-					obuf[scsc+lastoffset] == nbuf[scsc] {
-					oldscore++
-				}
-			}
-
-			if (length == oldscore && length != 0) || length > oldscore+8 {
-				break
-			}
-
-			if scan+lastoffset < len(obuf) && obuf[scan+lastoffset] == nbuf[scan] {
-				oldscore--
-			}
-		}
-
-		if length != oldscore || scan == len(nbuf) {
-			var s, Sf int
-			lenf = 0
-			for i := 0; lastscan+i < scan && lastpos+i < len(obuf); {
-				if obuf[lastpos+i] == nbuf[lastscan+i] {
-					s++
-				}
-				i++
-				if s*2-i > Sf*2-lenf {
-					Sf = s
-					lenf = i
-				}
-			}
-
-			lenb := 0
-			if scan < len(nbuf) {
-				var s, Sb int
-				for i := 1; (scan >= lastscan+i) && (pos >= i); i++ {
-					if obuf[pos-i] == nbuf[scan-i] {
-						s++
-					}
-					if s*2-i > Sb*2-lenb {
-						Sb = s
-						lenb = i
-					}
-				}
-			}
-
-			if lastscan+lenf > scan-lenb {
-				overlap := (lastscan + lenf) - (scan - lenb)
-				s := 0
-				Ss := 0
-				lens := 0
-				for i := 0; i < overlap; i++ {
-					if nbuf[lastscan+lenf-overlap+i] == obuf[lastpos+lenf-overlap+i] {
-						s++
-					}
-					if nbuf[scan-lenb+i] == obuf[pos-lenb+i] {
-						s--
-					}
-					if s > Ss {
-						Ss = s
-						lens = i + 1
-					}
-				}
-
-				lenf += lens - overlap
-				lenb -= lens
-			}
-
-			for i := 0; i < lenf; i++ {
-				db[dblen+i] = nbuf[lastscan+i] - obuf[lastpos+i]
-			}
-			for i := 0; i < (scan-lenb)-(lastscan+lenf); i++ {
-				eb[eblen+i] = nbuf[lastscan+lenf+i]
-			}
-
-			dblen += lenf
-			eblen += (scan - lenb) - (lastscan + lenf)
-
-			err = binary.Write(pfbz2, signMagLittleEndian{}, int64(lenf))
-			if err != nil {
-				pfbz2.Close()
-				return err
-			}
-
-			val := (scan - lenb) - (lastscan + lenf)
-			err = binary.Write(pfbz2, signMagLittleEndian{}, int64(val))
-			if err != nil {
-				pfbz2.Close()
-				return err
-			}
-
-			val = (pos - lenb) - (lastpos + lenf)
-			err = binary.Write(pfbz2, signMagLittleEndian{}, int64(val))
-			if err != nil {
-				pfbz2.Close()
-				return err
-			}
-
-			lastscan = scan - lenb
-			lastpos = pos - lenb
-			lastoffset = pos - scan
-		}
-	}
-	err = pfbz2.Close()
-	if err != nil {
-		return err
-	}
-
-	// Compute size of compressed ctrl data
-	l64, err := patch.Seek(0, 1)
-	if err != nil {
-		return err
-	}
-	hdr.CtrlLen = int64(l64 - 32)
-
-	// Write compressed diff data
-	pfbz2, err = newBzip2Writer(patch)
-	if err != nil {
-		return err
-	}
-	n, err := pfbz2.Write(db[:dblen])
-	if err != nil {
-		pfbz2.Close()
-		return err
-	}
-	if n != dblen {
-		pfbz2.Close()
-		return io.ErrShortWrite
-	}
-	err = pfbz2.Close()
-	if err != nil {
-		return err
-	}
-
-	// Compute size of compressed diff data
-	n64, err := patch.Seek(0, 1)
-	if err != nil {
-		return err
-	}
-	hdr.DiffLen = n64 - l64
-
-	// Write compressed extra data
-	pfbz2, err = newBzip2Writer(patch)
-	if err != nil {
-		return err
-	}
-	n, err = pfbz2.Write(eb[:eblen])
-	if err != nil {
-		pfbz2.Close()
-		return err
-	}
-	if n != eblen {
-		pfbz2.Close()
-		return io.ErrShortWrite
-	}
-	err = pfbz2.Close()
-	if err != nil {
-		return err
-	}
-
-	// Seek to the beginning, write the header, and close the file
-	_, err = patch.Seek(0, 0)
-	if err != nil {
-		return err
-	}
-	err = binary.Write(patch, signMagLittleEndian{}, &hdr)
-	if err != nil {
-		return err
-	}
-	return nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/doc.go

@@ -1,24 +0,0 @@
-// Package binarydist implements binary diff and patch as described on
-// http://www.daemonology.net/bsdiff/. It reads and writes files
-// compatible with the tools there.
-package binarydist
-
-var magic = [8]byte{'B', 'S', 'D', 'I', 'F', 'F', '4', '0'}
-
-// File format:
-//   0       8    "BSDIFF40"
-//   8       8    X
-//   16      8    Y
-//   24      8    sizeof(newfile)
-//   32      X    bzip2(control block)
-//   32+X    Y    bzip2(diff block)
-//   32+X+Y  ???  bzip2(extra block)
-// with control block a set of triples (x,y,z) meaning "add x bytes
-// from oldfile to x bytes from the diff block; copy y bytes from the
-// extra block; seek forwards in oldfile by z bytes".
-type header struct {
-	Magic   [8]byte
-	CtrlLen int64
-	DiffLen int64
-	NewSize int64
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/encoding.go

@@ -1,53 +0,0 @@
-package binarydist
-
-// SignMagLittleEndian is the numeric encoding used by the bsdiff tools.
-// It implements binary.ByteOrder using a sign-magnitude format
-// and little-endian byte order. Only methods Uint64 and String
-// have been written; the rest panic.
-type signMagLittleEndian struct{}
-
-func (signMagLittleEndian) Uint16(b []byte) uint16 { panic("unimplemented") }
-
-func (signMagLittleEndian) PutUint16(b []byte, v uint16) { panic("unimplemented") }
-
-func (signMagLittleEndian) Uint32(b []byte) uint32 { panic("unimplemented") }
-
-func (signMagLittleEndian) PutUint32(b []byte, v uint32) { panic("unimplemented") }
-
-func (signMagLittleEndian) Uint64(b []byte) uint64 {
-	y := int64(b[0]) |
-		int64(b[1])<<8 |
-		int64(b[2])<<16 |
-		int64(b[3])<<24 |
-		int64(b[4])<<32 |
-		int64(b[5])<<40 |
-		int64(b[6])<<48 |
-		int64(b[7]&0x7f)<<56
-
-	if b[7]&0x80 != 0 {
-		y = -y
-	}
-	return uint64(y)
-}
-
-func (signMagLittleEndian) PutUint64(b []byte, v uint64) {
-	x := int64(v)
-	neg := x < 0
-	if neg {
-		x = -x
-	}
-
-	b[0] = byte(x)
-	b[1] = byte(x >> 8)
-	b[2] = byte(x >> 16)
-	b[3] = byte(x >> 24)
-	b[4] = byte(x >> 32)
-	b[5] = byte(x >> 40)
-	b[6] = byte(x >> 48)
-	b[7] = byte(x >> 56)
-	if neg {
-		b[7] |= 0x80
-	}
-}
-
-func (signMagLittleEndian) String() string { return "signMagLittleEndian" }

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/patch.go

@@ -1,109 +0,0 @@
-package binarydist
-
-import (
-	"bytes"
-	"compress/bzip2"
-	"encoding/binary"
-	"errors"
-	"io"
-	"io/ioutil"
-)
-
-var ErrCorrupt = errors.New("corrupt patch")
-
-// Patch applies patch to old, according to the bspatch algorithm,
-// and writes the result to new.
-func Patch(old io.Reader, new io.Writer, patch io.Reader) error {
-	var hdr header
-	err := binary.Read(patch, signMagLittleEndian{}, &hdr)
-	if err != nil {
-		return err
-	}
-	if hdr.Magic != magic {
-		return ErrCorrupt
-	}
-	if hdr.CtrlLen < 0 || hdr.DiffLen < 0 || hdr.NewSize < 0 {
-		return ErrCorrupt
-	}
-
-	ctrlbuf := make([]byte, hdr.CtrlLen)
-	_, err = io.ReadFull(patch, ctrlbuf)
-	if err != nil {
-		return err
-	}
-	cpfbz2 := bzip2.NewReader(bytes.NewReader(ctrlbuf))
-
-	diffbuf := make([]byte, hdr.DiffLen)
-	_, err = io.ReadFull(patch, diffbuf)
-	if err != nil {
-		return err
-	}
-	dpfbz2 := bzip2.NewReader(bytes.NewReader(diffbuf))
-
-	// The entire rest of the file is the extra block.
-	epfbz2 := bzip2.NewReader(patch)
-
-	obuf, err := ioutil.ReadAll(old)
-	if err != nil {
-		return err
-	}
-
-	nbuf := make([]byte, hdr.NewSize)
-
-	var oldpos, newpos int64
-	for newpos < hdr.NewSize {
-		var ctrl struct{ Add, Copy, Seek int64 }
-		err = binary.Read(cpfbz2, signMagLittleEndian{}, &ctrl)
-		if err != nil {
-			return err
-		}
-
-		// Sanity-check
-		if newpos+ctrl.Add > hdr.NewSize {
-			return ErrCorrupt
-		}
-
-		// Read diff string
-		_, err = io.ReadFull(dpfbz2, nbuf[newpos:newpos+ctrl.Add])
-		if err != nil {
-			return ErrCorrupt
-		}
-
-		// Add old data to diff string
-		for i := int64(0); i < ctrl.Add; i++ {
-			if oldpos+i >= 0 && oldpos+i < int64(len(obuf)) {
-				nbuf[newpos+i] += obuf[oldpos+i]
-			}
-		}
-
-		// Adjust pointers
-		newpos += ctrl.Add
-		oldpos += ctrl.Add
-
-		// Sanity-check
-		if newpos+ctrl.Copy > hdr.NewSize {
-			return ErrCorrupt
-		}
-
-		// Read extra string
-		_, err = io.ReadFull(epfbz2, nbuf[newpos:newpos+ctrl.Copy])
-		if err != nil {
-			return ErrCorrupt
-		}
-
-		// Adjust pointers
-		newpos += ctrl.Copy
-		oldpos += ctrl.Seek
-	}
-
-	// Write the new file
-	for len(nbuf) > 0 {
-		n, err := new.Write(nbuf)
-		if err != nil {
-			return err
-		}
-		nbuf = nbuf[n:]
-	}
-
-	return nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/binarydist/seek.go

@@ -1,43 +0,0 @@
-package binarydist
-
-import (
-	"errors"
-)
-
-type seekBuffer struct {
-	buf []byte
-	pos int
-}
-
-func (b *seekBuffer) Write(p []byte) (n int, err error) {
-	n = copy(b.buf[b.pos:], p)
-	if n == len(p) {
-		b.pos += n
-		return n, nil
-	}
-	b.buf = append(b.buf, p[n:]...)
-	b.pos += len(p)
-	return len(p), nil
-}
-
-func (b *seekBuffer) Seek(offset int64, whence int) (ret int64, err error) {
-	var abs int64
-	switch whence {
-	case 0:
-		abs = offset
-	case 1:
-		abs = int64(b.pos) + offset
-	case 2:
-		abs = int64(len(b.buf)) + offset
-	default:
-		return 0, errors.New("binarydist: invalid whence")
-	}
-	if abs < 0 {
-		return 0, errors.New("binarydist: negative position")
-	}
-	if abs >= 1<<31 {
-		return 0, errors.New("binarydist: position out of range")
-	}
-	b.pos = int(abs)
-	return abs, nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/osext/LICENSE

@@ -1,27 +0,0 @@
-Copyright (c) 2012 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/equinox-io/equinox/internal/go-update/internal/osext/README.md

@@ -1,16 +0,0 @@
-### Extensions to the "os" package.
-
-## Find the current Executable and ExecutableFolder.
-
-There is sometimes utility in finding the current executable file
-that is running. This can be used for upgrading the current executable
-or finding resources located relative to the executable file. Both
-working directory and the os.Args[0] value are arbitrary and cannot
-be relied on; os.Args[0] can be "faked".
-
-Multi-platform and supports:
- * Linux
- * OS X
- * Windows
- * Plan 9
- * BSDs.

vendor/github.com/equinox-io/equinox/internal/go-update/internal/osext/osext.go

@@ -1,27 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Extensions to the standard "os" package.
-package osext
-
-import "path/filepath"
-
-// Executable returns an absolute path that can be used to
-// re-invoke the current program.
-// It may not be valid after the current program exits.
-func Executable() (string, error) {
-	p, err := executable()
-	return filepath.Clean(p), err
-}
-
-// Returns same path as Executable, returns just the folder
-// path. Excludes the executable name and any trailing slash.
-func ExecutableFolder() (string, error) {
-	p, err := Executable()
-	if err != nil {
-		return "", err
-	}
-
-	return filepath.Dir(p), nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/osext/osext_plan9.go

@@ -1,20 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package osext
-
-import (
-	"os"
-	"strconv"
-	"syscall"
-)
-
-func executable() (string, error) {
-	f, err := os.Open("/proc/" + strconv.Itoa(os.Getpid()) + "/text")
-	if err != nil {
-		return "", err
-	}
-	defer f.Close()
-	return syscall.Fd2path(int(f.Fd()))
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/osext/osext_procfs.go

@@ -1,36 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build linux netbsd openbsd solaris dragonfly
-
-package osext
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"runtime"
-	"strings"
-)
-
-func executable() (string, error) {
-	switch runtime.GOOS {
-	case "linux":
-		const deletedTag = " (deleted)"
-		execpath, err := os.Readlink("/proc/self/exe")
-		if err != nil {
-			return execpath, err
-		}
-		execpath = strings.TrimSuffix(execpath, deletedTag)
-		execpath = strings.TrimPrefix(execpath, deletedTag)
-		return execpath, nil
-	case "netbsd":
-		return os.Readlink("/proc/curproc/exe")
-	case "openbsd", "dragonfly":
-		return os.Readlink("/proc/curproc/file")
-	case "solaris":
-		return os.Readlink(fmt.Sprintf("/proc/%d/path/a.out", os.Getpid()))
-	}
-	return "", errors.New("ExecPath not implemented for " + runtime.GOOS)
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/osext/osext_sysctl.go

@@ -1,79 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin freebsd
-
-package osext
-
-import (
-	"os"
-	"path/filepath"
-	"runtime"
-	"syscall"
-	"unsafe"
-)
-
-var initCwd, initCwdErr = os.Getwd()
-
-func executable() (string, error) {
-	var mib [4]int32
-	switch runtime.GOOS {
-	case "freebsd":
-		mib = [4]int32{1 /* CTL_KERN */, 14 /* KERN_PROC */, 12 /* KERN_PROC_PATHNAME */, -1}
-	case "darwin":
-		mib = [4]int32{1 /* CTL_KERN */, 38 /* KERN_PROCARGS */, int32(os.Getpid()), -1}
-	}
-
-	n := uintptr(0)
-	// Get length.
-	_, _, errNum := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0)
-	if errNum != 0 {
-		return "", errNum
-	}
-	if n == 0 { // This shouldn't happen.
-		return "", nil
-	}
-	buf := make([]byte, n)
-	_, _, errNum = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0)
-	if errNum != 0 {
-		return "", errNum
-	}
-	if n == 0 { // This shouldn't happen.
-		return "", nil
-	}
-	for i, v := range buf {
-		if v == 0 {
-			buf = buf[:i]
-			break
-		}
-	}
-	var err error
-	execPath := string(buf)
-	// execPath will not be empty due to above checks.
-	// Try to get the absolute path if the execPath is not rooted.
-	if execPath[0] != '/' {
-		execPath, err = getAbs(execPath)
-		if err != nil {
-			return execPath, err
-		}
-	}
-	// For darwin KERN_PROCARGS may return the path to a symlink rather than the
-	// actual executable.
-	if runtime.GOOS == "darwin" {
-		if execPath, err = filepath.EvalSymlinks(execPath); err != nil {
-			return execPath, err
-		}
-	}
-	return execPath, nil
-}
-
-func getAbs(execPath string) (string, error) {
-	if initCwdErr != nil {
-		return execPath, initCwdErr
-	}
-	// The execPath may begin with a "../" or a "./" so clean it first.
-	// Join the two paths, trailing and starting slashes undetermined, so use
-	// the generic Join function.
-	return filepath.Join(initCwd, filepath.Clean(execPath)), nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/internal/osext/osext_windows.go

@@ -1,34 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package osext
-
-import (
-	"syscall"
-	"unicode/utf16"
-	"unsafe"
-)
-
-var (
-	kernel                = syscall.MustLoadDLL("kernel32.dll")
-	getModuleFileNameProc = kernel.MustFindProc("GetModuleFileNameW")
-)
-
-// GetModuleFileName() with hModule = NULL
-func executable() (exePath string, err error) {
-	return getModuleFileName()
-}
-
-func getModuleFileName() (string, error) {
-	var n uint32
-	b := make([]uint16, syscall.MAX_PATH)
-	size := uint32(len(b))
-
-	r0, _, e1 := getModuleFileNameProc.Call(0, uintptr(unsafe.Pointer(&b[0])), uintptr(size))
-	n = uint32(r0)
-	if n == 0 {
-		return "", e1
-	}
-	return string(utf16.Decode(b[0:n])), nil
-}

vendor/github.com/equinox-io/equinox/internal/go-update/patcher.go

@@ -1,24 +0,0 @@
-package update
-
-import (
-	"io"
-
-	"github.com/equinox-io/equinox/internal/go-update/internal/binarydist"
-)
-
-// Patcher defines an interface for applying binary patches to an old item to get an updated item.
-type Patcher interface {
-	Patch(old io.Reader, new io.Writer, patch io.Reader) error
-}
-
-type patchFn func(io.Reader, io.Writer, io.Reader) error
-
-func (fn patchFn) Patch(old io.Reader, new io.Writer, patch io.Reader) error {
-	return fn(old, new, patch)
-}
-
-// NewBSDifferPatcher returns a new Patcher that applies binary patches using
-// the bsdiff algorithm. See http://www.daemonology.net/bsdiff/
-func NewBSDiffPatcher() Patcher {
-	return patchFn(binarydist.Patch)
-}

vendor/github.com/equinox-io/equinox/internal/go-update/verifier.go

@@ -1,74 +0,0 @@
-package update
-
-import (
-	"crypto"
-	"crypto/dsa"
-	"crypto/ecdsa"
-	"crypto/rsa"
-	"encoding/asn1"
-	"errors"
-	"math/big"
-)
-
-// Verifier defines an interface for verfiying an update's signature with a public key.
-type Verifier interface {
-	VerifySignature(checksum, signature []byte, h crypto.Hash, publicKey crypto.PublicKey) error
-}
-
-type verifyFn func([]byte, []byte, crypto.Hash, crypto.PublicKey) error
-
-func (fn verifyFn) VerifySignature(checksum []byte, signature []byte, hash crypto.Hash, publicKey crypto.PublicKey) error {
-	return fn(checksum, signature, hash, publicKey)
-}
-
-// NewRSAVerifier returns a Verifier that uses the RSA algorithm to verify updates.
-func NewRSAVerifier() Verifier {
-	return verifyFn(func(checksum, signature []byte, hash crypto.Hash, publicKey crypto.PublicKey) error {
-		key, ok := publicKey.(*rsa.PublicKey)
-		if !ok {
-			return errors.New("not a valid RSA public key")
-		}
-		return rsa.VerifyPKCS1v15(key, hash, checksum, signature)
-	})
-}
-
-type rsDER struct {
-	R *big.Int
-	S *big.Int
-}
-
-// NewECDSAVerifier returns a Verifier that uses the ECDSA algorithm to verify updates.
-func NewECDSAVerifier() Verifier {
-	return verifyFn(func(checksum, signature []byte, hash crypto.Hash, publicKey crypto.PublicKey) error {
-		key, ok := publicKey.(*ecdsa.PublicKey)
-		if !ok {
-			return errors.New("not a valid ECDSA public key")
-		}
-		var rs rsDER
-		if _, err := asn1.Unmarshal(signature, &rs); err != nil {
-			return err
-		}
-		if !ecdsa.Verify(key, checksum, rs.R, rs.S) {
-			return errors.New("failed to verify ecsda signature")
-		}
-		return nil
-	})
-}
-
-// NewDSAVerifier returns a Verifier that uses the DSA algorithm to verify updates.
-func NewDSAVerifier() Verifier {
-	return verifyFn(func(checksum, signature []byte, hash crypto.Hash, publicKey crypto.PublicKey) error {
-		key, ok := publicKey.(*dsa.PublicKey)
-		if !ok {
-			return errors.New("not a valid DSA public key")
-		}
-		var rs rsDER
-		if _, err := asn1.Unmarshal(signature, &rs); err != nil {
-			return err
-		}
-		if !dsa.Verify(key, checksum, rs.R, rs.S) {
-			return errors.New("failed to verify ecsda signature")
-		}
-		return nil
-	})
-}

vendor/github.com/equinox-io/equinox/internal/osext/LICENSE

@@ -1,27 +0,0 @@
-Copyright (c) 2012 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/equinox-io/equinox/internal/osext/README.md

@@ -1,16 +0,0 @@
-### Extensions to the "os" package.
-
-## Find the current Executable and ExecutableFolder.
-
-There is sometimes utility in finding the current executable file
-that is running. This can be used for upgrading the current executable
-or finding resources located relative to the executable file. Both
-working directory and the os.Args[0] value are arbitrary and cannot
-be relied on; os.Args[0] can be "faked".
-
-Multi-platform and supports:
- * Linux
- * OS X
- * Windows
- * Plan 9
- * BSDs.

vendor/github.com/equinox-io/equinox/internal/osext/osext.go

@@ -1,27 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Extensions to the standard "os" package.
-package osext
-
-import "path/filepath"
-
-// Executable returns an absolute path that can be used to
-// re-invoke the current program.
-// It may not be valid after the current program exits.
-func Executable() (string, error) {
-	p, err := executable()
-	return filepath.Clean(p), err
-}
-
-// Returns same path as Executable, returns just the folder
-// path. Excludes the executable name and any trailing slash.
-func ExecutableFolder() (string, error) {
-	p, err := Executable()
-	if err != nil {
-		return "", err
-	}
-
-	return filepath.Dir(p), nil
-}

vendor/github.com/equinox-io/equinox/internal/osext/osext_plan9.go

@@ -1,20 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package osext
-
-import (
-	"os"
-	"strconv"
-	"syscall"
-)
-
-func executable() (string, error) {
-	f, err := os.Open("/proc/" + strconv.Itoa(os.Getpid()) + "/text")
-	if err != nil {
-		return "", err
-	}
-	defer f.Close()
-	return syscall.Fd2path(int(f.Fd()))
-}

vendor/github.com/equinox-io/equinox/internal/osext/osext_procfs.go

@@ -1,36 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build linux netbsd openbsd solaris dragonfly
-
-package osext
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"runtime"
-	"strings"
-)
-
-func executable() (string, error) {
-	switch runtime.GOOS {
-	case "linux":
-		const deletedTag = " (deleted)"
-		execpath, err := os.Readlink("/proc/self/exe")
-		if err != nil {
-			return execpath, err
-		}
-		execpath = strings.TrimSuffix(execpath, deletedTag)
-		execpath = strings.TrimPrefix(execpath, deletedTag)
-		return execpath, nil
-	case "netbsd":
-		return os.Readlink("/proc/curproc/exe")
-	case "openbsd", "dragonfly":
-		return os.Readlink("/proc/curproc/file")
-	case "solaris":
-		return os.Readlink(fmt.Sprintf("/proc/%d/path/a.out", os.Getpid()))
-	}
-	return "", errors.New("ExecPath not implemented for " + runtime.GOOS)
-}

vendor/github.com/equinox-io/equinox/internal/osext/osext_sysctl.go

@@ -1,79 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// +build darwin freebsd
-
-package osext
-
-import (
-	"os"
-	"path/filepath"
-	"runtime"
-	"syscall"
-	"unsafe"
-)
-
-var initCwd, initCwdErr = os.Getwd()
-
-func executable() (string, error) {
-	var mib [4]int32
-	switch runtime.GOOS {
-	case "freebsd":
-		mib = [4]int32{1 /* CTL_KERN */, 14 /* KERN_PROC */, 12 /* KERN_PROC_PATHNAME */, -1}
-	case "darwin":
-		mib = [4]int32{1 /* CTL_KERN */, 38 /* KERN_PROCARGS */, int32(os.Getpid()), -1}
-	}
-
-	n := uintptr(0)
-	// Get length.
-	_, _, errNum := syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, 0, uintptr(unsafe.Pointer(&n)), 0, 0)
-	if errNum != 0 {
-		return "", errNum
-	}
-	if n == 0 { // This shouldn't happen.
-		return "", nil
-	}
-	buf := make([]byte, n)
-	_, _, errNum = syscall.Syscall6(syscall.SYS___SYSCTL, uintptr(unsafe.Pointer(&mib[0])), 4, uintptr(unsafe.Pointer(&buf[0])), uintptr(unsafe.Pointer(&n)), 0, 0)
-	if errNum != 0 {
-		return "", errNum
-	}
-	if n == 0 { // This shouldn't happen.
-		return "", nil
-	}
-	for i, v := range buf {
-		if v == 0 {
-			buf = buf[:i]
-			break
-		}
-	}
-	var err error
-	execPath := string(buf)
-	// execPath will not be empty due to above checks.
-	// Try to get the absolute path if the execPath is not rooted.
-	if execPath[0] != '/' {
-		execPath, err = getAbs(execPath)
-		if err != nil {
-			return execPath, err
-		}
-	}
-	// For darwin KERN_PROCARGS may return the path to a symlink rather than the
-	// actual executable.
-	if runtime.GOOS == "darwin" {
-		if execPath, err = filepath.EvalSymlinks(execPath); err != nil {
-			return execPath, err
-		}
-	}
-	return execPath, nil
-}
-
-func getAbs(execPath string) (string, error) {
-	if initCwdErr != nil {
-		return execPath, initCwdErr
-	}
-	// The execPath may begin with a "../" or a "./" so clean it first.
-	// Join the two paths, trailing and starting slashes undetermined, so use
-	// the generic Join function.
-	return filepath.Join(initCwd, filepath.Clean(execPath)), nil
-}

vendor/github.com/equinox-io/equinox/internal/osext/osext_windows.go

@@ -1,34 +0,0 @@
-// Copyright 2012 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package osext
-
-import (
-	"syscall"
-	"unicode/utf16"
-	"unsafe"
-)
-
-var (
-	kernel                = syscall.MustLoadDLL("kernel32.dll")
-	getModuleFileNameProc = kernel.MustFindProc("GetModuleFileNameW")
-)
-
-// GetModuleFileName() with hModule = NULL
-func executable() (exePath string, err error) {
-	return getModuleFileName()
-}
-
-func getModuleFileName() (string, error) {
-	var n uint32
-	b := make([]uint16, syscall.MAX_PATH)
-	size := uint32(len(b))
-
-	r0, _, e1 := getModuleFileNameProc.Call(0, uintptr(unsafe.Pointer(&b[0])), uintptr(size))
-	n = uint32(r0)
-	if n == 0 {
-		return "", e1
-	}
-	return string(utf16.Decode(b[0:n])), nil
-}

vendor/github.com/equinox-io/equinox/proto/proto.go

@@ -1,42 +0,0 @@
-/*
-package proto defines a set of structures used to negotiate an update between an
-an application (the client) and an Equinox update service.
-*/
-package proto
-
-import "time"
-
-type PatchKind string
-
-const (
-	PatchNone   PatchKind = "none"
-	PatchBSDiff PatchKind = "bsdiff"
-)
-
-type Request struct {
-	AppID         string `json:"app_id"`
-	Channel       string `json:"channel"`
-	OS            string `json:"os"`
-	Arch          string `json:"arch"`
-	GoARM         string `json:"goarm"`
-	TargetVersion string `json:"target_version"`
-
-	CurrentVersion string `json:"current_version"`
-	CurrentSHA256  string `json:"current_sha256"`
-}
-
-type Response struct {
-	Available   bool      `json:"available"`
-	DownloadURL string    `json:"download_url"`
-	Checksum    string    `json:"checksum"`
-	Signature   string    `json:"signature"`
-	Patch       PatchKind `json:"patch_type"`
-	Release     Release   `json:"release"`
-}
-
-type Release struct {
-	Title       string    `json:"title"`
-	Version     string    `json:"version"`
-	Description string    `json:"description"`
-	CreateDate  time.Time `json:"create_date"`
-}

vendor/github.com/equinox-io/equinox/sdk.go

@@ -1,330 +0,0 @@
-package equinox
-
-import (
-	"bytes"
-	"crypto"
-	"crypto/sha256"
-	"crypto/x509"
-	"encoding/hex"
-	"encoding/json"
-	"encoding/pem"
-	"errors"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"os"
-	"runtime"
-	"time"
-
-	"github.com/equinox-io/equinox/internal/go-update"
-	"github.com/equinox-io/equinox/internal/osext"
-	"github.com/equinox-io/equinox/proto"
-)
-
-const protocolVersion = "1"
-const defaultCheckURL = "https://update.equinox.io/check"
-const userAgent = "EquinoxSDK/1.0"
-
-var NotAvailableErr = errors.New("No update available")
-
-type Options struct {
-	// Channel specifies the name of an Equinox release channel to check for
-	// a newer version of the application.
-	//
-	// If empty, defaults to 'stable'.
-	Channel string
-
-	// Version requests an update to a specific version of the application.
-	// If specified, `Channel` is ignored.
-	Version string
-
-	// TargetPath defines the path to the file to update.
-	// The emptry string means 'the executable file of the running program'.
-	TargetPath string
-
-	// Create TargetPath replacement with this file mode. If zero, defaults to 0755.
-	TargetMode os.FileMode
-
-	// Public key to use for signature verification. If nil, no signature
-	// verification is done. Use `SetPublicKeyPEM` to set this field with PEM data.
-	PublicKey crypto.PublicKey
-
-	// Target operating system of the update. Uses the same standard OS names used
-	// by Go build tags (windows, darwin, linux, etc).
-	// If empty, it will be populated by consulting runtime.GOOS
-	OS string
-
-	// Target architecture of the update. Uses the same standard Arch names used
-	// by Go build tags (amd64, 386, arm, etc).
-	// If empty, it will be populated by consulting runtime.GOARCH
-	Arch string
-
-	// Target ARM architecture, if a specific one if required. Uses the same names
-	// as the GOARM environment variable (5, 6, 7).
-	//
-	// GoARM is ignored if Arch != 'arm'.
-	// GoARM is ignored if it is the empty string. Omit it if you do not need
-	// to distinguish between ARM versions.
-	GoARM string
-
-	// The current application version. This is used for statistics and reporting only,
-	// it is optional.
-	CurrentVersion string
-
-	// CheckURL is the URL to request an update check from. You should only set
-	// this if you are running an on-prem Equinox server.
-	// If empty the default Equinox update service endpoint is used.
-	CheckURL string
-
-	// HTTPClient is used to make all HTTP requests necessary for the update check protocol.
-	// You may configure it to use custom timeouts, proxy servers or other behaviors.
-	HTTPClient *http.Client
-}
-
-// Response is returned by Check when an update is available. It may be
-// passed to Apply to perform the update.
-type Response struct {
-	// Version of the release that will be updated to if applied.
-	ReleaseVersion string
-
-	// Title of the the release
-	ReleaseTitle string
-
-	// Additional details about the release
-	ReleaseDescription string
-
-	// Creation date of the release
-	ReleaseDate time.Time
-
-	downloadURL string
-	checksum    []byte
-	signature   []byte
-	patch       proto.PatchKind
-	opts        Options
-}
-
-// SetPublicKeyPEM is a convenience method to set the PublicKey property
-// used for checking a completed update's signature by parsing a
-// Public Key formatted as PEM data.
-func (o *Options) SetPublicKeyPEM(pembytes []byte) error {
-	block, _ := pem.Decode(pembytes)
-	if block == nil {
-		return errors.New("couldn't parse PEM data")
-	}
-
-	pub, err := x509.ParsePKIXPublicKey(block.Bytes)
-	if err != nil {
-		return err
-	}
-	o.PublicKey = pub
-	return nil
-}
-
-// Check communicates with an Equinox update service to determine if
-// an update for the given application matching the specified options is
-// available. The returned error is nil only if an update is available.
-//
-// The appID is issued to you when creating an application at https://equinox.io
-//
-// You can compare the returned error to NotAvailableErr to differentiate between
-// a successful check that found no update from other errors like a failed
-// network connection.
-func Check(appID string, opts Options) (Response, error) {
-	var req, err = checkRequest(appID, &opts)
-
-	if err != nil {
-		return Response{}, err
-	}
-
-	return doCheckRequest(opts, req)
-}
-
-func checkRequest(appID string, opts *Options) (*http.Request, error) {
-	if opts.Channel == "" {
-		opts.Channel = "stable"
-	}
-	if opts.TargetPath == "" {
-		var err error
-		opts.TargetPath, err = osext.Executable()
-		if err != nil {
-			return nil, err
-		}
-	}
-	if opts.OS == "" {
-		opts.OS = runtime.GOOS
-	}
-	if opts.Arch == "" {
-		opts.Arch = runtime.GOARCH
-	}
-	if opts.CheckURL == "" {
-		opts.CheckURL = defaultCheckURL
-	}
-	if opts.HTTPClient == nil {
-		opts.HTTPClient = new(http.Client)
-	}
-	opts.HTTPClient.Transport = newUserAgentTransport(userAgent, opts.HTTPClient.Transport)
-
-	checksum := computeChecksum(opts.TargetPath)
-
-	payload, err := json.Marshal(proto.Request{
-		AppID:          appID,
-		Channel:        opts.Channel,
-		OS:             opts.OS,
-		Arch:           opts.Arch,
-		GoARM:          opts.GoARM,
-		TargetVersion:  opts.Version,
-		CurrentVersion: opts.CurrentVersion,
-		CurrentSHA256:  checksum,
-	})
-
-	req, err := http.NewRequest("POST", opts.CheckURL, bytes.NewReader(payload))
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Set("Accept", fmt.Sprintf("application/json; q=1; version=%s; charset=utf-8", protocolVersion))
-	req.Header.Set("Content-Type", "application/json; charset=utf-8")
-	req.Close = true
-
-	return req, err
-}
-
-func doCheckRequest(opts Options, req *http.Request) (r Response, err error) {
-	resp, err := opts.HTTPClient.Do(req)
-	if err != nil {
-		return r, err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != 200 {
-		body, _ := ioutil.ReadAll(resp.Body)
-		return r, fmt.Errorf("Server responded with %s: %s", resp.Status, body)
-	}
-
-	var protoResp proto.Response
-	err = json.NewDecoder(resp.Body).Decode(&protoResp)
-	if err != nil {
-		return r, err
-	}
-
-	if !protoResp.Available {
-		return r, NotAvailableErr
-	}
-
-	r.ReleaseVersion = protoResp.Release.Version
-	r.ReleaseTitle = protoResp.Release.Title
-	r.ReleaseDescription = protoResp.Release.Description
-	r.ReleaseDate = protoResp.Release.CreateDate
-	r.downloadURL = protoResp.DownloadURL
-	r.patch = protoResp.Patch
-	r.opts = opts
-	r.checksum, err = hex.DecodeString(protoResp.Checksum)
-	if err != nil {
-		return r, err
-	}
-	r.signature, err = hex.DecodeString(protoResp.Signature)
-	if err != nil {
-		return r, err
-	}
-
-	return r, nil
-}
-
-func computeChecksum(path string) string {
-	f, err := os.Open(path)
-	if err != nil {
-		return ""
-	}
-	defer f.Close()
-	h := sha256.New()
-	_, err = io.Copy(h, f)
-	if err != nil {
-		return ""
-	}
-	return hex.EncodeToString(h.Sum(nil))
-}
-
-// Apply performs an update of the current executable (or TargetFile, if it was
-// set on the Options) with the update specified by Response.
-//
-// Error is nil if and only if the entire update completes successfully.
-func (r Response) Apply() error {
-	var req, opts, err = r.applyRequest()
-
-	if err != nil {
-		return err
-	}
-
-	return r.applyUpdate(req, opts)
-}
-
-func (r Response) applyRequest() (*http.Request, update.Options, error) {
-	opts := update.Options{
-		TargetPath: r.opts.TargetPath,
-		TargetMode: r.opts.TargetMode,
-		Checksum:   r.checksum,
-		Signature:  r.signature,
-		Verifier:   update.NewECDSAVerifier(),
-		PublicKey:  r.opts.PublicKey,
-	}
-	switch r.patch {
-	case proto.PatchBSDiff:
-		opts.Patcher = update.NewBSDiffPatcher()
-	}
-
-	if err := opts.CheckPermissions(); err != nil {
-		return nil, opts, err
-	}
-
-	req, err := http.NewRequest("GET", r.downloadURL, nil)
-	return req, opts, err
-}
-
-func (r Response) applyUpdate(req *http.Request, opts update.Options) error {
-	// fetch the update
-	req.Close = true
-	resp, err := r.opts.HTTPClient.Do(req)
-	if err != nil {
-		return err
-	}
-
-	defer resp.Body.Close()
-
-	// check that we got a patch
-	if resp.StatusCode >= 400 {
-		msg := "error downloading patch"
-
-		id := resp.Header.Get("Request-Id")
-		if id != "" {
-			msg += ", request " + id
-		}
-
-		blob, err := ioutil.ReadAll(resp.Body)
-		if err == nil {
-			msg += ": " + string(bytes.TrimSpace(blob))
-		}
-		return fmt.Errorf(msg)
-	}
-
-	return update.Apply(resp.Body, opts)
-}
-
-type userAgentTransport struct {
-	userAgent string
-	http.RoundTripper
-}
-
-func newUserAgentTransport(userAgent string, rt http.RoundTripper) *userAgentTransport {
-	if rt == nil {
-		rt = http.DefaultTransport
-	}
-	return &userAgentTransport{userAgent, rt}
-}
-
-func (t *userAgentTransport) RoundTrip(r *http.Request) (*http.Response, error) {
-	if r.Header.Get("User-Agent") == "" {
-		r.Header.Set("User-Agent", t.userAgent)
-	}
-	return t.RoundTripper.RoundTrip(r)
-}

vendor/github.com/equinox-io/equinox/sdk_ctx.go

@@ -1,29 +0,0 @@
-// +build go1.7
-
-package equinox
-
-import (
-	"context"
-)
-
-// CheckContext is like Check but includes a context.
-func CheckContext(ctx context.Context, appID string, opts Options) (Response, error) {
-	var req, err = checkRequest(appID, &opts)
-
-	if err != nil {
-		return Response{}, err
-	}
-
-	return doCheckRequest(opts, req.WithContext(ctx))
-}
-
-// ApplyContext is like Apply but includes a context.
-func (r Response) ApplyContext(ctx context.Context) error {
-	var req, opts, err = r.applyRequest()
-
-	if err != nil {
-		return err
-	}
-
-	return r.applyUpdate(req.WithContext(ctx), opts)
-}

vendor/github.com/gobwas/httphead/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017 Sergey Kamardin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/gobwas/httphead/README.md

@@ -0,0 +1,63 @@
+# httphead.[go](https://golang.org)
+
+[![GoDoc][godoc-image]][godoc-url] 
+
+> Tiny HTTP header value parsing library in go.
+
+## Overview
+
+This library contains low-level functions for scanning HTTP RFC2616 compatible header value grammars.
+
+## Install
+
+```shell
+    go get github.com/gobwas/httphead
+```
+
+## Example
+
+The example below shows how multiple-choise HTTP header value could be parsed with this library:
+
+```go
+	options, ok := httphead.ParseOptions([]byte(`foo;bar=1,baz`), nil)
+	fmt.Println(options, ok)
+	// Output: [{foo map[bar:1]} {baz map[]}] true
+```
+
+The low-level example below shows how to optimize keys skipping and selection
+of some key:
+
+```go
+	// The right part of full header line like:
+	// X-My-Header: key;foo=bar;baz,key;baz
+	header := []byte(`foo;a=0,foo;a=1,foo;a=2,foo;a=3`)
+
+	// We want to search key "foo" with an "a" parameter that equal to "2".
+	var (
+		foo = []byte(`foo`)
+		a   = []byte(`a`)
+		v   = []byte(`2`)
+	)
+	var found bool
+	httphead.ScanOptions(header, func(i int, key, param, value []byte) Control {
+		if !bytes.Equal(key, foo) {
+			return ControlSkip
+		}
+		if !bytes.Equal(param, a) {
+			if bytes.Equal(value, v) {
+				// Found it!
+				found = true
+				return ControlBreak
+			}
+			return ControlSkip
+		}
+		return ControlContinue
+	})
+```
+
+For more usage examples please see [docs][godoc-url] or package tests.
+
+[godoc-image]: https://godoc.org/github.com/gobwas/httphead?status.svg
+[godoc-url]: https://godoc.org/github.com/gobwas/httphead
+[travis-image]: https://travis-ci.org/gobwas/httphead.svg?branch=master
+[travis-url]: https://travis-ci.org/gobwas/httphead

vendor/github.com/gobwas/httphead/cookie.go

@@ -0,0 +1,200 @@
+package httphead
+
+import (
+	"bytes"
+)
+
+// ScanCookie scans cookie pairs from data using DefaultCookieScanner.Scan()
+// method.
+func ScanCookie(data []byte, it func(key, value []byte) bool) bool {
+	return DefaultCookieScanner.Scan(data, it)
+}
+
+// DefaultCookieScanner is a CookieScanner which is used by ScanCookie().
+// Note that it is intended to have the same behavior as http.Request.Cookies()
+// has.
+var DefaultCookieScanner = CookieScanner{}
+
+// CookieScanner contains options for scanning cookie pairs.
+// See https://tools.ietf.org/html/rfc6265#section-4.1.1
+type CookieScanner struct {
+	// DisableNameValidation disables name validation of a cookie. If false,
+	// only RFC2616 "tokens" are accepted.
+	DisableNameValidation bool
+
+	// DisableValueValidation disables value validation of a cookie. If false,
+	// only RFC6265 "cookie-octet" characters are accepted.
+	//
+	// Note that Strict option also affects validation of a value.
+	//
+	// If Strict is false, then scanner begins to allow space and comma
+	// characters inside the value for better compatibility with non standard
+	// cookies implementations.
+	DisableValueValidation bool
+
+	// BreakOnPairError sets scanner to immediately return after first pair syntax
+	// validation error.
+	// If false, scanner will try to skip invalid pair bytes and go ahead.
+	BreakOnPairError bool
+
+	// Strict enables strict RFC6265 mode scanning. It affects name and value
+	// validation, as also some other rules.
+	// If false, it is intended to bring the same behavior as
+	// http.Request.Cookies().
+	Strict bool
+}
+
+// Scan maps data to name and value pairs. Usually data represents value of the
+// Cookie header.
+func (c CookieScanner) Scan(data []byte, it func(name, value []byte) bool) bool {
+	lexer := &Scanner{data: data}
+
+	const (
+		statePair = iota
+		stateBefore
+	)
+
+	state := statePair
+
+	for lexer.Buffered() > 0 {
+		switch state {
+		case stateBefore:
+			// Pairs separated by ";" and space, according to the RFC6265:
+			//   cookie-pair *( ";" SP cookie-pair )
+			//
+			// Cookie pairs MUST be separated by (";" SP). So our only option
+			// here is to fail as syntax error.
+			a, b := lexer.Peek2()
+			if a != ';' {
+				return false
+			}
+
+			state = statePair
+
+			advance := 1
+			if b == ' ' {
+				advance++
+			} else if c.Strict {
+				return false
+			}
+
+			lexer.Advance(advance)
+
+		case statePair:
+			if !lexer.FetchUntil(';') {
+				return false
+			}
+
+			var value []byte
+			name := lexer.Bytes()
+			if i := bytes.IndexByte(name, '='); i != -1 {
+				value = name[i+1:]
+				name = name[:i]
+			} else if c.Strict {
+				if !c.BreakOnPairError {
+					goto nextPair
+				}
+				return false
+			}
+
+			if !c.Strict {
+				trimLeft(name)
+			}
+			if !c.DisableNameValidation && !ValidCookieName(name) {
+				if !c.BreakOnPairError {
+					goto nextPair
+				}
+				return false
+			}
+
+			if !c.Strict {
+				value = trimRight(value)
+			}
+			value = stripQuotes(value)
+			if !c.DisableValueValidation && !ValidCookieValue(value, c.Strict) {
+				if !c.BreakOnPairError {
+					goto nextPair
+				}
+				return false
+			}
+
+			if !it(name, value) {
+				return true
+			}
+
+		nextPair:
+			state = stateBefore
+		}
+	}
+
+	return true
+}
+
+// ValidCookieValue reports whether given value is a valid RFC6265
+// "cookie-octet" bytes.
+//
+// cookie-octet = %x21 / %x23-2B / %x2D-3A / %x3C-5B / %x5D-7E
+//                ; US-ASCII characters excluding CTLs,
+//                ; whitespace DQUOTE, comma, semicolon,
+//                ; and backslash
+//
+// Note that the false strict parameter disables errors on space 0x20 and comma
+// 0x2c. This could be useful to bring some compatibility with non-compliant
+// clients/servers in the real world.
+// It acts the same as standard library cookie parser if strict is false.
+func ValidCookieValue(value []byte, strict bool) bool {
+	if len(value) == 0 {
+		return true
+	}
+	for _, c := range value {
+		switch c {
+		case '"', ';', '\\':
+			return false
+		case ',', ' ':
+			if strict {
+				return false
+			}
+		default:
+			if c <= 0x20 {
+				return false
+			}
+			if c >= 0x7f {
+				return false
+			}
+		}
+	}
+	return true
+}
+
+// ValidCookieName reports wheter given bytes is a valid RFC2616 "token" bytes.
+func ValidCookieName(name []byte) bool {
+	for _, c := range name {
+		if !OctetTypes[c].IsToken() {
+			return false
+		}
+	}
+	return true
+}
+
+func stripQuotes(bts []byte) []byte {
+	if last := len(bts) - 1; last > 0 && bts[0] == '"' && bts[last] == '"' {
+		return bts[1:last]
+	}
+	return bts
+}
+
+func trimLeft(p []byte) []byte {
+	var i int
+	for i < len(p) && OctetTypes[p[i]].IsSpace() {
+		i++
+	}
+	return p[i:]
+}
+
+func trimRight(p []byte) []byte {
+	j := len(p)
+	for j > 0 && OctetTypes[p[j-1]].IsSpace() {
+		j--
+	}
+	return p[:j]
+}

vendor/github.com/gobwas/httphead/head.go

@@ -0,0 +1,275 @@
+package httphead
+
+import (
+	"bufio"
+	"bytes"
+)
+
+// Version contains protocol major and minor version.
+type Version struct {
+	Major int
+	Minor int
+}
+
+// RequestLine contains parameters parsed from the first request line.
+type RequestLine struct {
+	Method  []byte
+	URI     []byte
+	Version Version
+}
+
+// ResponseLine contains parameters parsed from the first response line.
+type ResponseLine struct {
+	Version Version
+	Status  int
+	Reason  []byte
+}
+
+// SplitRequestLine splits given slice of bytes into three chunks without
+// parsing.
+func SplitRequestLine(line []byte) (method, uri, version []byte) {
+	return split3(line, ' ')
+}
+
+// ParseRequestLine parses http request line like "GET / HTTP/1.0".
+func ParseRequestLine(line []byte) (r RequestLine, ok bool) {
+	var i int
+	for i = 0; i < len(line); i++ {
+		c := line[i]
+		if !OctetTypes[c].IsToken() {
+			if i > 0 && c == ' ' {
+				break
+			}
+			return
+		}
+	}
+	if i == len(line) {
+		return
+	}
+
+	var proto []byte
+	r.Method = line[:i]
+	r.URI, proto = split2(line[i+1:], ' ')
+	if len(r.URI) == 0 {
+		return
+	}
+	if major, minor, ok := ParseVersion(proto); ok {
+		r.Version.Major = major
+		r.Version.Minor = minor
+		return r, true
+	}
+
+	return r, false
+}
+
+// SplitResponseLine splits given slice of bytes into three chunks without
+// parsing.
+func SplitResponseLine(line []byte) (version, status, reason []byte) {
+	return split3(line, ' ')
+}
+
+// ParseResponseLine parses first response line into ResponseLine struct.
+func ParseResponseLine(line []byte) (r ResponseLine, ok bool) {
+	var (
+		proto  []byte
+		status []byte
+	)
+	proto, status, r.Reason = split3(line, ' ')
+	if major, minor, ok := ParseVersion(proto); ok {
+		r.Version.Major = major
+		r.Version.Minor = minor
+	} else {
+		return r, false
+	}
+	if n, ok := IntFromASCII(status); ok {
+		r.Status = n
+	} else {
+		return r, false
+	}
+	// TODO(gobwas): parse here r.Reason fot TEXT rule:
+	//   TEXT = <any OCTET except CTLs,
+	//           but including LWS>
+	return r, true
+}
+
+var (
+	httpVersion10     = []byte("HTTP/1.0")
+	httpVersion11     = []byte("HTTP/1.1")
+	httpVersionPrefix = []byte("HTTP/")
+)
+
+// ParseVersion parses major and minor version of HTTP protocol.
+// It returns parsed values and true if parse is ok.
+func ParseVersion(bts []byte) (major, minor int, ok bool) {
+	switch {
+	case bytes.Equal(bts, httpVersion11):
+		return 1, 1, true
+	case bytes.Equal(bts, httpVersion10):
+		return 1, 0, true
+	case len(bts) < 8:
+		return
+	case !bytes.Equal(bts[:5], httpVersionPrefix):
+		return
+	}
+
+	bts = bts[5:]
+
+	dot := bytes.IndexByte(bts, '.')
+	if dot == -1 {
+		return
+	}
+	major, ok = IntFromASCII(bts[:dot])
+	if !ok {
+		return
+	}
+	minor, ok = IntFromASCII(bts[dot+1:])
+	if !ok {
+		return
+	}
+
+	return major, minor, true
+}
+
+// ReadLine reads line from br. It reads until '\n' and returns bytes without
+// '\n' or '\r\n' at the end.
+// It returns err if and only if line does not end in '\n'. Note that read
+// bytes returned in any case of error.
+//
+// It is much like the textproto/Reader.ReadLine() except the thing that it
+// returns raw bytes, instead of string. That is, it avoids copying bytes read
+// from br.
+//
+// textproto/Reader.ReadLineBytes() is also makes copy of resulting bytes to be
+// safe with future I/O operations on br.
+//
+// We could control I/O operations on br and do not need to make additional
+// copy for safety.
+func ReadLine(br *bufio.Reader) ([]byte, error) {
+	var line []byte
+	for {
+		bts, err := br.ReadSlice('\n')
+		if err == bufio.ErrBufferFull {
+			// Copy bytes because next read will discard them.
+			line = append(line, bts...)
+			continue
+		}
+		// Avoid copy of single read.
+		if line == nil {
+			line = bts
+		} else {
+			line = append(line, bts...)
+		}
+		if err != nil {
+			return line, err
+		}
+		// Size of line is at least 1.
+		// In other case bufio.ReadSlice() returns error.
+		n := len(line)
+		// Cut '\n' or '\r\n'.
+		if n > 1 && line[n-2] == '\r' {
+			line = line[:n-2]
+		} else {
+			line = line[:n-1]
+		}
+		return line, nil
+	}
+}
+
+// ParseHeaderLine parses HTTP header as key-value pair. It returns parsed
+// values and true if parse is ok.
+func ParseHeaderLine(line []byte) (k, v []byte, ok bool) {
+	colon := bytes.IndexByte(line, ':')
+	if colon == -1 {
+		return
+	}
+	k = trim(line[:colon])
+	for _, c := range k {
+		if !OctetTypes[c].IsToken() {
+			return nil, nil, false
+		}
+	}
+	v = trim(line[colon+1:])
+	return k, v, true
+}
+
+// IntFromASCII converts ascii encoded decimal numeric value from HTTP entities
+// to an integer.
+func IntFromASCII(bts []byte) (ret int, ok bool) {
+	// ASCII numbers all start with the high-order bits 0011.
+	// If you see that, and the next bits are 0-9 (0000 - 1001) you can grab those
+	// bits and interpret them directly as an integer.
+	var n int
+	if n = len(bts); n < 1 {
+		return 0, false
+	}
+	for i := 0; i < n; i++ {
+		if bts[i]&0xf0 != 0x30 {
+			return 0, false
+		}
+		ret += int(bts[i]&0xf) * pow(10, n-i-1)
+	}
+	return ret, true
+}
+
+const (
+	toLower = 'a' - 'A'      // for use with OR.
+	toUpper = ^byte(toLower) // for use with AND.
+)
+
+// CanonicalizeHeaderKey is like standard textproto/CanonicalMIMEHeaderKey,
+// except that it operates with slice of bytes and modifies it inplace without
+// copying.
+func CanonicalizeHeaderKey(k []byte) {
+	upper := true
+	for i, c := range k {
+		if upper && 'a' <= c && c <= 'z' {
+			k[i] &= toUpper
+		} else if !upper && 'A' <= c && c <= 'Z' {
+			k[i] |= toLower
+		}
+		upper = c == '-'
+	}
+}
+
+// pow for integers implementation.
+// See Donald Knuth, The Art of Computer Programming, Volume 2, Section 4.6.3
+func pow(a, b int) int {
+	p := 1
+	for b > 0 {
+		if b&1 != 0 {
+			p *= a
+		}
+		b >>= 1
+		a *= a
+	}
+	return p
+}
+
+func split3(p []byte, sep byte) (p1, p2, p3 []byte) {
+	a := bytes.IndexByte(p, sep)
+	b := bytes.IndexByte(p[a+1:], sep)
+	if a == -1 || b == -1 {
+		return p, nil, nil
+	}
+	b += a + 1
+	return p[:a], p[a+1 : b], p[b+1:]
+}
+
+func split2(p []byte, sep byte) (p1, p2 []byte) {
+	i := bytes.IndexByte(p, sep)
+	if i == -1 {
+		return p, nil
+	}
+	return p[:i], p[i+1:]
+}
+
+func trim(p []byte) []byte {
+	var i, j int
+	for i = 0; i < len(p) && (p[i] == ' ' || p[i] == '\t'); {
+		i++
+	}
+	for j = len(p); j > i && (p[j-1] == ' ' || p[j-1] == '\t'); {
+		j--
+	}
+	return p[i:j]
+}

vendor/github.com/gobwas/httphead/httphead.go

@@ -0,0 +1,331 @@
+// Package httphead contains utils for parsing HTTP and HTTP-grammar compatible
+// text protocols headers.
+//
+// That is, this package first aim is to bring ability to easily parse
+// constructions, described here https://tools.ietf.org/html/rfc2616#section-2
+package httphead
+
+import (
+	"bytes"
+	"strings"
+)
+
+// ScanTokens parses data in this form:
+//
+// list = 1#token
+//
+// It returns false if data is malformed.
+func ScanTokens(data []byte, it func([]byte) bool) bool {
+	lexer := &Scanner{data: data}
+
+	var ok bool
+	for lexer.Next() {
+		switch lexer.Type() {
+		case ItemToken:
+			ok = true
+			if !it(lexer.Bytes()) {
+				return true
+			}
+		case ItemSeparator:
+			if !isComma(lexer.Bytes()) {
+				return false
+			}
+		default:
+			return false
+		}
+	}
+
+	return ok && !lexer.err
+}
+
+// ParseOptions parses all header options and appends it to given slice of
+// Option. It returns flag of successful (wellformed input) parsing.
+//
+// Note that appended options are all consist of subslices of data. That is,
+// mutation of data will mutate appended options.
+func ParseOptions(data []byte, options []Option) ([]Option, bool) {
+	var i int
+	index := -1
+	return options, ScanOptions(data, func(idx int, name, attr, val []byte) Control {
+		if idx != index {
+			index = idx
+			i = len(options)
+			options = append(options, Option{Name: name})
+		}
+		if attr != nil {
+			options[i].Parameters.Set(attr, val)
+		}
+		return ControlContinue
+	})
+}
+
+// SelectFlag encodes way of options selection.
+type SelectFlag byte
+
+// String represetns flag as string.
+func (f SelectFlag) String() string {
+	var flags [2]string
+	var n int
+	if f&SelectCopy != 0 {
+		flags[n] = "copy"
+		n++
+	}
+	if f&SelectUnique != 0 {
+		flags[n] = "unique"
+		n++
+	}
+	return "[" + strings.Join(flags[:n], "|") + "]"
+}
+
+const (
+	// SelectCopy causes selector to copy selected option before appending it
+	// to resulting slice.
+	// If SelectCopy flag is not passed to selector, then appended options will
+	// contain sub-slices of the initial data.
+	SelectCopy SelectFlag = 1 << iota
+
+	// SelectUnique causes selector to append only not yet existing option to
+	// resulting slice. Unique is checked by comparing option names.
+	SelectUnique
+)
+
+// OptionSelector contains configuration for selecting Options from header value.
+type OptionSelector struct {
+	// Check is a filter function that applied to every Option that possibly
+	// could be selected.
+	// If Check is nil all options will be selected.
+	Check func(Option) bool
+
+	// Flags contains flags for options selection.
+	Flags SelectFlag
+
+	// Alloc used to allocate slice of bytes when selector is configured with
+	// SelectCopy flag. It will be called with number of bytes needed for copy
+	// of single Option.
+	// If Alloc is nil make is used.
+	Alloc func(n int) []byte
+}
+
+// Select parses header data and appends it to given slice of Option.
+// It also returns flag of successful (wellformed input) parsing.
+func (s OptionSelector) Select(data []byte, options []Option) ([]Option, bool) {
+	var current Option
+	var has bool
+	index := -1
+
+	alloc := s.Alloc
+	if alloc == nil {
+		alloc = defaultAlloc
+	}
+	check := s.Check
+	if check == nil {
+		check = defaultCheck
+	}
+
+	ok := ScanOptions(data, func(idx int, name, attr, val []byte) Control {
+		if idx != index {
+			if has && check(current) {
+				if s.Flags&SelectCopy != 0 {
+					current = current.Copy(alloc(current.Size()))
+				}
+				options = append(options, current)
+				has = false
+			}
+			if s.Flags&SelectUnique != 0 {
+				for i := len(options) - 1; i >= 0; i-- {
+					if bytes.Equal(options[i].Name, name) {
+						return ControlSkip
+					}
+				}
+			}
+			index = idx
+			current = Option{Name: name}
+			has = true
+		}
+		if attr != nil {
+			current.Parameters.Set(attr, val)
+		}
+
+		return ControlContinue
+	})
+	if has && check(current) {
+		if s.Flags&SelectCopy != 0 {
+			current = current.Copy(alloc(current.Size()))
+		}
+		options = append(options, current)
+	}
+
+	return options, ok
+}
+
+func defaultAlloc(n int) []byte { return make([]byte, n) }
+func defaultCheck(Option) bool  { return true }
+
+// Control represents operation that scanner should perform.
+type Control byte
+
+const (
+	// ControlContinue causes scanner to continue scan tokens.
+	ControlContinue Control = iota
+	// ControlBreak causes scanner to stop scan tokens.
+	ControlBreak
+	// ControlSkip causes scanner to skip current entity.
+	ControlSkip
+)
+
+// ScanOptions parses data in this form:
+//
+// values = 1#value
+// value = token *( ";" param )
+// param = token [ "=" (token | quoted-string) ]
+//
+// It calls given callback with the index of the option, option itself and its
+// parameter (attribute and its value, both could be nil). Index is useful when
+// header contains multiple choises for the same named option.
+//
+// Given callback should return one of the defined Control* values.
+// ControlSkip means that passed key is not in caller's interest. That is, all
+// parameters of that key will be skipped.
+// ControlBreak means that no more keys and parameters should be parsed. That
+// is, it must break parsing immediately.
+// ControlContinue means that caller want to receive next parameter and its
+// value or the next key.
+//
+// It returns false if data is malformed.
+func ScanOptions(data []byte, it func(index int, option, attribute, value []byte) Control) bool {
+	lexer := &Scanner{data: data}
+
+	var ok bool
+	var state int
+	const (
+		stateKey = iota
+		stateParamBeforeName
+		stateParamName
+		stateParamBeforeValue
+		stateParamValue
+	)
+
+	var (
+		index             int
+		key, param, value []byte
+		mustCall          bool
+	)
+	for lexer.Next() {
+		var (
+			call      bool
+			growIndex int
+		)
+
+		t := lexer.Type()
+		v := lexer.Bytes()
+
+		switch t {
+		case ItemToken:
+			switch state {
+			case stateKey, stateParamBeforeName:
+				key = v
+				state = stateParamBeforeName
+				mustCall = true
+			case stateParamName:
+				param = v
+				state = stateParamBeforeValue
+				mustCall = true
+			case stateParamValue:
+				value = v
+				state = stateParamBeforeName
+				call = true
+			default:
+				return false
+			}
+
+		case ItemString:
+			if state != stateParamValue {
+				return false
+			}
+			value = v
+			state = stateParamBeforeName
+			call = true
+
+		case ItemSeparator:
+			switch {
+			case isComma(v) && state == stateKey:
+				// Nothing to do.
+
+			case isComma(v) && state == stateParamBeforeName:
+				state = stateKey
+				// Make call only if we have not called this key yet.
+				call = mustCall
+				if !call {
+					// If we have already called callback with the key
+					// that just ended.
+					index++
+				} else {
+					// Else grow the index after calling callback.
+					growIndex = 1
+				}
+
+			case isComma(v) && state == stateParamBeforeValue:
+				state = stateKey
+				growIndex = 1
+				call = true
+
+			case isSemicolon(v) && state == stateParamBeforeName:
+				state = stateParamName
+
+			case isSemicolon(v) && state == stateParamBeforeValue:
+				state = stateParamName
+				call = true
+
+			case isEquality(v) && state == stateParamBeforeValue:
+				state = stateParamValue
+
+			default:
+				return false
+			}
+
+		default:
+			return false
+		}
+
+		if call {
+			switch it(index, key, param, value) {
+			case ControlBreak:
+				// User want to stop to parsing parameters.
+				return true
+
+			case ControlSkip:
+				// User want to skip current param.
+				state = stateKey
+				lexer.SkipEscaped(',')
+
+			case ControlContinue:
+				// User is interested in rest of parameters.
+				// Nothing to do.
+
+			default:
+				panic("unexpected control value")
+			}
+			ok = true
+			param = nil
+			value = nil
+			mustCall = false
+			index += growIndex
+		}
+	}
+	if mustCall {
+		ok = true
+		it(index, key, param, value)
+	}
+
+	return ok && !lexer.err
+}
+
+func isComma(b []byte) bool {
+	return len(b) == 1 && b[0] == ','
+}
+func isSemicolon(b []byte) bool {
+	return len(b) == 1 && b[0] == ';'
+}
+func isEquality(b []byte) bool {
+	return len(b) == 1 && b[0] == '='
+}

vendor/github.com/gobwas/httphead/lexer.go

@@ -0,0 +1,360 @@
+package httphead
+
+import (
+	"bytes"
+)
+
+// ItemType encodes type of the lexing token.
+type ItemType int
+
+const (
+	// ItemUndef reports that token is undefined.
+	ItemUndef ItemType = iota
+	// ItemToken reports that token is RFC2616 token.
+	ItemToken
+	// ItemSeparator reports that token is RFC2616 separator.
+	ItemSeparator
+	// ItemString reports that token is RFC2616 quouted string.
+	ItemString
+	// ItemComment reports that token is RFC2616 comment.
+	ItemComment
+	// ItemOctet reports that token is octet slice.
+	ItemOctet
+)
+
+// Scanner represents header tokens scanner.
+// See https://tools.ietf.org/html/rfc2616#section-2
+type Scanner struct {
+	data []byte
+	pos  int
+
+	itemType  ItemType
+	itemBytes []byte
+
+	err bool
+}
+
+// NewScanner creates new RFC2616 data scanner.
+func NewScanner(data []byte) *Scanner {
+	return &Scanner{data: data}
+}
+
+// Next scans for next token. It returns true on successful scanning, and false
+// on error or EOF.
+func (l *Scanner) Next() bool {
+	c, ok := l.nextChar()
+	if !ok {
+		return false
+	}
+	switch c {
+	case '"': // quoted-string;
+		return l.fetchQuotedString()
+
+	case '(': // comment;
+		return l.fetchComment()
+
+	case '\\', ')': // unexpected chars;
+		l.err = true
+		return false
+
+	default:
+		return l.fetchToken()
+	}
+}
+
+// FetchUntil fetches ItemOctet from current scanner position to first
+// occurence of the c or to the end of the underlying data.
+func (l *Scanner) FetchUntil(c byte) bool {
+	l.resetItem()
+	if l.pos == len(l.data) {
+		return false
+	}
+	return l.fetchOctet(c)
+}
+
+// Peek reads byte at current position without advancing it. On end of data it
+// returns 0.
+func (l *Scanner) Peek() byte {
+	if l.pos == len(l.data) {
+		return 0
+	}
+	return l.data[l.pos]
+}
+
+// Peek2 reads two first bytes at current position without advancing it.
+// If there not enough data it returs 0.
+func (l *Scanner) Peek2() (a, b byte) {
+	if l.pos == len(l.data) {
+		return 0, 0
+	}
+	if l.pos+1 == len(l.data) {
+		return l.data[l.pos], 0
+	}
+	return l.data[l.pos], l.data[l.pos+1]
+}
+
+// Buffered reporst how many bytes there are left to scan.
+func (l *Scanner) Buffered() int {
+	return len(l.data) - l.pos
+}
+
+// Advance moves current position index at n bytes. It returns true on
+// successful move.
+func (l *Scanner) Advance(n int) bool {
+	l.pos += n
+	if l.pos > len(l.data) {
+		l.pos = len(l.data)
+		return false
+	}
+	return true
+}
+
+// Skip skips all bytes until first occurence of c.
+func (l *Scanner) Skip(c byte) {
+	if l.err {
+		return
+	}
+	// Reset scanner state.
+	l.resetItem()
+
+	if i := bytes.IndexByte(l.data[l.pos:], c); i == -1 {
+		// Reached the end of data.
+		l.pos = len(l.data)
+	} else {
+		l.pos += i + 1
+	}
+}
+
+// SkipEscaped skips all bytes until first occurence of non-escaped c.
+func (l *Scanner) SkipEscaped(c byte) {
+	if l.err {
+		return
+	}
+	// Reset scanner state.
+	l.resetItem()
+
+	if i := ScanUntil(l.data[l.pos:], c); i == -1 {
+		// Reached the end of data.
+		l.pos = len(l.data)
+	} else {
+		l.pos += i + 1
+	}
+}
+
+// Type reports current token type.
+func (l *Scanner) Type() ItemType {
+	return l.itemType
+}
+
+// Bytes returns current token bytes.
+func (l *Scanner) Bytes() []byte {
+	return l.itemBytes
+}
+
+func (l *Scanner) nextChar() (byte, bool) {
+	// Reset scanner state.
+	l.resetItem()
+
+	if l.err {
+		return 0, false
+	}
+	l.pos += SkipSpace(l.data[l.pos:])
+	if l.pos == len(l.data) {
+		return 0, false
+	}
+	return l.data[l.pos], true
+}
+
+func (l *Scanner) resetItem() {
+	l.itemType = ItemUndef
+	l.itemBytes = nil
+}
+
+func (l *Scanner) fetchOctet(c byte) bool {
+	i := l.pos
+	if j := bytes.IndexByte(l.data[l.pos:], c); j == -1 {
+		// Reached the end of data.
+		l.pos = len(l.data)
+	} else {
+		l.pos += j
+	}
+
+	l.itemType = ItemOctet
+	l.itemBytes = l.data[i:l.pos]
+
+	return true
+}
+
+func (l *Scanner) fetchToken() bool {
+	n, t := ScanToken(l.data[l.pos:])
+	if n == -1 {
+		l.err = true
+		return false
+	}
+
+	l.itemType = t
+	l.itemBytes = l.data[l.pos : l.pos+n]
+	l.pos += n
+
+	return true
+}
+
+func (l *Scanner) fetchQuotedString() (ok bool) {
+	l.pos++
+
+	n := ScanUntil(l.data[l.pos:], '"')
+	if n == -1 {
+		l.err = true
+		return false
+	}
+
+	l.itemType = ItemString
+	l.itemBytes = RemoveByte(l.data[l.pos:l.pos+n], '\\')
+	l.pos += n + 1
+
+	return true
+}
+
+func (l *Scanner) fetchComment() (ok bool) {
+	l.pos++
+
+	n := ScanPairGreedy(l.data[l.pos:], '(', ')')
+	if n == -1 {
+		l.err = true
+		return false
+	}
+
+	l.itemType = ItemComment
+	l.itemBytes = RemoveByte(l.data[l.pos:l.pos+n], '\\')
+	l.pos += n + 1
+
+	return true
+}
+
+// ScanUntil scans for first non-escaped character c in given data.
+// It returns index of matched c and -1 if c is not found.
+func ScanUntil(data []byte, c byte) (n int) {
+	for {
+		i := bytes.IndexByte(data[n:], c)
+		if i == -1 {
+			return -1
+		}
+		n += i
+		if n == 0 || data[n-1] != '\\' {
+			break
+		}
+		n++
+	}
+	return
+}
+
+// ScanPairGreedy scans for complete pair of opening and closing chars in greedy manner.
+// Note that first opening byte must not be present in data.
+func ScanPairGreedy(data []byte, open, close byte) (n int) {
+	var m int
+	opened := 1
+	for {
+		i := bytes.IndexByte(data[n:], close)
+		if i == -1 {
+			return -1
+		}
+		n += i
+		// If found index is not escaped then it is the end.
+		if n == 0 || data[n-1] != '\\' {
+			opened--
+		}
+
+		for m < i {
+			j := bytes.IndexByte(data[m:i], open)
+			if j == -1 {
+				break
+			}
+			m += j + 1
+			opened++
+		}
+
+		if opened == 0 {
+			break
+		}
+
+		n++
+		m = n
+	}
+	return
+}
+
+// RemoveByte returns data without c. If c is not present in data it returns
+// the same slice. If not, it copies data without c.
+func RemoveByte(data []byte, c byte) []byte {
+	j := bytes.IndexByte(data, c)
+	if j == -1 {
+		return data
+	}
+
+	n := len(data) - 1
+
+	// If character is present, than allocate slice with n-1 capacity. That is,
+	// resulting bytes could be at most n-1 length.
+	result := make([]byte, n)
+	k := copy(result, data[:j])
+
+	for i := j + 1; i < n; {
+		j = bytes.IndexByte(data[i:], c)
+		if j != -1 {
+			k += copy(result[k:], data[i:i+j])
+			i = i + j + 1
+		} else {
+			k += copy(result[k:], data[i:])
+			break
+		}
+	}
+
+	return result[:k]
+}
+
+// SkipSpace skips spaces and lws-sequences from p.
+// It returns number ob bytes skipped.
+func SkipSpace(p []byte) (n int) {
+	for len(p) > 0 {
+		switch {
+		case len(p) >= 3 &&
+			p[0] == '\r' &&
+			p[1] == '\n' &&
+			OctetTypes[p[2]].IsSpace():
+			p = p[3:]
+			n += 3
+		case OctetTypes[p[0]].IsSpace():
+			p = p[1:]
+			n++
+		default:
+			return
+		}
+	}
+	return
+}
+
+// ScanToken scan for next token in p. It returns length of the token and its
+// type. It do not trim p.
+func ScanToken(p []byte) (n int, t ItemType) {
+	if len(p) == 0 {
+		return 0, ItemUndef
+	}
+
+	c := p[0]
+	switch {
+	case OctetTypes[c].IsSeparator():
+		return 1, ItemSeparator
+
+	case OctetTypes[c].IsToken():
+		for n = 1; n < len(p); n++ {
+			c := p[n]
+			if !OctetTypes[c].IsToken() {
+				break
+			}
+		}
+		return n, ItemToken
+
+	default:
+		return -1, ItemUndef
+	}
+}

vendor/github.com/gobwas/httphead/octet.go

@@ -0,0 +1,83 @@
+package httphead
+
+// OctetType desribes character type.
+//
+// From the "Basic Rules" chapter of RFC2616
+// See https://tools.ietf.org/html/rfc2616#section-2.2
+//
+// OCTET          = <any 8-bit sequence of data>
+// CHAR           = <any US-ASCII character (octets 0 - 127)>
+// UPALPHA        = <any US-ASCII uppercase letter "A".."Z">
+// LOALPHA        = <any US-ASCII lowercase letter "a".."z">
+// ALPHA          = UPALPHA | LOALPHA
+// DIGIT          = <any US-ASCII digit "0".."9">
+// CTL            = <any US-ASCII control character (octets 0 - 31) and DEL (127)>
+// CR             = <US-ASCII CR, carriage return (13)>
+// LF             = <US-ASCII LF, linefeed (10)>
+// SP             = <US-ASCII SP, space (32)>
+// HT             = <US-ASCII HT, horizontal-tab (9)>
+// <">            = <US-ASCII double-quote mark (34)>
+// CRLF           = CR LF
+// LWS            = [CRLF] 1*( SP | HT )
+//
+// Many HTTP/1.1 header field values consist of words separated by LWS
+// or special characters. These special characters MUST be in a quoted
+// string to be used within a parameter value (as defined in section
+// 3.6).
+//
+// token          = 1*<any CHAR except CTLs or separators>
+// separators     = "(" | ")" | "<" | ">" | "@"
+// | "," | ";" | ":" | "\" | <">
+// | "/" | "[" | "]" | "?" | "="
+// | "{" | "}" | SP | HT
+type OctetType byte
+
+// IsChar reports whether octet is CHAR.
+func (t OctetType) IsChar() bool { return t&octetChar != 0 }
+
+// IsControl reports whether octet is CTL.
+func (t OctetType) IsControl() bool { return t&octetControl != 0 }
+
+// IsSeparator reports whether octet is separator.
+func (t OctetType) IsSeparator() bool { return t&octetSeparator != 0 }
+
+// IsSpace reports whether octet is space (SP or HT).
+func (t OctetType) IsSpace() bool { return t&octetSpace != 0 }
+
+// IsToken reports whether octet is token.
+func (t OctetType) IsToken() bool { return t&octetToken != 0 }
+
+const (
+	octetChar OctetType = 1 << iota
+	octetControl
+	octetSpace
+	octetSeparator
+	octetToken
+)
+
+// OctetTypes is a table of octets.
+var OctetTypes [256]OctetType
+
+func init() {
+	for c := 32; c < 256; c++ {
+		var t OctetType
+		if c <= 127 {
+			t |= octetChar
+		}
+		if 0 <= c && c <= 31 || c == 127 {
+			t |= octetControl
+		}
+		switch c {
+		case '(', ')', '<', '>', '@', ',', ';', ':', '"', '/', '[', ']', '?', '=', '{', '}', '\\':
+			t |= octetSeparator
+		case ' ', '\t':
+			t |= octetSpace | octetSeparator
+		}
+
+		if t.IsChar() && !t.IsControl() && !t.IsSeparator() && !t.IsSpace() {
+			t |= octetToken
+		}
+
+		OctetTypes[c] = t
+	}
+}

vendor/github.com/gobwas/httphead/option.go

@@ -0,0 +1,193 @@
+package httphead
+
+import (
+	"bytes"
+	"sort"
+)
+
+// Option represents a header option.
+type Option struct {
+	Name       []byte
+	Parameters Parameters
+}
+
+// Size returns number of bytes need to be allocated for use in opt.Copy.
+func (opt Option) Size() int {
+	return len(opt.Name) + opt.Parameters.bytes
+}
+
+// Copy copies all underlying []byte slices into p and returns new Option.
+// Note that p must be at least of opt.Size() length.
+func (opt Option) Copy(p []byte) Option {
+	n := copy(p, opt.Name)
+	opt.Name = p[:n]
+	opt.Parameters, p = opt.Parameters.Copy(p[n:])
+	return opt
+}
+
+// Clone is a shorthand for making slice of opt.Size() sequenced with Copy()
+// call.
+func (opt Option) Clone() Option {
+	return opt.Copy(make([]byte, opt.Size()))
+}
+
+// String represents option as a string.
+func (opt Option) String() string {
+	return "{" + string(opt.Name) + " " + opt.Parameters.String() + "}"
+}
+
+// NewOption creates named option with given parameters.
+func NewOption(name string, params map[string]string) Option {
+	p := Parameters{}
+	for k, v := range params {
+		p.Set([]byte(k), []byte(v))
+	}
+	return Option{
+		Name:       []byte(name),
+		Parameters: p,
+	}
+}
+
+// Equal reports whether option is equal to b.
+func (opt Option) Equal(b Option) bool {
+	if bytes.Equal(opt.Name, b.Name) {
+		return opt.Parameters.Equal(b.Parameters)
+	}
+	return false
+}
+
+// Parameters represents option's parameters.
+type Parameters struct {
+	pos   int
+	bytes int
+	arr   [8]pair
+	dyn   []pair
+}
+
+// Equal reports whether a equal to b.
+func (p Parameters) Equal(b Parameters) bool {
+	switch {
+	case p.dyn == nil && b.dyn == nil:
+	case p.dyn != nil && b.dyn != nil:
+	default:
+		return false
+	}
+
+	ad, bd := p.data(), b.data()
+	if len(ad) != len(bd) {
+		return false
+	}
+
+	sort.Sort(pairs(ad))
+	sort.Sort(pairs(bd))
+
+	for i := 0; i < len(ad); i++ {
+		av, bv := ad[i], bd[i]
+		if !bytes.Equal(av.key, bv.key) || !bytes.Equal(av.value, bv.value) {
+			return false
+		}
+	}
+	return true
+}
+
+// Size returns number of bytes that needed to copy p.
+func (p *Parameters) Size() int {
+	return p.bytes
+}
+
+// Copy copies all underlying []byte slices into dst and returns new
+// Parameters.
+// Note that dst must be at least of p.Size() length.
+func (p *Parameters) Copy(dst []byte) (Parameters, []byte) {
+	ret := Parameters{
+		pos:   p.pos,
+		bytes: p.bytes,
+	}
+	if p.dyn != nil {
+		ret.dyn = make([]pair, len(p.dyn))
+		for i, v := range p.dyn {
+			ret.dyn[i], dst = v.copy(dst)
+		}
+	} else {
+		for i, p := range p.arr {
+			ret.arr[i], dst = p.copy(dst)
+		}
+	}
+	return ret, dst
+}
+
+// Get returns value by key and flag about existence such value.
+func (p *Parameters) Get(key string) (value []byte, ok bool) {
+	for _, v := range p.data() {
+		if string(v.key) == key {
+			return v.value, true
+		}
+	}
+	return nil, false
+}
+
+// Set sets value by key.
+func (p *Parameters) Set(key, value []byte) {
+	p.bytes += len(key) + len(value)
+
+	if p.pos < len(p.arr) {
+		p.arr[p.pos] = pair{key, value}
+		p.pos++
+		return
+	}
+
+	if p.dyn == nil {
+		p.dyn = make([]pair, len(p.arr), len(p.arr)+1)
+		copy(p.dyn, p.arr[:])
+	}
+	p.dyn = append(p.dyn, pair{key, value})
+}
+
+// ForEach iterates over parameters key-value pairs and calls cb for each one.
+func (p *Parameters) ForEach(cb func(k, v []byte) bool) {
+	for _, v := range p.data() {
+		if !cb(v.key, v.value) {
+			break
+		}
+	}
+}
+
+// String represents parameters as a string.
+func (p *Parameters) String() (ret string) {
+	ret = "["
+	for i, v := range p.data() {
+		if i > 0 {
+			ret += " "
+		}
+		ret += string(v.key) + ":" + string(v.value)
+	}
+	return ret + "]"
+}
+
+func (p *Parameters) data() []pair {
+	if p.dyn != nil {
+		return p.dyn
+	}
+	return p.arr[:p.pos]
+}
+
+type pair struct {
+	key, value []byte
+}
+
+func (p pair) copy(dst []byte) (pair, []byte) {
+	n := copy(dst, p.key)
+	p.key = dst[:n]
+	m := n + copy(dst[n:], p.value)
+	p.value = dst[n:m]
+
+	dst = dst[m:]
+
+	return p, dst
+}
+
+type pairs []pair
+
+func (p pairs) Len() int           { return len(p) }
+func (p pairs) Less(a, b int) bool { return bytes.Compare(p[a].key, p[b].key) == -1 }
+func (p pairs) Swap(a, b int)      { p[a], p[b] = p[b], p[a] }

vendor/github.com/gobwas/httphead/writer.go

@@ -0,0 +1,101 @@
+package httphead
+
+import "io"
+
+var (
+	comma     = []byte{','}
+	equality  = []byte{'='}
+	semicolon = []byte{';'}
+	quote     = []byte{'"'}
+	escape    = []byte{'\\'}
+)
+
+// WriteOptions write options list to the dest.
+// It uses the same form as {Scan,Parse}Options functions:
+// values = 1#value
+// value = token *( ";" param )
+// param = token [ "=" (token | quoted-string) ]
+//
+// It wraps valuse into the quoted-string sequence if it contains any
+// non-token characters.
+func WriteOptions(dest io.Writer, options []Option) (n int, err error) {
+	w := writer{w: dest}
+	for i, opt := range options {
+		if i > 0 {
+			w.write(comma)
+		}
+
+		writeTokenSanitized(&w, opt.Name)
+
+		for _, p := range opt.Parameters.data() {
+			w.write(semicolon)
+			writeTokenSanitized(&w, p.key)
+			if len(p.value) != 0 {
+				w.write(equality)
+				writeTokenSanitized(&w, p.value)
+			}
+		}
+	}
+	return w.result()
+}
+
+// writeTokenSanitized writes token as is or as quouted string if it contains
+// non-token characters.
+//
+// Note that is is not expects LWS sequnces be in s, cause LWS is used only as
+// header field continuation:
+// "A CRLF is allowed in the definition of TEXT only as part of a header field
+// continuation. It is expected that the folding LWS will be replaced with a
+// single SP before interpretation of the TEXT value."
+// See https://tools.ietf.org/html/rfc2616#section-2
+//
+// That is we sanitizing s for writing, so there could not be any header field
+// continuation.
+// That is any CRLF will be escaped as any other control characters not allowd in TEXT.
+func writeTokenSanitized(bw *writer, bts []byte) {
+	var qt bool
+	var pos int
+	for i := 0; i < len(bts); i++ {
+		c := bts[i]
+		if !OctetTypes[c].IsToken() && !qt {
+			qt = true
+			bw.write(quote)
+		}
+		if OctetTypes[c].IsControl() || c == '"' {
+			if !qt {
+				qt = true
+				bw.write(quote)
+			}
+			bw.write(bts[pos:i])
+			bw.write(escape)
+			bw.write(bts[i : i+1])
+			pos = i + 1
+		}
+	}
+	if !qt {
+		bw.write(bts)
+	} else {
+		bw.write(bts[pos:])
+		bw.write(quote)
+	}
+}
+
+type writer struct {
+	w   io.Writer
+	n   int
+	err error
+}
+
+func (w *writer) write(p []byte) {
+	if w.err != nil {
+		return
+	}
+	var n int
+	n, w.err = w.w.Write(p)
+	w.n += n
+	return
+}
+
+func (w *writer) result() (int, error) {
+	return w.n, w.err
+}

vendor/github.com/gobwas/pool/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2017-2019 Sergey Kamardin <gobwas@gmail.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/gobwas/pool/README.md

@@ -0,0 +1,107 @@
+# pool
+
+[![GoDoc][godoc-image]][godoc-url]
+
+> Tiny memory reuse helpers for Go.
+
+## generic
+
+Without use of subpackages, `pool` allows to reuse any struct distinguishable
+by size in generic way:
+
+```go
+package main
+
+import "github.com/gobwas/pool"
+
+func main() {
+	x, n := pool.Get(100) // Returns object with size 128 or nil.
+	if x == nil {
+		// Create x somehow with knowledge that n is 128.
+	}
+	defer pool.Put(x, n)
+	
+	// Work with x.
+}
+```
+
+Pool allows you to pass specific options for constructing custom pool:
+
+```go
+package main
+
+import "github.com/gobwas/pool"
+
+func main() {
+	p := pool.Custom(
+        pool.WithLogSizeMapping(),      // Will ceil size n passed to Get(n) to nearest power of two.
+        pool.WithLogSizeRange(64, 512), // Will reuse objects in logarithmic range [64, 512].
+        pool.WithSize(65536),           // Will reuse object with size 65536.
+    )
+	x, n := p.Get(1000)  // Returns nil and 1000 because mapped size 1000 => 1024 is not reusing by the pool.
+    defer pool.Put(x, n) // Will not reuse x.
+	
+	// Work with x.
+}
+```
+
+Note that there are few non-generic pooling implementations inside subpackages.
+
+## pbytes
+
+Subpackage `pbytes` is intended for `[]byte` reuse.
+
+```go
+package main
+
+import "github.com/gobwas/pool/pbytes"
+
+func main() {
+	bts := pbytes.GetCap(100) // Returns make([]byte, 0, 128).
+	defer pbytes.Put(bts)
+
+	// Work with bts.
+}
+```
+
+You can also create your own range for pooling:
+
+```go
+package main
+
+import "github.com/gobwas/pool/pbytes"
+
+func main() {
+	// Reuse only slices whose capacity is 128, 256, 512 or 1024.
+	pool := pbytes.New(128, 1024) 
+
+	bts := pool.GetCap(100) // Returns make([]byte, 0, 128).
+	defer pool.Put(bts)
+
+	// Work with bts.
+}
+```
+
+## pbufio
+
+Subpackage `pbufio` is intended for `*bufio.{Reader, Writer}` reuse.
+
+```go
+package main
+
+import "github.com/gobwas/pool/pbufio"
+
+func main() {
+	bw := pbufio.GetWriter(os.Stdout, 100) // Returns bufio.NewWriterSize(128).
+	defer pbufio.PutWriter(bw)
+
+	// Work with bw.
+}
+```
+
+Like with `pbytes`, you can also create pool with custom reuse bounds.
+
+
+
+[godoc-image]: https://godoc.org/github.com/gobwas/pool?status.svg
+[godoc-url]:   https://godoc.org/github.com/gobwas/pool

vendor/github.com/gobwas/pool/generic.go

@@ -0,0 +1,87 @@
+package pool
+
+import (
+	"sync"
+
+	"github.com/gobwas/pool/internal/pmath"
+)
+
+var DefaultPool = New(128, 65536)
+
+// Get pulls object whose generic size is at least of given size. It also
+// returns a real size of x for further pass to Put(). It returns -1 as real
+// size for nil x. Size >-1 does not mean that x is non-nil, so checks must be
+// done.
+//
+// Note that size could be ceiled to the next power of two.
+//
+// Get is a wrapper around DefaultPool.Get().
+func Get(size int) (interface{}, int) { return DefaultPool.Get(size) }
+
+// Put takes x and its size for future reuse.
+// Put is a wrapper around DefaultPool.Put().
+func Put(x interface{}, size int) { DefaultPool.Put(x, size) }
+
+// Pool contains logic of reusing objects distinguishable by size in generic
+// way.
+type Pool struct {
+	pool map[int]*sync.Pool
+	size func(int) int
+}
+
+// New creates new Pool that reuses objects which size is in logarithmic range
+// [min, max].
+//
+// Note that it is a shortcut for Custom() constructor with Options provided by
+// WithLogSizeMapping() and WithLogSizeRange(min, max) calls.
+func New(min, max int) *Pool {
+	return Custom(
+		WithLogSizeMapping(),
+		WithLogSizeRange(min, max),
+	)
+}
+
+// Custom creates new Pool with given options.
+func Custom(opts ...Option) *Pool {
+	p := &Pool{
+		pool: make(map[int]*sync.Pool),
+		size: pmath.Identity,
+	}
+
+	c := (*poolConfig)(p)
+	for _, opt := range opts {
+		opt(c)
+	}
+
+	return p
+}
+
+// Get pulls object whose generic size is at least of given size.
+// It also returns a real size of x for further pass to Put() even if x is nil.
+// Note that size could be ceiled to the next power of two.
+func (p *Pool) Get(size int) (interface{}, int) {
+	n := p.size(size)
+	if pool := p.pool[n]; pool != nil {
+		return pool.Get(), n
+	}
+	return nil, size
+}
+
+// Put takes x and its size for future reuse.
+func (p *Pool) Put(x interface{}, size int) {
+	if pool := p.pool[size]; pool != nil {
+		pool.Put(x)
+	}
+}
+
+type poolConfig Pool
+
+// AddSize adds size n to the map.
+func (p *poolConfig) AddSize(n int) {
+	p.pool[n] = new(sync.Pool)
+}
+
+// SetSizeMapping sets up incoming size mapping function.
+func (p *poolConfig) SetSizeMapping(size func(int) int) {
+	p.size = size
+}