Release 2018.12.0

TUN-1158: Fix segfault when carrier test case fails

Makefile

@@ -23,6 +23,10 @@ endif
 .PHONY: all
 all: cloudflared test
 
+.PHONY: clean
+clean:
+	go clean
+
 .PHONY: cloudflared
 cloudflared:
 	go build -v $(VERSION_FLAGS) $(IMPORT_PATH)/cmd/cloudflared

RELEASE_NOTES

@@ -1,14 +1,29 @@
-2018.10.4
-- 2018-09-21 AUTH-1070: added SSH/protocol forwarding
-- 2018-10-19 AUTH-1235: fixed packaging of deb dev file
-- 2018-10-19 TUN-1097: Host missing from WebSocket request
-- 2018-10-25 Release 2018.10.4
-- 2018-10-19 AUTH-1188: UX feedback, move warnings to debug
+2018.12.0
+- 2018-11-15 TUN-1196: Allow TLS config client CA and root CA to be constructed from multiple certificates
+- 2018-11-20 TUN-1209: TLS Config Certificates and GetCertificate can both be set
+- 2018-11-26 TUN-1212: Expose tunnel_id in metrics
+- 2018-11-30 TUN-1204: remove 'cloudflared hello' command
+- 2018-12-04 Fix license URL typo
+- 2018-12-07 TUN-1250: ValidateHTTPService shouldn't follow 302s
+
+2018.11.0
+- 2018-10-31 AUTH-1282: Fixed an issue where we were receiving as opposed sending on the channel.
+- 2018-11-06 TUN-1179: Fix log message in cmd/cloudflared/transfer.Run
+- 2018-11-13 AUTH-1308: get jwt even when you are already logged in
+- 2018-11-12 TUN-1190: check URL parse error when starting SSH proxy server
+- 2018-11-15 AUTH-1320: Fixed request issue and unhide the ssh command
+
+2018.10.5
+- 2018-10-18 TUN-968: Flow control for large requests/responses
+- 2018-10-26 TUN-1158: Windows: use process arguments rather than trivial service arguments
+- 2018-10-20 #30: Fix the Content-Length header for HTTP2->HTTP1
+- 2018-10-29 TUN-1160: pass Host header during origin url validation
 
 2018.10.4
 - 2018-09-21 AUTH-1070: added SSH/protocol forwarding
 - 2018-10-19 AUTH-1235: fixed packaging of deb dev file
 - 2018-10-19 TUN-1097: Host missing from WebSocket request
+- 2018-10-19 AUTH-1188: UX Review and Changes for CLI SSH Access
 
 2018.10.3
 - 2018-10-08 TUN-1099: Bring back changes in 2018.10.1

carrier/carrier.go

@@ -89,6 +89,7 @@ func createWebsocketStream(originURL string) (*websocket.Conn, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	wsConn, resp, err := websocket.ClientConnect(req, nil)
 	if err != nil && resp != nil && resp.StatusCode > 300 {
 		location, err := resp.Location()
@@ -125,7 +126,14 @@ func buildAccessRequest(originURL string) (*http.Request, error) {
 	if err != nil {
 		return nil, err
 	}
-	req.Header.Set("cf-access-token", token)
 
-	return req, nil
+	// We need to create a new request as FetchToken will modify req (boo mutable)
+	// as it has to follow redirect on the API and such, so here we init a new one
+	originRequest, err := http.NewRequest(http.MethodGet, originURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	originRequest.Header.Set("cf-access-token", token)
+
+	return originRequest, nil
 }

carrier/carrier_test.go

@@ -66,11 +66,16 @@ func TestStartServer(t *testing.T) {
 	defer ts.Close()
 
 	go func() {
-		StartServer(logger, listenerAddress, "http://"+ts.Listener.Addr().String(), shutdownC)
+		err := StartServer(logger, listenerAddress, "http://"+ts.Listener.Addr().String(), shutdownC)
+		if err != nil {
+			t.Fatalf("Error starting server: %v", err)
+		}
 	}()
 
 	conn, err := net.Dial("tcp", listenerAddress)
-	assert.NoError(t, err)
+	if err != nil {
+		t.Fatalf("Error connecting to server: %v", err)
+	}
 	conn.Write([]byte(message))
 
 	readBuffer := make([]byte, len(message))

cmd/cloudflared/access/cmd.go

@@ -89,14 +89,12 @@ func Commands() []*cli.Command {
 					Usage:       "",
 					ArgsUsage:   "",
 					Description: `The ssh subcommand sends data over a proxy to the Cloudflare edge.`,
-					Hidden:      true,
 					Flags: []cli.Flag{
 						&cli.StringFlag{
 							Name: "hostname",
 						},
 						&cli.StringFlag{
-							Name:   "url",
-							Hidden: true,
+							Name: "url",
 						},
 					},
 				},
@@ -105,7 +103,6 @@ func Commands() []*cli.Command {
 					Action:      sshConfig,
 					Usage:       "ssh-config",
 					Description: `Prints an example configuration ~/.ssh/config`,
-					Hidden:      true,
 				},
 			},
 		},

cmd/cloudflared/main.go

@@ -19,7 +19,7 @@ import (
 
 const (
 	developerPortal = "https://developers.cloudflare.com/argo-tunnel"
-	licenseUrl      = developerPortal + "/licence/"
+	licenseUrl      = developerPortal + "/license/"
 )
 
 var (

cmd/cloudflared/transfer/transfer.go

@@ -43,7 +43,7 @@ func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncr
 
 	err = shell.OpenBrowser(requestURL)
 	if err != nil {
-		fmt.Fprintf(os.Stdout, "Please open the following URL and log in with your Cloudflare account:\n\n%s\n\nLeave cloudflared running to download the %s automatically.\n", resourceName, requestURL)
+		fmt.Fprintf(os.Stdout, "Please open the following URL and log in with your Cloudflare account:\n\n%s\n\nLeave cloudflared running to download the %s automatically.\n", requestURL, resourceName)
 	} else {
 		fmt.Fprintf(os.Stdout, "A browser window should have opened at the following URL:\n\n%s\n\nIf the browser failed to open, open it yourself and visit the URL above.\n", requestURL)
 	}
@@ -83,21 +83,19 @@ func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncr
 
 // BuildRequestURL creates a request suitable for a resource transfer.
 // it will return a constructed url based off the base url and query key/value provided.
-// follow will follow redirects.
-func buildRequestURL(baseURL *url.URL, key, value string, follow bool) (string, error) {
+// cli will build a url for cli transfer request.
+func buildRequestURL(baseURL *url.URL, key, value string, cli bool) (string, error) {
 	q := baseURL.Query()
 	q.Set(key, value)
 	baseURL.RawQuery = q.Encode()
-	if !follow {
+	if !cli {
 		return baseURL.String(), nil
 	}
 
-	response, err := http.Get(baseURL.String())
-	if err != nil {
-		return "", err
-	}
-	return response.Request.URL.String(), nil
-
+	q.Set("redirect_url", baseURL.String()) // we add the token as a query param on both the redirect_url
+	baseURL.RawQuery = q.Encode()           // and this actual baseURL.
+	baseURL.Path = "cdn-cgi/access/cli"
+	return baseURL.String(), nil
 }
 
 // transferRequest downloads the requested resource from the request URL

cmd/cloudflared/tunnel/cmd.go

@@ -56,20 +56,6 @@ func Commands() []*cli.Command {
 			},
 			Hidden: true,
 		},
-		{
-			Name:   "hello",
-			Action: helloWorld,
-			Usage:  "Run a simple \"Hello World\" server for testing Argo Tunnel.",
-			Flags: []cli.Flag{
-				&cli.IntFlag{
-					Name:  "port",
-					Usage: "Listen on the selected port.",
-					Value: 8080,
-				},
-			},
-			ArgsUsage: " ", // can't be the empty string or we get the default output
-			Hidden:    true,
-		},
 		{
 			Name:   "proxy-dns",
 			Action: tunneldns.Run,
@@ -311,7 +297,7 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 		c.Set("url", "https://"+helloListener.Addr().String())
 	}
 
-	if uri, _ := url.Parse(c.String("url")); uri.Scheme == "ssh" {
+	if uri, err := url.Parse(c.String("url")); err == nil && uri.Scheme == "ssh" {
 		host := uri.Host
 		if uri.Port() == "" { // default to 22
 			host = uri.Hostname() + ":22"
@@ -438,7 +424,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    "cacert",
-			Usage:   "Certificate Authority authenticating the Cloudflare tunnel connection.",
+			Usage:   "Certificate Authority authenticating connections with Cloudflare's edge network.",
 			EnvVars: []string{"TUNNEL_CACERT"},
 			Hidden:  true,
 		}),

cmd/cloudflared/tunnel/configuration.go

@@ -192,18 +192,24 @@ func prepareTunnelConfig(c *cli.Context, buildInfo *origin.BuildInfo, version st
 		httpTransport.TLSClientConfig.ServerName = c.String("origin-server-name")
 	}
 
-	err = validation.ValidateHTTPService(originURL, httpTransport)
+	err = validation.ValidateHTTPService(originURL, hostname, httpTransport)
 	if err != nil {
 		logger.WithError(err).Error("unable to connect to the origin")
 		return nil, errors.Wrap(err, "unable to connect to the origin")
 	}
 
+	toEdgeTLSConfig, err := createTunnelConfig(c)
+	if err != nil {
+		logger.WithError(err).Error("unable to create TLS config to connect with edge")
+		return nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
+	}
+
 	return &origin.TunnelConfig{
 		EdgeAddrs:          c.StringSlice("edge"),
 		OriginUrl:          originURL,
 		Hostname:           hostname,
 		OriginCert:         originCert,
-		TlsConfig:          tlsconfig.CreateTunnelConfig(c, c.StringSlice("edge")),
+		TlsConfig:          toEdgeTLSConfig,
 		ClientTlsConfig:    httpTransport.TLSClientConfig,
 		Retries:            c.Uint("retries"),
 		HeartbeatInterval:  c.Duration("heartbeat-interval"),
@@ -240,7 +246,7 @@ func loadCertPool(c *cli.Context, logger *logrus.Logger) (*x509.CertPool, error)
 		}
 	}
 
-	originCertPool, err := tlsconfig.LoadOriginCertPool(originCustomCAPool)
+	originCertPool, err := loadOriginCertPool(originCustomCAPool)
 	if err != nil {
 		return nil, errors.Wrap(err, "error loading the certificate pool")
 	}
@@ -253,6 +259,86 @@ func loadCertPool(c *cli.Context, logger *logrus.Logger) (*x509.CertPool, error)
 	return originCertPool, nil
 }
 
+func loadOriginCertPool(originCAPoolPEM []byte) (*x509.CertPool, error) {
+	// Get the global pool
+	certPool, err := loadGlobalCertPool()
+	if err != nil {
+		return nil, err
+	}
+
+	// Then, add any custom origin CA pool the user may have passed
+	if originCAPoolPEM != nil {
+		if !certPool.AppendCertsFromPEM(originCAPoolPEM) {
+			logger.Warn("could not append the provided origin CA to the cloudflared certificate pool")
+		}
+	}
+
+	return certPool, nil
+}
+
+func loadGlobalCertPool() (*x509.CertPool, error) {
+	// First, obtain the system certificate pool
+	certPool, err := x509.SystemCertPool()
+	if err != nil {
+		if runtime.GOOS != "windows" {
+			logger.WithError(err).Warn("error obtaining the system certificates")
+		}
+		certPool = x509.NewCertPool()
+	}
+
+	// Next, append the Cloudflare CAs into the system pool
+	cfRootCA, err := tlsconfig.GetCloudflareRootCA()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not append Cloudflare Root CAs to cloudflared certificate pool")
+	}
+	for _, cert := range cfRootCA {
+		certPool.AddCert(cert)
+	}
+
+	// Finally, add the Hello certificate into the pool (since it's self-signed)
+	helloCert, err := tlsconfig.GetHelloCertificateX509()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not append Hello server certificate to cloudflared certificate pool")
+	}
+	certPool.AddCert(helloCert)
+
+	return certPool, nil
+}
+
+func createTunnelConfig(c *cli.Context) (*tls.Config, error) {
+	var rootCAs []string
+	if c.String("cacert") != "" {
+		rootCAs = append(rootCAs, c.String("cacert"))
+	}
+	edgeAddrs := c.StringSlice("edge")
+
+	userConfig := &tlsconfig.TLSParameters{RootCAs: rootCAs}
+	tlsConfig, err := tlsconfig.GetConfig(userConfig)
+	if err != nil {
+		return nil, err
+	}
+	if tlsConfig.RootCAs == nil {
+		rootCAPool := x509.NewCertPool()
+		cfRootCA, err := tlsconfig.GetCloudflareRootCA()
+		if err != nil {
+			return nil, errors.Wrap(err, "could not append Cloudflare Root CAs to cloudflared certificate pool")
+		}
+		for _, cert := range cfRootCA {
+			rootCAPool.AddCert(cert)
+		}
+		tlsConfig.RootCAs = rootCAPool
+		tlsConfig.ServerName = "cftunnel.com"
+	} else if len(edgeAddrs) > 0 {
+		// Set for development environments and for testing specific origintunneld instances
+		tlsConfig.ServerName, _, _ = net.SplitHostPort(edgeAddrs[0])
+	}
+
+	if tlsConfig.ServerName == "" && !tlsConfig.InsecureSkipVerify {
+		return nil, fmt.Errorf("either ServerName or InsecureSkipVerify must be specified in the tls.Config")
+	}
+	return tlsConfig, nil
+}
+
 func isRunningFromTerminal() bool {
 	return terminal.IsTerminal(int(os.Stdout.Fd()))
 }

cmd/cloudflared/tunnel/configuration_test.go

@@ -0,0 +1,214 @@
+// +build ignore
+// TODO: Remove the above build tag and include this test when we start compiling with Golang 1.10.0+
+
+package tunnel
+
+import (
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/asn1"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Generated using `openssl req -newkey rsa:512 -nodes -x509 -days 3650`
+var samplePEM = []byte(`
+-----BEGIN CERTIFICATE-----
+MIIB4DCCAYoCCQCb/H0EUrdXEjANBgkqhkiG9w0BAQsFADB3MQswCQYDVQQGEwJV
+UzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcGA1UECgwQQ2xv
+dWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVneTERMA8GA1UE
+AwwIVGVzdCBPbmUwHhcNMTgwNDI2MTYxMDUxWhcNMjgwNDIzMTYxMDUxWjB3MQsw
+CQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcG
+A1UECgwQQ2xvdWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVn
+eTERMA8GA1UEAwwIVGVzdCBPbmUwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAwVQD
+K0SJ25UFLznm2pU3zhzMEvpDEofHVNnCjk4mlDrtVop7PkKZ8pDEmuQANltUrxC8
+yHBE2wXMv+GlH+bDtwIDAQABMA0GCSqGSIb3DQEBCwUAA0EAjVYQzozIFPkt/HRY
+uUoZ8zEHIDICb0syFf5VAjm9AgTwIPzUmD+c5vl6LWDnxq7L45nLCzhhQ6YmiwDz
+X7Wcyg==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIB4DCCAYoCCQDZfCdAJ+mwzDANBgkqhkiG9w0BAQsFADB3MQswCQYDVQQGEwJV
+UzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcGA1UECgwQQ2xv
+dWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVneTERMA8GA1UE
+AwwIVGVzdCBUd28wHhcNMTgwNDI2MTYxMTIwWhcNMjgwNDIzMTYxMTIwWjB3MQsw
+CQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcG
+A1UECgwQQ2xvdWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVn
+eTERMA8GA1UEAwwIVGVzdCBUd28wXDANBgkqhkiG9w0BAQEFAANLADBIAkEAoHKp
+ROVK3zCSsH7ocYeyRAML4V7SFAbZcb4WIwDnE08oMBVRkQVcW5tqEkvG3RiClfzV
+wZIJ3CfqKIeSNSDU9wIDAQABMA0GCSqGSIb3DQEBCwUAA0EAJw2gUbnPiq4C2p5b
+iWzlA9Q7aKo+VQ4H7IZS7tTccr59nVjvH/TG3eWujpnocr4TOqW9M3CK1DF9mUGP
+3pQ3Jg==
+-----END CERTIFICATE-----
+`)
+
+var systemCertPoolSubjects []*pkix.Name
+
+type certificateFixture struct {
+	ou string
+	cn string
+}
+
+func TestMain(m *testing.M) {
+	systemCertPool, err := x509.SystemCertPool()
+	if isUnrecoverableError(err) {
+		os.Exit(1)
+	}
+
+	if systemCertPool == nil {
+		// On Windows, let's just assume the system cert pool was empty
+		systemCertPool = x509.NewCertPool()
+	}
+
+	systemCertPoolSubjects, err = getCertPoolSubjects(systemCertPool)
+	if err != nil {
+		os.Exit(1)
+	}
+
+	os.Exit(m.Run())
+}
+
+func TestLoadOriginCertPoolJustSystemPool(t *testing.T) {
+	certPoolSubjects := loadCertPoolSubjects(t, nil)
+	extraSubjects := subjectSubtract(systemCertPoolSubjects, certPoolSubjects)
+
+	// Remove extra subjects from the cert pool
+	var filteredSystemCertPoolSubjects []*pkix.Name
+
+	t.Log(extraSubjects)
+
+OUTER:
+	for _, subject := range certPoolSubjects {
+		for _, extraSubject := range extraSubjects {
+			if subject == extraSubject {
+				t.Log(extraSubject)
+				continue OUTER
+			}
+		}
+
+		filteredSystemCertPoolSubjects = append(filteredSystemCertPoolSubjects, subject)
+	}
+
+	assert.Equal(t, len(filteredSystemCertPoolSubjects), len(systemCertPoolSubjects))
+
+	difference := subjectSubtract(systemCertPoolSubjects, filteredSystemCertPoolSubjects)
+	assert.Equal(t, 0, len(difference))
+}
+
+func TestLoadOriginCertPoolCFCertificates(t *testing.T) {
+	certPoolSubjects := loadCertPoolSubjects(t, nil)
+
+	extraSubjects := subjectSubtract(systemCertPoolSubjects, certPoolSubjects)
+
+	expected := []*certificateFixture{
+		{ou: "CloudFlare Origin SSL ECC Certificate Authority"},
+		{ou: "CloudFlare Origin SSL Certificate Authority"},
+		{cn: "origin-pull.cloudflare.net"},
+		{cn: "Argo Tunnel Sample Hello Server Certificate"},
+	}
+
+	assertFixturesMatchSubjects(t, expected, extraSubjects)
+}
+
+func TestLoadOriginCertPoolWithExtraPEMs(t *testing.T) {
+	certPoolWithoutPEMSubjects := loadCertPoolSubjects(t, nil)
+	certPoolWithPEMSubjects := loadCertPoolSubjects(t, samplePEM)
+
+	difference := subjectSubtract(certPoolWithoutPEMSubjects, certPoolWithPEMSubjects)
+
+	assert.Equal(t, 2, len(difference))
+
+	expected := []*certificateFixture{
+		{cn: "Test One"},
+		{cn: "Test Two"},
+	}
+
+	assertFixturesMatchSubjects(t, expected, difference)
+}
+
+func loadCertPoolSubjects(t *testing.T, originCAPoolPEM []byte) []*pkix.Name {
+	certPool, err := loadOriginCertPool(originCAPoolPEM)
+	if isUnrecoverableError(err) {
+		t.Fatal(err)
+	}
+	assert.NotEmpty(t, certPool.Subjects())
+	certPoolSubjects, err := getCertPoolSubjects(certPool)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return certPoolSubjects
+}
+
+func assertFixturesMatchSubjects(t *testing.T, fixtures []*certificateFixture, subjects []*pkix.Name) {
+	assert.Equal(t, len(fixtures), len(subjects))
+
+	for _, fixture := range fixtures {
+		found := false
+		for _, subject := range subjects {
+			found = found || fixtureMatchesSubjectPredicate(fixture, subject)
+		}
+
+		if !found {
+			t.Fail()
+		}
+	}
+}
+
+func fixtureMatchesSubjectPredicate(fixture *certificateFixture, subject *pkix.Name) bool {
+	cnMatch := true
+	if fixture.cn != "" {
+		cnMatch = fixture.cn == subject.CommonName
+	}
+
+	ouMatch := true
+	if fixture.ou != "" {
+		ouMatch = len(subject.OrganizationalUnit) > 0 && fixture.ou == subject.OrganizationalUnit[0]
+	}
+
+	return cnMatch && ouMatch
+}
+
+func subjectSubtract(left []*pkix.Name, right []*pkix.Name) []*pkix.Name {
+	var difference []*pkix.Name
+
+	var found bool
+	for _, r := range right {
+		found = false
+		for _, l := range left {
+			if (*l).String() == (*r).String() {
+				found = true
+			}
+		}
+
+		if !found {
+			difference = append(difference, r)
+		}
+	}
+
+	return difference
+}
+
+func getCertPoolSubjects(certPool *x509.CertPool) ([]*pkix.Name, error) {
+	var subjects []*pkix.Name
+
+	for _, subject := range certPool.Subjects() {
+		var sequence pkix.RDNSequence
+		_, err := asn1.Unmarshal(subject, &sequence)
+		if err != nil {
+			return nil, err
+		}
+
+		name := pkix.Name{}
+		name.FillFromRDNSequence(&sequence)
+
+		subjects = append(subjects, &name)
+	}
+
+	return subjects, nil
+}
+
+func isUnrecoverableError(err error) bool {
+	return err != nil && err.Error() != "crypto/x509: system root pool is not available on Windows"
+}

cmd/cloudflared/tunnel/hello.go

@@ -1,20 +0,0 @@
-package tunnel
-
-import (
-	"fmt"
-
-	"gopkg.in/urfave/cli.v2"
-
-	"github.com/cloudflare/cloudflared/hello"
-)
-
-func helloWorld(c *cli.Context) error {
-	address := fmt.Sprintf(":%d", c.Int("port"))
-	listener, err := hello.CreateTLSListener(address)
-	if err != nil {
-		return err
-	}
-	defer listener.Close()
-	err = hello.StartHelloWorldServer(logger, listener, nil)
-	return err
-}

cmd/cloudflared/windows_service.go

@@ -87,7 +87,20 @@ type windowsService struct {
 }
 
 // called by the package code at the start of the service
-func (s *windowsService) Execute(args []string, r <-chan svc.ChangeRequest, statusChan chan<- svc.Status) (ssec bool, errno uint32) {
+func (s *windowsService) Execute(serviceArgs []string, r <-chan svc.ChangeRequest, statusChan chan<- svc.Status) (ssec bool, errno uint32) {
+	// the arguments passed here are only meaningful if they were manually
+	// specified by the user, e.g. using the Services console or `sc start`.
+	// https://docs.microsoft.com/en-us/windows/desktop/services/service-entry-point
+	// https://stackoverflow.com/a/6235139
+	var args []string
+	if len(serviceArgs) > 1 {
+		args = serviceArgs
+	} else {
+		// fall back to the arguments from ImagePath (or, as sc calls it, binPath)
+		args = os.Args
+	}
+	s.elog.Info(1, fmt.Sprintf("%s service arguments: %v", windowsServiceName, args))
+
 	const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown
 	statusChan <- svc.Status{State: svc.StartPending}
 	errC := make(chan error)

h2mux/h2mux.go

@@ -15,11 +15,12 @@ import (
 )
 
 const (
-	defaultFrameSize  uint32        = 1 << 14 // Minimum frame size in http2 spec
-	defaultWindowSize uint32        = 65535
-	maxWindowSize     uint32        = (1 << 31) - 1 // 2^31-1 = 2147483647, max window size specified in http2 spec
-	defaultTimeout    time.Duration = 5 * time.Second
-	defaultRetries    uint64        = 5
+	defaultFrameSize         uint32        = 1 << 14       // Minimum frame size in http2 spec
+	defaultWindowSize        uint32        = (1 << 16) - 1 // Minimum window size in http2 spec
+	maxWindowSize            uint32        = (1 << 31) - 1 // 2^31-1 = 2147483647, max window size in http2 spec
+	defaultTimeout           time.Duration = 5 * time.Second
+	defaultRetries           uint64        = 5
+	defaultWriteBufferMaxLen int           = 1024 * 1024 * 512 // 500mb
 
 	SettingMuxerMagic http2.SettingID = 0x42db
 	MuxerMagicOrigin  uint32          = 0xa2e43c8b
@@ -49,6 +50,12 @@ type MuxerConfig struct {
 	// Logger to use
 	Logger             *log.Entry
 	CompressionQuality CompressionSetting
+	// Initial size for HTTP2 flow control windows
+	DefaultWindowSize uint32
+	// Largest allowable size for HTTP2 flow control windows
+	MaxWindowSize uint32
+	// Largest allowable capacity for the buffer of data to be sent
+	StreamWriteBufferMaxLen int
 }
 
 type Muxer struct {
@@ -98,6 +105,15 @@ func Handshake(
 	if config.Timeout == 0 {
 		config.Timeout = defaultTimeout
 	}
+	if config.DefaultWindowSize == 0 {
+		config.DefaultWindowSize = defaultWindowSize
+	}
+	if config.MaxWindowSize == 0 {
+		config.MaxWindowSize = maxWindowSize
+	}
+	if config.StreamWriteBufferMaxLen == 0 {
+		config.StreamWriteBufferMaxLen = defaultWriteBufferMaxLen
+	}
 	// Initialise connection state fields
 	m := &Muxer{
 		f:             http2.NewFramer(w, r), // A framer that writes to w and reads from r
@@ -179,8 +195,9 @@ func Handshake(
 		abortChan:               m.abortChan,
 		pingTimestamp:           pingTimestamp,
 		connActive:              connActive,
-		initialStreamWindow:     defaultWindowSize,
-		streamWindowMax:         maxWindowSize,
+		initialStreamWindow:     m.config.DefaultWindowSize,
+		streamWindowMax:         m.config.MaxWindowSize,
+		streamWriteBufferMaxLen: m.config.StreamWriteBufferMaxLen,
 		r:                       m.r,
 		updateRTTChan:           updateRTTChan,
 		updateReceiveWindowChan: updateReceiveWindowChan,
@@ -375,10 +392,12 @@ func (m *Muxer) OpenStream(headers []Header, body io.Reader) (*MuxedStream, erro
 		responseHeadersReceived: make(chan struct{}),
 		readBuffer:              NewSharedBuffer(),
 		writeBuffer:             &bytes.Buffer{},
-		receiveWindow:           defaultWindowSize,
-		receiveWindowCurrentMax: defaultWindowSize, // Initial window size limit. exponentially increase it when receiveWindow is exhausted
-		receiveWindowMax:        maxWindowSize,
-		sendWindow:              defaultWindowSize,
+		writeBufferMaxLen:       m.config.StreamWriteBufferMaxLen,
+		writeBufferHasSpace:     make(chan struct{}, 1),
+		receiveWindow:           m.config.DefaultWindowSize,
+		receiveWindowCurrentMax: m.config.DefaultWindowSize,
+		receiveWindowMax:        m.config.MaxWindowSize,
+		sendWindow:              m.config.DefaultWindowSize,
 		readyList:               m.readyList,
 		writeHeaders:            headers,
 		dictionaries:            m.muxReader.dictionaries,

h2mux/h2mux_test.go

@@ -39,20 +39,26 @@ func NewDefaultMuxerPair() *DefaultMuxerPair {
 	origin, edge := net.Pipe()
 	return &DefaultMuxerPair{
 		OriginMuxConfig: MuxerConfig{
-			Timeout: time.Second,
-			IsClient: true,
-			Name: "origin",
-			Logger: log.NewEntry(log.New()),
+			Timeout:                 time.Second,
+			IsClient:                true,
+			Name:                    "origin",
+			Logger:                  log.NewEntry(log.New()),
+			DefaultWindowSize:       (1 << 8) - 1,
+			MaxWindowSize:           (1 << 15) - 1,
+			StreamWriteBufferMaxLen: 1024,
 		},
-		OriginConn:      origin,
-		EdgeMuxConfig:   MuxerConfig{
-			Timeout: time.Second,
-			IsClient: false,
-			Name: "edge",
-			Logger: log.NewEntry(log.New()),
+		OriginConn: origin,
+		EdgeMuxConfig: MuxerConfig{
+			Timeout:                 time.Second,
+			IsClient:                false,
+			Name:                    "edge",
+			Logger:                  log.NewEntry(log.New()),
+			DefaultWindowSize:       (1 << 8) - 1,
+			MaxWindowSize:           (1 << 15) - 1,
+			StreamWriteBufferMaxLen: 1024,
 		},
-		EdgeConn:        edge,
-		doneC:           make(chan struct{}),
+		EdgeConn: edge,
+		doneC:    make(chan struct{}),
 	}
 }
 
@@ -60,22 +66,22 @@ func NewCompressedMuxerPair(quality CompressionSetting) *DefaultMuxerPair {
 	origin, edge := net.Pipe()
 	return &DefaultMuxerPair{
 		OriginMuxConfig: MuxerConfig{
-			Timeout: time.Second,
-			IsClient: true,
-			Name: "origin",
+			Timeout:            time.Second,
+			IsClient:           true,
+			Name:               "origin",
 			CompressionQuality: quality,
-			Logger: log.NewEntry(log.New()),
+			Logger:             log.NewEntry(log.New()),
 		},
-		OriginConn:      origin,
-		EdgeMuxConfig:   MuxerConfig{
-			Timeout: time.Second,
-			IsClient: false,
-			Name: "edge",
+		OriginConn: origin,
+		EdgeMuxConfig: MuxerConfig{
+			Timeout:            time.Second,
+			IsClient:           false,
+			Name:               "edge",
 			CompressionQuality: quality,
-			Logger: log.NewEntry(log.New()),
+			Logger:             log.NewEntry(log.New()),
 		},
-		EdgeConn:        edge,
-		doneC:           make(chan struct{}),
+		EdgeConn: edge,
+		doneC:    make(chan struct{}),
 	}
 }
 
@@ -230,7 +236,6 @@ func TestSingleStream(t *testing.T) {
 func TestSingleStreamLargeResponseBody(t *testing.T) {
 	muxPair := NewDefaultMuxerPair()
 	bodySize := 1 << 24
-	streamReady := make(chan struct{})
 	muxPair.OriginMuxConfig.Handler = MuxedStreamFunc(func(stream *MuxedStream) error {
 		if len(stream.Headers) != 1 {
 			t.Fatalf("expected %d headers, got %d", 1, len(stream.Headers))
@@ -257,8 +262,6 @@ func TestSingleStreamLargeResponseBody(t *testing.T) {
 		if n != len(payload) {
 			t.Fatalf("origin short write: %d/%d bytes", n, len(payload))
 		}
-		t.Log("Payload written; signaling that the stream is ready")
-		streamReady <- struct{}{}
 
 		return nil
 	})
@@ -282,9 +285,6 @@ func TestSingleStreamLargeResponseBody(t *testing.T) {
 	}
 	responseBody := make([]byte, bodySize)
 
-	<-streamReady
-	t.Log("Received stream ready signal; resuming the test")
-
 	n, err := io.ReadFull(stream, responseBody)
 	if err != nil {
 		t.Fatalf("error from (*MuxedStream).Read: %s", err)
@@ -367,14 +367,13 @@ func TestMultipleStreams(t *testing.T) {
 		log.Error(err)
 	}
 	if testFail {
-		t.Fatalf("TestMultipleStreamsFlowControl failed")
+		t.Fatalf("TestMultipleStreams failed")
 	}
 }
 
 func TestMultipleStreamsFlowControl(t *testing.T) {
 	maxStreams := 32
 	errorsC := make(chan error, maxStreams)
-	streamReady := make(chan struct{})
 	responseSizes := make([]int32, maxStreams)
 	for i := 0; i < maxStreams; i++ {
 		responseSizes[i] = rand.Int31n(int32(defaultWindowSize << 4))
@@ -398,7 +397,6 @@ func TestMultipleStreamsFlowControl(t *testing.T) {
 			payload[i] = byte(i % 256)
 		}
 		n, err := stream.Write(payload)
-		streamReady <- struct{}{}
 		if err != nil {
 			t.Fatalf("origin write error: %s", err)
 		}
@@ -435,7 +433,6 @@ func TestMultipleStreamsFlowControl(t *testing.T) {
 				return
 			}
 
-			<-streamReady
 			responseBody := make([]byte, responseSizes[(stream.streamID-2)/2])
 			n, err := io.ReadFull(stream, responseBody)
 			if err != nil {
@@ -782,9 +779,11 @@ func TestMultipleStreamsWithDictionaries(t *testing.T) {
 		}
 
 		wg.Add(len(paths))
+		errorsC := make(chan error, len(paths))
 
 		for i, s := range paths {
 			go func(i int, path string) {
+				defer wg.Done()
 				stream, err := muxPair.EdgeMux.OpenStream(
 					[]Header{
 						{Name: ":method", Value: "GET"},
@@ -805,22 +804,30 @@ func TestMultipleStreamsWithDictionaries(t *testing.T) {
 				responseBody := make([]byte, len(expectBody)*2)
 				n, err := stream.Read(responseBody)
 				if err != nil {
-					log.Printf("error from (*MuxedStream).Read: %s", err)
-					t.Fatalf("error from (*MuxedStream).Read: %s", err)
+					errorsC <- fmt.Errorf("stream %d error from (*MuxedStream).Read: %s", stream.streamID, err)
+					return
 				}
 				if n != len(expectBody) {
-					log.Printf("expected response body to have %d bytes, got %d", len(expectBody), n)
-					t.Fatalf("expected response body to have %d bytes, got %d", len(expectBody), n)
+					errorsC <- fmt.Errorf("stream %d expected response body to have %d bytes, got %d", stream.streamID, len(expectBody), n)
+					return
 				}
 				if string(responseBody[:n]) != expectBody {
-					log.Printf("expected response body %s, got %s", expectBody, responseBody[:n])
-					t.Fatalf("expected response body %s, got %s", expectBody, responseBody[:n])
+					errorsC <- fmt.Errorf("stream %d expected response body %s, got %s", stream.streamID, expectBody, responseBody[:n])
+					return
 				}
-				wg.Done()
 			}(i, s)
-			time.Sleep(1 * time.Millisecond)
 		}
+
 		wg.Wait()
+		close(errorsC)
+		testFail := false
+		for err := range errorsC {
+			testFail = true
+			log.Error(err)
+		}
+		if testFail {
+			t.Fatalf("TestMultipleStreams failed")
+		}
 
 		if q > CompressionNone && muxPair.OriginMux.muxMetricsUpdater.compBytesBefore.Value() <= 10*muxPair.OriginMux.muxMetricsUpdater.compBytesAfter.Value() {
 			t.Fatalf("Cross-stream compression is expected to give a better compression ratio")

h2mux/muxedstream.go

@@ -17,32 +17,51 @@ type ReadWriteClosedCloser interface {
 	Closed() bool
 }
 
+// MuxedStream is logically an HTTP/2 stream, with an additional buffer for outgoing data.
 type MuxedStream struct {
-	Headers []Header
-
 	streamID uint32
 
+	// The "Receive" end of the stream
+	readBufferLock sync.RWMutex
+	readBuffer     ReadWriteClosedCloser
+	// This is the amount of bytes that are in our receive window
+	// (how much data we can receive into this stream).
+	receiveWindow uint32
+	// current receive window size limit. Exponentially increase it when it's exhausted
+	receiveWindowCurrentMax uint32
+	// hard limit set in http2 spec. 2^31-1
+	receiveWindowMax uint32
+	// The desired size increment for receiveWindow.
+	// If this is nonzero, a WINDOW_UPDATE frame needs to be sent.
+	windowUpdate uint32
+	// The headers that were most recently received.
+	// Particularly:
+	//     * for an eyeball-initiated stream (as passed to TunnelHandler::ServeStream),
+	//       these are the request headers
+	//     * for a cloudflared-initiated stream (as created by Register/UnregisterTunnel),
+	//       these are the response headers.
+	// They are useful in both of these contexts; hence `Headers` is public.
+	Headers []Header
+	// For use in the context of a cloudflared-initiated stream.
 	responseHeadersReceived chan struct{}
 
-	readBuffer    ReadWriteClosedCloser
-	receiveWindow uint32
-	// current window size limit. Exponentially increase it when it's exhausted
-	receiveWindowCurrentMax uint32
-	// limit set in http2 spec. 2^31-1
-	receiveWindowMax uint32
-
-	// nonzero if a WINDOW_UPDATE frame for a stream needs to be sent
-	windowUpdate uint32
-
-	writeLock sync.Mutex
-	// The zero value for Buffer is an empty buffer ready to use.
+	// The "Send" end of the stream
+	writeLock   sync.Mutex
 	writeBuffer ReadWriteLengther
-
+	// The maximum capacity that the send buffer should grow to.
+	writeBufferMaxLen int
+	// A channel to be notified when the send buffer is not full.
+	writeBufferHasSpace chan struct{}
+	// This is the amount of bytes that are in the peer's receive window
+	// (how much data we can send from this stream).
 	sendWindow uint32
-
-	readyList    *ReadyList
+	// Reference to the muxer's readyList; signal this for stream data to be sent.
+	readyList *ReadyList
+	// The headers that should be sent, and a flag so we only send them once.
 	headersSent  bool
 	writeHeaders []Header
+
+	// EOF-related fields
 	// true if the write end of this stream has been closed
 	writeEOF bool
 	// true if we have sent EOF to the peer
@@ -50,40 +69,63 @@ type MuxedStream struct {
 	// true if the peer sent us an EOF
 	receivedEOF bool
 
-	// dictionary that was used to compress the stream
+	// Compression-related fields
 	receivedUseDict bool
 	method          string
 	contentType     string
 	path            string
 	dictionaries    h2Dictionaries
-	readBufferLock  sync.RWMutex
 }
 
 func (s *MuxedStream) Read(p []byte) (n int, err error) {
+	var readBuffer ReadWriteClosedCloser
 	if s.dictionaries.read != nil {
 		s.readBufferLock.RLock()
-		b := s.readBuffer
+		readBuffer = s.readBuffer
 		s.readBufferLock.RUnlock()
-		return b.Read(p)
+	} else {
+		readBuffer = s.readBuffer
 	}
-	return s.readBuffer.Read(p)
+	n, err = readBuffer.Read(p)
+	s.replenishReceiveWindow(uint32(n))
+	return
 }
 
-func (s *MuxedStream) Write(p []byte) (n int, err error) {
+// Blocks until len(p) bytes have been written to the buffer
+func (s *MuxedStream) Write(p []byte) (int, error) {
+	// If assignDictToStream returns success, then it will have acquired the
+	// writeLock. Otherwise we must acquire it ourselves.
 	ok := assignDictToStream(s, p)
 	if !ok {
 		s.writeLock.Lock()
 	}
 	defer s.writeLock.Unlock()
+
 	if s.writeEOF {
 		return 0, io.EOF
 	}
-	n, err = s.writeBuffer.Write(p)
-	if n != len(p) || err != nil {
-		return n, err
+	totalWritten := 0
+	for totalWritten < len(p) {
+		// If the buffer is full, block till there is more room.
+		// Use a loop to recheck the buffer size after the lock is reacquired.
+		for s.writeBufferMaxLen <= s.writeBuffer.Len() {
+			s.writeLock.Unlock()
+			<-s.writeBufferHasSpace
+			s.writeLock.Lock()
+		}
+		amountToWrite := len(p) - totalWritten
+		spaceAvailable := s.writeBufferMaxLen - s.writeBuffer.Len()
+		if spaceAvailable < amountToWrite {
+			amountToWrite = spaceAvailable
+		}
+		amountWritten, err := s.writeBuffer.Write(p[totalWritten : totalWritten+amountToWrite])
+		totalWritten += amountWritten
+		if err != nil {
+			return totalWritten, err
+		}
+		s.writeNotify()
 	}
-	s.writeNotify()
-	return n, nil
+	return totalWritten, nil
 }
 
 func (s *MuxedStream) Close() error {
@@ -164,9 +206,9 @@ func (s *MuxedStream) writeNotify() {
 // receive window (how much data we can send).
 func (s *MuxedStream) replenishSendWindow(bytes uint32) {
 	s.writeLock.Lock()
+	defer s.writeLock.Unlock()
 	s.sendWindow += bytes
 	s.writeNotify()
-	s.writeLock.Unlock()
 }
 
 // Call by muxreader when it receives a data frame
@@ -178,17 +220,30 @@ func (s *MuxedStream) consumeReceiveWindow(bytes uint32) bool {
 		return false
 	}
 	s.receiveWindow -= bytes
-	if s.receiveWindow < s.receiveWindowCurrentMax/2 {
+	if s.receiveWindow < s.receiveWindowCurrentMax/2 && s.receiveWindowCurrentMax < s.receiveWindowMax {
 		// exhausting client send window (how much data client can send)
-		if s.receiveWindowCurrentMax < s.receiveWindowMax {
-			s.receiveWindowCurrentMax <<= 1
+		// and there is room to grow the receive window
+		newMax := s.receiveWindowCurrentMax << 1
+		if newMax > s.receiveWindowMax {
+			newMax = s.receiveWindowMax
 		}
-		s.windowUpdate += s.receiveWindowCurrentMax - s.receiveWindow
+		s.windowUpdate += newMax - s.receiveWindowCurrentMax
+		s.receiveWindowCurrentMax = newMax
+		// notify MuxWriter to write WINDOW_UPDATE frame
 		s.writeNotify()
 	}
 	return true
 }
 
+// Arranges for the MuxWriter to send a WINDOW_UPDATE
+// Called by MuxedStream::Read when data has left the read buffer.
+func (s *MuxedStream) replenishReceiveWindow(bytes uint32) {
+	s.writeLock.Lock()
+	defer s.writeLock.Unlock()
+	s.windowUpdate += bytes
+	s.writeNotify()
+}
+
 // receiveEOF should be called when the peer indicates no more data will be sent.
 // Returns true if the socket is now closed (i.e. the write side is already closed).
 func (s *MuxedStream) receiveEOF() (closed bool) {
@@ -226,7 +281,8 @@ type streamChunk struct {
 	// true if a HEADERS frame should be sent
 	sendHeaders bool
 	headers     []Header
-	// nonzero if a WINDOW_UPDATE frame should be sent
+	// nonzero if a WINDOW_UPDATE frame should be sent;
+	// in that case, it is the increment value to use
 	windowUpdate uint32
 	// true if data frames should be sent
 	sendData bool
@@ -249,11 +305,23 @@ func (s *MuxedStream) getChunk() *streamChunk {
 		eof:          s.writeEOF && uint32(s.writeBuffer.Len()) <= s.sendWindow,
 	}
 
-	// Copies at most s.sendWindow bytes
+	// Copy at most s.sendWindow bytes, adjust the sendWindow accordingly
 	writeLen, _ := io.CopyN(&chunk.buffer, s.writeBuffer, int64(s.sendWindow))
 	s.sendWindow -= uint32(writeLen)
+
+	// Non-blocking channel send. This will allow MuxedStream::Write() to continue, if needed
+	if s.writeBuffer.Len() < s.writeBufferMaxLen {
+		select {
+		case s.writeBufferHasSpace <- struct{}{}:
+		default:
+		}
+	}
+
+	// When we write the chunk, we'll write the WINDOW_UPDATE frame if needed
 	s.receiveWindow += s.windowUpdate
 	s.windowUpdate = 0
+
+	// When we write the chunk, we'll write the headers if needed
 	s.headersSent = true
 
 	// if this chunk contains the end of the stream, close the stream now

h2mux/muxedstream_test.go

@@ -23,47 +23,55 @@ func TestFlowControlSingleStream(t *testing.T) {
 		sendWindow:              testWindowSize,
 		readyList:               NewReadyList(),
 	}
+	var tempWindowUpdate uint32
+	var tempStreamChunk *streamChunk
+
 	assert.True(t, stream.consumeReceiveWindow(testWindowSize/2))
 	dataSent := testWindowSize / 2
 	assert.Equal(t, testWindowSize-dataSent, stream.receiveWindow)
 	assert.Equal(t, testWindowSize, stream.receiveWindowCurrentMax)
-	assert.Equal(t, uint32(0), stream.windowUpdate)
-	tempWindowUpdate := stream.windowUpdate
-
-	streamChunk := stream.getChunk()
-	assert.Equal(t, tempWindowUpdate, streamChunk.windowUpdate)
-	assert.Equal(t, testWindowSize-dataSent, stream.receiveWindow)
-	assert.Equal(t, uint32(0), stream.windowUpdate)
 	assert.Equal(t, testWindowSize, stream.sendWindow)
+	assert.Equal(t, uint32(0), stream.windowUpdate)
+
+	tempStreamChunk = stream.getChunk()
+	assert.Equal(t, uint32(0), tempStreamChunk.windowUpdate)
+	assert.Equal(t, testWindowSize-dataSent, stream.receiveWindow)
+	assert.Equal(t, testWindowSize, stream.receiveWindowCurrentMax)
+	assert.Equal(t, testWindowSize, stream.sendWindow)
+	assert.Equal(t, uint32(0), stream.windowUpdate)
 
 	assert.True(t, stream.consumeReceiveWindow(2))
 	dataSent += 2
 	assert.Equal(t, testWindowSize-dataSent, stream.receiveWindow)
 	assert.Equal(t, testWindowSize<<1, stream.receiveWindowCurrentMax)
-	assert.Equal(t, (testWindowSize<<1)-stream.receiveWindow, stream.windowUpdate)
+	assert.Equal(t, testWindowSize, stream.sendWindow)
+	assert.Equal(t, testWindowSize, stream.windowUpdate)
 	tempWindowUpdate = stream.windowUpdate
 
-	streamChunk = stream.getChunk()
-	assert.Equal(t, tempWindowUpdate, streamChunk.windowUpdate)
-	assert.Equal(t, testWindowSize<<1, stream.receiveWindow)
-	assert.Equal(t, uint32(0), stream.windowUpdate)
+	tempStreamChunk = stream.getChunk()
+	assert.Equal(t, tempWindowUpdate, tempStreamChunk.windowUpdate)
+	assert.Equal(t, (testWindowSize<<1)-dataSent, stream.receiveWindow)
+	assert.Equal(t, testWindowSize<<1, stream.receiveWindowCurrentMax)
 	assert.Equal(t, testWindowSize, stream.sendWindow)
+	assert.Equal(t, uint32(0), stream.windowUpdate)
 
 	assert.True(t, stream.consumeReceiveWindow(testWindowSize+10))
-	dataSent = testWindowSize + 10
+	dataSent += testWindowSize + 10
 	assert.Equal(t, (testWindowSize<<1)-dataSent, stream.receiveWindow)
 	assert.Equal(t, testWindowSize<<2, stream.receiveWindowCurrentMax)
-	assert.Equal(t, (testWindowSize<<2)-stream.receiveWindow, stream.windowUpdate)
+	assert.Equal(t, testWindowSize, stream.sendWindow)
+	assert.Equal(t, testWindowSize<<1, stream.windowUpdate)
 	tempWindowUpdate = stream.windowUpdate
 
-	streamChunk = stream.getChunk()
-	assert.Equal(t, tempWindowUpdate, streamChunk.windowUpdate)
-	assert.Equal(t, testWindowSize<<2, stream.receiveWindow)
-	assert.Equal(t, uint32(0), stream.windowUpdate)
+	tempStreamChunk = stream.getChunk()
+	assert.Equal(t, tempWindowUpdate, tempStreamChunk.windowUpdate)
+	assert.Equal(t, (testWindowSize<<2)-dataSent, stream.receiveWindow)
+	assert.Equal(t, testWindowSize<<2, stream.receiveWindowCurrentMax)
 	assert.Equal(t, testWindowSize, stream.sendWindow)
+	assert.Equal(t, uint32(0), stream.windowUpdate)
 
 	assert.False(t, stream.consumeReceiveWindow(testMaxWindowSize+1))
-	assert.Equal(t, testWindowSize<<2, stream.receiveWindow)
+	assert.Equal(t, (testWindowSize<<2)-dataSent, stream.receiveWindow)
 	assert.Equal(t, testMaxWindowSize, stream.receiveWindowCurrentMax)
 }
 

h2mux/muxreader.go

@@ -35,6 +35,8 @@ type MuxReader struct {
 	initialStreamWindow uint32
 	// The max value for the send window of a stream.
 	streamWindowMax uint32
+	// The max size for the write buffer of a stream
+	streamWriteBufferMaxLen int
 	// r is a reference to the underlying connection used when shutting down.
 	r io.Closer
 	// updateRTTChan is the channel to send new RTT measurement to muxerMetricsUpdater
@@ -153,6 +155,8 @@ func (r *MuxReader) newMuxedStream(streamID uint32) *MuxedStream {
 		streamID:                streamID,
 		readBuffer:              NewSharedBuffer(),
 		writeBuffer:             &bytes.Buffer{},
+		writeBufferMaxLen:       r.streamWriteBufferMaxLen,
+		writeBufferHasSpace:     make(chan struct{}, 1),
 		receiveWindow:           r.initialStreamWindow,
 		receiveWindowCurrentMax: r.initialStreamWindow,
 		receiveWindowMax:        r.streamWindowMax,

origin/metrics.go

@@ -52,6 +52,7 @@ type TunnelMetrics struct {
 	oldServerLocations map[string]string
 
 	muxerMetrics *muxerMetrics
+	tunnelsHA    tunnelsForHA
 }
 
 func newMuxerMetrics() *muxerMetrics {
@@ -355,6 +356,7 @@ func NewTunnelMetrics() *TunnelMetrics {
 		serverLocations:                serverLocations,
 		oldServerLocations:             make(map[string]string),
 		muxerMetrics:                   newMuxerMetrics(),
+		tunnelsHA:                      NewTunnelsForHA(),
 	}
 }
 
@@ -375,7 +377,7 @@ func (t *TunnelMetrics) incrementRequests(connectionID string) {
 	var concurrentRequests uint64
 	var ok bool
 	if concurrentRequests, ok = t.concurrentRequests[connectionID]; ok {
-		t.concurrentRequests[connectionID] += 1
+		t.concurrentRequests[connectionID]++
 		concurrentRequests++
 	} else {
 		t.concurrentRequests[connectionID] = 1
@@ -395,7 +397,7 @@ func (t *TunnelMetrics) incrementRequests(connectionID string) {
 func (t *TunnelMetrics) decrementConcurrentRequests(connectionID string) {
 	t.concurrentRequestsLock.Lock()
 	if _, ok := t.concurrentRequests[connectionID]; ok {
-		t.concurrentRequests[connectionID] -= 1
+		t.concurrentRequests[connectionID]--
 	}
 	t.concurrentRequestsLock.Unlock()
 

origin/tunnel.go

@@ -292,7 +292,13 @@ func IsRPCStreamResponse(headers []h2mux.Header) bool {
 	return true
 }
 
-func RegisterTunnel(ctx context.Context, muxer *h2mux.Muxer, config *TunnelConfig, connectionID uint8, originLocalIP string) error {
+func RegisterTunnel(
+	ctx context.Context,
+	muxer *h2mux.Muxer,
+	config *TunnelConfig,
+	connectionID uint8,
+	originLocalIP string,
+) error {
 	config.Logger.Debug("initiating RPC stream to register")
 	stream, err := muxer.OpenStream([]h2mux.Header{
 		{Name: ":method", Value: "RPC"},
@@ -346,7 +352,8 @@ func RegisterTunnel(ctx context.Context, muxer *h2mux.Muxer, config *TunnelConfi
 	}
 
 	if registration.TunnelID != "" {
-		config.Logger.Info("Tunnel ID: " + registration.TunnelID)
+		config.Metrics.tunnelsHA.AddTunnelID(connectionID, registration.TunnelID)
+		config.Logger.Infof("Each HA connection's tunnel IDs: %v", config.Metrics.tunnelsHA.String())
 	}
 
 	// Print out the user's trial zone URL in a nice box (if they requested and got one)
@@ -431,6 +438,12 @@ func H2RequestHeadersToH1Request(h2 []h2mux.Header, h1 *http.Request) error {
 				return fmt.Errorf("invalid path")
 			}
 			h1.URL = resolved
+		case "content-length":
+			contentLength, err := strconv.ParseInt(header.Value, 10, 64)
+			if err != nil {
+				return fmt.Errorf("unparseable content length")
+			}
+			h1.ContentLength = contentLength
 		default:
 			h1.Header.Add(http.CanonicalHeaderKey(header.Name), header.Value)
 		}

origin/tunnelsforha.go

@@ -0,0 +1,49 @@
+package origin
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// tunnelsForHA maps this cloudflared instance's HA connections to the tunnel IDs they serve.
+type tunnelsForHA struct {
+	sync.Mutex
+	metrics *prometheus.GaugeVec
+	entries map[uint8]string
+}
+
+// NewTunnelsForHA initializes the Prometheus metrics etc for a tunnelsForHA.
+func NewTunnelsForHA() tunnelsForHA {
+	metrics := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "tunnel_ids",
+			Help: "The ID of all tunnels (and their corresponding HA connection ID) running in this instance of cloudflared.",
+		},
+		[]string{"tunnel_id", "ha_conn_id"},
+	)
+	prometheus.MustRegister(metrics)
+
+	return tunnelsForHA{
+		metrics: metrics,
+		entries: make(map[uint8]string),
+	}
+}
+
+// Track a new tunnel ID, removing the disconnected tunnel (if any) and update metrics.
+func (t *tunnelsForHA) AddTunnelID(haConn uint8, tunnelID string) {
+	t.Lock()
+	defer t.Unlock()
+	if oldTunnelID, ok := t.entries[haConn]; ok {
+		t.metrics.WithLabelValues(oldTunnelID).Dec()
+	}
+	t.entries[haConn] = tunnelID
+	t.metrics.WithLabelValues(tunnelID, fmt.Sprintf("%v", haConn)).Inc()
+}
+
+func (t *tunnelsForHA) String() string {
+	t.Lock()
+	defer t.Unlock()
+	return fmt.Sprintf("%v", t.entries)
+}

tlsconfig/certreloader.go

@@ -2,14 +2,10 @@ package tlsconfig
 
 import (
 	"crypto/tls"
-	"errors"
 	"fmt"
 	"sync"
 
-	tunnellog "github.com/cloudflare/cloudflared/log"
 	"github.com/getsentry/raven-go"
-	log "github.com/sirupsen/logrus"
-	"gopkg.in/urfave/cli.v2"
 )
 
 // CertReloader can load and reload a TLS certificate from a particular filepath.
@@ -21,18 +17,14 @@ type CertReloader struct {
 	keyPath     string
 }
 
-// NewCertReloader makes a CertReloader, memorizing the filepaths in the context/flags.
-func NewCertReloader(c *cli.Context, f CLIFlags) (*CertReloader, error) {
-	if !c.IsSet(f.Cert) {
-		return nil, errors.New("CertReloader: cert not provided")
-	}
-	if !c.IsSet(f.Key) {
-		return nil, errors.New("CertReloader: key not provided")
-	}
+// NewCertReloader makes a CertReloader. It loads the cert during initialization to make sure certPath and keyPath are valid
+func NewCertReloader(certPath, keyPath string) (*CertReloader, error) {
 	cr := new(CertReloader)
-	cr.certPath = c.String(f.Cert)
-	cr.keyPath = c.String(f.Key)
-	cr.LoadCert()
+	cr.certPath = certPath
+	cr.keyPath = keyPath
+	if err := cr.LoadCert(); err != nil {
+		return nil, err
+	}
 	return cr, nil
 }
 
@@ -45,18 +37,17 @@ func (cr *CertReloader) Cert(clientHello *tls.ClientHelloInfo) (*tls.Certificate
 
 // LoadCert loads a TLS certificate from the CertReloader's specified filepath.
 // Call this after writing a new certificate to the disk (e.g. after renewing a certificate)
-func (cr *CertReloader) LoadCert() {
+func (cr *CertReloader) LoadCert() error {
 	cr.Lock()
 	defer cr.Unlock()
 
-	log.SetFormatter(&tunnellog.JSONFormatter{})
-	log.Info("Reloading certificate")
 	cert, err := tls.LoadX509KeyPair(cr.certPath, cr.keyPath)
 
 	// Keep the old certificate if there's a problem reading the new one.
 	if err != nil {
 		raven.CaptureError(fmt.Errorf("Error parsing X509 key pair: %v", err), nil)
-		return
+		return err
 	}
 	cr.certificate = &cert
+	return nil
 }

tlsconfig/cloudflare_ca.go

@@ -2,6 +2,7 @@ package tlsconfig
 
 import (
 	"crypto/x509"
+	"encoding/pem"
 )
 
 // TODO: remove the Origin CA root certs when migrated to Authenticated Origin Pull certs
@@ -85,11 +86,26 @@ QzMmZpRpIBB321ZBlcnlxiTJvWxvbCPHKHj20VwwAz7LONF59s84ZsOqfoBv8gKM
 s0s5dsq5zpLeaw==
 -----END CERTIFICATE-----`)
 
-func GetCloudflareRootCA() *x509.CertPool {
-	ca := x509.NewCertPool()
-	if !ca.AppendCertsFromPEM([]byte(cloudflareRootCA)) {
-		// should never happen
-		panic("failure loading Cloudflare origin CA pem")
+func GetCloudflareRootCA() ([]*x509.Certificate, error) {
+	var certs []*x509.Certificate
+	pemBlocks := cloudflareRootCA
+	for len(pemBlocks) > 0 {
+		var block *pem.Block
+		block, pemBlocks = pem.Decode(pemBlocks)
+		if block == nil {
+			break
+		}
+		if block.Type != "CERTIFICATE" {
+			continue
+		}
+
+		cert, err := x509.ParseCertificate(block.Bytes)
+		if err != nil {
+			return nil, err
+		}
+
+		certs = append(certs, cert)
 	}
-	return ca
+
+	return certs, nil
 }

tlsconfig/testcert.pem

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIICBjCCAbCgAwIBAgIJAPKk4bYMrSFMMA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV
+BAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEPMA0GA1UEBwwGQXVzdGluMRkwFwYDVQQK
+DBBDbG91ZGZsYXJlLCBJbmMuMRIwEAYDVQQDDAlsb2NhbGhvc3QwHhcNMTgxMTE1
+MjA1NzU3WhcNMjgxMTEyMjA1NzU3WjBdMQswCQYDVQQGEwJVUzEOMAwGA1UECAwF
+VGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcGA1UECgwQQ2xvdWRmbGFyZSwgSW5j
+LjESMBAGA1UEAwwJbG9jYWxob3N0MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAOQN
+pTRn5wLf8SSI5x2kpDvbdDy7lfhamJ2En4Q+wy1cSKp8bn8/oyhVF7QTsimDGTI4
+45pV9nDfNJPYB3IW0x0CAwEAAaNTMFEwHQYDVR0OBBYEFE4jIa97mIEiYFa02X++
+uu5mCEn+MB8GA1UdIwQYMBaAFE4jIa97mIEiYFa02X++uu5mCEn+MA8GA1UdEwEB
+/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADQQAkE+pDee0o5cNcZRUszy8sTQzB1Wlp
+J6ucfmo16crqRaK7uGvhkMyibIc4D8z2Cxw3aI3IMMFoIIlYoYKiUcbd
+-----END CERTIFICATE-----

tlsconfig/testcert2.pem

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIICBjCCAbCgAwIBAgIJAN6cXRTbJtFnMA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV
+BAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEPMA0GA1UEBwwGQXVzdGluMRgwFgYDVQQK
+DA9DbG91ZGZsYXJlLCBJbmMxEzARBgNVBAMMCmxvY2FsaG9zdDIwHhcNMTgxMTE1
+MjExMTU4WhcNMjgxMTEyMjExMTU4WjBdMQswCQYDVQQGEwJVUzEOMAwGA1UECAwF
+VGV4YXMxDzANBgNVBAcMBkF1c3RpbjEYMBYGA1UECgwPQ2xvdWRmbGFyZSwgSW5j
+MRMwEQYDVQQDDApsb2NhbGhvc3QyMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKQx
+IMZ6QgXoul2ITF/7sly4fW2Ol+a/AYw42zCWhVqOXv8AhY21I0Q8lkRR6wOroQwZ
+O7jKKOcE5TnR/NRcZr8CAwEAAaNTMFEwHQYDVR0OBBYEFONKxLZc2RUD0KTHkAz4
+8nrb5688MB8GA1UdIwQYMBaAFONKxLZc2RUD0KTHkAz48nrb5688MA8GA1UdEwEB
+/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADQQA56pwhvGpNPjyLcWfJHu/vI3ZjdoLB
+LnrkRaMjJmv0H0Beh4upJhoz8u6lhMACerKQrrdQhEPB2u+maFrEBtmN
+-----END CERTIFICATE-----

tlsconfig/testkey.pem

@@ -0,0 +1,10 @@
+-----BEGIN PRIVATE KEY-----
+MIIBVQIBADANBgkqhkiG9w0BAQEFAASCAT8wggE7AgEAAkEA5A2lNGfnAt/xJIjn
+HaSkO9t0PLuV+FqYnYSfhD7DLVxIqnxufz+jKFUXtBOyKYMZMjjjmlX2cN80k9gH
+chbTHQIDAQABAkAoeDtu91lJa1AxuZG58vOqI6GW/Xr5naojmdts7m5YaAhDa7DE
+zJUp4d8SP5cGBf1/PB3x6Cu9UviFNQ16wmzJAiEA8gUm4UYpWZD4Ze2l/xb+BK8D
+IglSUIy1VxW+X1G55wMCIQDxOfXiFzPqnv/e5avKGv6CU11Dhmbi1OpiyybZTjGz
+XwIhAM3bE/cJdqJ4bNBGE6umIupY8pFA3IMnLBempwbsvPOBAiEAgzJ+5OSxu92W
+VGidsmJUIhWtF9i1hJFAmVLcYjwBFAkCICtjP/vv0qOZWk4mAAn2zz9UVWp45DSR
+p/FA8V77ohXD
+-----END PRIVATE KEY-----

tlsconfig/tlsconfig.go

@@ -6,150 +6,86 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"io/ioutil"
-	"net"
 
-	"github.com/cloudflare/cloudflared/log"
 	"github.com/pkg/errors"
-	"gopkg.in/urfave/cli.v2"
-	"runtime"
 )
 
-var logger = log.CreateLogger()
-
-// CLIFlags names the flags used to configure TLS for a command or subsystem.
-// The nil value for a field means the flag is ignored.
-type CLIFlags struct {
-	Cert       string
-	Key        string
-	ClientCert string
-	RootCA     string
+// Config is the user provided parameters to create a tls.Config
+type TLSParameters struct {
+	Cert             string
+	Key              string
+	GetCertificate   *CertReloader
+	ClientCAs        []string
+	RootCAs          []string
+	ServerName       string
+	CurvePreferences []tls.CurveID
 }
 
-// GetConfig returns a TLS configuration according to the flags defined in f and
-// set by the user.
-func (f CLIFlags) GetConfig(c *cli.Context) *tls.Config {
-	config := &tls.Config{}
-
-	if c.IsSet(f.Cert) && c.IsSet(f.Key) {
-		cert, err := tls.LoadX509KeyPair(c.String(f.Cert), c.String(f.Key))
+// GetConfig returns a TLS configuration according to the Config set by the user.
+func GetConfig(p *TLSParameters) (*tls.Config, error) {
+	tlsconfig := &tls.Config{}
+	if p.Cert != "" && p.Key != "" {
+		cert, err := tls.LoadX509KeyPair(p.Cert, p.Key)
 		if err != nil {
-			logger.WithError(err).Fatal("Error parsing X509 key pair")
+			return nil, errors.Wrap(err, "Error parsing X509 key pair")
 		}
-		config.Certificates = []tls.Certificate{cert}
-		config.BuildNameToCertificate()
+		tlsconfig.Certificates = []tls.Certificate{cert}
+		// BuildNameToCertificate parses Certificates and builds NameToCertificate from common name
+		// and SAN fields of leaf certificates
+		tlsconfig.BuildNameToCertificate()
 	}
-	return f.finishGettingConfig(c, config)
-}
 
-func (f CLIFlags) GetConfigReloadableCert(c *cli.Context, cr *CertReloader) *tls.Config {
-	config := &tls.Config{
-		GetCertificate: cr.Cert,
+	if p.GetCertificate != nil {
+		// GetCertificate is called when client supplies SNI info or Certificates is empty.
+		// Order of retrieving certificate is GetCertificate, NameToCertificate and lastly first element of Certificates
+		tlsconfig.GetCertificate = p.GetCertificate.Cert
 	}
-	config.BuildNameToCertificate()
-	return f.finishGettingConfig(c, config)
-}
 
-func (f CLIFlags) finishGettingConfig(c *cli.Context, config *tls.Config) *tls.Config {
-	if c.IsSet(f.ClientCert) {
+	if len(p.ClientCAs) > 0 {
 		// set of root certificate authorities that servers use if required to verify a client certificate
 		// by the policy in ClientAuth
-		config.ClientCAs = LoadCert(c.String(f.ClientCert))
+		clientCAs, err := LoadCert(p.ClientCAs)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error loading client CAs")
+		}
+		tlsconfig.ClientCAs = clientCAs
 		// server's policy for TLS Client Authentication. Default is no client cert
-		config.ClientAuth = tls.RequireAndVerifyClientCert
+		tlsconfig.ClientAuth = tls.RequireAndVerifyClientCert
 	}
-	// set of root certificate authorities that clients use when verifying server certificates
-	if c.IsSet(f.RootCA) {
-		config.RootCAs = LoadCert(c.String(f.RootCA))
+
+	if len(p.RootCAs) > 0 {
+		rootCAs, err := LoadCert(p.RootCAs)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error loading root CAs")
+		}
+		tlsconfig.RootCAs = rootCAs
 	}
-	// we optimize CurveP256
-	config.CurvePreferences = []tls.CurveID{tls.CurveP256}
-	return config
+
+	if p.ServerName != "" {
+		tlsconfig.ServerName = p.ServerName
+	}
+
+	if len(p.CurvePreferences) > 0 {
+		tlsconfig.CurvePreferences = p.CurvePreferences
+	} else {
+		// Cloudflare optimize CurveP256
+		tlsconfig.CurvePreferences = []tls.CurveID{tls.CurveP256}
+	}
+
+	return tlsconfig, nil
 }
 
 // LoadCert creates a CertPool containing all certificates in a PEM-format file.
-func LoadCert(certPath string) *x509.CertPool {
-	caCert, err := ioutil.ReadFile(certPath)
-	if err != nil {
-		logger.WithError(err).Fatalf("Error reading certificate %s", certPath)
-	}
+func LoadCert(certPaths []string) (*x509.CertPool, error) {
 	ca := x509.NewCertPool()
-	if !ca.AppendCertsFromPEM(caCert) {
-		logger.WithError(err).Fatalf("Error parsing certificate %s", certPath)
-	}
-	return ca
-}
-
-func LoadGlobalCertPool() (*x509.CertPool, error) {
-	success := false
-
-	// First, obtain the system certificate pool
-	certPool, systemCertPoolErr := x509.SystemCertPool()
-	if systemCertPoolErr != nil {
-		if runtime.GOOS != "windows" {
-			logger.Warnf("error obtaining the system certificates: %s", systemCertPoolErr)
+	for _, certPath := range certPaths {
+		caCert, err := ioutil.ReadFile(certPath)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Error reading certificate %s", certPath)
 		}
-		certPool = x509.NewCertPool()
-	} else {
-		success = true
-	}
-
-	// Next, append the Cloudflare CA pool into the system pool
-	if !certPool.AppendCertsFromPEM(cloudflareRootCA) {
-		logger.Warn("could not append the CF certificate to the cloudflared certificate pool")
-	} else {
-		success = true
-	}
-
-	if success != true { // Obtaining any of the CAs has failed; this is a fatal error
-		return nil, errors.New("error loading any of the CAs into the global certificate pool")
-	}
-
-	// Finally, add the Hello certificate into the pool (since it's self-signed)
-	helloCertificate, err := GetHelloCertificateX509()
-	if err != nil {
-		logger.Warn("error obtaining the Hello server certificate")
-	}
-
-	certPool.AddCert(helloCertificate)
-
-	return certPool, nil
-}
-
-func LoadOriginCertPool(originCAPoolPEM []byte) (*x509.CertPool, error) {
-	success := false
-
-	// Get the global pool
-	certPool, globalPoolErr := LoadGlobalCertPool()
-	if globalPoolErr != nil {
-		certPool = x509.NewCertPool()
-	} else {
-		success = true
-	}
-
-	// Then, add any custom origin CA pool the user may have passed
-	if originCAPoolPEM != nil {
-		if !certPool.AppendCertsFromPEM(originCAPoolPEM) {
-			logger.Warn("could not append the provided origin CA to the cloudflared certificate pool")
-		} else {
-			success = true
+		if !ca.AppendCertsFromPEM(caCert) {
+			return nil, errors.Wrapf(err, "Error parsing certificate %s", certPath)
 		}
 	}
-
-	if success != true {
-		return nil, errors.New("error loading any of the CAs into the origin certificate pool")
-	}
-
-	return certPool, nil
-}
-
-func CreateTunnelConfig(c *cli.Context, addrs []string) *tls.Config {
-	tlsConfig := CLIFlags{RootCA: "cacert"}.GetConfig(c)
-	if tlsConfig.RootCAs == nil {
-		tlsConfig.RootCAs = GetCloudflareRootCA()
-		tlsConfig.ServerName = "cftunnel.com"
-	} else if len(addrs) > 0 {
-		// Set for development environments and for testing specific origintunneld instances
-		tlsConfig.ServerName, _, _ = net.SplitHostPort(addrs[0])
-	}
-	return tlsConfig
+	return ca, nil
 }

tlsconfig/tlsconfig_test.go

@@ -1,214 +1,84 @@
-// +build ignore
 // TODO: Remove the above build tag and include this test when we start compiling with Golang 1.10.0+
 
 package tlsconfig
 
 import (
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/asn1"
-	"os"
+	"crypto/tls"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-// Generated using `openssl req -newkey rsa:512 -nodes -x509 -days 3650`
-var samplePEM = []byte(`
------BEGIN CERTIFICATE-----
-MIIB4DCCAYoCCQCb/H0EUrdXEjANBgkqhkiG9w0BAQsFADB3MQswCQYDVQQGEwJV
-UzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcGA1UECgwQQ2xv
-dWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVneTERMA8GA1UE
-AwwIVGVzdCBPbmUwHhcNMTgwNDI2MTYxMDUxWhcNMjgwNDIzMTYxMDUxWjB3MQsw
-CQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcG
-A1UECgwQQ2xvdWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVn
-eTERMA8GA1UEAwwIVGVzdCBPbmUwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAwVQD
-K0SJ25UFLznm2pU3zhzMEvpDEofHVNnCjk4mlDrtVop7PkKZ8pDEmuQANltUrxC8
-yHBE2wXMv+GlH+bDtwIDAQABMA0GCSqGSIb3DQEBCwUAA0EAjVYQzozIFPkt/HRY
-uUoZ8zEHIDICb0syFf5VAjm9AgTwIPzUmD+c5vl6LWDnxq7L45nLCzhhQ6YmiwDz
-X7Wcyg==
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-MIIB4DCCAYoCCQDZfCdAJ+mwzDANBgkqhkiG9w0BAQsFADB3MQswCQYDVQQGEwJV
-UzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcGA1UECgwQQ2xv
-dWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVneTERMA8GA1UE
-AwwIVGVzdCBUd28wHhcNMTgwNDI2MTYxMTIwWhcNMjgwNDIzMTYxMTIwWjB3MQsw
-CQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcG
-A1UECgwQQ2xvdWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVn
-eTERMA8GA1UEAwwIVGVzdCBUd28wXDANBgkqhkiG9w0BAQEFAANLADBIAkEAoHKp
-ROVK3zCSsH7ocYeyRAML4V7SFAbZcb4WIwDnE08oMBVRkQVcW5tqEkvG3RiClfzV
-wZIJ3CfqKIeSNSDU9wIDAQABMA0GCSqGSIb3DQEBCwUAA0EAJw2gUbnPiq4C2p5b
-iWzlA9Q7aKo+VQ4H7IZS7tTccr59nVjvH/TG3eWujpnocr4TOqW9M3CK1DF9mUGP
-3pQ3Jg==
------END CERTIFICATE-----
-`)
+// testcert.pem and testcert2.pem are Generated using `openssl req -newkey rsa:512 -nodes -x509 -days 3650`
+const (
+	testcertCommonName = "localhost"
+)
 
-var systemCertPoolSubjects []*pkix.Name
+func TestGetFromEmptyConfig(t *testing.T) {
+	c := &TLSParameters{}
 
-type certificateFixture struct {
-	ou string
-	cn string
+	tlsConfig, err := GetConfig(c)
+	assert.NoError(t, err)
+	assert.Empty(t, tlsConfig.Certificates)
+
+	assert.Empty(t, tlsConfig.NameToCertificate)
+
+	assert.Nil(t, tlsConfig.ClientCAs)
+	assert.Equal(t, tls.NoClientCert, tlsConfig.ClientAuth)
+
+	assert.Nil(t, tlsConfig.RootCAs)
+
+	assert.Len(t, tlsConfig.CurvePreferences, 1)
+	assert.Equal(t, tls.CurveP256, tlsConfig.CurvePreferences[0])
 }
 
-func TestMain(m *testing.M) {
-	systemCertPool, err := x509.SystemCertPool()
-	if isUnrecoverableError(err) {
-		os.Exit(1)
-	}
+func TestGetConfig(t *testing.T) {
+	cert, err := tls.LoadX509KeyPair("testcert.pem", "testkey.pem")
+	assert.NoError(t, err)
 
-	if systemCertPool == nil {
-		// On Windows, let's just assume the system cert pool was empty
-		systemCertPool = x509.NewCertPool()
+	c := &TLSParameters{
+		Cert:             "testcert.pem",
+		Key:              "testkey.pem",
+		ClientCAs:        []string{"testcert.pem", "testcert2.pem"},
+		RootCAs:          []string{"testcert.pem", "testcert2.pem"},
+		ServerName:       "test",
+		CurvePreferences: []tls.CurveID{tls.CurveP384},
 	}
+	tlsConfig, err := GetConfig(c)
+	assert.NoError(t, err)
+	assert.Len(t, tlsConfig.Certificates, 1)
+	assert.Equal(t, cert, tlsConfig.Certificates[0])
 
-	systemCertPoolSubjects, err = getCertPoolSubjects(systemCertPool)
-	if err != nil {
-		os.Exit(1)
-	}
+	assert.Equal(t, cert, *tlsConfig.NameToCertificate[testcertCommonName])
 
-	os.Exit(m.Run())
+	assert.NotNil(t, tlsConfig.ClientCAs)
+	assert.Equal(t, tls.RequireAndVerifyClientCert, tlsConfig.ClientAuth)
+
+	assert.NotNil(t, tlsConfig.RootCAs)
+
+	assert.Len(t, tlsConfig.CurvePreferences, 1)
+	assert.Equal(t, tls.CurveP384, tlsConfig.CurvePreferences[0])
 }
 
-func TestLoadOriginCertPoolJustSystemPool(t *testing.T) {
-	certPoolSubjects := loadCertPoolSubjects(t, nil)
-	extraSubjects := subjectSubtract(systemCertPoolSubjects, certPoolSubjects)
+func TestCertReloader(t *testing.T) {
+	expectedCert, err := tls.LoadX509KeyPair("testcert.pem", "testkey.pem")
+	assert.NoError(t, err)
 
-	// Remove extra subjects from the cert pool
-	var filteredSystemCertPoolSubjects []*pkix.Name
+	certReloader, err := NewCertReloader("testcert.pem", "testkey.pem")
+	assert.NoError(t, err)
 
-	t.Log(extraSubjects)
+	chi := &tls.ClientHelloInfo{ServerName: testcertCommonName}
+	cert, err := certReloader.Cert(chi)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedCert, *cert)
 
-OUTER:
-	for _, subject := range certPoolSubjects {
-		for _, extraSubject := range extraSubjects {
-			if subject == extraSubject {
-				t.Log(extraSubject)
-				continue OUTER
-			}
-		}
-
-		filteredSystemCertPoolSubjects = append(filteredSystemCertPoolSubjects, subject)
+	c := &TLSParameters{
+		GetCertificate: certReloader,
 	}
+	tlsConfig, err := GetConfig(c)
+	assert.NoError(t, err)
 
-	assert.Equal(t, len(filteredSystemCertPoolSubjects), len(systemCertPoolSubjects))
-
-	difference := subjectSubtract(systemCertPoolSubjects, filteredSystemCertPoolSubjects)
-	assert.Equal(t, 0, len(difference))
-}
-
-func TestLoadOriginCertPoolCFCertificates(t *testing.T) {
-	certPoolSubjects := loadCertPoolSubjects(t, nil)
-
-	extraSubjects := subjectSubtract(systemCertPoolSubjects, certPoolSubjects)
-
-	expected := []*certificateFixture{
-		{ou: "CloudFlare Origin SSL ECC Certificate Authority"},
-		{ou: "CloudFlare Origin SSL Certificate Authority"},
-		{cn: "origin-pull.cloudflare.net"},
-		{cn: "Argo Tunnel Sample Hello Server Certificate"},
-	}
-
-	assertFixturesMatchSubjects(t, expected, extraSubjects)
-}
-
-func TestLoadOriginCertPoolWithExtraPEMs(t *testing.T) {
-	certPoolWithoutPEMSubjects := loadCertPoolSubjects(t, nil)
-	certPoolWithPEMSubjects := loadCertPoolSubjects(t, samplePEM)
-
-	difference := subjectSubtract(certPoolWithoutPEMSubjects, certPoolWithPEMSubjects)
-
-	assert.Equal(t, 2, len(difference))
-
-	expected := []*certificateFixture{
-		{cn: "Test One"},
-		{cn: "Test Two"},
-	}
-
-	assertFixturesMatchSubjects(t, expected, difference)
-}
-
-func loadCertPoolSubjects(t *testing.T, originCAPoolPEM []byte) []*pkix.Name {
-	certPool, err := LoadOriginCertPool(originCAPoolPEM)
-	if isUnrecoverableError(err) {
-		t.Fatal(err)
-	}
-	assert.NotEmpty(t, certPool.Subjects())
-	certPoolSubjects, err := getCertPoolSubjects(certPool)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	return certPoolSubjects
-}
-
-func assertFixturesMatchSubjects(t *testing.T, fixtures []*certificateFixture, subjects []*pkix.Name) {
-	assert.Equal(t, len(fixtures), len(subjects))
-
-	for _, fixture := range fixtures {
-		found := false
-		for _, subject := range subjects {
-			found = found || fixtureMatchesSubjectPredicate(fixture, subject)
-		}
-
-		if !found {
-			t.Fail()
-		}
-	}
-}
-
-func fixtureMatchesSubjectPredicate(fixture *certificateFixture, subject *pkix.Name) bool {
-	cnMatch := true
-	if fixture.cn != "" {
-		cnMatch = fixture.cn == subject.CommonName
-	}
-
-	ouMatch := true
-	if fixture.ou != "" {
-		ouMatch = len(subject.OrganizationalUnit) > 0 && fixture.ou == subject.OrganizationalUnit[0]
-	}
-
-	return cnMatch && ouMatch
-}
-
-func subjectSubtract(left []*pkix.Name, right []*pkix.Name) []*pkix.Name {
-	var difference []*pkix.Name
-
-	var found bool
-	for _, r := range right {
-		found = false
-		for _, l := range left {
-			if (*l).String() == (*r).String() {
-				found = true
-			}
-		}
-
-		if !found {
-			difference = append(difference, r)
-		}
-	}
-
-	return difference
-}
-
-func getCertPoolSubjects(certPool *x509.CertPool) ([]*pkix.Name, error) {
-	var subjects []*pkix.Name
-
-	for _, subject := range certPool.Subjects() {
-		var sequence pkix.RDNSequence
-		_, err := asn1.Unmarshal(subject, &sequence)
-		if err != nil {
-			return nil, err
-		}
-
-		name := pkix.Name{}
-		name.FillFromRDNSequence(&sequence)
-
-		subjects = append(subjects, &name)
-	}
-
-	return subjects, nil
-}
-
-func isUnrecoverableError(err error) bool {
-	return err != nil && err.Error() != "crypto/x509: system root pool is not available on Windows"
+	cert, err = tlsConfig.GetCertificate(chi)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedCert, *cert)
 }

validation/validation.go

@@ -6,9 +6,10 @@ import (
 	"net/url"
 	"strings"
 
+	"net/http"
+
 	"github.com/pkg/errors"
 	"golang.org/x/net/idna"
-	"net/http"
 )
 
 const defaultScheme = "http"
@@ -137,26 +138,40 @@ func validateIP(scheme, host, port string) (string, error) {
 	return fmt.Sprintf("%s://%s", scheme, host), nil
 }
 
-func ValidateHTTPService(originURL string, transport http.RoundTripper) error {
+func ValidateHTTPService(originURL string, hostname string, transport http.RoundTripper) error {
 	parsedURL, err := url.Parse(originURL)
 	if err != nil {
 		return err
 	}
 
-	client := &http.Client{Transport: transport}
+	client := &http.Client{
+		Transport: transport,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
 
-	initialResponse, initialErr := client.Get(parsedURL.String())
-	if initialErr != nil || initialResponse.StatusCode != http.StatusOK {
+	initialRequest, err := http.NewRequest("GET", parsedURL.String(), nil)
+	if err != nil {
+		return err
+	}
+	initialRequest.Host = hostname
+	_, initialErr := client.Do(initialRequest)
+	if initialErr != nil {
 		// Attempt the same endpoint via the other protocol (http/https); maybe we have better luck?
 		oldScheme := parsedURL.Scheme
 		parsedURL.Scheme = toggleProtocol(parsedURL.Scheme)
 
-		secondResponse, _ := client.Get(parsedURL.String())
-
-		if secondResponse != nil && secondResponse.StatusCode == http.StatusOK { // Worked this time--advise the user to switch protocols
+		secondRequest, err := http.NewRequest("GET", parsedURL.String(), nil)
+		if err != nil {
+			return err
+		}
+		secondRequest.Host = hostname
+		_, secondErr := client.Do(secondRequest)
+		if secondErr == nil { // Worked this time--advise the user to switch protocols
 			return errors.Errorf(
 				"%s doesn't seem to work over %s, but does seem to work over %s. Consider changing the origin URL to %s",
-				parsedURL.Hostname(),
+				parsedURL.Host,
 				oldScheme,
 				parsedURL.Scheme,
 				parsedURL,

validation/validation_test.go

@@ -1,18 +1,21 @@
 package validation
 
 import (
+	"bytes"
 	"fmt"
+	"io/ioutil"
 	"testing"
 
 	"context"
 	"crypto/tls"
 	"crypto/x509"
-	"github.com/stretchr/testify/assert"
 	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"strings"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestValidateHostname(t *testing.T) {
@@ -150,58 +153,248 @@ func TestToggleProtocol(t *testing.T) {
 	assert.Equal(t, "", toggleProtocol(""))
 }
 
+// Happy path 1: originURL is HTTP, and HTTP connections work
 func TestValidateHTTPService_HTTP2HTTP(t *testing.T) {
+	originURL := "http://127.0.0.1/"
+	hostname := "example.com"
+
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			return emptyResponse(200), nil
+		}
+		if req.URL.Scheme == "https" {
+			t.Fatal("http works, shouldn't have tried with https")
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			return emptyResponse(503), nil
+		}
+		if req.URL.Scheme == "https" {
+			t.Fatal("http works, shouldn't have tried with https")
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	// Integration-style test with a mock server
 	server, client, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, hostname, r.Host)
 		w.WriteHeader(200)
 	}))
 	assert.NoError(t, err)
 	defer server.Close()
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, client.Transport))
 
-	assert.Equal(t, nil, ValidateHTTPService("http://example.com/", client.Transport))
-}
-
-func TestValidateHTTPService_ServerNonOKResponse(t *testing.T) {
-	server, client, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(400)
+	// this will fail if the client follows the 302
+	redirectServer, redirectClient, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/followedRedirect" {
+			t.Fatal("shouldn't have followed the 302")
+		}
+		if r.Method == "CONNECT" {
+			assert.Equal(t, "127.0.0.1:443", r.Host)
+		} else {
+			assert.Equal(t, hostname, r.Host)
+		}
+		w.Header().Set("Location", "/followedRedirect")
+		w.WriteHeader(302)
 	}))
 	assert.NoError(t, err)
-	defer server.Close()
+	defer redirectServer.Close()
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, redirectClient.Transport))
 
-	assert.Equal(t, nil, ValidateHTTPService("http://example.com/", client.Transport))
-}
-
-func TestValidateHTTPService_HTTPS2HTTP(t *testing.T) {
-	server, client, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.WriteHeader(200)
-	}))
-	assert.NoError(t, err)
-	defer server.Close()
-
-	assert.Equal(t,
-		"example.com doesn't seem to work over https, but does seem to work over http. Consider changing the origin URL to http://example.com:1234/",
-		ValidateHTTPService("https://example.com:1234/", client.Transport).Error())
 }
 
+// Happy path 2: originURL is HTTPS, and HTTPS connections work
 func TestValidateHTTPService_HTTPS2HTTPS(t *testing.T) {
+	originURL := "https://127.0.0.1/"
+	hostname := "example.com"
+
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			t.Fatal("https works, shouldn't have tried with http")
+		}
+		if req.URL.Scheme == "https" {
+			return emptyResponse(200), nil
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			t.Fatal("https works, shouldn't have tried with http")
+		}
+		if req.URL.Scheme == "https" {
+			return emptyResponse(503), nil
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	// Integration-style test with a mock server
 	server, client, err := createSecureMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == "CONNECT" {
+			assert.Equal(t, "127.0.0.1:443", r.Host)
+		} else {
+			assert.Equal(t, hostname, r.Host)
+		}
 		w.WriteHeader(200)
 	}))
 	assert.NoError(t, err)
 	defer server.Close()
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, client.Transport))
 
-	assert.Equal(t, nil, ValidateHTTPService("https://example.com/", client.Transport))
+	// this will fail if the client follows the 302
+	redirectServer, redirectClient, err := createSecureMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/followedRedirect" {
+			t.Fatal("shouldn't have followed the 302")
+		}
+		if r.Method == "CONNECT" {
+			assert.Equal(t, "127.0.0.1:443", r.Host)
+		} else {
+			assert.Equal(t, hostname, r.Host)
+		}
+		w.Header().Set("Location", "/followedRedirect")
+		w.WriteHeader(302)
+	}))
+	assert.NoError(t, err)
+	defer redirectServer.Close()
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, redirectClient.Transport))
 }
 
-func TestValidateHTTPService_HTTP2HTTPS(t *testing.T) {
-	server, client, err := createSecureMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+// Error path 1: originURL is HTTPS, but HTTP connections work
+func TestValidateHTTPService_HTTPS2HTTP(t *testing.T) {
+	originURL := "https://127.0.0.1:1234/"
+	hostname := "example.com"
+
+	assert.Error(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			return emptyResponse(200), nil
+		}
+		if req.URL.Scheme == "https" {
+			return nil, assert.AnError
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	assert.Error(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			return emptyResponse(503), nil
+		}
+		if req.URL.Scheme == "https" {
+			return nil, assert.AnError
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	// Integration-style test with a mock server
+	server, client, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == "CONNECT" {
+			assert.Equal(t, "127.0.0.1:1234", r.Host)
+		} else {
+			assert.Equal(t, hostname, r.Host)
+		}
 		w.WriteHeader(200)
 	}))
 	assert.NoError(t, err)
 	defer server.Close()
+	assert.Error(t, ValidateHTTPService(originURL, hostname, client.Transport))
 
-	assert.Equal(t,
-		"example.com doesn't seem to work over http, but does seem to work over https. Consider changing the origin URL to https://example.com:1234/",
-		ValidateHTTPService("http://example.com:1234/", client.Transport).Error())
+	// this will fail if the client follows the 302
+	redirectServer, redirectClient, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/followedRedirect" {
+			t.Fatal("shouldn't have followed the 302")
+		}
+		if r.Method == "CONNECT" {
+			assert.Equal(t, "127.0.0.1:1234", r.Host)
+		} else {
+			assert.Equal(t, hostname, r.Host)
+		}
+		w.Header().Set("Location", "/followedRedirect")
+		w.WriteHeader(302)
+	}))
+	assert.NoError(t, err)
+	defer redirectServer.Close()
+	assert.Error(t, ValidateHTTPService(originURL, hostname, redirectClient.Transport))
+
+}
+
+// Error path 2: originURL is HTTP, but HTTPS connections work
+func TestValidateHTTPService_HTTP2HTTPS(t *testing.T) {
+	originURL := "http://127.0.0.1:1234/"
+	hostname := "example.com"
+
+	assert.Error(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			return nil, assert.AnError
+		}
+		if req.URL.Scheme == "https" {
+			return emptyResponse(200), nil
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	assert.Error(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			return nil, assert.AnError
+		}
+		if req.URL.Scheme == "https" {
+			return emptyResponse(503), nil
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	// Integration-style test with a mock server
+	server, client, err := createSecureMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == "CONNECT" {
+			assert.Equal(t, "127.0.0.1:1234", r.Host)
+		} else {
+			assert.Equal(t, hostname, r.Host)
+		}
+		w.WriteHeader(200)
+	}))
+	assert.NoError(t, err)
+	defer server.Close()
+	assert.Error(t, ValidateHTTPService(originURL, hostname, client.Transport))
+
+	// this will fail if the client follows the 302
+	redirectServer, redirectClient, err := createSecureMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/followedRedirect" {
+			t.Fatal("shouldn't have followed the 302")
+		}
+		if r.Method == "CONNECT" {
+			assert.Equal(t, "127.0.0.1:443", r.Host)
+		} else {
+			assert.Equal(t, hostname, r.Host)
+		}
+		w.Header().Set("Location", "/followedRedirect")
+		w.WriteHeader(302)
+	}))
+	assert.NoError(t, err)
+	defer redirectServer.Close()
+	assert.Error(t, ValidateHTTPService(originURL, hostname, redirectClient.Transport))
+}
+
+type testRoundTripper func(req *http.Request) (*http.Response, error)
+
+func (f testRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req)
+}
+
+func emptyResponse(statusCode int) *http.Response {
+	return &http.Response{
+		StatusCode: statusCode,
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		Header:     make(http.Header),
+	}
 }
 
 func createMockServerAndClient(handler http.Handler) (*httptest.Server, *http.Client, error) {

websocket/websocket.go

@@ -150,7 +150,7 @@ func StartProxyServer(logger *logrus.Logger, listener net.Listener, remote strin
 		done := make(chan struct{})
 		go pinger(logger, conn, done)
 		defer func() {
-			<-done
+			done <- struct{}{}
 			conn.Close()
 		}()
 		Stream(&Conn{conn}, stream)

websocket/websocket_test.go

@@ -2,18 +2,17 @@ package websocket
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"io"
 	"math/rand"
 	"net/http"
 	"testing"
 
-	"github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/assert"
-
-	"golang.org/x/net/websocket"
-
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/tlsconfig"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/websocket"
 )
 
 const (
@@ -40,8 +39,10 @@ func testRequest(t *testing.T, url string, stream io.ReadWriter) *http.Request {
 }
 
 func websocketClientTLSConfig(t *testing.T) *tls.Config {
-	certPool, err := tlsconfig.LoadOriginCertPool(nil)
+	certPool := x509.NewCertPool()
+	helloCert, err := tlsconfig.GetHelloCertificateX509()
 	assert.NoError(t, err)
+	certPool.AddCert(helloCert)
 	assert.NotNil(t, certPool)
 	return &tls.Config{RootCAs: certPool}
 }