Release 2018.12.0

RELEASE_NOTES

@@ -1,3 +1,18 @@
+2018.12.0
+- 2018-11-15 TUN-1196: Allow TLS config client CA and root CA to be constructed from multiple certificates
+- 2018-11-20 TUN-1209: TLS Config Certificates and GetCertificate can both be set
+- 2018-11-26 TUN-1212: Expose tunnel_id in metrics
+- 2018-11-30 TUN-1204: remove 'cloudflared hello' command
+- 2018-12-04 Fix license URL typo
+- 2018-12-07 TUN-1250: ValidateHTTPService shouldn't follow 302s
+
+2018.11.0
+- 2018-10-31 AUTH-1282: Fixed an issue where we were receiving as opposed sending on the channel.
+- 2018-11-06 TUN-1179: Fix log message in cmd/cloudflared/transfer.Run
+- 2018-11-13 AUTH-1308: get jwt even when you are already logged in
+- 2018-11-12 TUN-1190: check URL parse error when starting SSH proxy server
+- 2018-11-15 AUTH-1320: Fixed request issue and unhide the ssh command
+
 2018.10.5
 - 2018-10-18 TUN-968: Flow control for large requests/responses
 - 2018-10-26 TUN-1158: Windows: use process arguments rather than trivial service arguments

carrier/carrier.go

@@ -89,6 +89,7 @@ func createWebsocketStream(originURL string) (*websocket.Conn, error) {
 	if err != nil {
 		return nil, err
 	}
+
 	wsConn, resp, err := websocket.ClientConnect(req, nil)
 	if err != nil && resp != nil && resp.StatusCode > 300 {
 		location, err := resp.Location()
@@ -125,7 +126,14 @@ func buildAccessRequest(originURL string) (*http.Request, error) {
 	if err != nil {
 		return nil, err
 	}
-	req.Header.Set("cf-access-token", token)
 
-	return req, nil
+	// We need to create a new request as FetchToken will modify req (boo mutable)
+	// as it has to follow redirect on the API and such, so here we init a new one
+	originRequest, err := http.NewRequest(http.MethodGet, originURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	originRequest.Header.Set("cf-access-token", token)
+
+	return originRequest, nil
 }

cmd/cloudflared/access/cmd.go

@@ -89,14 +89,12 @@ func Commands() []*cli.Command {
 					Usage:       "",
 					ArgsUsage:   "",
 					Description: `The ssh subcommand sends data over a proxy to the Cloudflare edge.`,
-					Hidden:      true,
 					Flags: []cli.Flag{
 						&cli.StringFlag{
 							Name: "hostname",
 						},
 						&cli.StringFlag{
-							Name:   "url",
-							Hidden: true,
+							Name: "url",
 						},
 					},
 				},
@@ -105,7 +103,6 @@ func Commands() []*cli.Command {
 					Action:      sshConfig,
 					Usage:       "ssh-config",
 					Description: `Prints an example configuration ~/.ssh/config`,
-					Hidden:      true,
 				},
 			},
 		},

cmd/cloudflared/main.go

@@ -19,7 +19,7 @@ import (
 
 const (
 	developerPortal = "https://developers.cloudflare.com/argo-tunnel"
-	licenseUrl      = developerPortal + "/licence/"
+	licenseUrl      = developerPortal + "/license/"
 )
 
 var (

cmd/cloudflared/transfer/transfer.go

@@ -43,7 +43,7 @@ func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncr
 
 	err = shell.OpenBrowser(requestURL)
 	if err != nil {
-		fmt.Fprintf(os.Stdout, "Please open the following URL and log in with your Cloudflare account:\n\n%s\n\nLeave cloudflared running to download the %s automatically.\n", resourceName, requestURL)
+		fmt.Fprintf(os.Stdout, "Please open the following URL and log in with your Cloudflare account:\n\n%s\n\nLeave cloudflared running to download the %s automatically.\n", requestURL, resourceName)
 	} else {
 		fmt.Fprintf(os.Stdout, "A browser window should have opened at the following URL:\n\n%s\n\nIf the browser failed to open, open it yourself and visit the URL above.\n", requestURL)
 	}
@@ -83,21 +83,19 @@ func Run(transferURL *url.URL, resourceName, key, value, path string, shouldEncr
 
 // BuildRequestURL creates a request suitable for a resource transfer.
 // it will return a constructed url based off the base url and query key/value provided.
-// follow will follow redirects.
-func buildRequestURL(baseURL *url.URL, key, value string, follow bool) (string, error) {
+// cli will build a url for cli transfer request.
+func buildRequestURL(baseURL *url.URL, key, value string, cli bool) (string, error) {
 	q := baseURL.Query()
 	q.Set(key, value)
 	baseURL.RawQuery = q.Encode()
-	if !follow {
+	if !cli {
 		return baseURL.String(), nil
 	}
 
-	response, err := http.Get(baseURL.String())
-	if err != nil {
-		return "", err
-	}
-	return response.Request.URL.String(), nil
-
+	q.Set("redirect_url", baseURL.String()) // we add the token as a query param on both the redirect_url
+	baseURL.RawQuery = q.Encode()           // and this actual baseURL.
+	baseURL.Path = "cdn-cgi/access/cli"
+	return baseURL.String(), nil
 }
 
 // transferRequest downloads the requested resource from the request URL

cmd/cloudflared/tunnel/cmd.go

@@ -56,20 +56,6 @@ func Commands() []*cli.Command {
 			},
 			Hidden: true,
 		},
-		{
-			Name:   "hello",
-			Action: helloWorld,
-			Usage:  "Run a simple \"Hello World\" server for testing Argo Tunnel.",
-			Flags: []cli.Flag{
-				&cli.IntFlag{
-					Name:  "port",
-					Usage: "Listen on the selected port.",
-					Value: 8080,
-				},
-			},
-			ArgsUsage: " ", // can't be the empty string or we get the default output
-			Hidden:    true,
-		},
 		{
 			Name:   "proxy-dns",
 			Action: tunneldns.Run,
@@ -311,7 +297,7 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 		c.Set("url", "https://"+helloListener.Addr().String())
 	}
 
-	if uri, _ := url.Parse(c.String("url")); uri.Scheme == "ssh" {
+	if uri, err := url.Parse(c.String("url")); err == nil && uri.Scheme == "ssh" {
 		host := uri.Host
 		if uri.Port() == "" { // default to 22
 			host = uri.Hostname() + ":22"
@@ -438,7 +424,7 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 		}),
 		altsrc.NewStringFlag(&cli.StringFlag{
 			Name:    "cacert",
-			Usage:   "Certificate Authority authenticating the Cloudflare tunnel connection.",
+			Usage:   "Certificate Authority authenticating connections with Cloudflare's edge network.",
 			EnvVars: []string{"TUNNEL_CACERT"},
 			Hidden:  true,
 		}),

cmd/cloudflared/tunnel/configuration.go

@@ -198,12 +198,18 @@ func prepareTunnelConfig(c *cli.Context, buildInfo *origin.BuildInfo, version st
 		return nil, errors.Wrap(err, "unable to connect to the origin")
 	}
 
+	toEdgeTLSConfig, err := createTunnelConfig(c)
+	if err != nil {
+		logger.WithError(err).Error("unable to create TLS config to connect with edge")
+		return nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
+	}
+
 	return &origin.TunnelConfig{
 		EdgeAddrs:          c.StringSlice("edge"),
 		OriginUrl:          originURL,
 		Hostname:           hostname,
 		OriginCert:         originCert,
-		TlsConfig:          tlsconfig.CreateTunnelConfig(c, c.StringSlice("edge")),
+		TlsConfig:          toEdgeTLSConfig,
 		ClientTlsConfig:    httpTransport.TLSClientConfig,
 		Retries:            c.Uint("retries"),
 		HeartbeatInterval:  c.Duration("heartbeat-interval"),
@@ -240,7 +246,7 @@ func loadCertPool(c *cli.Context, logger *logrus.Logger) (*x509.CertPool, error)
 		}
 	}
 
-	originCertPool, err := tlsconfig.LoadOriginCertPool(originCustomCAPool)
+	originCertPool, err := loadOriginCertPool(originCustomCAPool)
 	if err != nil {
 		return nil, errors.Wrap(err, "error loading the certificate pool")
 	}
@@ -253,6 +259,86 @@ func loadCertPool(c *cli.Context, logger *logrus.Logger) (*x509.CertPool, error)
 	return originCertPool, nil
 }
 
+func loadOriginCertPool(originCAPoolPEM []byte) (*x509.CertPool, error) {
+	// Get the global pool
+	certPool, err := loadGlobalCertPool()
+	if err != nil {
+		return nil, err
+	}
+
+	// Then, add any custom origin CA pool the user may have passed
+	if originCAPoolPEM != nil {
+		if !certPool.AppendCertsFromPEM(originCAPoolPEM) {
+			logger.Warn("could not append the provided origin CA to the cloudflared certificate pool")
+		}
+	}
+
+	return certPool, nil
+}
+
+func loadGlobalCertPool() (*x509.CertPool, error) {
+	// First, obtain the system certificate pool
+	certPool, err := x509.SystemCertPool()
+	if err != nil {
+		if runtime.GOOS != "windows" {
+			logger.WithError(err).Warn("error obtaining the system certificates")
+		}
+		certPool = x509.NewCertPool()
+	}
+
+	// Next, append the Cloudflare CAs into the system pool
+	cfRootCA, err := tlsconfig.GetCloudflareRootCA()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not append Cloudflare Root CAs to cloudflared certificate pool")
+	}
+	for _, cert := range cfRootCA {
+		certPool.AddCert(cert)
+	}
+
+	// Finally, add the Hello certificate into the pool (since it's self-signed)
+	helloCert, err := tlsconfig.GetHelloCertificateX509()
+	if err != nil {
+		return nil, errors.Wrap(err, "could not append Hello server certificate to cloudflared certificate pool")
+	}
+	certPool.AddCert(helloCert)
+
+	return certPool, nil
+}
+
+func createTunnelConfig(c *cli.Context) (*tls.Config, error) {
+	var rootCAs []string
+	if c.String("cacert") != "" {
+		rootCAs = append(rootCAs, c.String("cacert"))
+	}
+	edgeAddrs := c.StringSlice("edge")
+
+	userConfig := &tlsconfig.TLSParameters{RootCAs: rootCAs}
+	tlsConfig, err := tlsconfig.GetConfig(userConfig)
+	if err != nil {
+		return nil, err
+	}
+	if tlsConfig.RootCAs == nil {
+		rootCAPool := x509.NewCertPool()
+		cfRootCA, err := tlsconfig.GetCloudflareRootCA()
+		if err != nil {
+			return nil, errors.Wrap(err, "could not append Cloudflare Root CAs to cloudflared certificate pool")
+		}
+		for _, cert := range cfRootCA {
+			rootCAPool.AddCert(cert)
+		}
+		tlsConfig.RootCAs = rootCAPool
+		tlsConfig.ServerName = "cftunnel.com"
+	} else if len(edgeAddrs) > 0 {
+		// Set for development environments and for testing specific origintunneld instances
+		tlsConfig.ServerName, _, _ = net.SplitHostPort(edgeAddrs[0])
+	}
+
+	if tlsConfig.ServerName == "" && !tlsConfig.InsecureSkipVerify {
+		return nil, fmt.Errorf("either ServerName or InsecureSkipVerify must be specified in the tls.Config")
+	}
+	return tlsConfig, nil
+}
+
 func isRunningFromTerminal() bool {
 	return terminal.IsTerminal(int(os.Stdout.Fd()))
 }

cmd/cloudflared/tunnel/configuration_test.go

@@ -0,0 +1,214 @@
+// +build ignore
+// TODO: Remove the above build tag and include this test when we start compiling with Golang 1.10.0+
+
+package tunnel
+
+import (
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/asn1"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Generated using `openssl req -newkey rsa:512 -nodes -x509 -days 3650`
+var samplePEM = []byte(`
+-----BEGIN CERTIFICATE-----
+MIIB4DCCAYoCCQCb/H0EUrdXEjANBgkqhkiG9w0BAQsFADB3MQswCQYDVQQGEwJV
+UzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcGA1UECgwQQ2xv
+dWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVneTERMA8GA1UE
+AwwIVGVzdCBPbmUwHhcNMTgwNDI2MTYxMDUxWhcNMjgwNDIzMTYxMDUxWjB3MQsw
+CQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcG
+A1UECgwQQ2xvdWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVn
+eTERMA8GA1UEAwwIVGVzdCBPbmUwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAwVQD
+K0SJ25UFLznm2pU3zhzMEvpDEofHVNnCjk4mlDrtVop7PkKZ8pDEmuQANltUrxC8
+yHBE2wXMv+GlH+bDtwIDAQABMA0GCSqGSIb3DQEBCwUAA0EAjVYQzozIFPkt/HRY
+uUoZ8zEHIDICb0syFf5VAjm9AgTwIPzUmD+c5vl6LWDnxq7L45nLCzhhQ6YmiwDz
+X7Wcyg==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIB4DCCAYoCCQDZfCdAJ+mwzDANBgkqhkiG9w0BAQsFADB3MQswCQYDVQQGEwJV
+UzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcGA1UECgwQQ2xv
+dWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVneTERMA8GA1UE
+AwwIVGVzdCBUd28wHhcNMTgwNDI2MTYxMTIwWhcNMjgwNDIzMTYxMTIwWjB3MQsw
+CQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcG
+A1UECgwQQ2xvdWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVn
+eTERMA8GA1UEAwwIVGVzdCBUd28wXDANBgkqhkiG9w0BAQEFAANLADBIAkEAoHKp
+ROVK3zCSsH7ocYeyRAML4V7SFAbZcb4WIwDnE08oMBVRkQVcW5tqEkvG3RiClfzV
+wZIJ3CfqKIeSNSDU9wIDAQABMA0GCSqGSIb3DQEBCwUAA0EAJw2gUbnPiq4C2p5b
+iWzlA9Q7aKo+VQ4H7IZS7tTccr59nVjvH/TG3eWujpnocr4TOqW9M3CK1DF9mUGP
+3pQ3Jg==
+-----END CERTIFICATE-----
+`)
+
+var systemCertPoolSubjects []*pkix.Name
+
+type certificateFixture struct {
+	ou string
+	cn string
+}
+
+func TestMain(m *testing.M) {
+	systemCertPool, err := x509.SystemCertPool()
+	if isUnrecoverableError(err) {
+		os.Exit(1)
+	}
+
+	if systemCertPool == nil {
+		// On Windows, let's just assume the system cert pool was empty
+		systemCertPool = x509.NewCertPool()
+	}
+
+	systemCertPoolSubjects, err = getCertPoolSubjects(systemCertPool)
+	if err != nil {
+		os.Exit(1)
+	}
+
+	os.Exit(m.Run())
+}
+
+func TestLoadOriginCertPoolJustSystemPool(t *testing.T) {
+	certPoolSubjects := loadCertPoolSubjects(t, nil)
+	extraSubjects := subjectSubtract(systemCertPoolSubjects, certPoolSubjects)
+
+	// Remove extra subjects from the cert pool
+	var filteredSystemCertPoolSubjects []*pkix.Name
+
+	t.Log(extraSubjects)
+
+OUTER:
+	for _, subject := range certPoolSubjects {
+		for _, extraSubject := range extraSubjects {
+			if subject == extraSubject {
+				t.Log(extraSubject)
+				continue OUTER
+			}
+		}
+
+		filteredSystemCertPoolSubjects = append(filteredSystemCertPoolSubjects, subject)
+	}
+
+	assert.Equal(t, len(filteredSystemCertPoolSubjects), len(systemCertPoolSubjects))
+
+	difference := subjectSubtract(systemCertPoolSubjects, filteredSystemCertPoolSubjects)
+	assert.Equal(t, 0, len(difference))
+}
+
+func TestLoadOriginCertPoolCFCertificates(t *testing.T) {
+	certPoolSubjects := loadCertPoolSubjects(t, nil)
+
+	extraSubjects := subjectSubtract(systemCertPoolSubjects, certPoolSubjects)
+
+	expected := []*certificateFixture{
+		{ou: "CloudFlare Origin SSL ECC Certificate Authority"},
+		{ou: "CloudFlare Origin SSL Certificate Authority"},
+		{cn: "origin-pull.cloudflare.net"},
+		{cn: "Argo Tunnel Sample Hello Server Certificate"},
+	}
+
+	assertFixturesMatchSubjects(t, expected, extraSubjects)
+}
+
+func TestLoadOriginCertPoolWithExtraPEMs(t *testing.T) {
+	certPoolWithoutPEMSubjects := loadCertPoolSubjects(t, nil)
+	certPoolWithPEMSubjects := loadCertPoolSubjects(t, samplePEM)
+
+	difference := subjectSubtract(certPoolWithoutPEMSubjects, certPoolWithPEMSubjects)
+
+	assert.Equal(t, 2, len(difference))
+
+	expected := []*certificateFixture{
+		{cn: "Test One"},
+		{cn: "Test Two"},
+	}
+
+	assertFixturesMatchSubjects(t, expected, difference)
+}
+
+func loadCertPoolSubjects(t *testing.T, originCAPoolPEM []byte) []*pkix.Name {
+	certPool, err := loadOriginCertPool(originCAPoolPEM)
+	if isUnrecoverableError(err) {
+		t.Fatal(err)
+	}
+	assert.NotEmpty(t, certPool.Subjects())
+	certPoolSubjects, err := getCertPoolSubjects(certPool)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return certPoolSubjects
+}
+
+func assertFixturesMatchSubjects(t *testing.T, fixtures []*certificateFixture, subjects []*pkix.Name) {
+	assert.Equal(t, len(fixtures), len(subjects))
+
+	for _, fixture := range fixtures {
+		found := false
+		for _, subject := range subjects {
+			found = found || fixtureMatchesSubjectPredicate(fixture, subject)
+		}
+
+		if !found {
+			t.Fail()
+		}
+	}
+}
+
+func fixtureMatchesSubjectPredicate(fixture *certificateFixture, subject *pkix.Name) bool {
+	cnMatch := true
+	if fixture.cn != "" {
+		cnMatch = fixture.cn == subject.CommonName
+	}
+
+	ouMatch := true
+	if fixture.ou != "" {
+		ouMatch = len(subject.OrganizationalUnit) > 0 && fixture.ou == subject.OrganizationalUnit[0]
+	}
+
+	return cnMatch && ouMatch
+}
+
+func subjectSubtract(left []*pkix.Name, right []*pkix.Name) []*pkix.Name {
+	var difference []*pkix.Name
+
+	var found bool
+	for _, r := range right {
+		found = false
+		for _, l := range left {
+			if (*l).String() == (*r).String() {
+				found = true
+			}
+		}
+
+		if !found {
+			difference = append(difference, r)
+		}
+	}
+
+	return difference
+}
+
+func getCertPoolSubjects(certPool *x509.CertPool) ([]*pkix.Name, error) {
+	var subjects []*pkix.Name
+
+	for _, subject := range certPool.Subjects() {
+		var sequence pkix.RDNSequence
+		_, err := asn1.Unmarshal(subject, &sequence)
+		if err != nil {
+			return nil, err
+		}
+
+		name := pkix.Name{}
+		name.FillFromRDNSequence(&sequence)
+
+		subjects = append(subjects, &name)
+	}
+
+	return subjects, nil
+}
+
+func isUnrecoverableError(err error) bool {
+	return err != nil && err.Error() != "crypto/x509: system root pool is not available on Windows"
+}

cmd/cloudflared/tunnel/hello.go

@@ -1,20 +0,0 @@
-package tunnel
-
-import (
-	"fmt"
-
-	"gopkg.in/urfave/cli.v2"
-
-	"github.com/cloudflare/cloudflared/hello"
-)
-
-func helloWorld(c *cli.Context) error {
-	address := fmt.Sprintf(":%d", c.Int("port"))
-	listener, err := hello.CreateTLSListener(address)
-	if err != nil {
-		return err
-	}
-	defer listener.Close()
-	err = hello.StartHelloWorldServer(logger, listener, nil)
-	return err
-}

origin/metrics.go

@@ -52,6 +52,7 @@ type TunnelMetrics struct {
 	oldServerLocations map[string]string
 
 	muxerMetrics *muxerMetrics
+	tunnelsHA    tunnelsForHA
 }
 
 func newMuxerMetrics() *muxerMetrics {
@@ -355,6 +356,7 @@ func NewTunnelMetrics() *TunnelMetrics {
 		serverLocations:                serverLocations,
 		oldServerLocations:             make(map[string]string),
 		muxerMetrics:                   newMuxerMetrics(),
+		tunnelsHA:                      NewTunnelsForHA(),
 	}
 }
 
@@ -375,7 +377,7 @@ func (t *TunnelMetrics) incrementRequests(connectionID string) {
 	var concurrentRequests uint64
 	var ok bool
 	if concurrentRequests, ok = t.concurrentRequests[connectionID]; ok {
-		t.concurrentRequests[connectionID] += 1
+		t.concurrentRequests[connectionID]++
 		concurrentRequests++
 	} else {
 		t.concurrentRequests[connectionID] = 1
@@ -395,7 +397,7 @@ func (t *TunnelMetrics) incrementRequests(connectionID string) {
 func (t *TunnelMetrics) decrementConcurrentRequests(connectionID string) {
 	t.concurrentRequestsLock.Lock()
 	if _, ok := t.concurrentRequests[connectionID]; ok {
-		t.concurrentRequests[connectionID] -= 1
+		t.concurrentRequests[connectionID]--
 	}
 	t.concurrentRequestsLock.Unlock()
 

origin/tunnel.go

@@ -292,7 +292,13 @@ func IsRPCStreamResponse(headers []h2mux.Header) bool {
 	return true
 }
 
-func RegisterTunnel(ctx context.Context, muxer *h2mux.Muxer, config *TunnelConfig, connectionID uint8, originLocalIP string) error {
+func RegisterTunnel(
+	ctx context.Context,
+	muxer *h2mux.Muxer,
+	config *TunnelConfig,
+	connectionID uint8,
+	originLocalIP string,
+) error {
 	config.Logger.Debug("initiating RPC stream to register")
 	stream, err := muxer.OpenStream([]h2mux.Header{
 		{Name: ":method", Value: "RPC"},
@@ -346,7 +352,8 @@ func RegisterTunnel(ctx context.Context, muxer *h2mux.Muxer, config *TunnelConfi
 	}
 
 	if registration.TunnelID != "" {
-		config.Logger.Info("Tunnel ID: " + registration.TunnelID)
+		config.Metrics.tunnelsHA.AddTunnelID(connectionID, registration.TunnelID)
+		config.Logger.Infof("Each HA connection's tunnel IDs: %v", config.Metrics.tunnelsHA.String())
 	}
 
 	// Print out the user's trial zone URL in a nice box (if they requested and got one)

origin/tunnelsforha.go

@@ -0,0 +1,49 @@
+package origin
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// tunnelsForHA maps this cloudflared instance's HA connections to the tunnel IDs they serve.
+type tunnelsForHA struct {
+	sync.Mutex
+	metrics *prometheus.GaugeVec
+	entries map[uint8]string
+}
+
+// NewTunnelsForHA initializes the Prometheus metrics etc for a tunnelsForHA.
+func NewTunnelsForHA() tunnelsForHA {
+	metrics := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "tunnel_ids",
+			Help: "The ID of all tunnels (and their corresponding HA connection ID) running in this instance of cloudflared.",
+		},
+		[]string{"tunnel_id", "ha_conn_id"},
+	)
+	prometheus.MustRegister(metrics)
+
+	return tunnelsForHA{
+		metrics: metrics,
+		entries: make(map[uint8]string),
+	}
+}
+
+// Track a new tunnel ID, removing the disconnected tunnel (if any) and update metrics.
+func (t *tunnelsForHA) AddTunnelID(haConn uint8, tunnelID string) {
+	t.Lock()
+	defer t.Unlock()
+	if oldTunnelID, ok := t.entries[haConn]; ok {
+		t.metrics.WithLabelValues(oldTunnelID).Dec()
+	}
+	t.entries[haConn] = tunnelID
+	t.metrics.WithLabelValues(tunnelID, fmt.Sprintf("%v", haConn)).Inc()
+}
+
+func (t *tunnelsForHA) String() string {
+	t.Lock()
+	defer t.Unlock()
+	return fmt.Sprintf("%v", t.entries)
+}

tlsconfig/certreloader.go

@@ -2,14 +2,10 @@ package tlsconfig
 
 import (
 	"crypto/tls"
-	"errors"
 	"fmt"
 	"sync"
 
-	tunnellog "github.com/cloudflare/cloudflared/log"
 	"github.com/getsentry/raven-go"
-	log "github.com/sirupsen/logrus"
-	"gopkg.in/urfave/cli.v2"
 )
 
 // CertReloader can load and reload a TLS certificate from a particular filepath.
@@ -21,18 +17,14 @@ type CertReloader struct {
 	keyPath     string
 }
 
-// NewCertReloader makes a CertReloader, memorizing the filepaths in the context/flags.
-func NewCertReloader(c *cli.Context, f CLIFlags) (*CertReloader, error) {
-	if !c.IsSet(f.Cert) {
-		return nil, errors.New("CertReloader: cert not provided")
-	}
-	if !c.IsSet(f.Key) {
-		return nil, errors.New("CertReloader: key not provided")
-	}
+// NewCertReloader makes a CertReloader. It loads the cert during initialization to make sure certPath and keyPath are valid
+func NewCertReloader(certPath, keyPath string) (*CertReloader, error) {
 	cr := new(CertReloader)
-	cr.certPath = c.String(f.Cert)
-	cr.keyPath = c.String(f.Key)
-	cr.LoadCert()
+	cr.certPath = certPath
+	cr.keyPath = keyPath
+	if err := cr.LoadCert(); err != nil {
+		return nil, err
+	}
 	return cr, nil
 }
 
@@ -45,18 +37,17 @@ func (cr *CertReloader) Cert(clientHello *tls.ClientHelloInfo) (*tls.Certificate
 
 // LoadCert loads a TLS certificate from the CertReloader's specified filepath.
 // Call this after writing a new certificate to the disk (e.g. after renewing a certificate)
-func (cr *CertReloader) LoadCert() {
+func (cr *CertReloader) LoadCert() error {
 	cr.Lock()
 	defer cr.Unlock()
 
-	log.SetFormatter(&tunnellog.JSONFormatter{})
-	log.Info("Reloading certificate")
 	cert, err := tls.LoadX509KeyPair(cr.certPath, cr.keyPath)
 
 	// Keep the old certificate if there's a problem reading the new one.
 	if err != nil {
 		raven.CaptureError(fmt.Errorf("Error parsing X509 key pair: %v", err), nil)
-		return
+		return err
 	}
 	cr.certificate = &cert
+	return nil
 }

tlsconfig/cloudflare_ca.go

@@ -2,6 +2,7 @@ package tlsconfig
 
 import (
 	"crypto/x509"
+	"encoding/pem"
 )
 
 // TODO: remove the Origin CA root certs when migrated to Authenticated Origin Pull certs
@@ -85,11 +86,26 @@ QzMmZpRpIBB321ZBlcnlxiTJvWxvbCPHKHj20VwwAz7LONF59s84ZsOqfoBv8gKM
 s0s5dsq5zpLeaw==
 -----END CERTIFICATE-----`)
 
-func GetCloudflareRootCA() *x509.CertPool {
-	ca := x509.NewCertPool()
-	if !ca.AppendCertsFromPEM([]byte(cloudflareRootCA)) {
-		// should never happen
-		panic("failure loading Cloudflare origin CA pem")
+func GetCloudflareRootCA() ([]*x509.Certificate, error) {
+	var certs []*x509.Certificate
+	pemBlocks := cloudflareRootCA
+	for len(pemBlocks) > 0 {
+		var block *pem.Block
+		block, pemBlocks = pem.Decode(pemBlocks)
+		if block == nil {
+			break
+		}
+		if block.Type != "CERTIFICATE" {
+			continue
+		}
+
+		cert, err := x509.ParseCertificate(block.Bytes)
+		if err != nil {
+			return nil, err
+		}
+
+		certs = append(certs, cert)
 	}
-	return ca
+
+	return certs, nil
 }

tlsconfig/testcert.pem

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIICBjCCAbCgAwIBAgIJAPKk4bYMrSFMMA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV
+BAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEPMA0GA1UEBwwGQXVzdGluMRkwFwYDVQQK
+DBBDbG91ZGZsYXJlLCBJbmMuMRIwEAYDVQQDDAlsb2NhbGhvc3QwHhcNMTgxMTE1
+MjA1NzU3WhcNMjgxMTEyMjA1NzU3WjBdMQswCQYDVQQGEwJVUzEOMAwGA1UECAwF
+VGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcGA1UECgwQQ2xvdWRmbGFyZSwgSW5j
+LjESMBAGA1UEAwwJbG9jYWxob3N0MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAOQN
+pTRn5wLf8SSI5x2kpDvbdDy7lfhamJ2En4Q+wy1cSKp8bn8/oyhVF7QTsimDGTI4
+45pV9nDfNJPYB3IW0x0CAwEAAaNTMFEwHQYDVR0OBBYEFE4jIa97mIEiYFa02X++
+uu5mCEn+MB8GA1UdIwQYMBaAFE4jIa97mIEiYFa02X++uu5mCEn+MA8GA1UdEwEB
+/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADQQAkE+pDee0o5cNcZRUszy8sTQzB1Wlp
+J6ucfmo16crqRaK7uGvhkMyibIc4D8z2Cxw3aI3IMMFoIIlYoYKiUcbd
+-----END CERTIFICATE-----

tlsconfig/testcert2.pem

@@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIICBjCCAbCgAwIBAgIJAN6cXRTbJtFnMA0GCSqGSIb3DQEBCwUAMF0xCzAJBgNV
+BAYTAlVTMQ4wDAYDVQQIDAVUZXhhczEPMA0GA1UEBwwGQXVzdGluMRgwFgYDVQQK
+DA9DbG91ZGZsYXJlLCBJbmMxEzARBgNVBAMMCmxvY2FsaG9zdDIwHhcNMTgxMTE1
+MjExMTU4WhcNMjgxMTEyMjExMTU4WjBdMQswCQYDVQQGEwJVUzEOMAwGA1UECAwF
+VGV4YXMxDzANBgNVBAcMBkF1c3RpbjEYMBYGA1UECgwPQ2xvdWRmbGFyZSwgSW5j
+MRMwEQYDVQQDDApsb2NhbGhvc3QyMFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAKQx
+IMZ6QgXoul2ITF/7sly4fW2Ol+a/AYw42zCWhVqOXv8AhY21I0Q8lkRR6wOroQwZ
+O7jKKOcE5TnR/NRcZr8CAwEAAaNTMFEwHQYDVR0OBBYEFONKxLZc2RUD0KTHkAz4
+8nrb5688MB8GA1UdIwQYMBaAFONKxLZc2RUD0KTHkAz48nrb5688MA8GA1UdEwEB
+/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADQQA56pwhvGpNPjyLcWfJHu/vI3ZjdoLB
+LnrkRaMjJmv0H0Beh4upJhoz8u6lhMACerKQrrdQhEPB2u+maFrEBtmN
+-----END CERTIFICATE-----

tlsconfig/testkey.pem

@@ -0,0 +1,10 @@
+-----BEGIN PRIVATE KEY-----
+MIIBVQIBADANBgkqhkiG9w0BAQEFAASCAT8wggE7AgEAAkEA5A2lNGfnAt/xJIjn
+HaSkO9t0PLuV+FqYnYSfhD7DLVxIqnxufz+jKFUXtBOyKYMZMjjjmlX2cN80k9gH
+chbTHQIDAQABAkAoeDtu91lJa1AxuZG58vOqI6GW/Xr5naojmdts7m5YaAhDa7DE
+zJUp4d8SP5cGBf1/PB3x6Cu9UviFNQ16wmzJAiEA8gUm4UYpWZD4Ze2l/xb+BK8D
+IglSUIy1VxW+X1G55wMCIQDxOfXiFzPqnv/e5avKGv6CU11Dhmbi1OpiyybZTjGz
+XwIhAM3bE/cJdqJ4bNBGE6umIupY8pFA3IMnLBempwbsvPOBAiEAgzJ+5OSxu92W
+VGidsmJUIhWtF9i1hJFAmVLcYjwBFAkCICtjP/vv0qOZWk4mAAn2zz9UVWp45DSR
+p/FA8V77ohXD
+-----END PRIVATE KEY-----

tlsconfig/tlsconfig.go

@@ -6,150 +6,86 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"io/ioutil"
-	"net"
 
-	"github.com/cloudflare/cloudflared/log"
 	"github.com/pkg/errors"
-	"gopkg.in/urfave/cli.v2"
-	"runtime"
 )
 
-var logger = log.CreateLogger()
-
-// CLIFlags names the flags used to configure TLS for a command or subsystem.
-// The nil value for a field means the flag is ignored.
-type CLIFlags struct {
-	Cert       string
-	Key        string
-	ClientCert string
-	RootCA     string
+// Config is the user provided parameters to create a tls.Config
+type TLSParameters struct {
+	Cert             string
+	Key              string
+	GetCertificate   *CertReloader
+	ClientCAs        []string
+	RootCAs          []string
+	ServerName       string
+	CurvePreferences []tls.CurveID
 }
 
-// GetConfig returns a TLS configuration according to the flags defined in f and
-// set by the user.
-func (f CLIFlags) GetConfig(c *cli.Context) *tls.Config {
-	config := &tls.Config{}
-
-	if c.IsSet(f.Cert) && c.IsSet(f.Key) {
-		cert, err := tls.LoadX509KeyPair(c.String(f.Cert), c.String(f.Key))
+// GetConfig returns a TLS configuration according to the Config set by the user.
+func GetConfig(p *TLSParameters) (*tls.Config, error) {
+	tlsconfig := &tls.Config{}
+	if p.Cert != "" && p.Key != "" {
+		cert, err := tls.LoadX509KeyPair(p.Cert, p.Key)
 		if err != nil {
-			logger.WithError(err).Fatal("Error parsing X509 key pair")
+			return nil, errors.Wrap(err, "Error parsing X509 key pair")
 		}
-		config.Certificates = []tls.Certificate{cert}
-		config.BuildNameToCertificate()
+		tlsconfig.Certificates = []tls.Certificate{cert}
+		// BuildNameToCertificate parses Certificates and builds NameToCertificate from common name
+		// and SAN fields of leaf certificates
+		tlsconfig.BuildNameToCertificate()
 	}
-	return f.finishGettingConfig(c, config)
-}
 
-func (f CLIFlags) GetConfigReloadableCert(c *cli.Context, cr *CertReloader) *tls.Config {
-	config := &tls.Config{
-		GetCertificate: cr.Cert,
+	if p.GetCertificate != nil {
+		// GetCertificate is called when client supplies SNI info or Certificates is empty.
+		// Order of retrieving certificate is GetCertificate, NameToCertificate and lastly first element of Certificates
+		tlsconfig.GetCertificate = p.GetCertificate.Cert
 	}
-	config.BuildNameToCertificate()
-	return f.finishGettingConfig(c, config)
-}
 
-func (f CLIFlags) finishGettingConfig(c *cli.Context, config *tls.Config) *tls.Config {
-	if c.IsSet(f.ClientCert) {
+	if len(p.ClientCAs) > 0 {
 		// set of root certificate authorities that servers use if required to verify a client certificate
 		// by the policy in ClientAuth
-		config.ClientCAs = LoadCert(c.String(f.ClientCert))
+		clientCAs, err := LoadCert(p.ClientCAs)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error loading client CAs")
+		}
+		tlsconfig.ClientCAs = clientCAs
 		// server's policy for TLS Client Authentication. Default is no client cert
-		config.ClientAuth = tls.RequireAndVerifyClientCert
+		tlsconfig.ClientAuth = tls.RequireAndVerifyClientCert
 	}
-	// set of root certificate authorities that clients use when verifying server certificates
-	if c.IsSet(f.RootCA) {
-		config.RootCAs = LoadCert(c.String(f.RootCA))
+
+	if len(p.RootCAs) > 0 {
+		rootCAs, err := LoadCert(p.RootCAs)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error loading root CAs")
+		}
+		tlsconfig.RootCAs = rootCAs
 	}
-	// we optimize CurveP256
-	config.CurvePreferences = []tls.CurveID{tls.CurveP256}
-	return config
+
+	if p.ServerName != "" {
+		tlsconfig.ServerName = p.ServerName
+	}
+
+	if len(p.CurvePreferences) > 0 {
+		tlsconfig.CurvePreferences = p.CurvePreferences
+	} else {
+		// Cloudflare optimize CurveP256
+		tlsconfig.CurvePreferences = []tls.CurveID{tls.CurveP256}
+	}
+
+	return tlsconfig, nil
 }
 
 // LoadCert creates a CertPool containing all certificates in a PEM-format file.
-func LoadCert(certPath string) *x509.CertPool {
-	caCert, err := ioutil.ReadFile(certPath)
-	if err != nil {
-		logger.WithError(err).Fatalf("Error reading certificate %s", certPath)
-	}
+func LoadCert(certPaths []string) (*x509.CertPool, error) {
 	ca := x509.NewCertPool()
-	if !ca.AppendCertsFromPEM(caCert) {
-		logger.WithError(err).Fatalf("Error parsing certificate %s", certPath)
-	}
-	return ca
-}
-
-func LoadGlobalCertPool() (*x509.CertPool, error) {
-	success := false
-
-	// First, obtain the system certificate pool
-	certPool, systemCertPoolErr := x509.SystemCertPool()
-	if systemCertPoolErr != nil {
-		if runtime.GOOS != "windows" {
-			logger.Warnf("error obtaining the system certificates: %s", systemCertPoolErr)
+	for _, certPath := range certPaths {
+		caCert, err := ioutil.ReadFile(certPath)
+		if err != nil {
+			return nil, errors.Wrapf(err, "Error reading certificate %s", certPath)
 		}
-		certPool = x509.NewCertPool()
-	} else {
-		success = true
-	}
-
-	// Next, append the Cloudflare CA pool into the system pool
-	if !certPool.AppendCertsFromPEM(cloudflareRootCA) {
-		logger.Warn("could not append the CF certificate to the cloudflared certificate pool")
-	} else {
-		success = true
-	}
-
-	if success != true { // Obtaining any of the CAs has failed; this is a fatal error
-		return nil, errors.New("error loading any of the CAs into the global certificate pool")
-	}
-
-	// Finally, add the Hello certificate into the pool (since it's self-signed)
-	helloCertificate, err := GetHelloCertificateX509()
-	if err != nil {
-		logger.Warn("error obtaining the Hello server certificate")
-	}
-
-	certPool.AddCert(helloCertificate)
-
-	return certPool, nil
-}
-
-func LoadOriginCertPool(originCAPoolPEM []byte) (*x509.CertPool, error) {
-	success := false
-
-	// Get the global pool
-	certPool, globalPoolErr := LoadGlobalCertPool()
-	if globalPoolErr != nil {
-		certPool = x509.NewCertPool()
-	} else {
-		success = true
-	}
-
-	// Then, add any custom origin CA pool the user may have passed
-	if originCAPoolPEM != nil {
-		if !certPool.AppendCertsFromPEM(originCAPoolPEM) {
-			logger.Warn("could not append the provided origin CA to the cloudflared certificate pool")
-		} else {
-			success = true
+		if !ca.AppendCertsFromPEM(caCert) {
+			return nil, errors.Wrapf(err, "Error parsing certificate %s", certPath)
 		}
 	}
-
-	if success != true {
-		return nil, errors.New("error loading any of the CAs into the origin certificate pool")
-	}
-
-	return certPool, nil
-}
-
-func CreateTunnelConfig(c *cli.Context, addrs []string) *tls.Config {
-	tlsConfig := CLIFlags{RootCA: "cacert"}.GetConfig(c)
-	if tlsConfig.RootCAs == nil {
-		tlsConfig.RootCAs = GetCloudflareRootCA()
-		tlsConfig.ServerName = "cftunnel.com"
-	} else if len(addrs) > 0 {
-		// Set for development environments and for testing specific origintunneld instances
-		tlsConfig.ServerName, _, _ = net.SplitHostPort(addrs[0])
-	}
-	return tlsConfig
+	return ca, nil
 }

tlsconfig/tlsconfig_test.go

@@ -1,214 +1,84 @@
-// +build ignore
 // TODO: Remove the above build tag and include this test when we start compiling with Golang 1.10.0+
 
 package tlsconfig
 
 import (
-	"crypto/x509"
-	"crypto/x509/pkix"
-	"encoding/asn1"
-	"os"
+	"crypto/tls"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 )
 
-// Generated using `openssl req -newkey rsa:512 -nodes -x509 -days 3650`
-var samplePEM = []byte(`
------BEGIN CERTIFICATE-----
-MIIB4DCCAYoCCQCb/H0EUrdXEjANBgkqhkiG9w0BAQsFADB3MQswCQYDVQQGEwJV
-UzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcGA1UECgwQQ2xv
-dWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVneTERMA8GA1UE
-AwwIVGVzdCBPbmUwHhcNMTgwNDI2MTYxMDUxWhcNMjgwNDIzMTYxMDUxWjB3MQsw
-CQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcG
-A1UECgwQQ2xvdWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVn
-eTERMA8GA1UEAwwIVGVzdCBPbmUwXDANBgkqhkiG9w0BAQEFAANLADBIAkEAwVQD
-K0SJ25UFLznm2pU3zhzMEvpDEofHVNnCjk4mlDrtVop7PkKZ8pDEmuQANltUrxC8
-yHBE2wXMv+GlH+bDtwIDAQABMA0GCSqGSIb3DQEBCwUAA0EAjVYQzozIFPkt/HRY
-uUoZ8zEHIDICb0syFf5VAjm9AgTwIPzUmD+c5vl6LWDnxq7L45nLCzhhQ6YmiwDz
-X7Wcyg==
------END CERTIFICATE-----
------BEGIN CERTIFICATE-----
-MIIB4DCCAYoCCQDZfCdAJ+mwzDANBgkqhkiG9w0BAQsFADB3MQswCQYDVQQGEwJV
-UzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcGA1UECgwQQ2xv
-dWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVneTERMA8GA1UE
-AwwIVGVzdCBUd28wHhcNMTgwNDI2MTYxMTIwWhcNMjgwNDIzMTYxMTIwWjB3MQsw
-CQYDVQQGEwJVUzEOMAwGA1UECAwFVGV4YXMxDzANBgNVBAcMBkF1c3RpbjEZMBcG
-A1UECgwQQ2xvdWRmbGFyZSwgSW5jLjEZMBcGA1UECwwQUHJvZHVjdCBTdHJhdGVn
-eTERMA8GA1UEAwwIVGVzdCBUd28wXDANBgkqhkiG9w0BAQEFAANLADBIAkEAoHKp
-ROVK3zCSsH7ocYeyRAML4V7SFAbZcb4WIwDnE08oMBVRkQVcW5tqEkvG3RiClfzV
-wZIJ3CfqKIeSNSDU9wIDAQABMA0GCSqGSIb3DQEBCwUAA0EAJw2gUbnPiq4C2p5b
-iWzlA9Q7aKo+VQ4H7IZS7tTccr59nVjvH/TG3eWujpnocr4TOqW9M3CK1DF9mUGP
-3pQ3Jg==
------END CERTIFICATE-----
-`)
+// testcert.pem and testcert2.pem are Generated using `openssl req -newkey rsa:512 -nodes -x509 -days 3650`
+const (
+	testcertCommonName = "localhost"
+)
 
-var systemCertPoolSubjects []*pkix.Name
+func TestGetFromEmptyConfig(t *testing.T) {
+	c := &TLSParameters{}
 
-type certificateFixture struct {
-	ou string
-	cn string
+	tlsConfig, err := GetConfig(c)
+	assert.NoError(t, err)
+	assert.Empty(t, tlsConfig.Certificates)
+
+	assert.Empty(t, tlsConfig.NameToCertificate)
+
+	assert.Nil(t, tlsConfig.ClientCAs)
+	assert.Equal(t, tls.NoClientCert, tlsConfig.ClientAuth)
+
+	assert.Nil(t, tlsConfig.RootCAs)
+
+	assert.Len(t, tlsConfig.CurvePreferences, 1)
+	assert.Equal(t, tls.CurveP256, tlsConfig.CurvePreferences[0])
 }
 
-func TestMain(m *testing.M) {
-	systemCertPool, err := x509.SystemCertPool()
-	if isUnrecoverableError(err) {
-		os.Exit(1)
-	}
+func TestGetConfig(t *testing.T) {
+	cert, err := tls.LoadX509KeyPair("testcert.pem", "testkey.pem")
+	assert.NoError(t, err)
 
-	if systemCertPool == nil {
-		// On Windows, let's just assume the system cert pool was empty
-		systemCertPool = x509.NewCertPool()
+	c := &TLSParameters{
+		Cert:             "testcert.pem",
+		Key:              "testkey.pem",
+		ClientCAs:        []string{"testcert.pem", "testcert2.pem"},
+		RootCAs:          []string{"testcert.pem", "testcert2.pem"},
+		ServerName:       "test",
+		CurvePreferences: []tls.CurveID{tls.CurveP384},
 	}
+	tlsConfig, err := GetConfig(c)
+	assert.NoError(t, err)
+	assert.Len(t, tlsConfig.Certificates, 1)
+	assert.Equal(t, cert, tlsConfig.Certificates[0])
 
-	systemCertPoolSubjects, err = getCertPoolSubjects(systemCertPool)
-	if err != nil {
-		os.Exit(1)
-	}
+	assert.Equal(t, cert, *tlsConfig.NameToCertificate[testcertCommonName])
 
-	os.Exit(m.Run())
+	assert.NotNil(t, tlsConfig.ClientCAs)
+	assert.Equal(t, tls.RequireAndVerifyClientCert, tlsConfig.ClientAuth)
+
+	assert.NotNil(t, tlsConfig.RootCAs)
+
+	assert.Len(t, tlsConfig.CurvePreferences, 1)
+	assert.Equal(t, tls.CurveP384, tlsConfig.CurvePreferences[0])
 }
 
-func TestLoadOriginCertPoolJustSystemPool(t *testing.T) {
-	certPoolSubjects := loadCertPoolSubjects(t, nil)
-	extraSubjects := subjectSubtract(systemCertPoolSubjects, certPoolSubjects)
+func TestCertReloader(t *testing.T) {
+	expectedCert, err := tls.LoadX509KeyPair("testcert.pem", "testkey.pem")
+	assert.NoError(t, err)
 
-	// Remove extra subjects from the cert pool
-	var filteredSystemCertPoolSubjects []*pkix.Name
+	certReloader, err := NewCertReloader("testcert.pem", "testkey.pem")
+	assert.NoError(t, err)
 
-	t.Log(extraSubjects)
+	chi := &tls.ClientHelloInfo{ServerName: testcertCommonName}
+	cert, err := certReloader.Cert(chi)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedCert, *cert)
 
-OUTER:
-	for _, subject := range certPoolSubjects {
-		for _, extraSubject := range extraSubjects {
-			if subject == extraSubject {
-				t.Log(extraSubject)
-				continue OUTER
-			}
-		}
-
-		filteredSystemCertPoolSubjects = append(filteredSystemCertPoolSubjects, subject)
+	c := &TLSParameters{
+		GetCertificate: certReloader,
 	}
+	tlsConfig, err := GetConfig(c)
+	assert.NoError(t, err)
 
-	assert.Equal(t, len(filteredSystemCertPoolSubjects), len(systemCertPoolSubjects))
-
-	difference := subjectSubtract(systemCertPoolSubjects, filteredSystemCertPoolSubjects)
-	assert.Equal(t, 0, len(difference))
-}
-
-func TestLoadOriginCertPoolCFCertificates(t *testing.T) {
-	certPoolSubjects := loadCertPoolSubjects(t, nil)
-
-	extraSubjects := subjectSubtract(systemCertPoolSubjects, certPoolSubjects)
-
-	expected := []*certificateFixture{
-		{ou: "CloudFlare Origin SSL ECC Certificate Authority"},
-		{ou: "CloudFlare Origin SSL Certificate Authority"},
-		{cn: "origin-pull.cloudflare.net"},
-		{cn: "Argo Tunnel Sample Hello Server Certificate"},
-	}
-
-	assertFixturesMatchSubjects(t, expected, extraSubjects)
-}
-
-func TestLoadOriginCertPoolWithExtraPEMs(t *testing.T) {
-	certPoolWithoutPEMSubjects := loadCertPoolSubjects(t, nil)
-	certPoolWithPEMSubjects := loadCertPoolSubjects(t, samplePEM)
-
-	difference := subjectSubtract(certPoolWithoutPEMSubjects, certPoolWithPEMSubjects)
-
-	assert.Equal(t, 2, len(difference))
-
-	expected := []*certificateFixture{
-		{cn: "Test One"},
-		{cn: "Test Two"},
-	}
-
-	assertFixturesMatchSubjects(t, expected, difference)
-}
-
-func loadCertPoolSubjects(t *testing.T, originCAPoolPEM []byte) []*pkix.Name {
-	certPool, err := LoadOriginCertPool(originCAPoolPEM)
-	if isUnrecoverableError(err) {
-		t.Fatal(err)
-	}
-	assert.NotEmpty(t, certPool.Subjects())
-	certPoolSubjects, err := getCertPoolSubjects(certPool)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	return certPoolSubjects
-}
-
-func assertFixturesMatchSubjects(t *testing.T, fixtures []*certificateFixture, subjects []*pkix.Name) {
-	assert.Equal(t, len(fixtures), len(subjects))
-
-	for _, fixture := range fixtures {
-		found := false
-		for _, subject := range subjects {
-			found = found || fixtureMatchesSubjectPredicate(fixture, subject)
-		}
-
-		if !found {
-			t.Fail()
-		}
-	}
-}
-
-func fixtureMatchesSubjectPredicate(fixture *certificateFixture, subject *pkix.Name) bool {
-	cnMatch := true
-	if fixture.cn != "" {
-		cnMatch = fixture.cn == subject.CommonName
-	}
-
-	ouMatch := true
-	if fixture.ou != "" {
-		ouMatch = len(subject.OrganizationalUnit) > 0 && fixture.ou == subject.OrganizationalUnit[0]
-	}
-
-	return cnMatch && ouMatch
-}
-
-func subjectSubtract(left []*pkix.Name, right []*pkix.Name) []*pkix.Name {
-	var difference []*pkix.Name
-
-	var found bool
-	for _, r := range right {
-		found = false
-		for _, l := range left {
-			if (*l).String() == (*r).String() {
-				found = true
-			}
-		}
-
-		if !found {
-			difference = append(difference, r)
-		}
-	}
-
-	return difference
-}
-
-func getCertPoolSubjects(certPool *x509.CertPool) ([]*pkix.Name, error) {
-	var subjects []*pkix.Name
-
-	for _, subject := range certPool.Subjects() {
-		var sequence pkix.RDNSequence
-		_, err := asn1.Unmarshal(subject, &sequence)
-		if err != nil {
-			return nil, err
-		}
-
-		name := pkix.Name{}
-		name.FillFromRDNSequence(&sequence)
-
-		subjects = append(subjects, &name)
-	}
-
-	return subjects, nil
-}
-
-func isUnrecoverableError(err error) bool {
-	return err != nil && err.Error() != "crypto/x509: system root pool is not available on Windows"
+	cert, err = tlsConfig.GetCertificate(chi)
+	assert.NoError(t, err)
+	assert.Equal(t, expectedCert, *cert)
 }

validation/validation.go

@@ -144,15 +144,20 @@ func ValidateHTTPService(originURL string, hostname string, transport http.Round
 		return err
 	}
 
-	client := &http.Client{Transport: transport}
+	client := &http.Client{
+		Transport: transport,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
 
 	initialRequest, err := http.NewRequest("GET", parsedURL.String(), nil)
 	if err != nil {
 		return err
 	}
 	initialRequest.Host = hostname
-	initialResponse, initialErr := client.Do(initialRequest)
-	if initialErr != nil || initialResponse.StatusCode != http.StatusOK {
+	_, initialErr := client.Do(initialRequest)
+	if initialErr != nil {
 		// Attempt the same endpoint via the other protocol (http/https); maybe we have better luck?
 		oldScheme := parsedURL.Scheme
 		parsedURL.Scheme = toggleProtocol(parsedURL.Scheme)
@@ -162,8 +167,8 @@ func ValidateHTTPService(originURL string, hostname string, transport http.Round
 			return err
 		}
 		secondRequest.Host = hostname
-		secondResponse, _ := client.Do(secondRequest)
-		if secondResponse != nil && secondResponse.StatusCode == http.StatusOK { // Worked this time--advise the user to switch protocols
+		_, secondErr := client.Do(secondRequest)
+		if secondErr == nil { // Worked this time--advise the user to switch protocols
 			return errors.Errorf(
 				"%s doesn't seem to work over %s, but does seem to work over %s. Consider changing the origin URL to %s",
 				parsedURL.Host,

validation/validation_test.go

@@ -1,7 +1,9 @@
 package validation
 
 import (
+	"bytes"
 	"fmt"
+	"io/ioutil"
 	"testing"
 
 	"context"
@@ -151,58 +153,89 @@ func TestToggleProtocol(t *testing.T) {
 	assert.Equal(t, "", toggleProtocol(""))
 }
 
+// Happy path 1: originURL is HTTP, and HTTP connections work
 func TestValidateHTTPService_HTTP2HTTP(t *testing.T) {
 	originURL := "http://127.0.0.1/"
 	hostname := "example.com"
+
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			return emptyResponse(200), nil
+		}
+		if req.URL.Scheme == "https" {
+			t.Fatal("http works, shouldn't have tried with https")
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			return emptyResponse(503), nil
+		}
+		if req.URL.Scheme == "https" {
+			t.Fatal("http works, shouldn't have tried with https")
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	// Integration-style test with a mock server
 	server, client, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		assert.Equal(t, hostname, r.Host)
 		w.WriteHeader(200)
 	}))
 	assert.NoError(t, err)
 	defer server.Close()
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, client.Transport))
 
-	assert.Equal(t, nil, ValidateHTTPService(originURL, hostname, client.Transport))
-}
-
-func TestValidateHTTPService_ServerNonOKResponse(t *testing.T) {
-	originURL := "http://127.0.0.1/"
-	hostname := "example.com"
-	server, client, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	// this will fail if the client follows the 302
+	redirectServer, redirectClient, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/followedRedirect" {
+			t.Fatal("shouldn't have followed the 302")
+		}
 		if r.Method == "CONNECT" {
 			assert.Equal(t, "127.0.0.1:443", r.Host)
 		} else {
 			assert.Equal(t, hostname, r.Host)
 		}
-		w.WriteHeader(400)
+		w.Header().Set("Location", "/followedRedirect")
+		w.WriteHeader(302)
 	}))
 	assert.NoError(t, err)
-	defer server.Close()
+	defer redirectServer.Close()
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, redirectClient.Transport))
 
-	assert.Equal(t, nil, ValidateHTTPService(originURL, hostname, client.Transport))
-}
-
-func TestValidateHTTPService_HTTPS2HTTP(t *testing.T) {
-	originURL := "https://127.0.0.1:1234/"
-	hostname := "example.com"
-	server, client, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		if r.Method == "CONNECT" {
-			assert.Equal(t, "127.0.0.1:1234", r.Host)
-		} else {
-			assert.Equal(t, hostname, r.Host)
-		}
-		w.WriteHeader(200)
-	}))
-	assert.NoError(t, err)
-	defer server.Close()
-
-	assert.Equal(t,
-		"127.0.0.1:1234 doesn't seem to work over https, but does seem to work over http. Consider changing the origin URL to http://127.0.0.1:1234/",
-		ValidateHTTPService(originURL, hostname, client.Transport).Error())
 }
 
+// Happy path 2: originURL is HTTPS, and HTTPS connections work
 func TestValidateHTTPService_HTTPS2HTTPS(t *testing.T) {
 	originURL := "https://127.0.0.1/"
 	hostname := "example.com"
+
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			t.Fatal("https works, shouldn't have tried with http")
+		}
+		if req.URL.Scheme == "https" {
+			return emptyResponse(200), nil
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			t.Fatal("https works, shouldn't have tried with http")
+		}
+		if req.URL.Scheme == "https" {
+			return emptyResponse(503), nil
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	// Integration-style test with a mock server
 	server, client, err := createSecureMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.Method == "CONNECT" {
 			assert.Equal(t, "127.0.0.1:443", r.Host)
@@ -213,13 +246,113 @@ func TestValidateHTTPService_HTTPS2HTTPS(t *testing.T) {
 	}))
 	assert.NoError(t, err)
 	defer server.Close()
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, client.Transport))
 
-	assert.Equal(t, nil, ValidateHTTPService(originURL, hostname, client.Transport))
+	// this will fail if the client follows the 302
+	redirectServer, redirectClient, err := createSecureMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/followedRedirect" {
+			t.Fatal("shouldn't have followed the 302")
+		}
+		if r.Method == "CONNECT" {
+			assert.Equal(t, "127.0.0.1:443", r.Host)
+		} else {
+			assert.Equal(t, hostname, r.Host)
+		}
+		w.Header().Set("Location", "/followedRedirect")
+		w.WriteHeader(302)
+	}))
+	assert.NoError(t, err)
+	defer redirectServer.Close()
+	assert.Nil(t, ValidateHTTPService(originURL, hostname, redirectClient.Transport))
 }
 
+// Error path 1: originURL is HTTPS, but HTTP connections work
+func TestValidateHTTPService_HTTPS2HTTP(t *testing.T) {
+	originURL := "https://127.0.0.1:1234/"
+	hostname := "example.com"
+
+	assert.Error(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			return emptyResponse(200), nil
+		}
+		if req.URL.Scheme == "https" {
+			return nil, assert.AnError
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	assert.Error(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			return emptyResponse(503), nil
+		}
+		if req.URL.Scheme == "https" {
+			return nil, assert.AnError
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	// Integration-style test with a mock server
+	server, client, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == "CONNECT" {
+			assert.Equal(t, "127.0.0.1:1234", r.Host)
+		} else {
+			assert.Equal(t, hostname, r.Host)
+		}
+		w.WriteHeader(200)
+	}))
+	assert.NoError(t, err)
+	defer server.Close()
+	assert.Error(t, ValidateHTTPService(originURL, hostname, client.Transport))
+
+	// this will fail if the client follows the 302
+	redirectServer, redirectClient, err := createMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/followedRedirect" {
+			t.Fatal("shouldn't have followed the 302")
+		}
+		if r.Method == "CONNECT" {
+			assert.Equal(t, "127.0.0.1:1234", r.Host)
+		} else {
+			assert.Equal(t, hostname, r.Host)
+		}
+		w.Header().Set("Location", "/followedRedirect")
+		w.WriteHeader(302)
+	}))
+	assert.NoError(t, err)
+	defer redirectServer.Close()
+	assert.Error(t, ValidateHTTPService(originURL, hostname, redirectClient.Transport))
+
+}
+
+// Error path 2: originURL is HTTP, but HTTPS connections work
 func TestValidateHTTPService_HTTP2HTTPS(t *testing.T) {
 	originURL := "http://127.0.0.1:1234/"
 	hostname := "example.com"
+
+	assert.Error(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			return nil, assert.AnError
+		}
+		if req.URL.Scheme == "https" {
+			return emptyResponse(200), nil
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	assert.Error(t, ValidateHTTPService(originURL, hostname, testRoundTripper(func(req *http.Request) (*http.Response, error) {
+		assert.Equal(t, req.Host, hostname)
+		if req.URL.Scheme == "http" {
+			return nil, assert.AnError
+		}
+		if req.URL.Scheme == "https" {
+			return emptyResponse(503), nil
+		}
+		panic("Shouldn't reach here")
+	})))
+
+	// Integration-style test with a mock server
 	server, client, err := createSecureMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.Method == "CONNECT" {
 			assert.Equal(t, "127.0.0.1:1234", r.Host)
@@ -230,10 +363,38 @@ func TestValidateHTTPService_HTTP2HTTPS(t *testing.T) {
 	}))
 	assert.NoError(t, err)
 	defer server.Close()
+	assert.Error(t, ValidateHTTPService(originURL, hostname, client.Transport))
 
-	assert.Equal(t,
-		"127.0.0.1:1234 doesn't seem to work over http, but does seem to work over https. Consider changing the origin URL to https://127.0.0.1:1234/",
-		ValidateHTTPService(originURL, hostname, client.Transport).Error())
+	// this will fail if the client follows the 302
+	redirectServer, redirectClient, err := createSecureMockServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/followedRedirect" {
+			t.Fatal("shouldn't have followed the 302")
+		}
+		if r.Method == "CONNECT" {
+			assert.Equal(t, "127.0.0.1:443", r.Host)
+		} else {
+			assert.Equal(t, hostname, r.Host)
+		}
+		w.Header().Set("Location", "/followedRedirect")
+		w.WriteHeader(302)
+	}))
+	assert.NoError(t, err)
+	defer redirectServer.Close()
+	assert.Error(t, ValidateHTTPService(originURL, hostname, redirectClient.Transport))
+}
+
+type testRoundTripper func(req *http.Request) (*http.Response, error)
+
+func (f testRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req)
+}
+
+func emptyResponse(statusCode int) *http.Response {
+	return &http.Response{
+		StatusCode: statusCode,
+		Body:       ioutil.NopCloser(bytes.NewReader(nil)),
+		Header:     make(http.Header),
+	}
 }
 
 func createMockServerAndClient(handler http.Handler) (*httptest.Server, *http.Client, error) {

websocket/websocket.go

@@ -150,7 +150,7 @@ func StartProxyServer(logger *logrus.Logger, listener net.Listener, remote strin
 		done := make(chan struct{})
 		go pinger(logger, conn, done)
 		defer func() {
-			<-done
+			done <- struct{}{}
 			conn.Close()
 		}()
 		Stream(&Conn{conn}, stream)

websocket/websocket_test.go

@@ -2,18 +2,17 @@ package websocket
 
 import (
 	"crypto/tls"
+	"crypto/x509"
 	"io"
 	"math/rand"
 	"net/http"
 	"testing"
 
-	"github.com/sirupsen/logrus"
-	"github.com/stretchr/testify/assert"
-
-	"golang.org/x/net/websocket"
-
 	"github.com/cloudflare/cloudflared/hello"
 	"github.com/cloudflare/cloudflared/tlsconfig"
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"golang.org/x/net/websocket"
 )
 
 const (
@@ -40,8 +39,10 @@ func testRequest(t *testing.T, url string, stream io.ReadWriter) *http.Request {
 }
 
 func websocketClientTLSConfig(t *testing.T) *tls.Config {
-	certPool, err := tlsconfig.LoadOriginCertPool(nil)
+	certPool := x509.NewCertPool()
+	helloCert, err := tlsconfig.GetHelloCertificateX509()
 	assert.NoError(t, err)
+	certPool.AddCert(helloCert)
 	assert.NotNil(t, certPool)
 	return &tls.Config{RootCAs: certPool}
 }