Release 2019.10.4

This drops the default size auf the h2mux write buffer from 512 MB to 1 MB.
This massively reduces memory usage, since each stream has its own buffer.

RELEASE_NOTES

@@ -1,3 +1,15 @@
+2019.10.4
+- 2019-10-21 TUN-2450: Remove Brew publishing formula
+
+2019.10.3
+- 2019-10-18 Fix #129: Excessive memory usage streaming large files (#142)
+
+2019.10.2
+- 2019-10-17 AUTH-2167: Adds CLI option for host key directory
+
+2019.10.1
+- 2019-10-17 Adds variable to fix windows build
+
 2019.10.0
 - 2019-10-11 AUTH-2105: Dont require --destination arg
 - 2019-10-14 TUN-2344: log more details: http2.Framer.ErrorDetail() if available, connectionID

cmd/cloudflared/tunnel/cmd.go

@@ -74,6 +74,9 @@ const (
 	// s3URLFlag is the S3 URL of SSH log uploader (e.g. don't use AWS s3 and use google storage bucket instead)
 	s3URLFlag = "s3-url-host"
 
+	// hostKeyPath is the path of the dir to save SSH host keys too
+	hostKeyPath = "host-key-path"
+
 	noIntentMsg = "The --intent argument is required. Cloudflared looks up an Intent to determine what configuration to use (i.e. which tunnels to start). If you don't have any Intents yet, you can use a placeholder Intent Label for now. Then, when you make an Intent with that label, cloudflared will get notified and open the tunnels you specified in that Intent."
 )
 
@@ -396,7 +399,7 @@ func StartServer(c *cli.Context, version string, shutdownC, graceShutdownC chan
 		}
 
 		localServerAddress := "127.0.0.1:" + c.String(sshPortFlag)
-		server, err := sshserver.New(logManager, logger, version, localServerAddress, c.String("hostname"), shutdownC, c.Duration(sshIdleTimeoutFlag), c.Duration(sshMaxTimeoutFlag))
+		server, err := sshserver.New(logManager, logger, version, localServerAddress, c.String("hostname"), c.Path(hostKeyPath), shutdownC, c.Duration(sshIdleTimeoutFlag), c.Duration(sshMaxTimeoutFlag))
 		if err != nil {
 			msg := "Cannot create new SSH Server"
 			logger.WithError(err).Error(msg)
@@ -1019,5 +1022,12 @@ func tunnelFlags(shouldHide bool) []cli.Flag {
 			EnvVars: []string{"S3_URL"},
 			Hidden:  true,
 		}),
+		altsrc.NewPathFlag(&cli.PathFlag{
+			Name:    hostKeyPath,
+			Usage:   "Absolute path of directory to save SSH host keys in",
+			EnvVars: []string{"HOST_KEY_PATH"},
+			Hidden:  true,
+
+		}),
 	}
-}
+}

h2mux/h2mux.go

@@ -19,7 +19,7 @@ const (
 	maxWindowSize            uint32        = (1 << 31) - 1 // 2^31-1 = 2147483647, max window size in http2 spec
 	defaultTimeout           time.Duration = 5 * time.Second
 	defaultRetries           uint64        = 5
-	defaultWriteBufferMaxLen int           = 1024 * 1024 * 512 // 500mb
+	defaultWriteBufferMaxLen int           = 1024 * 1024 // 1mb
 
 	SettingMuxerMagic http2.SettingID = 0x42db
 	MuxerMagicOrigin  uint32          = 0xa2e43c8b

publish-homebrew-formula.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-FILENAME=$1
-VERSION=$2
-TAP_ROOT=$3
-URL="https://developers.cloudflare.com/argo-tunnel/dl/cloudflared-${VERSION}-darwin-amd64.tgz"
-SHA256=$(sha256sum -b "${FILENAME}" | cut -b1-64)
-
-cd "${TAP_ROOT}" || exit 1
-git checkout -f master
-git reset --hard origin/master
-
-tee cloudflared.rb <<EOF
-class Cloudflared < Formula
-  desc 'Argo Tunnel'
-  homepage 'https://developers.cloudflare.com/argo-tunnel/'
-  url '${URL}'
-  sha256 '${SHA256}'
-  version '${VERSION}'
-  def install
-    bin.install 'cloudflared'
-  end
-end
-EOF
-git add cloudflared.rb
-
-git config user.name "cloudflare-warp-bot"
-git config user.email "warp-bot@cloudflare.com"
-git commit -m "Release Argo Tunnel ${VERSION}"
-git version
-GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=../github_known_hosts" git push -v origin master

sshserver/host_keys.go

@@ -19,47 +19,44 @@ import (
 )
 
 const (
-	systemConfigPath = "/usr/local/etc/cloudflared/"
-	rsaFilename      = "ssh_host_rsa_key"
-	ecdsaFilename    = "ssh_host_ecdsa_key"
+	rsaFilename       = "ssh_host_rsa_key"
+	ecdsaFilename     = "ssh_host_ecdsa_key"
 )
 
-func (s *SSHProxy) configureHostKeys() error {
-	if _, err := os.Stat(systemConfigPath); os.IsNotExist(err) {
-		if err := os.MkdirAll(systemConfigPath, 0755); err != nil {
-			return errors.Wrap(err, fmt.Sprintf("Error creating %s directory", systemConfigPath))
+var defaultHostKeyDir = filepath.Join(".cloudflared", "host_keys")
+
+func (s *SSHProxy) configureHostKeys(hostKeyDir string) error {
+	if hostKeyDir == "" {
+		homeDir, err := os.UserHomeDir()
+		if err != nil {
+			return err
+		}
+		hostKeyDir = filepath.Join(homeDir, defaultHostKeyDir)
+	}
+
+	if _, err := os.Stat(hostKeyDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(hostKeyDir, 0755); err != nil {
+			return errors.Wrap(err, fmt.Sprintf("Error creating %s directory", hostKeyDir))
 		}
 	}
 
-	if err := s.configureHostKey(s.ensureECDSAKeyExists); err != nil {
+	if err := s.configureECDSAKey(hostKeyDir); err != nil {
 		return err
 	}
 
-	if err := s.configureHostKey(s.ensureRSAKeyExists); err != nil {
+	if err := s.configureRSAKey(hostKeyDir); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (s *SSHProxy) configureHostKey(keyFunc func() (string, error)) error {
-	path, err := keyFunc()
-	if err != nil {
-		return err
-	}
-
-	if err := s.SetOption(ssh.HostKeyFile(path)); err != nil {
-		return errors.Wrap(err, "Could not set SSH host key")
-	}
-	return nil
-}
-
-func (s *SSHProxy) ensureRSAKeyExists() (string, error) {
-	keyPath := filepath.Join(systemConfigPath, rsaFilename)
+func (s *SSHProxy) configureRSAKey(basePath string) error {
+	keyPath := filepath.Join(basePath, rsaFilename)
 	if _, err := os.Stat(keyPath); os.IsNotExist(err) {
 		key, err := rsa.GenerateKey(rand.Reader, 2048)
 		if err != nil {
-			return "", errors.Wrap(err, "Error generating RSA host key")
+			return errors.Wrap(err, "Error generating RSA host key")
 		}
 
 		privateKey := &pem.Block{
@@ -68,25 +65,28 @@ func (s *SSHProxy) ensureRSAKeyExists() (string, error) {
 		}
 
 		if err = writePrivateKey(keyPath, privateKey); err != nil {
-			return "", err
+			return err
 		}
 
 		s.logger.Debug("Created new RSA SSH host key: ", keyPath)
 	}
-	return keyPath, nil
+	if err := s.SetOption(ssh.HostKeyFile(keyPath)); err != nil {
+		return errors.Wrap(err, "Could not set SSH RSA host key")
+	}
+	return nil
 }
 
-func (s *SSHProxy) ensureECDSAKeyExists() (string, error) {
-	keyPath := filepath.Join(systemConfigPath, ecdsaFilename)
+func (s *SSHProxy) configureECDSAKey(basePath string) error {
+	keyPath := filepath.Join(basePath, ecdsaFilename)
 	if _, err := os.Stat(keyPath); os.IsNotExist(err) {
 		key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
 		if err != nil {
-			return "", errors.Wrap(err, "Error generating ECDSA host key")
+			return errors.Wrap(err, "Error generating ECDSA host key")
 		}
 
 		keyBytes, err := x509.MarshalECPrivateKey(key)
 		if err != nil {
-			return "", errors.Wrap(err, "Error marshalling ECDSA key")
+			return errors.Wrap(err, "Error marshalling ECDSA key")
 		}
 
 		privateKey := &pem.Block{
@@ -95,12 +95,15 @@ func (s *SSHProxy) ensureECDSAKeyExists() (string, error) {
 		}
 
 		if err = writePrivateKey(keyPath, privateKey); err != nil {
-			return "", err
+			return err
 		}
 
 		s.logger.Debug("Created new ECDSA SSH host key: ", keyPath)
 	}
-	return keyPath, nil
+	if err := s.SetOption(ssh.HostKeyFile(keyPath)); err != nil {
+		return errors.Wrap(err, "Could not set SSH ECDSA host key")
+	}
+	return nil
 }
 
 func writePrivateKey(keyPath string, privateKey *pem.Block) error {

sshserver/sshserver_unix.go

@@ -78,7 +78,7 @@ type SSHPreamble struct {
 }
 
 // New creates a new SSHProxy and configures its host keys and authentication by the data provided
-func New(logManager sshlog.Manager, logger *logrus.Logger, version, localAddress, hostname string, shutdownC chan struct{}, idleTimeout, maxTimeout time.Duration) (*SSHProxy, error) {
+func New(logManager sshlog.Manager, logger *logrus.Logger, version, localAddress, hostname, hostKeyDir string, shutdownC chan struct{}, idleTimeout, maxTimeout time.Duration) (*SSHProxy, error) {
 	sshProxy := SSHProxy{
 		hostname:   hostname,
 		logger:     logger,
@@ -98,7 +98,7 @@ func New(logManager sshlog.Manager, logger *logrus.Logger, version, localAddress
 		},
 	}
 
-	if err := sshProxy.configureHostKeys(); err != nil {
+	if err := sshProxy.configureHostKeys(hostKeyDir); err != nil {
 		return nil, err
 	}
 

sshserver/sshserver_windows.go

@@ -11,6 +11,8 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+const SSHPreambleLength = 2
+
 type SSHServer struct{}
 
 type SSHPreamble struct {
@@ -18,7 +20,7 @@ type SSHPreamble struct {
 	JWT         string
 }
 
-func New(_ sshlog.Manager, _ *logrus.Logger, _, _, _ string, _ chan struct{}, _, _ time.Duration) (*SSHServer, error) {
+func New(_ sshlog.Manager, _ *logrus.Logger, _, _, _, _ string, _ chan struct{}, _, _ time.Duration) (*SSHServer, error) {
 	return nil, errors.New("cloudflared ssh server is not supported on windows")
 }