Release 2020.10.1

.gitignore

@@ -5,6 +5,7 @@ guide/public
 /.GOPATH
 /bin
 .idea
+.build
 .vscode
 \#*\#
 cscope.*
@@ -12,6 +13,7 @@ cloudflared
 cloudflared.pkg
 cloudflared.exe
 cloudflared.msi
+cloudflared-x86-64*
 !cmd/cloudflared/
 .DS_Store
 *-session.log

.teamcity/build-macos.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-set -euo pipefail
-
 if [[ "$(uname)" != "Darwin" ]] ; then
     echo "This should be run on macOS"
     exit 1
@@ -18,85 +16,138 @@ TARGET_DIRECTORY=".build"
 BINARY_NAME="cloudflared"
 VERSION=$(git describe --tags --always --dirty="-dev")
 PRODUCT="cloudflared"
-CODE_SIGN_PRIV="code_sign.pk12"
+CODE_SIGN_PRIV="code_sign.p12"
 CODE_SIGN_CERT="code_sign.cer"
-INSTALLER_PRIV="installer.pk12"
+INSTALLER_PRIV="installer.p12"
 INSTALLER_CERT="installer.cer"
+BUNDLE_ID="com.cloudflare.cloudflared"
+SEC_DUP_MSG="security: SecKeychainItemImport: The specified item already exists in the keychain."
 export PATH="$PATH:/usr/local/bin"
 mkdir -p ../src/github.com/cloudflare/    
 cp -r . ../src/github.com/cloudflare/cloudflared
 cd ../src/github.com/cloudflare/cloudflared 
 GOCACHE="$PWD/../../../../" GOPATH="$PWD/../../../../" CGO_ENABLED=1 make cloudflared
 
-# TODO: AUTH-2653 - The CFD_CODE_SIGN_KEY and CFD_INSTALLER_KEY are "doubly" gpg encrypted.
-# this needs to be fixed, but I don't have access to the keys to do it. 
-# The private keys are on from Dane's laptop
+# Add code signing private key to the key chain
+if [[ ! -z "$CFD_CODE_SIGN_KEY" ]]; then
+  if [[ ! -z "$CFD_CODE_SIGN_PASS" ]]; then
+    # write private key to disk and then import it keychain
+    echo -n -e ${CFD_CODE_SIGN_KEY} | base64 -D > ${CODE_SIGN_PRIV}
+    out=$(security import ${CODE_SIGN_PRIV} -A -P "${CFD_CODE_SIGN_PASS}" 2>&1)
+    exitcode=$?
+    if [ -n "$out" ]; then
+      if [ $exitcode -eq 0 ]; then
+          echo "$out"
+      else
+          if [ "$out" != "${SEC_DUP_MSG}" ]; then
+              echo "$out" >&2
+              exit $exitcode
+          fi
+      fi
+    fi
+    rm ${CODE_SIGN_PRIV}
+  fi
+fi
 
-# # Add code signing private key to the key chain
-# if [[ -n "${CFD_CODE_SIGN_KEY:-}" ]]; then
-#   if [[ -n "${CFD_CODE_SIGN_PASS:-}" ]]; then
-#     # write private key to disk and then import it keychain
-#     echo -n -e ${CFD_CODE_SIGN_KEY} | base64 -D > ${CODE_SIGN_PRIV}
-#     security import ${CODE_SIGN_PRIV} -A -P "${CFD_CODE_SIGN_PASS}"
-#     rm ${CODE_SIGN_PRIV}
-#   fi
-# fi
+# Add code signing certificate to the key chain
+if [[ ! -z "$CFD_CODE_SIGN_CERT" ]]; then
+  # write certificate to disk and then import it keychain
+  echo -n -e ${CFD_CODE_SIGN_CERT} | base64 -D > ${CODE_SIGN_CERT}
+  out1=$(security import ${CODE_SIGN_CERT} -A 2>&1)
+  exitcode1=$?
+  if [ -n "$out1" ]; then
+    if [ $exitcode1 -eq 0 ]; then
+        echo "$out1"
+    else
+        if [ "$out1" != "${SEC_DUP_MSG}" ]; then
+            echo "$out1" >&2
+            exit $exitcode1
+        else 
+            echo "already imported code signing certificate"
+        fi
+    fi
+  fi
+  rm ${CODE_SIGN_CERT}
+fi
 
-# # Add code signing certificate to the key chain
-# if [[ -n "${CFD_CODE_SIGN_CERT:-}" ]]; then
-#   # write certificate to disk and then import it keychain
-#   echo -n -e ${CFD_CODE_SIGN_CERT} | base64 -D > ${CODE_SIGN_CERT}
-#   security import ${CODE_SIGN_CERT}
-#   rm ${CODE_SIGN_CERT}
-# fi
+# Add package signing private key to the key chain
+if [[ ! -z "$CFD_INSTALLER_KEY" ]]; then
+  if [[ ! -z "$CFD_INSTALLER_PASS" ]]; then
+    # write private key to disk and then import it into the keychain
+    echo -n -e ${CFD_INSTALLER_KEY} | base64 -D > ${INSTALLER_PRIV}
+    out2=$(security import ${INSTALLER_PRIV} -A -P "${CFD_INSTALLER_PASS}" 2>&1)
+    exitcode2=$?
+    if [ -n "$out2" ]; then
+      if [ $exitcode2 -eq 0 ]; then
+          echo "$out2"
+      else
+          if [ "$out2" != "${SEC_DUP_MSG}" ]; then
+              echo "$out2" >&2
+              exit $exitcode2
+          fi
+      fi
+    fi
+    rm ${INSTALLER_PRIV}
+  fi
+fi
 
-# # Add package signing private key to the key chain
-# if [[ -n "${CFD_INSTALLER_KEY:-}" ]]; then
-#   if [[ -n "${CFD_INSTALLER_PASS:-}" ]]; then
-#     # write private key to disk and then import it into the keychain
-#     echo -n -e ${CFD_INSTALLER_KEY} | base64 -D > ${INSTALLER_PRIV}
-#     security import ${INSTALLER_PRIV} -A -P "${CFD_INSTALLER_PASS}"
-#     rm ${INSTALLER_PRIV}
-#   fi
-# fi
+# Add package signing certificate to the key chain
+if [[ ! -z "$CFD_INSTALLER_CERT" ]]; then
+  # write certificate to disk and then import it keychain
+  echo -n -e ${CFD_INSTALLER_CERT} | base64 -D > ${INSTALLER_CERT}
+  out3=$(security import ${INSTALLER_CERT} -A 2>&1)
+  exitcode3=$?
+  if [ -n "$out3" ]; then
+    if [ $exitcode3 -eq 0 ]; then
+        echo "$out3"
+    else
+        if [ "$out3" != "${SEC_DUP_MSG}" ]; then
+            echo "$out3" >&2
+            exit $exitcode3
+        else 
+            echo "already imported installer certificate"
+        fi
+    fi
+  fi
+  rm ${INSTALLER_CERT}
+fi
 
-# # Add package signing certificate to the key chain
-# if [[ -n "${CFD_INSTALLER_CERT:-}" ]]; then
-#   # write certificate to disk and then import it keychain
-#   echo -n -e ${CFD_INSTALLER_CERT} | base64 -D > ${INSTALLER_CERT}
-#   security import ${INSTALLER_CERT}
-#   rm ${INSTALLER_CERT}
-# fi
+# get the code signing certificate name
+if [[ ! -z "$CFD_CODE_SIGN_NAME" ]]; then
+  CODE_SIGN_NAME="${CFD_CODE_SIGN_NAME}"
+else
+  if [[ -n "$(security find-certificate -c "Developer ID Application" | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)" ]]; then
+    CODE_SIGN_NAME=$(security find-certificate -c "Developer ID Application" | cut -d'"' -f 4 -s | grep "Developer ID Application:" | head -1)
+  else
+    CODE_SIGN_NAME=""
+  fi
+fi
 
-# # get the code signing certificate name
-# if [[ -n "${CFD_CODE_SIGN_NAME:-}" ]]; then
-#   CODE_SIGN_NAME="${CFD_CODE_SIGN_NAME}"
-# else
-#   if [[ -n "$(security find-identity -v | cut -d'"' -f 2 -s | grep "Developer ID Application:")" ]]; then
-#     CODE_SIGN_NAME=$(security find-identity -v | cut -d'"' -f 2 -s | grep "Developer ID Application:")
-#   else
-#     CODE_SIGN_NAME=""
-#   fi
-# fi
+# get the package signing certificate name
+if [[ ! -z "$CFD_INSTALLER_NAME" ]]; then
+  PKG_SIGN_NAME="${CFD_INSTALLER_NAME}"
+else
+  if [[ -n "$(security find-certificate -c "Developer ID Installer" | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)" ]]; then
+    PKG_SIGN_NAME=$(security find-certificate -c "Developer ID Installer" | cut -d'"' -f 4 -s | grep "Developer ID Installer:" | head -1)
+  else
+    PKG_SIGN_NAME=""
+  fi
+fi
 
-# # get the package signing certificate name
-# if [[ -n "${CFD_INSTALLER_NAME:-}" ]]; then
-#   PKG_SIGN_NAME="${CFD_INSTALLER_NAME}"
-# else
-#   if [[ -n "$(security find-identity -v | cut -d'"' -f 2 -s | grep "Developer ID Installer:")" ]]; then
-#     PKG_SIGN_NAME=$(security find-identity -v | cut -d'"' -f 2 -s | grep "Developer ID Installer:")
-#   else
-#     PKG_SIGN_NAME=""
-#   fi
-# fi
-
-# # sign the cloudflared binary
-# if [[ -n "${CODE_SIGN_NAME:-}" ]]; then
-#   codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME}
-# fi
+# sign the cloudflared binary
+if [[ ! -z "$CODE_SIGN_NAME" ]]; then
+  codesign -s "${CODE_SIGN_NAME}" -f -v --timestamp --options runtime ${BINARY_NAME}
+  
+  # notarize the binary
+  if [[ ! -z "$CFD_NOTE_PASSWORD" ]]; then
+    zip "${BINARY_NAME}.zip" ${BINARY_NAME} 
+    xcrun altool --notarize-app -f "${BINARY_NAME}.zip" -t osx -u ${CFD_NOTE_USERNAME} -p ${CFD_NOTE_PASSWORD} --primary-bundle-id ${BUNDLE_ID}
+  fi
+fi
 
 
 # creating build directory
+rm -rf $TARGET_DIRECTORY
 mkdir "${TARGET_DIRECTORY}"
 mkdir "${TARGET_DIRECTORY}/contents"
 cp -r ".mac_resources/scripts" "${TARGET_DIRECTORY}/scripts"
@@ -108,7 +159,7 @@ cp ${BINARY_NAME} "${TARGET_DIRECTORY}/contents/${PRODUCT}"
 tar czf "$FILENAME" "${BINARY_NAME}"
 
 # build the installer package
-if [[ -n "${PKG_SIGN_NAME:-}" ]]; then
+if [[ ! -z "$PKG_SIGN_NAME" ]]; then
   pkgbuild --identifier com.cloudflare.${PRODUCT} \
       --version ${VERSION} \
       --scripts ${TARGET_DIRECTORY}/scripts \
@@ -116,6 +167,12 @@ if [[ -n "${PKG_SIGN_NAME:-}" ]]; then
       --install-location /usr/local/bin \
       --sign "${PKG_SIGN_NAME}" \
       ${PKGNAME}
+
+      # notarize the package
+      if [[ ! -z "$CFD_NOTE_PASSWORD" ]]; then
+        xcrun altool --notarize-app -f ${PKGNAME} -t osx -u ${CFD_NOTE_USERNAME} -p ${CFD_NOTE_PASSWORD} --primary-bundle-id ${BUNDLE_ID}
+        xcrun stapler staple ${PKGNAME}
+      fi
 else
     pkgbuild --identifier com.cloudflare.${PRODUCT} \
       --version ${VERSION} \

Dockerfile

@@ -1,7 +1,7 @@
 # use a builder image for building cloudflare
 ARG TARGET_GOOS
 ARG TARGET_GOARCH
-FROM golang:1.13.3 as builder
+FROM golang:1.15.3 as builder
 ENV GO111MODULE=on \
     CGO_ENABLED=0 \
     TARGET_GOOS=${TARGET_GOOS} \

Makefile

@@ -2,7 +2,7 @@ VERSION       := $(shell git describe --tags --always --dirty="-dev" --match "[0
 DATE          := $(shell date -u '+%Y-%m-%d-%H%M UTC')
 VERSION_FLAGS := -ldflags='-X "main.Version=$(VERSION)" -X "main.BuildTime=$(DATE)"'
 MSI_VERSION   := $(shell git tag -l --sort=v:refname | grep "w" | tail -1 | cut -c2-)
-#MSI_VERSION expects the format of the tag to be: (wX.X.X). Starts with the w character to not break cfsetup. 
+#MSI_VERSION expects the format of the tag to be: (wX.X.X). Starts with the w character to not break cfsetup.
 #e.g. w3.0.1 or w4.2.10. It trims off the w character when creating the MSI.
 
 IMPORT_PATH   := github.com/cloudflare/cloudflared
@@ -44,6 +44,8 @@ else ifeq ($(LOCAL_OS),darwin)
     TARGET_OS ?= darwin
 else ifeq ($(LOCAL_OS),windows)
     TARGET_OS ?= windows
+else ifeq ($(LOCAL_OS),freebsd)
+    TARGET_OS ?= freebsd
 else
     $(error This system's OS $(LOCAL_OS) isn't supported)
 endif
@@ -125,6 +127,58 @@ cloudflared-darwin-amd64.tgz: cloudflared
 	tar czf cloudflared-darwin-amd64.tgz cloudflared
 	rm cloudflared
 
+.PHONY: cloudflared-junos
+cloudflared-junos: cloudflared jetez-certificate.pem jetez-key.pem
+	jetez --source . \
+		  -j jet.yaml \
+		  --key jetez-key.pem \
+		  --cert jetez-certificate.pem \
+		  --version $(VERSION)
+	rm jetez-*.pem
+
+jetez-certificate.pem:
+ifndef JETEZ_CERT
+	$(error JETEZ_CERT not defined)
+endif
+	@echo "Writing JetEZ certificate"
+	@echo "$$JETEZ_CERT" > jetez-certificate.pem
+
+jetez-key.pem:
+ifndef JETEZ_KEY
+	$(error JETEZ_KEY not defined)
+endif
+	@echo "Writing JetEZ key"
+	@echo "$$JETEZ_KEY" > jetez-key.pem
+
+.PHONY: publish-cloudflared-junos
+publish-cloudflared-junos: cloudflared-junos cloudflared-x86-64.latest.s3
+ifndef S3_ENDPOINT
+	$(error S3_HOST not defined)
+endif
+ifndef S3_URI
+	$(error S3_URI not defined)
+endif
+ifndef S3_ACCESS_KEY
+	$(error S3_ACCESS_KEY not defined)
+endif
+ifndef S3_SECRET_KEY
+	$(error S3_SECRET_KEY not defined)
+endif
+	sha256sum cloudflared-x86-64-$(VERSION).tgz | awk '{printf $$1}' > cloudflared-x86-64-$(VERSION).tgz.shasum
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+		put cloudflared-x86-64-$(VERSION).tgz $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+		put cloudflared-x86-64-$(VERSION).tgz.shasum $(S3_URI)/cloudflared-x86-64-$(VERSION).tgz.shasum
+	dpkg --compare-versions "$(VERSION)" gt "$(shell cat cloudflared-x86-64.latest.s3)" && \
+		echo -n "$(VERSION)" > cloudflared-x86-64.latest && \
+		s4cmd --endpoint-url $(S3_ENDPOINT) --force --API-GrantRead=uri=http://acs.amazonaws.com/groups/global/AllUsers \
+			put cloudflared-x86-64.latest $(S3_URI)/cloudflared-x86-64.latest || \
+		echo "Latest version not updated"
+
+cloudflared-x86-64.latest.s3:
+	s4cmd --endpoint-url $(S3_ENDPOINT) --force \
+		get $(S3_URI)/cloudflared-x86-64.latest cloudflared-x86-64.latest.s3
+
 .PHONY: homebrew-upload
 homebrew-upload: cloudflared-darwin-amd64.tgz
 	aws s3 --endpoint-url $(S3_ENDPOINT) cp --acl public-read $$^ $(S3_URI)/cloudflared-$$(VERSION)-$1.tgz
@@ -148,8 +202,8 @@ github-message:
 
 .PHONY: github-mac-upload
 github-mac-upload:
-	python3 github_release.py --path .artifacts/cloudflared-darwin-amd64.tgz --release-version $(VERSION) --name cloudflared-darwin-amd64.tgz
-	python3 github_release.py --path .artifacts/cloudflared-amd64.pkg --release-version $(VERSION) --name cloudflared-amd64.pkg
+	python3 github_release.py --path artifacts/cloudflared-darwin-amd64.tgz --release-version $(VERSION) --name cloudflared-darwin-amd64.tgz
+	python3 github_release.py --path artifacts/cloudflared-amd64.pkg --release-version $(VERSION) --name cloudflared-amd64.pkg
 
 bin/equinox:
 	mkdir -p bin

RELEASE_NOTES

@@ -1,3 +1,83 @@
+2020.10.1
+- 2020-10-20 Split out typed config from legacy command-line switches; refactor ingress commands and fix tests
+- 2020-10-20 Move raw ingress rules to config package
+- 2020-10-21 TUN-3476: Fix conversion to string and int slice
+- 2020-10-12 TUN-3441:  Multiple-origin routing via ingress rules
+- 2020-10-15 TUN-3464: Newtype to wrap []ingress.Rule
+- 2020-10-15 TUN-3465: Use Go 1.15.3
+- 2020-10-15 TUN-3463: Let users run a named tunnel via config file setting
+- 2020-10-19 TUN-3475: Unify config file handling with typed config for new fields
+- 2020-10-19 TUN-3459: Make service install on linux use named tunnels
+- 2020-10-06 AUTH-3148 fixed cloudflared copy and match all the files in the checksum upload
+- 2020-10-06 TUN-3436, TUN-3437: Parse ingress from YAML, ensure last rule catches everything
+- 2020-10-06 TUN-3446: Use go 1.15.2 and add a step to build cloudflared in the dev Dockerfile
+- 2020-10-07 TUN-3439: 'tunnel validate' command to check ingress rules
+- 2020-10-07 TUN-3440: 'tunnel rule' command to test ingress rules
+- 2020-10-08 TUN-3451: Cloudflared tunnel ingress command
+- 2020-10-09 TUN-3452: Fix loading of flags from config file for tunnel run subcommand. This change also cleans up building of tunnel subcommand list, hides deprecated subcommands and improves help.
+- 2020-10-08 TUN-3438: move ingress into own package, read into TunnelConfig
+
+2020.10.0
+- 2020-10-02 AUTH-2993 cleaned up worker service tests
+- 2020-10-02 TUN-3443: Decode as v4api response on non-200 status
+- 2020-09-24 TRAFFIC-448: allow the user to specify the proxy address and port to bind to, falling back to 127.0.0.1 and random port if not specified
+- 2020-09-28 TUN-3427: Define a struct that only implements RegistrationServer in tunnelpogs
+- 2020-09-29 TUN-3430: Copy flags to configure proxy to run subcommand, print relevant tunnel flags in help
+- 2020-08-12 AUTH-2993 added workers updater logic
+
+2020.9.3
+- 2020-09-22 TRAFFIC-448: build cloudflare for junos and publish to s3
+- 2020-09-22 TUN-3410: Request the v1 Tunnelstore API
+- 2020-09-23 Release 2020.9.2
+- 2020-09-17 updater service exit code should be 11
+- 2020-09-18 AUTH-3109 upload the checksum to workers kv on github releases
+
+2020.9.2
+- 2020-09-22 TRAFFIC-448: build cloudflare for junos and publish to s3
+- 2020-09-22 TUN-3410: Request the v1 Tunnelstore API
+- 2020-09-17 AUTH-3103 CI build fixes
+- 2020-09-18 AUTH-3110-use-cfsetup-precache
+- 2020-09-17 TUN-3295: Show route command results
+- 2020-09-16 TUN-3291: cloudflared tunnel run -h explains how to use flags from parent command
+- 2020-09-18 AUTH-3109 upload the checksum to workers kv on github releases
+- 2020-09-17 updater service exit code should be 11
+- 2020-09-01 TUN-3216: UI improvements
+- 2020-08-25 Rebased and passed TunnelEventChan to LogServerInfo in new ReconnectTunnel function
+- 2020-08-25 TUN-3321: Add box around logs on UI
+- 2020-08-26 TUN-3328: Filter out free tunnel has started log from UI
+- 2020-08-27 TUN-3333: Add text to UI explaining how to exit
+- 2020-08-27 TUN-3335: Dynamically set connection table size for UI
+- 2020-08-10 TUN-3238: Update UI when connection re-connects
+- 2020-08-17 TUN-3261: Display connections on UI for free classic tunnels
+- 2020-07-24 TUN-3201: Create base cloudflared UI structure
+- 2020-07-29 TUN-3200: Add connection information to UI
+- 2020-07-24 TUN-3255: Update UI to display URL instead of hostname
+- 2020-07-29 TUN-3198: Handle errors while running tunnel UI
+
+2020.9.1
+- 2020-09-14 TUN-3395: Unhide named tunnel subcommands, tweak help
+- 2020-09-15 TUN-3395: Improve help for list command
+- 2020-09-14 TUN-3294: Perform basic validation on arguments of route command; remove default pool name which wasn't valid
+- 2020-09-15 TUN-3395: Improve help for list command
+- 2020-09-16 Use Go 1.15.2
+
+2020.9.0
+- 2020-09-11 TUN-3293: Try to use error information from the body of a failed tunnelstore reresponse if available
+- 2020-09-04 AUTH-2653 renabled signing
+- 2020-09-04 TUN-3377: Tunnel route should check dns/lb before checking tunnel ID
+- 2020-09-04 AUTH-2653 changed to proper file extension
+- 2020-09-04 AUTH-2653 handle duplicate key import errors
+- 2020-09-04 TUN-3345: tunnel run accepts name of tunnel as argument
+- 2020-09-08 AUTH-2653 disble error pipe to see what is failing
+- 2020-09-08 AUTH-2653 search for the certificate and not the identity
+- 2020-09-09 TUN-3284: Use cloudflared/<version> as user agent of tunnelstore client
+- 2020-09-09 TUN-3375: Upgrade x/text and gorilla websocket deps
+- 2020-09-09 TUN-3375: Upgrade coredns and prometheus dependencies
+- 2020-09-09 AUTH-2653 add notarization to mac build
+- 2020-09-08 TUN-3292: Mention cleanup in tunnel run help.
+- 2020-08-20 AUTH-2016 fixed variable fail
+- 2020-08-12 TUN-3352 extra debug logging for websockets
+
 2020.8.2
 - 2020-08-20 AUTH-3021 fixed the git version call by using the older flag
 - 2020-08-18 TUN-3268: Each connection has its own event digest to reconnect

carrier/websocket.go

@@ -86,6 +86,7 @@ func createWebsocketStream(options *StartOptions, logger logger.Service) (*cfweb
 
 	wsConn, resp, err := cfwebsocket.ClientConnect(req, nil)
 	defer closeRespBody(resp)
+
 	if err != nil && IsAccessResponse(resp) {
 		wsConn, err = createAccessAuthenticatedStream(options, logger)
 		if err != nil {
@@ -141,5 +142,16 @@ func createAccessWebSocketStream(options *StartOptions, logger logger.Service) (
 	dump, err := httputil.DumpRequest(req, false)
 	logger.Debugf("Access Websocket request: %s", string(dump))
 
-	return cfwebsocket.ClientConnect(req, nil)
+	conn, resp, err := cfwebsocket.ClientConnect(req, nil)
+
+	if resp != nil {
+		r, err := httputil.DumpResponse(resp, true)
+		if r != nil {
+			logger.Debugf("Websocket response: %q", r)
+		} else if err != nil {
+			logger.Debugf("Websocket response error: %v", err)
+		}
+	}
+
+	return conn, resp, err
 }

cfsetup.yaml

@@ -1,4 +1,4 @@
-pinned_go: &pinned_go go=1.14.7-1
+pinned_go: &pinned_go go=1.15.3-1
 build_dir: &build_dir /cfsetup_build
 default-flavor: stretch
 stretch: &stretch
@@ -61,8 +61,9 @@ stretch: &stretch
       - build-essential
       - python3-setuptools
       - python3-pip
-    post-cache:
+    pre-cache: &install_pygithub
       - pip3 install pygithub
+    post-cache:
       - export GOOS=linux
       - export GOARCH=amd64
       - make github-release
@@ -85,8 +86,8 @@ stretch: &stretch
       - gcc-arm-linux-gnueabihf
       - python3-setuptools
       - python3-pip
+    pre-cache: *install_pygithub
     post-cache:
-      - pip3 install pygithub
       - export GOOS=linux
       - export GOARCH=arm
       - export CC=arm-linux-gnueabihf-gcc
@@ -107,8 +108,8 @@ stretch: &stretch
       - gcc-multilib
       - python3-setuptools
       - python3-pip
+    pre-cache: *install_pygithub
     post-cache:
-      - pip3 install pygithub
       - export GOOS=linux
       - export GOARCH=386
       - make github-release
@@ -129,8 +130,8 @@ stretch: &stretch
       - gcc-mingw-w64
       - python3-setuptools
       - python3-pip
+    pre-cache: *install_pygithub
     post-cache:
-      - pip3 install pygithub
       - export GOOS=windows
       - export GOARCH=amd64
       - export CC=x86_64-w64-mingw32-gcc
@@ -152,8 +153,8 @@ stretch: &stretch
       - gcc-mingw-w64
       - python3-setuptools
       - python3-pip
+    pre-cache: *install_pygithub
     post-cache:
-      - pip3 install pygithub
       - export GOOS=windows
       - export GOARCH=386
       - export CC=i686-w64-mingw32-gcc-win32
@@ -166,8 +167,8 @@ stretch: &stretch
       - g++-aarch64-linux-gnu
       - python3-setuptools
       - python3-pip
+    pre-cache: *install_pygithub
     post-cache:
-      - pip3 install pygithub
       - export GOOS=linux
       - export GOARCH=arm64
       - export CC=aarch64-linux-gnu-gcc
@@ -178,8 +179,8 @@ stretch: &stretch
       - *pinned_go
       - python3-setuptools
       - python3-pip
+    pre-cache: *install_pygithub
     post-cache:
-      - pip3 install pygithub
       - make github-mac-upload
   test:
     build_dir: *build_dir
@@ -205,9 +206,38 @@ stretch: &stretch
       - *pinned_go
       - python3-setuptools
       - python3-pip
+    pre-cache: *install_pygithub
     post-cache:
-      - pip3 install pygithub
       - make github-message
+  build-junos:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - python3
+      - genisoimage
+      - jetez
+    pre-cache:
+      - ln -s /usr/bin/genisoimage /usr/bin/mkisofs
+    post-cache:
+      - export GOOS=freebsd
+      - export GOARCH=amd64
+      - make cloudflared-junos
+  publish-junos:
+    build_dir: *build_dir
+    builddeps:
+      - *pinned_go
+      - build-essential
+      - python3
+      - genisoimage
+      - jetez
+      - s4cmd
+    pre-cache:
+      - ln -s /usr/bin/genisoimage /usr/bin/mkisofs
+    post-cache:
+      - export GOOS=freebsd
+      - export GOARCH=amd64
+      - make publish-cloudflared-junos
 
 jessie: *stretch
 
@@ -218,10 +248,11 @@ centos-7:
     build_dir: *build_dir
     builddeps: &el7_builddeps
       - https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
+    pre-cache:
+      - yum install -y fakeroot
+      - wget https://golang.org/dl/go1.15.3.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.15.3.linux-amd64.tar.gz
     post-cache:
-      - sudo yum install -y fakeroot
-      - wget https://golang.org/dl/go1.14.7.linux-amd64.tar.gz -P /tmp/
-      - sudo tar -C /usr/local -xzf /tmp/go1.14.7.linux-amd64.tar.gz
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
       - export GOARCH=amd64
@@ -229,10 +260,11 @@ centos-7:
   build-rpm:
     build_dir: *build_dir
     builddeps: *el7_builddeps
+    pre-cache:
+      - yum install -y fakeroot
+      - wget https://golang.org/dl/go1.15.3.linux-amd64.tar.gz -P /tmp/
+      - tar -C /usr/local -xzf /tmp/go1.15.3.linux-amd64.tar.gz
     post-cache:
-      - sudo yum install -y fakeroot
-      - wget https://golang.org/dl/go1.14.7.linux-amd64.tar.gz -P /tmp/
-      - sudo tar -C /usr/local -xzf /tmp/go1.14.7.linux-amd64.tar.gz
       - export PATH=$PATH:/usr/local/go/bin
       - export GOOS=linux
       - export GOARCH=amd64

cmd/cloudflared/config/configuration.go

@@ -2,15 +2,17 @@ package config
 
 import (
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
+	"time"
 
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
-	"github.com/urfave/cli/v2/altsrc"
 	"gopkg.in/yaml.v2"
 
+	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/validation"
 )
 
@@ -28,6 +30,8 @@ var (
 	// Windows default config dir was ~/cloudflare-warp in documentation; let's keep it compatible
 	defaultUserConfigDirs = []string{"~/.cloudflared", "~/.cloudflare-warp", "~/cloudflare-warp"}
 	defaultNixConfigDirs  = []string{"/etc/cloudflared", DefaultUnixConfigLocation}
+
+	ErrNoConfigFile = fmt.Errorf("Cannot determine default configuration path. No file %v in %v", DefaultConfigFiles, DefaultConfigSearchDirectories())
 )
 
 const DefaultCredentialFile = "cert.pem"
@@ -88,14 +92,6 @@ func FileExists(path string) (bool, error) {
 	return true, nil
 }
 
-// FindInputSourceContext pulls the input source from the config flag.
-func FindInputSourceContext(context *cli.Context) (altsrc.InputSourceContext, error) {
-	if context.String("config") != "" {
-		return altsrc.NewYamlSourceFromFile(context.String("config"))
-	}
-	return nil, nil
-}
-
 // FindDefaultConfigPath returns the first path that contains a config file.
 // If none of the combination of DefaultConfigSearchDirectories() and DefaultConfigFiles
 // contains a config file, return empty string.
@@ -199,3 +195,155 @@ func ValidateUrl(c *cli.Context, allowFromArgs bool) (string, error) {
 	validUrl, err := validation.ValidateUrl(url)
 	return validUrl, err
 }
+
+type UnvalidatedIngressRule struct {
+	Hostname string
+	Path     string
+	Service  string
+}
+
+type Configuration struct {
+	TunnelID   string `yaml:"tunnel"`
+	Ingress    []UnvalidatedIngressRule
+	sourceFile string
+}
+
+type configFileSettings struct {
+	Configuration `yaml:",inline"`
+	// older settings will be aggregated into the generic map, should be read via cli.Context
+	Settings map[string]interface{} `yaml:",inline"`
+}
+
+func (c *Configuration) Source() string {
+	return c.sourceFile
+}
+
+func (c *configFileSettings) Int(name string) (int, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(int); ok {
+			return v, nil
+		}
+		return 0, fmt.Errorf("expected int found %T for %s", raw, name)
+	}
+	return 0, nil
+}
+
+func (c *configFileSettings) Duration(name string) (time.Duration, error) {
+	if raw, ok := c.Settings[name]; ok {
+		switch v := raw.(type) {
+		case time.Duration:
+			return v, nil
+		case string:
+			return time.ParseDuration(v)
+		}
+		return 0, fmt.Errorf("expected duration found %T for %s", raw, name)
+	}
+	return 0, nil
+}
+
+func (c *configFileSettings) Float64(name string) (float64, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(float64); ok {
+			return v, nil
+		}
+		return 0, fmt.Errorf("expected float found %T for %s", raw, name)
+	}
+	return 0, nil
+}
+
+func (c *configFileSettings) String(name string) (string, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(string); ok {
+			return v, nil
+		}
+		return "", fmt.Errorf("expected string found %T for %s", raw, name)
+	}
+	return "", nil
+}
+
+func (c *configFileSettings) StringSlice(name string) ([]string, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if slice, ok := raw.([]interface{}); ok {
+			strSlice := make([]string, len(slice))
+			for i, v := range slice {
+				str, ok := v.(string)
+				if !ok {
+					return nil, fmt.Errorf("expected string, found %T for %v", i, v)
+				}
+				strSlice[i] = str
+			}
+			return strSlice, nil
+		}
+		return nil, fmt.Errorf("expected string slice found %T for %s", raw, name)
+	}
+	return nil, nil
+}
+
+func (c *configFileSettings) IntSlice(name string) ([]int, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if slice, ok := raw.([]interface{}); ok {
+			intSlice := make([]int, len(slice))
+			for i, v := range slice {
+				str, ok := v.(int)
+				if !ok {
+					return nil, fmt.Errorf("expected int, found %T for %v ", v, v)
+				}
+				intSlice[i] = str
+			}
+			return intSlice, nil
+		}
+		if v, ok := raw.([]int); ok {
+			return v, nil
+		}
+		return nil, fmt.Errorf("expected int slice found %T for %s", raw, name)
+	}
+	return nil, nil
+}
+
+func (c *configFileSettings) Generic(name string) (cli.Generic, error) {
+	return nil, errors.New("option type Generic not supported")
+}
+
+func (c *configFileSettings) Bool(name string) (bool, error) {
+	if raw, ok := c.Settings[name]; ok {
+		if v, ok := raw.(bool); ok {
+			return v, nil
+		}
+		return false, fmt.Errorf("expected boolean found %T for %s", raw, name)
+	}
+	return false, nil
+}
+
+var configuration configFileSettings
+
+func GetConfiguration() *Configuration {
+	return &configuration.Configuration
+}
+
+// ReadConfigFile returns InputSourceContext initialized from the configuration file.
+// On repeat calls returns with the same file, returns without reading the file again; however,
+// if value of "config" flag changes, will read the new config file
+func ReadConfigFile(c *cli.Context, log logger.Service) (*configFileSettings, error) {
+	configFile := c.String("config")
+	if configuration.Source() == configFile || configFile == "" {
+		if configuration.Source() == "" {
+			return nil, ErrNoConfigFile
+		}
+		return &configuration, nil
+	}
+
+	log.Debugf("Loading configuration from %s", configFile)
+	file, err := os.Open(configFile)
+	if err != nil {
+		if os.IsNotExist(err) {
+			err = ErrNoConfigFile
+		}
+		return nil, err
+	}
+	defer file.Close()
+	if err := yaml.NewDecoder(file).Decode(&configuration); err != nil {
+		return nil, err
+	}
+	configuration.sourceFile = configFile
+	return &configuration, nil
+}

cmd/cloudflared/config/configuration_test.go

@@ -0,0 +1,76 @@
+package config
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v2"
+)
+
+func TestConfigFileSettings(t *testing.T) {
+	var (
+		firstIngress = UnvalidatedIngressRule{
+			Hostname: "tunnel1.example.com",
+			Path:     "/id",
+			Service:  "https://localhost:8000",
+		}
+		secondIngress = UnvalidatedIngressRule{
+			Hostname: "*",
+			Path:     "",
+			Service:  "https://localhost:8001",
+		}
+	)
+	rawYAML := `
+tunnel: config-file-test
+ingress:
+ - hostname: tunnel1.example.com
+   path: /id
+   service: https://localhost:8000
+ - hostname: "*"
+   service: https://localhost:8001
+retries: 5
+grace-period: 30s
+percentage: 3.14
+hostname: example.com
+tag:
+ - test
+ - central-1
+counters:
+ - 123
+ - 456
+`
+	var config configFileSettings
+	err := yaml.Unmarshal([]byte(rawYAML), &config)
+	assert.NoError(t, err)
+
+	assert.Equal(t, "config-file-test", config.TunnelID)
+	assert.Equal(t, firstIngress, config.Ingress[0])
+	assert.Equal(t, secondIngress, config.Ingress[1])
+
+	retries, err := config.Int("retries")
+	assert.NoError(t, err)
+	assert.Equal(t, 5, retries)
+
+	gracePeriod, err := config.Duration("grace-period")
+	assert.NoError(t, err)
+	assert.Equal(t, time.Second*30, gracePeriod)
+
+	percentage, err := config.Float64("percentage")
+	assert.NoError(t, err)
+	assert.Equal(t, 3.14, percentage)
+
+	hostname, err := config.String("hostname")
+	assert.NoError(t, err)
+	assert.Equal(t, "example.com", hostname)
+
+	tags, err := config.StringSlice("tag")
+	assert.NoError(t, err)
+	assert.Equal(t, "test", tags[0])
+	assert.Equal(t, "central-1", tags[1])
+
+	counters, err := config.IntSlice("counters")
+	assert.NoError(t, err)
+	assert.Equal(t, 123, counters[0])
+	assert.Equal(t, 456, counters[1])
+}

cmd/cloudflared/config/manager_test.go

@@ -4,9 +4,10 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/watcher"
-	"github.com/stretchr/testify/assert"
 )
 
 type mockNotifier struct {

cmd/cloudflared/linux_service.go

@@ -7,11 +7,12 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/pkg/errors"
+	cli "github.com/urfave/cli/v2"
+
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/pkg/errors"
-	cli "github.com/urfave/cli/v2"
 )
 
 func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
@@ -23,6 +24,12 @@ func runApp(app *cli.App, shutdownC, graceShutdownC chan struct{}) {
 				Name:   "install",
 				Usage:  "Install Argo Tunnel as a system service",
 				Action: cliutil.ErrorHandler(installLinuxService),
+				Flags: []cli.Flag{
+					&cli.BoolFlag{
+						Name:  "legacy",
+						Usage: "Generate service file for non-named tunnels",
+					},
+				},
 			},
 			&cli.Command{
 				Name:   "uninstall",
@@ -40,6 +47,7 @@ const (
 	serviceConfigDir      = "/etc/cloudflared"
 	serviceConfigFile     = "config.yml"
 	serviceCredentialFile = "cert.pem"
+	serviceConfigPath     = serviceConfigDir + "/" + serviceConfigFile
 )
 
 var systemdTemplates = []ServiceTemplate{
@@ -52,7 +60,7 @@ After=network.target
 [Service]
 TimeoutStartSec=0
 Type=notify
-ExecStart={{ .Path }} --config /etc/cloudflared/config.yml --origincert /etc/cloudflared/cert.pem --no-autoupdate
+ExecStart={{ .Path }} --config /etc/cloudflared/config.yml --no-autoupdate{{ range .ExtraArgs }} {{ . }}{{ end }}
 Restart=on-failure
 RestartSec=5s
 
@@ -67,7 +75,7 @@ Description=Update Argo Tunnel
 After=network.target
 
 [Service]
-ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 64 ]; then systemctl restart cloudflared; exit 0; fi; exit $code'
+ExecStart=/bin/bash -c '{{ .Path }} update; code=$?; if [ $code -eq 11 ]; then systemctl restart cloudflared; exit 0; fi; exit $code'
 `,
 	},
 	{
@@ -102,7 +110,7 @@ var sysvTemplate = ServiceTemplate{
 # Description:       Argo Tunnel agent
 ### END INIT INFO
 name=$(basename $(readlink -f $0))
-cmd="{{.Path}} --config /etc/cloudflared/config.yml --origincert /etc/cloudflared/cert.pem --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s"
+cmd="{{.Path}} --config /etc/cloudflared/config.yml --pidfile /var/run/$name.pid --autoupdate-freq 24h0m0s{{ range .ExtraArgs }} {{ . }}{{ end }}"
 pid_file="/var/run/$name.pid"
 stdout_log="/var/log/$name.log"
 stderr_log="/var/log/$name.err"
@@ -182,10 +190,6 @@ func isSystemd() bool {
 }
 
 func copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile string, logger logger.Service) error {
-	if err := ensureConfigDirExists(serviceConfigDir); err != nil {
-		return err
-	}
-
 	srcCredentialPath := filepath.Join(userConfigDir, userCredentialFile)
 	destCredentialPath := filepath.Join(serviceConfigDir, serviceCredentialFile)
 	if srcCredentialPath != destCredentialPath {
@@ -216,23 +220,63 @@ func installLinuxService(c *cli.Context) error {
 	if err != nil {
 		return fmt.Errorf("error determining executable path: %v", err)
 	}
-	templateArgs := ServiceTemplateArgs{Path: etPath}
+	templateArgs := ServiceTemplateArgs{
+		Path: etPath,
+	}
 
-	userConfigDir := filepath.Dir(c.String("config"))
-	userConfigFile := filepath.Base(c.String("config"))
-	userCredentialFile := config.DefaultCredentialFile
-	if err = copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile, logger); err != nil {
-		logger.Errorf("Failed to copy user configuration: %s. Before running the service, ensure that %s contains two files, %s and %s", err,
-			serviceConfigDir, serviceCredentialFile, serviceConfigFile)
+	if err := ensureConfigDirExists(serviceConfigDir); err != nil {
 		return err
 	}
+	if c.Bool("legacy") {
+		userConfigDir := filepath.Dir(c.String("config"))
+		userConfigFile := filepath.Base(c.String("config"))
+		userCredentialFile := config.DefaultCredentialFile
+		if err = copyUserConfiguration(userConfigDir, userConfigFile, userCredentialFile, logger); err != nil {
+			logger.Errorf("Failed to copy user configuration: %s. Before running the service, ensure that %s contains two files, %s and %s", err,
+				serviceConfigDir, serviceCredentialFile, serviceConfigFile)
+			return err
+		}
+		templateArgs.ExtraArgs = []string{
+			"--origincert", serviceConfigDir + "/" + serviceCredentialFile,
+		}
+	} else {
+		src, err := config.ReadConfigFile(c, logger)
+		if err != nil {
+			return err
+		}
+
+		// can't use context because this command doesn't define "credentials-file" flag
+		configPresent := func(s string) bool {
+			val, err := src.String(s)
+			return err == nil && val != ""
+		}
+		if src.TunnelID == "" || !configPresent("credentials-file") {
+			return fmt.Errorf(`Configuration file %s must contain entries for the tunnel to run and its associated credentials:
+tunnel: TUNNEL-UUID
+credentials-file: CREDENTIALS-FILE
+`, src.Source())
+		}
+		if src.Source() != serviceConfigPath {
+			if exists, err := config.FileExists(serviceConfigPath); err != nil || exists {
+				return fmt.Errorf("Possible conflicting configuration in %[1]s and %[2]s. Either remove %[2]s or run `cloudflared --config %[2]s service install`", src.Source(), serviceConfigPath)
+			}
+
+			if err := copyFile(src.Source(), serviceConfigPath); err != nil {
+				return fmt.Errorf("failed to copy %s to %s: %w", src.Source(), serviceConfigPath, err)
+			}
+		}
+
+		templateArgs.ExtraArgs = []string{
+			"tunnel", "run",
+		}
+	}
 
 	switch {
 	case isSystemd():
 		logger.Infof("Using Systemd")
 		return installSystemd(&templateArgs, logger)
 	default:
-		logger.Infof("Using Sysv")
+		logger.Infof("Using SysV")
 		return installSysv(&templateArgs, logger)
 	}
 }
@@ -291,7 +335,7 @@ func uninstallLinuxService(c *cli.Context) error {
 		logger.Infof("Using Systemd")
 		return uninstallSystemd(logger)
 	default:
-		logger.Infof("Using Sysv")
+		logger.Infof("Using SysV")
 		return uninstallSysv(logger)
 	}
 }

cmd/cloudflared/main.go

@@ -13,6 +13,7 @@ import (
 	log "github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/metrics"
 	"github.com/cloudflare/cloudflared/overwatch"
+	"github.com/cloudflare/cloudflared/tunneldns"
 	"github.com/cloudflare/cloudflared/watcher"
 
 	raven "github.com/getsentry/raven-go"
@@ -76,21 +77,42 @@ func main() {
 	and configure access control.`
 	app.Flags = flags()
 	app.Action = action(Version, shutdownC, graceShutdownC)
-	app.Before = tunnel.Before
+	app.Before = tunnel.SetFlagsFromConfigFile
 	app.Commands = commands(cli.ShowVersion)
 
 	tunnel.Init(Version, shutdownC, graceShutdownC) // we need this to support the tunnel sub command...
 	access.Init(shutdownC, graceShutdownC)
+	updater.Init(Version)
 	runApp(app, shutdownC, graceShutdownC)
 }
 
 func commands(version func(c *cli.Context)) []*cli.Command {
 	cmds := []*cli.Command{
 		{
-			Name:      "update",
-			Action:    cliutil.ErrorHandler(updater.Update),
-			Usage:     "Update the agent if a new version exists",
-			ArgsUsage: " ",
+			Name:   "update",
+			Action: cliutil.ErrorHandler(updater.Update),
+			Usage:  "Update the agent if a new version exists",
+			Flags: []cli.Flag{
+				&cli.BoolFlag{
+					Name:  "beta",
+					Usage: "specify if you wish to update to the latest beta version",
+				},
+				&cli.BoolFlag{
+					Name:   "force",
+					Usage:  "specify if you wish to force an upgrade to the latest version regardless of the current version",
+					Hidden: true,
+				},
+				&cli.BoolFlag{
+					Name:   "staging",
+					Usage:  "specify if you wish to use the staging url for updating",
+					Hidden: true,
+				},
+				&cli.StringFlag{
+					Name:   "version",
+					Usage:  "specify a version you wish to upgrade or downgrade to",
+					Hidden: false,
+				},
+			},
 			Description: `Looks for a new version on the official download server.
 If a new version exists, updates the agent binary and quits.
 Otherwise, does nothing.
@@ -108,6 +130,7 @@ To determine if an update happened in a script, check for error code 64.`,
 		},
 	}
 	cmds = append(cmds, tunnel.Commands()...)
+	cmds = append(cmds, tunneldns.Command(false))
 	cmds = append(cmds, access.Commands()...)
 	return cmds
 }

cmd/cloudflared/service_template.go

@@ -10,8 +10,9 @@ import (
 	"os/exec"
 	"text/template"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/mitchellh/go-homedir"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 )
 
 type ServiceTemplate struct {
@@ -21,7 +22,8 @@ type ServiceTemplate struct {
 }
 
 type ServiceTemplateArgs struct {
-	Path string
+	Path      string
+	ExtraArgs []string
 }
 
 func (st *ServiceTemplate) ResolvePath() (string, error) {
@@ -80,7 +82,7 @@ func runCommand(command string, args ...string) error {
 		return fmt.Errorf("error starting %s: %v", command, err)
 	}
 
-	ioutil.ReadAll(stderr)
+	_, _ = ioutil.ReadAll(stderr)
 	err = cmd.Wait()
 	if err != nil {
 		return fmt.Errorf("%s returned with error: %v", command, err)
@@ -139,6 +141,33 @@ func copyCredential(srcCredentialPath, destCredentialPath string) error {
 	return nil
 }
 
+func copyFile(src, dest string) error {
+	srcFile, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+
+	destFile, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	ok := false
+	defer func() {
+		destFile.Close()
+		if !ok {
+			_ = os.Remove(dest)
+		}
+	}()
+
+	if _, err := io.Copy(destFile, srcFile); err != nil {
+		return err
+	}
+
+	ok = true
+	return nil
+}
+
 func copyConfig(srcConfigPath, destConfigPath string) error {
 	// Copy or create config
 	destFile, exists, err := openFile(destConfigPath, true)

cmd/cloudflared/tunnel/configuration.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/tlsconfig"
@@ -184,12 +185,6 @@ func prepareTunnelConfig(
 
 	tags = append(tags, tunnelpogs.Tag{Name: "ID", Value: clientID})
 
-	originURL, err := config.ValidateUrl(c, compatibilityMode)
-	if err != nil {
-		logger.Errorf("Error validating origin URL: %s", err)
-		return nil, errors.Wrap(err, "Error validating origin URL")
-	}
-
 	var originCert []byte
 	if !isFreeTunnel {
 		originCert, err = getOriginCert(c, logger)
@@ -224,6 +219,36 @@ func prepareTunnelConfig(
 	}
 	dialContext := dialer.DialContext
 
+	var ingressRules ingress.Ingress
+	if namedTunnel != nil {
+		clientUUID, err := uuid.NewRandom()
+		if err != nil {
+			return nil, errors.Wrap(err, "can't generate clientUUID")
+		}
+		namedTunnel.Client = tunnelpogs.ClientInfo{
+			ClientID: clientUUID[:],
+			Features: []string{origin.FeatureSerializedHeaders},
+			Version:  version,
+			Arch:     fmt.Sprintf("%s_%s", buildInfo.GoOS, buildInfo.GoArch),
+		}
+		ingressRules, err = ingress.ParseIngress(config.GetConfiguration())
+		if err != nil && err != ingress.ErrNoIngressRules {
+			return nil, err
+		}
+		if !ingressRules.IsEmpty() && c.IsSet("url") {
+			return nil, ingress.ErrURLIncompatibleWithIngress
+		}
+	}
+
+	var originURL string
+	if ingressRules.IsEmpty() {
+		originURL, err = config.ValidateUrl(c, compatibilityMode)
+		if err != nil {
+			logger.Errorf("Error validating origin URL: %s", err)
+			return nil, errors.Wrap(err, "Error validating origin URL")
+		}
+	}
+
 	if c.IsSet("unix-socket") {
 		unixSocket, err := config.ValidateUnixSocket(c)
 		if err != nil {
@@ -246,8 +271,22 @@ func prepareTunnelConfig(
 	}
 	// If tunnel running in bastion mode, a connection to origin will not exist until initiated by the client.
 	if !c.IsSet(bastionFlag) {
-		if err = validation.ValidateHTTPService(originURL, hostname, httpTransport); err != nil {
-			logger.Errorf("unable to connect to the origin: %s", err)
+
+		// List all origin URLs that require validation
+		var originURLs []string
+		if ingressRules.IsEmpty() {
+			originURLs = append(originURLs, originURL)
+		} else {
+			for _, rule := range ingressRules.Rules {
+				originURLs = append(originURLs, rule.Service.String())
+			}
+		}
+
+		// Validate each origin URL
+		for _, u := range originURLs {
+			if err = validation.ValidateHTTPService(u, hostname, httpTransport); err != nil {
+				logger.Errorf("unable to connect to the origin: %s", err)
+			}
 		}
 	}
 
@@ -256,20 +295,6 @@ func prepareTunnelConfig(
 		logger.Errorf("unable to create TLS config to connect with edge: %s", err)
 		return nil, errors.Wrap(err, "unable to create TLS config to connect with edge")
 	}
-
-	if namedTunnel != nil {
-		clientUUID, err := uuid.NewRandom()
-		if err != nil {
-			return nil, errors.Wrap(err, "can't generate clientUUID")
-		}
-		namedTunnel.Client = tunnelpogs.ClientInfo{
-			ClientID: clientUUID[:],
-			Features: []string{origin.FeatureSerializedHeaders},
-			Version:  version,
-			Arch:     fmt.Sprintf("%s_%s", buildInfo.GoOS, buildInfo.GoArch),
-		}
-	}
-
 	return &origin.TunnelConfig{
 		BuildInfo:          buildInfo,
 		ClientID:           clientID,
@@ -301,6 +326,7 @@ func prepareTunnelConfig(
 		TlsConfig:          toEdgeTLSConfig,
 		NamedTunnel:        namedTunnel,
 		ReplaceExisting:    c.Bool("force"),
+		IngressRules:       ingressRules,
 		// turn off use of reconnect token and auth refresh when using named tunnels
 		UseReconnectToken: compatibilityMode && c.Bool("use-reconnect-token"),
 	}, nil

cmd/cloudflared/tunnel/ingress_subcommands.go

@@ -0,0 +1,110 @@
+package tunnel
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/ingress"
+)
+
+func buildIngressSubcommand() *cli.Command {
+	return &cli.Command{
+		Name:      "ingress",
+		Category:  "Tunnel",
+		Usage:     "Validate and test cloudflared tunnel's ingress configuration",
+		UsageText: "cloudflared tunnel [--config FILEPATH] ingress COMMAND [arguments...]",
+		Hidden:    true,
+		Description: ` Cloudflared lets you route traffic from the internet to multiple different addresses on your
+		origin. Multiple-origin routing is configured by a set of rules. Each rule matches traffic
+		by its hostname or path, and routes it to an address. These rules are configured under the
+		'ingress' key of your config.yaml, for example:
+
+		ingress:
+		  - hostname: www.example.com
+		    service: https://localhost:8000
+		  - hostname: *.example.xyz
+		    path: /[a-zA-Z]+.html
+		    service: https://localhost:8001
+		  - hostname: *
+		    service: https://localhost:8002
+
+		To ensure cloudflared can route all incoming requests, the last rule must be a catch-all
+		rule that matches all traffic. You can validate these rules with the 'ingress validate'
+		command, and test which rule matches a particular URL with 'ingress rule <URL>'.
+
+		Multiple-origin routing is incompatible with the --url flag.`,
+		Subcommands: []*cli.Command{buildValidateIngressCommand(), buildTestURLCommand()},
+	}
+}
+
+func buildValidateIngressCommand() *cli.Command {
+	return &cli.Command{
+		Name:        "validate",
+		Action:      cliutil.ErrorHandler(validateIngressCommand),
+		Usage:       "Validate the ingress configuration ",
+		UsageText:   "cloudflared tunnel [--config FILEPATH] ingress validate",
+		Description: "Validates the configuration file, ensuring your ingress rules are OK.",
+	}
+}
+
+func buildTestURLCommand() *cli.Command {
+	return &cli.Command{
+		Name:      "rule",
+		Action:    cliutil.ErrorHandler(testURLCommand),
+		Usage:     "Check which ingress rule matches a given request URL",
+		UsageText: "cloudflared tunnel [--config FILEPATH] ingress rule URL",
+		ArgsUsage: "URL",
+		Description: "Check which ingress rule matches a given request URL. " +
+			"Ingress rules match a request's hostname and path. Hostname is " +
+			"optional and is either a full hostname like `www.example.com` or a " +
+			"hostname with a `*` for its subdomains, e.g. `*.example.com`. Path " +
+			"is optional and matches a regular expression, like `/[a-zA-Z0-9_]+.html`",
+	}
+}
+
+// validateIngressCommand check the syntax of the ingress rules in the cloudflared config file
+func validateIngressCommand(c *cli.Context) error {
+	conf := config.GetConfiguration()
+	fmt.Println("Validating rules from", conf.Source())
+	if _, err := ingress.ParseIngress(conf); err != nil {
+		return errors.Wrap(err, "Validation failed")
+	}
+	if c.IsSet("url") {
+		return ingress.ErrURLIncompatibleWithIngress
+	}
+	fmt.Println("OK")
+	return nil
+}
+
+// testURLCommand checks which ingress rule matches the given URL.
+func testURLCommand(c *cli.Context) error {
+	requestArg := c.Args().First()
+	if requestArg == "" {
+		return errors.New("cloudflared tunnel rule expects a single argument, the URL to test")
+	}
+
+	requestURL, err := url.Parse(requestArg)
+	if err != nil {
+		return fmt.Errorf("%s is not a valid URL", requestArg)
+	}
+	if requestURL.Hostname() == "" && requestURL.Scheme == "" {
+		return fmt.Errorf("%s doesn't have a hostname, consider adding a scheme", requestArg)
+	}
+
+	conf := config.GetConfiguration()
+	fmt.Println("Using rules from", conf.Source())
+	ing, err := ingress.ParseIngress(conf)
+	if err != nil {
+		return errors.Wrap(err, "Validation failed")
+	}
+
+	i := ing.FindMatchingRule(requestURL.Hostname(), requestURL.Path)
+	fmt.Printf("Matched rule #%d\n", i+1)
+	fmt.Println(ing.Rules[i].MultiLineString())
+	return nil
+}

cmd/cloudflared/tunnel/login.go

@@ -7,12 +7,14 @@ import (
 	"path/filepath"
 	"syscall"
 
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
-	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
-	"github.com/cloudflare/cloudflared/logger"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	cli "github.com/urfave/cli/v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/transfer"
+	"github.com/cloudflare/cloudflared/logger"
 )
 
 const (
@@ -20,6 +22,22 @@ const (
 	callbackStoreURL = "https://login.argotunnel.com/"
 )
 
+func buildLoginSubcommand(hidden bool) *cli.Command {
+	return &cli.Command{
+		Name:      "login",
+		Action:    cliutil.ErrorHandler(login),
+		Usage:     "Generate a configuration file with your login details",
+		ArgsUsage: " ",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:   "url",
+				Hidden: true,
+			},
+		},
+		Hidden: hidden,
+	}
+}
+
 func login(c *cli.Context) error {
 	logger, err := logger.New()
 	if err != nil {

cmd/cloudflared/tunnel/subcommand_context.go

@@ -8,19 +8,20 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/google/uuid"
+	"github.com/pkg/errors"
+	"github.com/urfave/cli/v2"
+
 	"github.com/cloudflare/cloudflared/certutil"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/origin"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/tunnelstore"
-	"github.com/google/uuid"
-	"github.com/pkg/errors"
-	"github.com/urfave/cli/v2"
 )
 
-// subcommandContext carries structs shared between subcommands, to reduce number of arguments needed to pass between subcommands,
-// and make sure they are only initialized once
+// subcommandContext carries structs shared between subcommands, to reduce number of arguments needed to
+// pass between subcommands, and make sure they are only initialized once
 type subcommandContext struct {
 	c      *cli.Context
 	logger logger.Service
@@ -28,16 +29,23 @@ type subcommandContext struct {
 	// These fields should be accessed using their respective Getter
 	tunnelstoreClient tunnelstore.Client
 	userCredential    *userCredential
+
+	isUIEnabled bool
 }
 
 func newSubcommandContext(c *cli.Context) (*subcommandContext, error) {
-	logger, err := createLogger(c, false)
+	isUIEnabled := c.IsSet(uiFlag) && c.String("name") != ""
+
+	// If UI is enabled, terminal log output should be disabled -- log should be written into a UI log window instead
+	logger, err := createLogger(c, false, isUIEnabled)
 	if err != nil {
 		return nil, errors.Wrap(err, "error setting up logger")
 	}
+
 	return &subcommandContext{
-		c:      c,
-		logger: logger,
+		c:           c,
+		logger:      logger,
+		isUIEnabled: isUIEnabled,
 	}, nil
 }
 
@@ -54,7 +62,8 @@ func (sc *subcommandContext) client() (tunnelstore.Client, error) {
 	if err != nil {
 		return nil, err
 	}
-	client, err := tunnelstore.NewRESTClient(sc.c.String("api-url"), credential.cert.AccountID, credential.cert.ZoneID, credential.cert.ServiceKey, sc.logger)
+	userAgent := fmt.Sprintf("cloudflared/%s", version)
+	client, err := tunnelstore.NewRESTClient(sc.c.String("api-url"), credential.cert.AccountID, credential.cert.ZoneID, credential.cert.ServiceKey, userAgent, sc.logger)
 	if err != nil {
 		return nil, err
 	}
@@ -112,6 +121,7 @@ func (sc *subcommandContext) tunnelCredentialsPath(tunnelID uuid.UUID) (string,
 		if validFilePath(filePath) {
 			return filePath, nil
 		}
+		return "", fmt.Errorf("Tunnel credentials file %s doesn't exist or is not a file", filePath)
 	}
 
 	// Fallback to look for tunnel credentials in the origin cert directory
@@ -236,7 +246,16 @@ func (sc *subcommandContext) run(tunnelID uuid.UUID) error {
 	if err != nil {
 		return err
 	}
-	return StartServer(sc.c, version, shutdownC, graceShutdownC, &origin.NamedTunnelConfig{Auth: *credentials, ID: tunnelID}, sc.logger)
+
+	return StartServer(
+		sc.c,
+		version,
+		shutdownC,
+		graceShutdownC,
+		&origin.NamedTunnelConfig{Auth: *credentials, ID: tunnelID},
+		sc.logger,
+		sc.isUIEnabled,
+	)
 }
 
 func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
@@ -253,17 +272,13 @@ func (sc *subcommandContext) cleanupConnections(tunnelIDs []uuid.UUID) error {
 	return nil
 }
 
-func (sc *subcommandContext) route(tunnelID uuid.UUID, r tunnelstore.Route) error {
+func (sc *subcommandContext) route(tunnelID uuid.UUID, r tunnelstore.Route) (tunnelstore.RouteResult, error) {
 	client, err := sc.client()
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	if err := client.RouteTunnel(tunnelID, r); err != nil {
-		return err
-	}
-
-	return nil
+	return client.RouteTunnel(tunnelID, r)
 }
 
 func (sc *subcommandContext) tunnelActive(name string) (*tunnelstore.Tunnel, bool, error) {

cmd/cloudflared/tunnel/subcommands.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 	"text/tabwriter"
@@ -16,9 +17,12 @@ import (
 	"github.com/mitchellh/go-homedir"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
+	"github.com/urfave/cli/v2/altsrc"
+	"golang.org/x/net/idna"
 	"gopkg.in/yaml.v2"
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/cliutil"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 	"github.com/cloudflare/cloudflared/tunnelstore"
@@ -37,42 +41,43 @@ var (
 	listNameFlag = &cli.StringFlag{
 		Name:    "name",
 		Aliases: []string{"n"},
-		Usage:   "List tunnels with the given name",
+		Usage:   "List tunnels with the given `NAME`",
 	}
 	listExistedAtFlag = &cli.TimestampFlag{
-		Name:    "when",
-		Aliases: []string{"w"},
-		Usage:   fmt.Sprintf("List tunnels that are active at the given time, expect format in RFC3339 (%s)", time.Now().Format(tunnelstore.TimeLayout)),
-		Layout:  tunnelstore.TimeLayout,
+		Name:        "when",
+		Aliases:     []string{"w"},
+		Usage:       "List tunnels that are active at the given `TIME` in RFC3339 format",
+		Layout:      tunnelstore.TimeLayout,
+		DefaultText: fmt.Sprintf("current time, %s", time.Now().Format(tunnelstore.TimeLayout)),
 	}
 	listIDFlag = &cli.StringFlag{
 		Name:    "id",
 		Aliases: []string{"i"},
-		Usage:   "List tunnel by ID",
+		Usage:   "List tunnel by `ID`",
 	}
 	showRecentlyDisconnected = &cli.BoolFlag{
 		Name:    "show-recently-disconnected",
 		Aliases: []string{"rd"},
 		Usage:   "Include connections that have recently disconnected in the list",
 	}
-	outputFormatFlag = &cli.StringFlag{
+	outputFormatFlag = altsrc.NewStringFlag(&cli.StringFlag{
 		Name:    "output",
 		Aliases: []string{"o"},
 		Usage:   "Render output using given `FORMAT`. Valid options are 'json' or 'yaml'",
-	}
-	forceFlag = &cli.BoolFlag{
+	})
+	forceFlag = altsrc.NewBoolFlag(&cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
 		Usage: "By default, if a tunnel is currently being run from a cloudflared, you can't " +
 			"simultaneously rerun it again from a second cloudflared. The --force flag lets you " +
 			"overwrite the previous tunnel. If you want to use a single hostname with multiple " +
 			"tunnels, you can do so with Cloudflare's Load Balancer product.",
-	}
-	credentialsFileFlag = &cli.StringFlag{
+	})
+	credentialsFileFlag = altsrc.NewStringFlag(&cli.StringFlag{
 		Name:    "credentials-file",
 		Aliases: []string{credFileFlagAlias},
 		Usage:   "File path of tunnel credentials",
-	}
+	})
 	forceDeleteFlag = &cli.BoolFlag{
 		Name:    "force",
 		Aliases: []string{"f"},
@@ -80,16 +85,20 @@ var (
 	}
 )
 
-const hideSubcommands = true
-
 func buildCreateCommand() *cli.Command {
 	return &cli.Command{
-		Name:      "create",
-		Action:    cliutil.ErrorHandler(createCommand),
-		Usage:     "Create a new tunnel with given name",
-		ArgsUsage: "TUNNEL-NAME",
-		Hidden:    hideSubcommands,
-		Flags:     []cli.Flag{outputFormatFlag},
+		Name:               "create",
+		Action:             cliutil.ErrorHandler(createCommand),
+		Usage:              "Create a new tunnel with given name",
+		UsageText:          "cloudflared tunnel [tunnel command options] create [subcommand options] NAME",
+		Description:        `Creates a tunnel, registers it with Cloudflare edge and generates credential file used to run this tunnel.
+  Use "cloudflared tunnel route" subcommand to map a DNS name to this tunnel and "cloudflared tunnel run" to start the connection. 
+
+  For example, to create a tunnel named 'my-tunnel' run:
+
+  $ cloudflared tunnel create my-tunnel`,
+		Flags:              []cli.Flag{outputFormatFlag},
+		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
 
@@ -149,12 +158,13 @@ func validFilePath(path string) bool {
 
 func buildListCommand() *cli.Command {
 	return &cli.Command{
-		Name:      "list",
-		Action:    cliutil.ErrorHandler(listCommand),
-		Usage:     "List existing tunnels",
-		ArgsUsage: " ",
-		Hidden:    hideSubcommands,
-		Flags:     []cli.Flag{outputFormatFlag, showDeletedFlag, listNameFlag, listExistedAtFlag, listIDFlag, showRecentlyDisconnected},
+		Name:               "list",
+		Action:             cliutil.ErrorHandler(listCommand),
+		Usage:              "List existing tunnels",
+		UsageText:          "cloudflared tunnel [tunnel command options] list [subcommand options]",
+		Description:        "cloudflared tunnel list will display all active tunnels, their created time and associated connections. Use -d flag to include deleted tunnels. See the list of options to filter the list",
+		Flags:              []cli.Flag{outputFormatFlag, showDeletedFlag, listNameFlag, listExistedAtFlag, listIDFlag, showRecentlyDisconnected},
+		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
 
@@ -209,6 +219,7 @@ func fmtAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconnect
 	)
 
 	writer := tabwriter.NewWriter(os.Stdout, minWidth, tabWidth, padding, padChar, flags)
+	defer writer.Flush()
 
 	// Print column headers with tabbed columns
 	fmt.Fprintln(writer, "ID\tNAME\tCREATED\tCONNECTIONS\t")
@@ -224,9 +235,6 @@ func fmtAndPrintTunnelList(tunnels []*tunnelstore.Tunnel, showRecentlyDisconnect
 		)
 		fmt.Fprintln(writer, formattedStr)
 	}
-
-	// Write data buffered in tabwriter to output
-	writer.Flush()
 }
 
 func fmtConnections(connections []tunnelstore.Connection, showRecentlyDisconnected bool) string {
@@ -256,12 +264,13 @@ func fmtConnections(connections []tunnelstore.Connection, showRecentlyDisconnect
 
 func buildDeleteCommand() *cli.Command {
 	return &cli.Command{
-		Name:      "delete",
-		Action:    cliutil.ErrorHandler(deleteCommand),
-		Usage:     "Delete existing tunnel with given IDs",
-		ArgsUsage: "TUNNEL-ID",
-		Hidden:    hideSubcommands,
-		Flags:     []cli.Flag{credentialsFileFlag, forceDeleteFlag},
+		Name:               "delete",
+		Action:             cliutil.ErrorHandler(deleteCommand),
+		Usage:              "Delete existing tunnel by UUID or name",
+		UsageText:          "cloudflared tunnel [tunnel command options] delete [subcommand options] TUNNEL",
+		Description:        "cloudflared tunnel delete will delete tunnels with the given tunnel UUIDs or names. A tunnel cannot be deleted if it has active connections. To delete the tunnel unconditionally, use -f flag.",
+		Flags:              []cli.Flag{credentialsFileFlag, forceDeleteFlag},
+		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
 
@@ -297,13 +306,28 @@ func renderOutput(format string, v interface{}) error {
 }
 
 func buildRunCommand() *cli.Command {
+	flags := []cli.Flag{
+		forceFlag,
+		credentialsFileFlag,
+	}
+	flags = append(flags, configureProxyFlags(false)...)
 	return &cli.Command{
 		Name:      "run",
 		Action:    cliutil.ErrorHandler(runCommand),
+		Before:    SetFlagsFromConfigFile,
 		Usage:     "Proxy a local web server by running the given tunnel",
-		ArgsUsage: "TUNNEL-ID",
-		Hidden:    hideSubcommands,
-		Flags:     []cli.Flag{forceFlag, credentialsFileFlag},
+		UsageText: "cloudflared tunnel [tunnel command options] run [subcommand options] [TUNNEL]",
+		Description: `Runs the tunnel identified by name or UUUD, creating highly available connections 
+  between your server and the Cloudflare edge. You can provide name or UUID of tunnel to run either as the
+  last command line argument or in the configuration file using "tunnel: TUNNEL".
+
+  This command requires the tunnel credentials file created when "cloudflared tunnel create" was run, 
+  however it does not need access to cert.pem from "cloudflared login" if you identify the tunnel by UUID.
+  If you experience other problems running the tunnel, "cloudflared tunnel cleanup" may help by removing 
+  any old connection records.
+`,
+		Flags:              flags,
+		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
 
@@ -313,24 +337,40 @@ func runCommand(c *cli.Context) error {
 		return err
 	}
 
-	if c.NArg() != 1 {
-		return cliutil.UsageError(`"cloudflared tunnel run" requires exactly 1 argument, the ID or name of the tunnel to run.`)
+	if c.NArg() > 1 {
+		return cliutil.UsageError(`"cloudflared tunnel run" accepts only one argument, the ID or name of the tunnel to run.`)
 	}
-	tunnelID, err := uuid.Parse(c.Args().First())
+	tunnelRef := c.Args().First()
+	if tunnelRef == "" {
+		// see if tunnel id was in the config file
+		tunnelRef = config.GetConfiguration().TunnelID
+		if tunnelRef == "" {
+			return cliutil.UsageError(`"cloudflared tunnel run" requires the ID or name of the tunnel to run as the last command line argument or in the configuration file.`)
+		}
+	}
+
+	return runNamedTunnel(sc, tunnelRef)
+}
+
+func runNamedTunnel(sc *subcommandContext, tunnelRef string) error {
+	tunnelID, err := sc.findID(tunnelRef)
 	if err != nil {
 		return errors.Wrap(err, "error parsing tunnel ID")
 	}
 
+	sc.logger.Infof("Starting tunnel %s", tunnelID.String())
+
 	return sc.run(tunnelID)
 }
 
 func buildCleanupCommand() *cli.Command {
 	return &cli.Command{
-		Name:      "cleanup",
-		Action:    cliutil.ErrorHandler(cleanupCommand),
-		Usage:     "Cleanup connections for the tunnel with given IDs",
-		ArgsUsage: "TUNNEL-IDS",
-		Hidden:    hideSubcommands,
+		Name:               "cleanup",
+		Action:             cliutil.ErrorHandler(cleanupCommand),
+		Usage:              "Cleanup tunnel connections",
+		UsageText:          "cloudflared tunnel [tunnel command options] cleanup [subcommand options] TUNNEL",
+		Description:        "Delete connections for tunnels with the given UUIDs or names.",
+		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
 
@@ -354,54 +394,79 @@ func cleanupCommand(c *cli.Context) error {
 
 func buildRouteCommand() *cli.Command {
 	return &cli.Command{
-		Name:   "route",
-		Action: cliutil.ErrorHandler(routeCommand),
-		Usage:  "Define what hostname or load balancer can route to this tunnel",
-		Description: `The route defines what hostname or load balancer can route to this tunnel.
-	 To route a hostname: cloudflared tunnel route dns <tunnel ID> <hostname>
-	 To route a load balancer: cloudflared tunnel route lb <tunnel ID> <load balancer name> <load balancer pool>
-	 If you don't specify a load balancer pool, we will create a new pool called tunnel:<tunnel ID>`,
-		ArgsUsage: "dns|lb TUNNEL-ID HOSTNAME [LB-POOL]",
-		Hidden:    hideSubcommands,
+		Name:      "route",
+		Action:    cliutil.ErrorHandler(routeCommand),
+		Usage:     "Define what hostname or load balancer can route to this tunnel",
+		UsageText: "cloudflared tunnel [tunnel command options] route [subcommand options] dns|lb TUNNEL HOSTNAME [LB-POOL]",
+		Description: `The route defines what hostname or load balancer will proxy requests to this tunnel.
+
+   To route a hostname by creating a CNAME to tunnel's address:
+      cloudflared tunnel route dns <tunnel ID> <hostname>
+   To use this tunnel as a load balancer origin, creating pool and load balancer if necessary:
+      cloudflared tunnel route lb <tunnel ID> <load balancer name> <load balancer pool>`,
+		CustomHelpTemplate: commandHelpTemplate(),
 	}
 }
 
-func dnsRouteFromArg(c *cli.Context, tunnelID uuid.UUID) (tunnelstore.Route, error) {
+func dnsRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 	const (
 		userHostnameIndex = 2
-		expectArgs        = 3
+		expectedNArgs     = 3
 	)
-	if c.NArg() != expectArgs {
-		return nil, cliutil.UsageError("Expect %d arguments, got %d", expectArgs, c.NArg())
+	if c.NArg() != expectedNArgs {
+		return nil, cliutil.UsageError("Expected %d arguments, got %d", expectedNArgs, c.NArg())
 	}
 	userHostname := c.Args().Get(userHostnameIndex)
 	if userHostname == "" {
 		return nil, cliutil.UsageError("The third argument should be the hostname")
+	} else if !validateHostname(userHostname) {
+		return nil, errors.Errorf("%s is not a valid hostname", userHostname)
 	}
 	return tunnelstore.NewDNSRoute(userHostname), nil
 }
 
-func lbRouteFromArg(c *cli.Context, tunnelID uuid.UUID) (tunnelstore.Route, error) {
+func lbRouteFromArg(c *cli.Context) (tunnelstore.Route, error) {
 	const (
 		lbNameIndex   = 2
 		lbPoolIndex   = 3
-		expectMinArgs = 3
+		expectedNArgs = 4
 	)
-	if c.NArg() < expectMinArgs {
-		return nil, cliutil.UsageError("Expect at least %d arguments, got %d", expectMinArgs, c.NArg())
+	if c.NArg() != expectedNArgs {
+		return nil, cliutil.UsageError("Expected %d arguments, got %d", expectedNArgs, c.NArg())
 	}
 	lbName := c.Args().Get(lbNameIndex)
 	if lbName == "" {
 		return nil, cliutil.UsageError("The third argument should be the load balancer name")
+	} else if !validateHostname(lbName) {
+		return nil, errors.Errorf("%s is not a valid load balancer name", lbName)
 	}
+
 	lbPool := c.Args().Get(lbPoolIndex)
 	if lbPool == "" {
-		lbPool = defaultPoolName(tunnelID)
+		return nil, cliutil.UsageError("The fourth argument should be the pool name")
+	} else if !validateName(lbPool) {
+		return nil, errors.Errorf("%s is not a valid pool name", lbPool)
 	}
 
 	return tunnelstore.NewLBRoute(lbName, lbPool), nil
 }
 
+var nameRegex = regexp.MustCompile("^[_a-zA-Z0-9][-_.a-zA-Z0-9]*$")
+
+func validateName(s string) bool {
+	return nameRegex.MatchString(s)
+}
+
+func validateHostname(s string) bool {
+	// Slightly stricter than PunyCodeProfile
+	idnaProfile := idna.New(
+		idna.ValidateLabels(true),
+		idna.VerifyDNSLength(true))
+
+	puny, err := idnaProfile.ToASCII(s)
+	return err == nil && validateName(puny)
+}
+
 func routeCommand(c *cli.Context) error {
 	if c.NArg() < 2 {
 		return cliutil.UsageError(`"cloudflared tunnel route" requires the first argument to be the route type(dns or lb), followed by the ID or name of the tunnel`)
@@ -412,21 +477,26 @@ func routeCommand(c *cli.Context) error {
 	}
 
 	const tunnelIDIndex = 1
-	tunnelID, err := sc.findID(c.Args().Get(tunnelIDIndex))
-	if err != nil {
-		return err
-	}
 
 	routeType := c.Args().First()
-	var r tunnelstore.Route
+	var route tunnelstore.Route
+	var tunnelID uuid.UUID
 	switch routeType {
 	case "dns":
-		r, err = dnsRouteFromArg(c, tunnelID)
+		tunnelID, err = sc.findID(c.Args().Get(tunnelIDIndex))
+		if err != nil {
+			return err
+		}
+		route, err = dnsRouteFromArg(c)
 		if err != nil {
 			return err
 		}
 	case "lb":
-		r, err = lbRouteFromArg(c, tunnelID)
+		tunnelID, err = sc.findID(c.Args().Get(tunnelIDIndex))
+		if err != nil {
+			return err
+		}
+		route, err = lbRouteFromArg(c)
 		if err != nil {
 			return err
 		}
@@ -434,13 +504,37 @@ func routeCommand(c *cli.Context) error {
 		return cliutil.UsageError("%s is not a recognized route type. Supported route types are dns and lb", routeType)
 	}
 
-	if err := sc.route(tunnelID, r); err != nil {
+	res, err := sc.route(tunnelID, route)
+	if err != nil {
 		return err
 	}
-	sc.logger.Infof(r.SuccessSummary())
+
+	sc.logger.Infof(res.SuccessSummary())
 	return nil
 }
 
-func defaultPoolName(tunnelID uuid.UUID) string {
-	return fmt.Sprintf("tunnel:%v", tunnelID)
+func commandHelpTemplate() string {
+	var parentFlagsHelp string
+	for _, f := range configureCloudflaredFlags(false) {
+		parentFlagsHelp += fmt.Sprintf(" %s\n\t", f)
+	}
+	for _, f := range configureLoggingFlags(false) {
+		parentFlagsHelp += fmt.Sprintf(" %s\n\t", f)
+	}
+	const template = `NAME:
+	{{.HelpName}} - {{.Usage}}
+	
+USAGE:
+	{{.UsageText}}
+ 
+DESCRIPTION:
+	{{.Description}}
+ 
+TUNNEL COMMAND OPTIONS:
+	%s
+SUBCOMMAND OPTIONS:
+	{{range .VisibleFlags}}{{.}}
+	{{end}}
+`
+	return fmt.Sprintf(template, parentFlagsHelp)
 }

cmd/cloudflared/tunnel/subcommands_test.go

@@ -98,3 +98,27 @@ func TestTunnelfilePath(t *testing.T) {
 	expected := fmt.Sprintf("%s/.cloudflared/%v.json", homeDir, tunnelID)
 	assert.Equal(t, expected, actual)
 }
+
+func TestValidateName(t *testing.T) {
+	tests := []struct {
+		name string
+		want bool
+	}{
+		{name: "", want: false},
+		{name: "-", want: false},
+		{name: ".", want: false},
+		{name: "a b", want: false},
+		{name: "a+b", want: false},
+		{name: "-ab", want: false},
+
+		{name: "ab", want: true},
+		{name: "ab-c", want: true},
+		{name: "abc.def", want: true},
+		{name: "_ab_c.-d-ef", want: true},
+	}
+	for _, tt := range tests {
+		if got := validateName(tt.name); got != tt.want {
+			t.Errorf("validateName() = %v, want %v", got, tt.want)
+		}
+	}
+}

cmd/cloudflared/ui/launch_ui.go

@@ -0,0 +1,236 @@
+package ui
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/cloudflare/cloudflared/logger"
+
+	"github.com/gdamore/tcell"
+	"github.com/rivo/tview"
+)
+
+type connState struct {
+	location string
+	state    status
+}
+
+type status int
+
+const (
+	Disconnected status = iota
+	Connected
+	Reconnecting
+	SetUrl
+	RegisteringTunnel
+)
+
+type TunnelEvent struct {
+	Index     uint8
+	EventType status
+	Location  string
+	Url       string
+}
+
+type uiModel struct {
+	version     string
+	edgeURL     string
+	metricsURL  string
+	proxyURL    string
+	connections []connState
+}
+
+type palette struct {
+	url          string
+	connected    string
+	defaultText  string
+	disconnected string
+	reconnecting string
+}
+
+func NewUIModel(version, hostname, metricsURL, proxyURL string, haConnections int) *uiModel {
+	return &uiModel{
+		version:     version,
+		edgeURL:     hostname,
+		metricsURL:  metricsURL,
+		proxyURL:    proxyURL,
+		connections: make([]connState, haConnections),
+	}
+}
+
+func (data *uiModel) LaunchUI(
+	ctx context.Context,
+	log logger.Service,
+	logLevels []logger.Level,
+	tunnelEventChan <-chan TunnelEvent,
+) {
+	// Configure the logger to stream logs into the textview
+
+	// Add TextView as a group to write output to
+	logTextView := NewDynamicColorTextView()
+	log.Add(logTextView, logger.NewUIFormatter(time.RFC3339), logLevels...)
+
+	// Construct the UI
+	palette := palette{
+		url:          "lightblue",
+		connected:    "lime",
+		defaultText:  "white",
+		disconnected: "red",
+		reconnecting: "orange",
+	}
+
+	app := tview.NewApplication()
+
+	grid := tview.NewGrid().SetGap(1, 0)
+	frame := tview.NewFrame(grid)
+	header := fmt.Sprintf("cloudflared [::b]%s", data.version)
+
+	frame.AddText(header, true, tview.AlignLeft, tcell.ColorWhite)
+
+	// Create table to store connection info and status
+	connTable := tview.NewTable()
+	// SetColumns takes a value for each column, representing the size of the column
+	// Numbers <= 0 represent proportional widths and positive numbers represent absolute widths
+	grid.SetColumns(20, 0)
+
+	// SetRows takes a value for each row, representing the size of the row
+	grid.SetRows(1, 1, len(data.connections), 1, 0)
+
+	// AddItem takes a primitive tview type, row, column, rowSpan, columnSpan, minGridHeight, minGridWidth, and focus
+	grid.AddItem(tview.NewTextView().SetText("Tunnel:"), 0, 0, 1, 1, 0, 0, false)
+	grid.AddItem(tview.NewTextView().SetText("Status:"), 1, 0, 1, 1, 0, 0, false)
+	grid.AddItem(tview.NewTextView().SetText("Connections:"), 2, 0, 1, 1, 0, 0, false)
+
+	grid.AddItem(tview.NewTextView().SetText("Metrics:"), 3, 0, 1, 1, 0, 0, false)
+
+	tunnelHostText := tview.NewTextView().SetText(data.edgeURL)
+
+	grid.AddItem(tunnelHostText, 0, 1, 1, 1, 0, 0, false)
+	grid.AddItem(NewDynamicColorTextView().SetText(fmt.Sprintf("[%s]\u2022[%s] Proxying to [%s::b]%s", palette.connected, palette.defaultText, palette.url, data.proxyURL)), 1, 1, 1, 1, 0, 0, false)
+
+	grid.AddItem(connTable, 2, 1, 1, 1, 0, 0, false)
+
+	grid.AddItem(NewDynamicColorTextView().SetText(fmt.Sprintf("Metrics at [%s::b]http://%s/metrics", palette.url, data.metricsURL)), 3, 1, 1, 1, 0, 0, false)
+
+	// Add TextView to stream logs
+	// Logs are displayed in a new grid so a border can be set around them
+	logGrid := tview.NewGrid().SetBorders(true).AddItem(logTextView.SetChangedFunc(handleNewText(app, logTextView)), 0, 0, 5, 2, 0, 0, false)
+	// LogFrame holds the Logs header as well as the grid with the textView for streamed logs
+	logFrame := tview.NewFrame(logGrid).AddText("[::b]Logs:[::-]", true, tview.AlignLeft, tcell.ColorWhite).SetBorders(0, 0, 0, 0, 0, 0)
+	// Footer for log frame
+	logFrame.AddText("[::d]Use Ctrl+C to exit[::-]", false, tview.AlignRight, tcell.ColorWhite)
+	grid.AddItem(logFrame, 4, 0, 5, 2, 0, 0, false)
+
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				app.Stop()
+				return
+			case event := <-tunnelEventChan:
+				switch event.EventType {
+				case Connected:
+					data.setConnTableCell(event, connTable, palette)
+				case Disconnected, Reconnecting:
+					data.changeConnStatus(event, connTable, log, palette)
+				case SetUrl:
+					tunnelHostText.SetText(event.Url)
+					data.edgeURL = event.Url
+				case RegisteringTunnel:
+					if data.edgeURL == "" {
+						tunnelHostText.SetText("Registering tunnel...")
+					}
+				}
+			}
+			app.Draw()
+		}
+	}()
+
+	go func() {
+		if err := app.SetRoot(frame, true).Run(); err != nil {
+			log.Errorf("Error launching UI: %s", err)
+		}
+	}()
+}
+
+func NewDynamicColorTextView() *tview.TextView {
+	return tview.NewTextView().SetDynamicColors(true)
+}
+
+// Re-draws application when new logs are streamed to UI
+func handleNewText(app *tview.Application, logTextView *tview.TextView) func() {
+	return func() {
+		app.Draw()
+		// SetFocus to enable scrolling in textview
+		app.SetFocus(logTextView)
+	}
+}
+
+func (data *uiModel) changeConnStatus(event TunnelEvent, table *tview.Table, logger logger.Service, palette palette) {
+	index := int(event.Index)
+	// Get connection location and state
+	connState := data.getConnState(index)
+	// Check if connection is already displayed in UI
+	if connState == nil {
+		logger.Info("Connection is not in the UI table")
+		return
+	}
+
+	locationState := event.Location
+
+	if event.EventType == Disconnected {
+		connState.state = Disconnected
+	} else if event.EventType == Reconnecting {
+		connState.state = Reconnecting
+		locationState = "Reconnecting..."
+	}
+
+	connectionNum := index + 1
+	// Get table cell
+	cell := table.GetCell(index, 0)
+	// Change dot color in front of text as well as location state
+	text := newCellText(palette, connectionNum, locationState, event.EventType)
+	cell.SetText(text)
+}
+
+// Return connection location and row in UI table
+func (data *uiModel) getConnState(connID int) *connState {
+	if connID < len(data.connections) {
+		return &data.connections[connID]
+	}
+
+	return nil
+}
+
+func (data *uiModel) setConnTableCell(event TunnelEvent, table *tview.Table, palette palette) {
+	index := int(event.Index)
+	connectionNum := index + 1
+
+	// Update slice to keep track of connection location and state in UI table
+	data.connections[index].state = Connected
+	data.connections[index].location = event.Location
+
+	// Update text in table cell to show disconnected state
+	text := newCellText(palette, connectionNum, event.Location, event.EventType)
+	cell := tview.NewTableCell(text)
+	table.SetCell(index, 0, cell)
+}
+
+func newCellText(palette palette, connectionNum int, location string, connectedStatus status) string {
+	// HA connection indicator formatted as: "• #<CONNECTION_INDEX>: <COLO>",
+	//  where the left middle dot's color depends on the status of the connection
+	const connFmtString = "[%s]\u2022[%s] #%d: %s"
+
+	var dotColor string
+	switch connectedStatus {
+	case Connected:
+		dotColor = palette.connected
+	case Disconnected:
+		dotColor = palette.disconnected
+	case Reconnecting:
+		dotColor = palette.reconnecting
+	}
+
+	return fmt.Sprintf(connFmtString, dotColor, palette.defaultText, connectionNum, location)
+}

cmd/cloudflared/updater/service.go

@@ -0,0 +1,26 @@
+package updater
+
+// Version is the functions needed to perform an update
+type Version interface {
+	Apply() error
+	String() string
+}
+
+// Service is the functions to get check for new updates
+type Service interface {
+	Check() (Version, error)
+}
+
+const (
+	// OSKeyName is the url parameter key to send to the checkin API for the operating system of the local cloudflared (e.g. windows, darwin, linux)
+	OSKeyName = "os"
+
+	// ArchitectureKeyName is the url parameter key to send to the checkin API for the architecture of the local cloudflared (e.g. amd64, x86)
+	ArchitectureKeyName = "arch"
+
+	// BetaKeyName is the url parameter key to send to the checkin API to signal if the update should be a beta version or not
+	BetaKeyName = "beta"
+
+	// VersionKeyName is the url parameter key to send to the checkin API to specific what version to upgrade or downgrade to
+	VersionKeyName = "version"
+)

cmd/cloudflared/updater/update.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
 	"github.com/cloudflare/cloudflared/logger"
-	"github.com/equinox-io/equinox"
 	"github.com/facebookgo/grace/gracenet"
 	"github.com/pkg/errors"
 )
@@ -25,16 +24,12 @@ const (
 	noUpdateOnWindowsMessage      = "cloudflared will not automatically update on Windows systems."
 	noUpdateManagedPackageMessage = "cloudflared will not automatically update if installed by a package manager."
 	isManagedInstallFile          = ".installedFromPackageManager"
+	UpdateURL                     = "https://update.argotunnel.com"
+	StagingUpdateURL              = "https://staging-update.argotunnel.com"
 )
 
 var (
-	publicKey = []byte(`
------BEGIN ECDSA PUBLIC KEY-----
-MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE4OWZocTVZ8Do/L6ScLdkV+9A0IYMHoOf
-dsCmJ/QZ6aw0w9qkkwEpne1Lmo6+0pGexZzFZOH6w5amShn+RXt7qkSid9iWlzGq
-EKx0BZogHSor9Wy5VztdFaAaVbsJiCbO
------END ECDSA PUBLIC KEY-----
-`)
+	version string
 )
 
 // BinaryUpdated implements ExitCoder interface, the app will exit with status code 11
@@ -64,6 +59,13 @@ func (e *statusErr) ExitCode() int {
 	return 10
 }
 
+type updateOptions struct {
+	isBeta    bool
+	isStaging bool
+	isForced  bool
+	version   string
+}
+
 type UpdateOutcome struct {
 	Updated bool
 	Version string
@@ -74,29 +76,44 @@ func (uo *UpdateOutcome) noUpdate() bool {
 	return uo.Error == nil && uo.Updated == false
 }
 
-func checkForUpdateAndApply() UpdateOutcome {
-	var opts equinox.Options
-	if err := opts.SetPublicKeyPEM(publicKey); err != nil {
-		return UpdateOutcome{Error: err}
-	}
+func Init(v string) {
+	version = v
+}
 
-	resp, err := equinox.Check(appID, opts)
-	switch {
-	case err == equinox.NotAvailableErr:
-		return UpdateOutcome{}
-	case err != nil:
-		return UpdateOutcome{Error: err}
-	}
-
-	err = resp.Apply()
+func checkForUpdateAndApply(options updateOptions) UpdateOutcome {
+	cfdPath, err := os.Executable()
 	if err != nil {
 		return UpdateOutcome{Error: err}
 	}
 
-	return UpdateOutcome{Updated: true, Version: resp.ReleaseVersion}
+	url := UpdateURL
+	if options.isStaging {
+		url = StagingUpdateURL
+	}
+
+	s := NewWorkersService(version, url, cfdPath, Options{IsBeta: options.isBeta,
+		IsForced: options.isForced, RequestedVersion: options.version})
+
+	v, err := s.Check()
+	if err != nil {
+		return UpdateOutcome{Error: err}
+	}
+
+	//already on the latest version
+	if v == nil {
+		return UpdateOutcome{}
+	}
+
+	err = v.Apply()
+	if err != nil {
+		return UpdateOutcome{Error: err}
+	}
+
+	return UpdateOutcome{Updated: true, Version: v.String()}
 }
 
-func Update(_ *cli.Context) error {
+// Update is the handler for the update command from the command line
+func Update(c *cli.Context) error {
 	logger, err := logger.New()
 	if err != nil {
 		return errors.Wrap(err, "error setting up logger")
@@ -107,7 +124,22 @@ func Update(_ *cli.Context) error {
 		return nil
 	}
 
-	updateOutcome := loggedUpdate(logger)
+	isBeta := c.Bool("beta")
+	if isBeta {
+		logger.Info("cloudflared is set to update to the latest beta version")
+	}
+
+	isStaging := c.Bool("staging")
+	if isStaging {
+		logger.Info("cloudflared is set to update from staging")
+	}
+
+	isForced := c.Bool("force")
+	if isForced {
+		logger.Info("cloudflared is set to upgrade to the latest publish version regardless of the current version")
+	}
+
+	updateOutcome := loggedUpdate(logger, updateOptions{isBeta: isBeta, isStaging: isStaging, isForced: isForced, version: c.String("version")})
 	if updateOutcome.Error != nil {
 		return &statusErr{updateOutcome.Error}
 	}
@@ -121,8 +153,8 @@ func Update(_ *cli.Context) error {
 }
 
 // Checks for an update and applies it if one is available
-func loggedUpdate(logger logger.Service) UpdateOutcome {
-	updateOutcome := checkForUpdateAndApply()
+func loggedUpdate(logger logger.Service, options updateOptions) UpdateOutcome {
+	updateOutcome := checkForUpdateAndApply(options)
 	if updateOutcome.Updated {
 		logger.Infof("cloudflared has been updated to version %s", updateOutcome.Version)
 	}
@@ -168,7 +200,7 @@ func (a *AutoUpdater) Run(ctx context.Context) error {
 	ticker := time.NewTicker(a.configurable.freq)
 	for {
 		if a.configurable.enabled {
-			updateOutcome := loggedUpdate(a.logger)
+			updateOutcome := loggedUpdate(a.logger, updateOptions{})
 			if updateOutcome.Updated {
 				os.Args = append(os.Args, "--is-autoupdated=true")
 				if IsSysV() {

cmd/cloudflared/updater/workers_service.go

@@ -0,0 +1,154 @@
+package updater
+
+import (
+	"encoding/json"
+	"errors"
+	"net/http"
+	"runtime"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// Options are the update options supported by the
+type Options struct {
+	// IsBeta is for beta updates to be installed if available
+	IsBeta bool
+
+	// IsForced is to forcibly download the latest version regardless of the current version
+	IsForced bool
+
+	// RequestedVersion is the specific version to upgrade or downgrade to
+	RequestedVersion string
+}
+
+// VersionResponse is the JSON response from the Workers API endpoint
+type VersionResponse struct {
+	URL          string `json:"url"`
+	Version      string `json:"version"`
+	Checksum     string `json:"checksum"`
+	IsCompressed bool   `json:"compressed"`
+	Error        string `json:"error"`
+}
+
+// WorkersService implements Service.
+// It contains everything needed to check in with the WorkersAPI and download and apply the updates
+type WorkersService struct {
+	currentVersion string
+	url            string
+	targetPath     string
+	opts           Options
+}
+
+// NewWorkersService creates a new updater Service object.
+func NewWorkersService(currentVersion, url, targetPath string, opts Options) Service {
+	return &WorkersService{
+		currentVersion: currentVersion,
+		url:            url,
+		targetPath:     targetPath,
+		opts:           opts,
+	}
+}
+
+// Check does a check in with the Workers API to get a new version update
+func (s *WorkersService) Check() (Version, error) {
+	client := &http.Client{
+		Timeout: time.Second * 5,
+	}
+
+	req, err := http.NewRequest(http.MethodGet, s.url, nil)
+	q := req.URL.Query()
+	q.Add(OSKeyName, runtime.GOOS)
+	q.Add(ArchitectureKeyName, runtime.GOARCH)
+
+	if s.opts.IsBeta {
+		q.Add(BetaKeyName, "true")
+	}
+
+	if s.opts.RequestedVersion != "" {
+		q.Add(VersionKeyName, s.opts.RequestedVersion)
+	}
+
+	req.URL.RawQuery = q.Encode()
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var v VersionResponse
+	if err := json.NewDecoder(resp.Body).Decode(&v); err != nil {
+		return nil, err
+	}
+
+	if v.Error != "" {
+		return nil, errors.New(v.Error)
+	}
+
+	if !s.opts.IsForced && !IsNewerVersion(s.currentVersion, v.Version) {
+		return nil, nil
+	}
+
+	return NewWorkersVersion(v.URL, v.Version, v.Checksum, s.targetPath, v.IsCompressed), nil
+}
+
+// IsNewerVersion checks semantic versioning for the latest version
+// cloudflared tagging is more of a date than a semantic version,
+// but the same comparision logic still holds for major.minor.patch
+// e.g. 2020.8.2 is newer than 2020.8.1.
+func IsNewerVersion(current string, check string) bool {
+	if strings.Contains(strings.ToLower(current), "dev") {
+		return false // dev builds shouldn't update
+	}
+
+	cMajor, cMinor, cPatch, err := SemanticParts(current)
+	if err != nil {
+		return false
+	}
+
+	nMajor, nMinor, nPatch, err := SemanticParts(check)
+	if err != nil {
+		return false
+	}
+
+	if nMajor > cMajor {
+		return true
+	}
+
+	if nMajor == cMajor && nMinor > cMinor {
+		return true
+	}
+
+	if nMajor == cMajor && nMinor == cMinor && nPatch > cPatch {
+		return true
+	}
+	return false
+}
+
+// SemanticParts gets the major, minor, and patch version of a semantic version string
+// e.g. 3.1.2 would return 3, 1, 2, nil
+func SemanticParts(version string) (major int, minor int, patch int, err error) {
+	major = 0
+	minor = 0
+	patch = 0
+	parts := strings.Split(version, ".")
+	if len(parts) != 3 {
+		err = errors.New("invalid version")
+		return
+	}
+	major, err = strconv.Atoi(parts[0])
+	if err != nil {
+		return
+	}
+
+	minor, err = strconv.Atoi(parts[1])
+	if err != nil {
+		return
+	}
+
+	patch, err = strconv.Atoi(parts[2])
+	if err != nil {
+		return
+	}
+	return
+}

cmd/cloudflared/updater/workers_service_test.go

@@ -0,0 +1,273 @@
+package updater
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"crypto/sha256"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func respondWithJSON(w http.ResponseWriter, v interface{}, status int) {
+	data, _ := json.Marshal(v)
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(status)
+	w.Write(data)
+}
+
+func respondWithData(w http.ResponseWriter, b []byte, status int) {
+	w.Header().Set("Content-Type", "application/octet-stream")
+	w.WriteHeader(status)
+	w.Write(b)
+}
+
+func updateHandler(w http.ResponseWriter, r *http.Request) {
+	version := "2020.08.05"
+	host := fmt.Sprintf("http://%s", r.Host)
+	url := host + "/download"
+
+	query := r.URL.Query()
+
+	if query.Get(BetaKeyName) == "true" {
+		version = "2020.08.06"
+		url = host + "/beta"
+	}
+
+	requestedVersion := query.Get(VersionKeyName)
+	if requestedVersion != "" {
+		version = requestedVersion
+		url = fmt.Sprintf("%s?version=%s", url, requestedVersion)
+	}
+
+	if query.Get(ArchitectureKeyName) != runtime.GOARCH || query.Get(OSKeyName) != runtime.GOOS {
+		respondWithJSON(w, VersionResponse{Error: "unsupported os and architecture"}, http.StatusBadRequest)
+		return
+	}
+
+	h := sha256.New()
+	fmt.Fprint(h, version)
+	checksum := fmt.Sprintf("%x", h.Sum(nil))
+
+	v := VersionResponse{URL: url, Version: version, Checksum: checksum}
+	respondWithJSON(w, v, http.StatusOK)
+}
+
+func gzipUpdateHandler(w http.ResponseWriter, r *http.Request) {
+	log.Println("got a request!")
+	version := "2020.09.02"
+	h := sha256.New()
+	fmt.Fprint(h, version)
+	checksum := fmt.Sprintf("%x", h.Sum(nil))
+
+	url := fmt.Sprintf("http://%s/gzip-download.tgz", r.Host)
+	v := VersionResponse{URL: url, Version: version, Checksum: checksum}
+	respondWithJSON(w, v, http.StatusOK)
+}
+
+func compressedDownloadHandler(w http.ResponseWriter, r *http.Request) {
+	version := "2020.09.02"
+	buf := new(bytes.Buffer)
+
+	gw := gzip.NewWriter(buf)
+	tw := tar.NewWriter(gw)
+
+	header := &tar.Header{
+		Size: int64(len(version)),
+		Name: "download",
+	}
+	tw.WriteHeader(header)
+	tw.Write([]byte(version))
+
+	tw.Close()
+	gw.Close()
+
+	respondWithData(w, buf.Bytes(), http.StatusOK)
+}
+
+func downloadHandler(w http.ResponseWriter, r *http.Request) {
+	version := "2020.08.05"
+	requestedVersion := r.URL.Query().Get(VersionKeyName)
+	if requestedVersion != "" {
+		version = requestedVersion
+	}
+	respondWithData(w, []byte(version), http.StatusOK)
+}
+
+func betaHandler(w http.ResponseWriter, r *http.Request) {
+	respondWithData(w, []byte("2020.08.06"), http.StatusOK)
+}
+
+func failureHandler(w http.ResponseWriter, r *http.Request) {
+	respondWithJSON(w, VersionResponse{Error: "unsupported os and architecture"}, http.StatusBadRequest)
+}
+
+func createServer() *httptest.Server {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/updater", updateHandler)
+	mux.HandleFunc("/download", downloadHandler)
+	mux.HandleFunc("/beta", betaHandler)
+	mux.HandleFunc("/fail", failureHandler)
+	mux.HandleFunc("/compressed", gzipUpdateHandler)
+	mux.HandleFunc("/gzip-download.tgz", compressedDownloadHandler)
+	return httptest.NewServer(mux)
+}
+
+func createTestFile(t *testing.T, path string) {
+	f, err := os.Create("tmpfile")
+	require.NoError(t, err)
+	fmt.Fprint(f, "2020.08.04")
+	f.Close()
+}
+
+func TestUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+	log.Println("server url: ", ts.URL)
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, v.String(), "2020.08.05")
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, string(dat), "2020.08.05")
+}
+
+func TestBetaUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{IsBeta: true})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, v.String(), "2020.08.06")
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, string(dat), "2020.08.06")
+}
+
+func TestFailUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/fail", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.Error(t, err)
+	require.Nil(t, v)
+}
+
+func TestNoUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService("2020.8.5", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Nil(t, v)
+}
+
+func TestForcedUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService("2020.8.5", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{IsForced: true})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, v.String(), "2020.08.05")
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, string(dat), "2020.08.05")
+}
+
+func TestUpdateSpecificVersionService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+	reqVersion := "2020.9.1"
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/updater", ts.URL), testFilePath, Options{RequestedVersion: reqVersion})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, reqVersion, v.String())
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, reqVersion, string(dat))
+}
+
+func TestCompressedUpdateService(t *testing.T) {
+	ts := createServer()
+	defer ts.Close()
+
+	testFilePath := "tmpfile"
+	createTestFile(t, testFilePath)
+	defer os.Remove(testFilePath)
+
+	s := NewWorkersService("2020.8.2", fmt.Sprintf("%s/compressed", ts.URL), testFilePath, Options{})
+	v, err := s.Check()
+	require.NoError(t, err)
+	require.Equal(t, "2020.09.02", v.String())
+
+	require.NoError(t, v.Apply())
+	dat, err := ioutil.ReadFile(testFilePath)
+	require.NoError(t, err)
+
+	require.Equal(t, "2020.09.02", string(dat))
+}
+
+func TestVersionParsing(t *testing.T) {
+	require.False(t, IsNewerVersion("2020.8.2", "2020.8.2"))
+	require.True(t, IsNewerVersion("2020.8.2", "2020.8.3"))
+	require.True(t, IsNewerVersion("2020.8.2", "2021.1.2"))
+	require.True(t, IsNewerVersion("2020.8.2", "2020.9.1"))
+	require.True(t, IsNewerVersion("2020.8.2", "2020.12.45"))
+	require.False(t, IsNewerVersion("2020.8.2", "2020.6.3"))
+	require.False(t, IsNewerVersion("DEV", "2020.8.5"))
+	require.False(t, IsNewerVersion("2020.8.2", "asdlkfjasdf"))
+	require.True(t, IsNewerVersion("3.0.1", "4.2.1"))
+}

cmd/cloudflared/updater/workers_update.go

@@ -0,0 +1,227 @@
+package updater
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"crypto/sha256"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"text/template"
+	"time"
+)
+
+// stop the service
+// rename cloudflared.exe to cloudflared.exe.old
+// rename cloudflared.exe.new to cloudflared.exe
+// delete cloudflared.exe.old
+// start the service
+// delete the batch file
+const windowsUpdateCommandTemplate = `@echo off
+sc stop cloudflared >nul 2>&1
+rename "{{.TargetPath}}" {{.OldName}}
+rename "{{.NewPath}}" {{.BinaryName}}
+del "{{.OldPath}}"
+sc start cloudflared >nul 2>&1
+del {{.BatchName}}`
+const batchFileName = "cfd_update.bat"
+
+// Prepare some data to insert into the template.
+type batchData struct {
+	TargetPath string
+	OldName    string
+	NewPath    string
+	OldPath    string
+	BinaryName string
+	BatchName  string
+}
+
+// WorkersVersion implements the Version interface.
+// It contains everything needed to preform a version upgrade
+type WorkersVersion struct {
+	downloadURL  string
+	checksum     string
+	version      string
+	targetPath   string
+	isCompressed bool
+}
+
+// NewWorkersVersion creates a new Version object. This is normally created by a WorkersService JSON checkin response
+// url is where to download the file
+// version is the version of this update
+// checksum is the expected checksum of the downloaded file
+// target path is where the file should be replace. Normally this the running cloudflared's path
+func NewWorkersVersion(url, version, checksum, targetPath string, isCompressed bool) Version {
+	return &WorkersVersion{downloadURL: url, version: version, checksum: checksum, targetPath: targetPath, isCompressed: isCompressed}
+}
+
+// Apply does the actual verification and update logic.
+// This includes signature and checksum validation,
+// replacing the binary, etc
+func (v *WorkersVersion) Apply() error {
+	newFilePath := fmt.Sprintf("%s.new", v.targetPath)
+	os.Remove(newFilePath) //remove any failed updates before download
+
+	// download the file
+	if err := download(v.downloadURL, newFilePath, v.isCompressed); err != nil {
+		return err
+	}
+
+	// check that the file is what is expected
+	if err := isValidChecksum(v.checksum, newFilePath); err != nil {
+		return err
+	}
+
+	oldFilePath := fmt.Sprintf("%s.old", v.targetPath)
+	// Windows requires more effort to self update, especially when it is running as a service:
+	// you have to stop the service (if running as one) in order to move/rename the binary
+	// but now the binary isn't running though, so an external process
+	// has to move the old binary out and the new one in then start the service
+	// the easiest way to do this is with a batch file (or with a DLL, but that gets ugly for a cross compiled binary like cloudflared)
+	// a batch file isn't ideal, but it is the simplest path forward for the constraints Windows creates
+	if runtime.GOOS == "windows" {
+		if err := writeBatchFile(v.targetPath, newFilePath, oldFilePath); err != nil {
+			return err
+		}
+		rootDir := filepath.Dir(v.targetPath)
+		batchPath := filepath.Join(rootDir, batchFileName)
+		return runWindowsBatch(batchPath)
+	}
+
+	// now move the current file out, move the new file in and delete the old file
+	if err := os.Rename(v.targetPath, oldFilePath); err != nil {
+		return err
+	}
+
+	if err := os.Rename(newFilePath, v.targetPath); err != nil {
+		//attempt rollback
+		os.Rename(oldFilePath, v.targetPath)
+		return err
+	}
+	os.Remove(oldFilePath)
+
+	return nil
+}
+
+// String returns the version number of this update/release (e.g. 2020.08.05)
+func (v *WorkersVersion) String() string {
+	return v.version
+}
+
+// download the file from the link in the json
+func download(url, filepath string, isCompressed bool) error {
+	client := &http.Client{
+		Timeout: time.Second * 5,
+	}
+	resp, err := client.Get(url)
+
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	var r io.Reader
+	r = resp.Body
+
+	// compressed macos binary, need to decompress
+	if isCompressed || isCompressedFile(url) {
+		// first the gzip reader
+		gr, err := gzip.NewReader(resp.Body)
+		if err != nil {
+			return err
+		}
+		defer gr.Close()
+
+		// now the tar
+		tr := tar.NewReader(gr)
+
+		// advance the reader pass the header, which will be the single binary file
+		tr.Next()
+
+		r = tr
+	}
+
+	out, err := os.OpenFile(filepath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0755)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, r)
+	return err
+}
+
+// isCompressedFile is a really simple file extension check to see if this is a macos tar and gzipped
+func isCompressedFile(urlstring string) bool {
+	if strings.HasSuffix(urlstring, ".tgz") {
+		return true
+	}
+
+	u, err := url.Parse(urlstring)
+	if err != nil {
+		return false
+	}
+	return strings.HasSuffix(u.Path, ".tgz")
+}
+
+// checks if the checksum in the json response matches the checksum of the file download
+func isValidChecksum(checksum, filePath string) error {
+	f, err := os.Open(filePath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return err
+	}
+
+	hash := fmt.Sprintf("%x", h.Sum(nil))
+
+	if checksum != hash {
+		return errors.New("checksum validation failed")
+	}
+	return nil
+}
+
+// writeBatchFile writes a batch file out to disk
+// see the dicussion on why it has to be done this way
+func writeBatchFile(targetPath string, newPath string, oldPath string) error {
+	os.Remove(batchFileName) //remove any failed updates before download
+	f, err := os.Create(batchFileName)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	cfdName := filepath.Base(targetPath)
+	oldName := filepath.Base(oldPath)
+
+	data := batchData{
+		TargetPath: targetPath,
+		OldName:    oldName,
+		NewPath:    newPath,
+		OldPath:    oldPath,
+		BinaryName: cfdName,
+		BatchName:  batchFileName,
+	}
+
+	t, err := template.New("batch").Parse(windowsUpdateCommandTemplate)
+	if err != nil {
+		return err
+	}
+	return t.Execute(f, data)
+}
+
+// run each OS command for windows
+func runWindowsBatch(batchFile string) error {
+	cmd := exec.Command("cmd", "/c", batchFile)
+	return cmd.Start()
+}

connection/rpc.go

@@ -2,48 +2,40 @@ package connection
 
 import (
 	"context"
-	"fmt"
-	"time"
+	"io"
 
 	rpc "zombiezen.com/go/capnproto2/rpc"
 
-	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
 	tunnelpogs "github.com/cloudflare/cloudflared/tunnelrpc/pogs"
 )
 
-// NewRPCClient creates and returns a new RPC client, which will communicate
+// NewTunnelRPCClient creates and returns a new RPC client, which will communicate
 // using a stream on the given muxer
-func NewRPCClient(
+func NewTunnelRPCClient(
 	ctx context.Context,
-	muxer *h2mux.Muxer,
+	stream io.ReadWriteCloser,
 	logger logger.Service,
-	openStreamTimeout time.Duration,
 ) (client tunnelpogs.TunnelServer_PogsClient, err error) {
-	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
-	defer openStreamCancel()
-	stream, err := muxer.OpenRPCStream(openStreamCtx)
-	if err != nil {
-		return
-	}
-
-	if !isRPCStreamResponse(stream.Headers) {
-		stream.Close()
-		err = fmt.Errorf("rpc: bad response headers: %v", stream.Headers)
-		return
-	}
-
 	conn := rpc.NewConn(
 		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
 		tunnelrpc.ConnLog(logger),
 	)
-	client = tunnelpogs.TunnelServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
+	registrationClient := tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
+	client = tunnelpogs.TunnelServer_PogsClient{RegistrationServer_PogsClient: registrationClient, Client: conn.Bootstrap(ctx), Conn: conn}
 	return client, nil
 }
 
-func isRPCStreamResponse(headers []h2mux.Header) bool {
-	return len(headers) == 1 &&
-		headers[0].Name == ":status" &&
-		headers[0].Value == "200"
+func NewRegistrationRPCClient(
+	ctx context.Context,
+	stream io.ReadWriteCloser,
+	logger logger.Service,
+) (client tunnelpogs.RegistrationServer_PogsClient, err error) {
+	conn := rpc.NewConn(
+		tunnelrpc.NewTransportLogger(logger, rpc.StreamTransport(stream)),
+		tunnelrpc.ConnLog(logger),
+	)
+	client = tunnelpogs.RegistrationServer_PogsClient{Client: conn.Bootstrap(ctx), Conn: conn}
+	return client, nil
 }

dev.Dockerfile

@@ -1,4 +1,9 @@
-FROM golang:1.12 as builder
+FROM golang:1.15.3 as builder
+ENV GO111MODULE=on \
+    CGO_ENABLED=0
 WORKDIR /go/src/github.com/cloudflare/cloudflared/
 RUN apt-get update
 COPY . .
+# compile cloudflared
+RUN make cloudflared
+RUN cp /go/src/github.com/cloudflare/cloudflared/cloudflared /usr/local/bin/

github_message.py

@@ -97,17 +97,18 @@ def main():
 
         msg = release.body
 
-        for filename in glob.glob(".artifacts/*.*"):
-            pkg_hash = get_sha256(filename)
+        for filepath in glob.glob("artifacts/*"):
+            pkg_hash = get_sha256(filepath)
             # add the sha256 of the new artifact to the release message body
-            msg = update_or_add_message(msg, filename, pkg_hash)
+            name = os.path.basename(filepath)
+            msg = update_or_add_message(msg, name, pkg_hash)
 
         if args.dry_run:
             logging.info("Skipping asset upload because of dry-run")
             return
 
         # update the release body text
-        release.update_release(version, version, msg)
+        release.update_release(args.release_version, msg)
 
     except Exception as e:
         logging.exception(e)

github_release.py

@@ -7,6 +7,9 @@ import argparse
 import logging
 import os
 import shutil
+import hashlib
+import requests
+import tarfile
 
 from github import Github, GithubException, UnknownObjectException
 
@@ -15,6 +18,37 @@ logging.basicConfig(format=FORMAT)
 
 CLOUDFLARED_REPO = os.environ.get("GITHUB_REPO", "cloudflare/cloudflared")
 GITHUB_CONFLICT_CODE = "already_exists"
+BASE_KV_URL = 'https://api.cloudflare.com/client/v4/accounts/'
+UPDATER_PREFIX = 'update'
+
+def get_sha256(filename):
+    """ get the sha256 of a file """
+    sha256_hash = hashlib.sha256()
+    with open(filename,"rb") as f:
+        for byte_block in iter(lambda: f.read(4096),b""):
+            sha256_hash.update(byte_block)
+        return sha256_hash.hexdigest()
+
+def send_hash(pkg_hash, name, version, account, namespace, api_token):
+    """ send the checksum of a file to workers kv """
+    key = '{0}_{1}_{2}'.format(UPDATER_PREFIX, version, name)
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": "Bearer " + api_token,
+    }
+    response = requests.put(
+            BASE_KV_URL + account + "/storage/kv/namespaces/" + namespace + "/values/" + key,
+            headers=headers,
+            data=pkg_hash
+    )
+
+    if response.status_code != 200:
+        jsonResponse = response.json()
+        errors = jsonResponse["errors"]
+        if len(errors) > 0:
+            raise Exception("failed to upload checksum: {0}", errors[0])
+
+
 
 def assert_tag_exists(repo, version):
     """ Raise exception if repo does not contain a tag matching version """
@@ -78,6 +112,15 @@ def parse_args():
     parser.add_argument(
         "--name", default=os.environ.get("ASSET_NAME"), help="Asset Name"
     )
+    parser.add_argument(
+        "--namespace-id", default=os.environ.get("KV_NAMESPACE"), help="workersKV namespace id"
+    )
+    parser.add_argument(
+        "--kv-account-id", default=os.environ.get("KV_ACCOUNT"), help="workersKV account id"
+    )
+    parser.add_argument(
+        "--kv-api-token", default=os.environ.get("KV_API_TOKEN"), help="workersKV API Token"
+    )
     parser.add_argument(
         "--dry-run", action="store_true", help="Do not create release or upload asset"
     )
@@ -99,6 +142,18 @@ def parse_args():
     if not args.api_key:
         logging.error("Missing API key")
         is_valid = False
+    
+    if not args.namespace_id:
+        logging.error("Missing KV namespace id")
+        is_valid = False
+
+    if not args.kv_account_id:
+        logging.error("Missing KV account id")
+        is_valid = False
+
+    if not args.kv_api_token:
+        logging.error("Missing KV API token")
+        is_valid = False
 
     if is_valid:
         return args
@@ -121,6 +176,23 @@ def main():
 
         release.upload_asset(args.path, name=args.name)
 
+        # check and extract if the file is a tar and gzipped file (as is the case with the macos builds)
+        binary_path = args.path
+        if binary_path.endswith("tgz"):
+            try:
+                shutil.rmtree('cfd') 
+            except OSError as e:
+                pass
+            zipfile = tarfile.open(binary_path, "r:gz")
+            zipfile.extractall('cfd') # specify which folder to extract to
+            zipfile.close()
+
+            binary_path = os.path.join(os.getcwd(), 'cfd', 'cloudflared') 
+
+        # send the sha256 (the checksum) to workers kv
+        pkg_hash = get_sha256(binary_path)
+        send_hash(pkg_hash, args.name, args.release_version, args.kv_account_id, args.namespace_id, args.kv_api_token)
+
         # create the artifacts directory if it doesn't exist 
         artifact_path = os.path.join(os.getcwd(), 'artifacts') 
         if not os.path.isdir(artifact_path):
@@ -128,9 +200,10 @@ def main():
 
         # copy the binary to the path
         copy_path = os.path.join(artifact_path, args.name)
-        shutil.copy(args.path, copy_path)
-
-
+        try:
+            shutil.copy(args.path, copy_path)
+       except shutil.SameFileError:
+            pass # the macOS release copy fails with being the same file (already in the artifacts directory). Catching to ignore.
 
     except Exception as e:
         logging.exception(e)

go.mod

@@ -1,22 +1,19 @@
 module github.com/cloudflare/cloudflared
 
-go 1.12
+go 1.15
 
 require (
-	github.com/BurntSushi/go-sumtype v0.0.0-20190304192233-fcb4a6205bdc // indirect
 	github.com/DATA-DOG/go-sqlmock v1.3.3
 	github.com/acmacalister/skittles v0.0.0-20160609003031-7423546701e1
-	github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4
+	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d
 	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
-	github.com/aws/aws-sdk-go v1.25.8
-	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/aws/aws-sdk-go v1.34.19
 	github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect
 	github.com/cloudflare/brotli-go v0.0.0-20191101163834-d34379f7ff93
-	github.com/cloudflare/cfssl v0.0.0-20141119014638-2f7f44e802e2 // indirect
 	github.com/cloudflare/golibs v0.0.0-20170913112048-333127dbecfc
-	github.com/coredns/coredns v1.2.0
+	github.com/coredns/coredns v1.7.0
 	github.com/coreos/go-oidc v0.0.0-20171002155002-a93f71fdfe73
-	github.com/coreos/go-systemd v0.0.0-20190620071333-e64a0ec8b42a
+	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/denisenkom/go-mssqldb v0.0.0-20191001013358-cfbb681360f0
 	github.com/equinox-io/equinox v1.2.0
 	github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
@@ -24,56 +21,48 @@ require (
 	github.com/facebookgo/grace v0.0.0-20180706040059-75cf19382434
 	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
 	github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
-	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
+	github.com/frankban/quicktest v1.10.0 // indirect
 	github.com/fsnotify/fsnotify v1.4.9
+	github.com/gdamore/tcell v1.3.0
 	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10
 	github.com/gliderlabs/ssh v0.0.0-20191009160644-63518b5243e0
-	github.com/go-sql-driver/mysql v1.4.1
+	github.com/go-sql-driver/mysql v1.5.0
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
-	github.com/google/uuid v1.1.1
+	github.com/google/go-cmp v0.5.2 // indirect
+	github.com/google/uuid v1.1.2
 	github.com/gorilla/mux v1.7.3
-	github.com/gorilla/websocket v1.4.0
-	github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 // indirect
+	github.com/gorilla/websocket v1.4.2
 	github.com/jmoiron/sqlx v1.2.0
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/kshvakov/clickhouse v1.3.11
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lib/pq v1.2.0
-	github.com/mattn/go-colorable v0.1.4 // indirect
-	github.com/mattn/go-isatty v0.0.10 // indirect
 	github.com/mattn/go-sqlite3 v1.11.0
-	github.com/mholt/caddy v0.0.0-20180807230124-d3b731e9255b // indirect
-	github.com/miekg/dns v1.1.27
+	github.com/miekg/dns v1.1.31
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/opentracing/opentracing-go v1.1.0 // indirect
-	github.com/philhofer/fwd v1.0.0 // indirect
-	github.com/pkg/errors v0.8.1
+	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
+	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
+	github.com/pkg/errors v0.9.1
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
-	github.com/prometheus/client_golang v1.0.0
-	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
-	github.com/prometheus/common v0.7.0 // indirect
-	github.com/prometheus/procfs v0.0.5 // indirect
+	github.com/prometheus/client_golang v1.7.1
+	github.com/prometheus/common v0.13.0 // indirect
 	github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5 // indirect
-	github.com/stretchr/testify v1.3.0
-	github.com/tinylib/msgp v1.1.0 // indirect
+	github.com/rivo/tview v0.0.0-20200712113419-c65badfc3d92
+	github.com/stretchr/testify v1.6.0
 	github.com/urfave/cli/v2 v2.2.0
 	github.com/xo/dburl v0.0.0-20191005012637-293c3298d6c0
-	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
-	golang.org/x/net v0.0.0-20200625001655-4c5254603344
-	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 // indirect
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae
-	google.golang.org/appengine v1.5.0 // indirect
-	google.golang.org/genproto v0.0.0-20191007204434-a023cd5227bd // indirect
-	google.golang.org/grpc v1.24.0 // indirect
+	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a
+	golang.org/x/net v0.0.0-20200904194848-62affa334b73
+	golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43 // indirect
+	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
+	golang.org/x/sys v0.0.0-20200909081042-eff7692f9009
+	google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d // indirect
+	google.golang.org/grpc v1.32.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/coreos/go-oidc.v2 v2.1.0
 	gopkg.in/square/go-jose.v2 v2.4.0 // indirect
-	gopkg.in/urfave/cli.v2 v2.0.0-20180128181224-d604b6ffeee8 // indirect
-	gopkg.in/yaml.v2 v2.2.4
+	gopkg.in/yaml.v2 v2.3.0
+	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 // indirect
 	zombiezen.com/go/capnproto2 v2.18.0+incompatible
 )
-
-// ../../go/pkg/mod/github.com/coredns/coredns@v1.2.0/plugin/metrics/metrics.go:40:49: too many arguments in call to prometheus.NewProcessCollector
-// have (int, string)
-// want (prometheus.ProcessCollectorOpts)
-replace github.com/prometheus/client_golang => github.com/prometheus/client_golang v0.9.0-pre1

h2mux/error.go

@@ -19,6 +19,7 @@ var (
 	ErrUnexpectedFrameType = MuxerProtocolError{"2001 unexpected frame type", http2.ErrCodeProtocol}
 	ErrUnknownStream       = MuxerProtocolError{"2002 unknown stream", http2.ErrCodeProtocol}
 	ErrInvalidStream       = MuxerProtocolError{"2003 invalid stream", http2.ErrCodeProtocol}
+	ErrNotRPCStream        = MuxerProtocolError{"2004 not RPC stream", http2.ErrCodeProtocol}
 
 	ErrStreamHeadersSent               = MuxerApplicationError{"3000 headers already sent"}
 	ErrStreamRequestConnectionClosed   = MuxerApplicationError{"3001 connection closed while opening stream"}

h2mux/h2mux.go

@@ -22,7 +22,7 @@ const (
 	defaultTimeout           time.Duration = 5 * time.Second
 	defaultRetries           uint64        = 5
 	defaultWriteBufferMaxLen int           = 1024 * 1024 // 1mb
-	writeBufferInitialSize   int           = 16 * 1024 // 16KB
+	writeBufferInitialSize   int           = 16 * 1024   // 16KB
 
 	SettingMuxerMagic http2.SettingID = 0x42db
 	MuxerMagicOrigin  uint32          = 0xa2e43c8b
@@ -441,11 +441,17 @@ func (m *Muxer) OpenStream(ctx context.Context, headers []Header, body io.Reader
 func (m *Muxer) OpenRPCStream(ctx context.Context) (*MuxedStream, error) {
 	stream := m.NewStream(RPCHeaders())
 	if err := m.MakeMuxedStreamRequest(ctx, NewMuxedStreamRequest(stream, nil)); err != nil {
+		stream.Close()
 		return nil, err
 	}
 	if err := m.AwaitResponseHeaders(ctx, stream); err != nil {
+		stream.Close()
 		return nil, err
 	}
+	if !IsRPCStreamResponse(stream) {
+		stream.Close()
+		return nil, ErrNotRPCStream
+	}
 	return stream, nil
 }
 
@@ -499,3 +505,10 @@ func (m *Muxer) abort() {
 func (m *Muxer) TimerRetries() uint64 {
 	return m.muxWriter.idleTimer.RetryCount()
 }
+
+func IsRPCStreamResponse(stream *MuxedStream) bool {
+	headers := stream.Headers
+	return len(headers) == 1 &&
+		headers[0].Name == ":status" &&
+		headers[0].Value == "200"
+}

ingress/ingress.go

@@ -0,0 +1,159 @@
+package ingress
+
+import (
+	"fmt"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+)
+
+var (
+	ErrNoIngressRules             = errors.New("No ingress rules were specified in the config file")
+	errLastRuleNotCatchAll        = errors.New("The last ingress rule must match all hostnames (i.e. it must be missing, or must be \"*\")")
+	errBadWildcard                = errors.New("Hostname patterns can have at most one wildcard character (\"*\") and it can only be used for subdomains, e.g. \"*.example.com\"")
+	ErrURLIncompatibleWithIngress = errors.New("You can't set the --url flag (or $TUNNEL_URL) when using multiple-origin ingress rules")
+)
+
+// Each rule route traffic from a hostname/path on the public
+// internet to the service running on the given URL.
+type Rule struct {
+	// Requests for this hostname will be proxied to this rule's service.
+	Hostname string
+
+	// Path is an optional regex that can specify path-driven ingress rules.
+	Path *regexp.Regexp
+
+	// A (probably local) address. Requests for a hostname which matches this
+	// rule's hostname pattern will be proxied to the service running on this
+	// address.
+	Service *url.URL
+}
+
+func (r Rule) MultiLineString() string {
+	var out strings.Builder
+	if r.Hostname != "" {
+		out.WriteString("\thostname: ")
+		out.WriteString(r.Hostname)
+		out.WriteRune('\n')
+	}
+	if r.Path != nil {
+		out.WriteString("\tpath: ")
+		out.WriteString(r.Path.String())
+		out.WriteRune('\n')
+	}
+	out.WriteString("\tservice: ")
+	out.WriteString(r.Service.String())
+	return out.String()
+}
+
+func (r *Rule) Matches(hostname, path string) bool {
+	hostMatch := r.Hostname == "" || r.Hostname == "*" || matchHost(r.Hostname, hostname)
+	pathMatch := r.Path == nil || r.Path.MatchString(path)
+	return hostMatch && pathMatch
+}
+
+// FindMatchingRule returns the index of the Ingress Rule which matches the given
+// hostname and path. This function assumes the last rule matches everything,
+// which is the case if the rules were instantiated via the ingress#Validate method
+func (ing Ingress) FindMatchingRule(hostname, path string) int {
+	for i, rule := range ing.Rules {
+		if rule.Matches(hostname, path) {
+			return i
+		}
+	}
+	return len(ing.Rules) - 1
+}
+
+func matchHost(ruleHost, reqHost string) bool {
+	if ruleHost == reqHost {
+		return true
+	}
+
+	// Validate hostnames that use wildcards at the start
+	if strings.HasPrefix(ruleHost, "*.") {
+		toMatch := strings.TrimPrefix(ruleHost, "*.")
+		return strings.HasSuffix(reqHost, toMatch)
+	}
+	return false
+}
+
+// Ingress maps eyeball requests to origins.
+type Ingress struct {
+	Rules []Rule
+}
+
+// IsEmpty checks if there are any ingress rules.
+func (ing Ingress) IsEmpty() bool {
+	return len(ing.Rules) == 0
+}
+
+func validate(ingress []config.UnvalidatedIngressRule) (Ingress, error) {
+	rules := make([]Rule, len(ingress))
+	for i, r := range ingress {
+		service, err := url.Parse(r.Service)
+		if err != nil {
+			return Ingress{}, err
+		}
+		if service.Scheme == "" || service.Hostname() == "" {
+			return Ingress{}, fmt.Errorf("The service %s must have a scheme and a hostname", r.Service)
+		}
+
+		if service.Path != "" {
+			return Ingress{}, fmt.Errorf("%s is an invalid address, ingress rules don't support proxying to a different path on the origin service. The path will be the same as the eyeball request's path.", r.Service)
+		}
+
+		// Ensure that there are no wildcards anywhere except the first character
+		// of the hostname.
+		if strings.LastIndex(r.Hostname, "*") > 0 {
+			return Ingress{}, errBadWildcard
+		}
+
+		// The last rule should catch all hostnames.
+		isCatchAllRule := (r.Hostname == "" || r.Hostname == "*") && r.Path == ""
+		isLastRule := i == len(ingress)-1
+		if isLastRule && !isCatchAllRule {
+			return Ingress{}, errLastRuleNotCatchAll
+		}
+		// ONLY the last rule should catch all hostnames.
+		if !isLastRule && isCatchAllRule {
+			return Ingress{}, errRuleShouldNotBeCatchAll{i: i, hostname: r.Hostname}
+		}
+
+		var pathRegex *regexp.Regexp
+		if r.Path != "" {
+			pathRegex, err = regexp.Compile(r.Path)
+			if err != nil {
+				return Ingress{}, errors.Wrapf(err, "Rule #%d has an invalid regex", i+1)
+			}
+		}
+
+		rules[i] = Rule{
+			Hostname: r.Hostname,
+			Service:  service,
+			Path:     pathRegex,
+		}
+	}
+	return Ingress{Rules: rules}, nil
+}
+
+type errRuleShouldNotBeCatchAll struct {
+	i        int
+	hostname string
+}
+
+func (e errRuleShouldNotBeCatchAll) Error() string {
+	return fmt.Sprintf("Rule #%d is matching the hostname '%s', but "+
+		"this will match every hostname, meaning the rules which follow it "+
+		"will never be triggered.", e.i+1, e.hostname)
+}
+
+func ParseIngress(conf *config.Configuration) (Ingress, error) {
+	if len(conf.Ingress) == 0 {
+		return Ingress{}, ErrNoIngressRules
+	}
+	return validate(conf.Ingress)
+}

ingress/ingress_test.go

@@ -0,0 +1,312 @@
+package ingress
+
+import (
+	"net/url"
+	"regexp"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v2"
+
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/config"
+)
+
+func Test_parseIngress(t *testing.T) {
+	localhost8000 := MustParseURL(t, "https://localhost:8000")
+	localhost8001 := MustParseURL(t, "https://localhost:8001")
+	type args struct {
+		rawYAML string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    Ingress
+		wantErr bool
+	}{
+		{
+			name:    "Empty file",
+			args:    args{rawYAML: ""},
+			wantErr: true,
+		},
+		{
+			name: "Multiple rules",
+			args: args{rawYAML: `
+ingress:
+ - hostname: tunnel1.example.com
+   service: https://localhost:8000
+ - hostname: "*"
+   service: https://localhost:8001
+`},
+			want: Ingress{Rules: []Rule{
+				{
+					Hostname: "tunnel1.example.com",
+					Service:  localhost8000,
+				},
+				{
+					Hostname: "*",
+					Service:  localhost8001,
+				},
+			}},
+		},
+		{
+			name: "Extra keys",
+			args: args{rawYAML: `
+ingress:
+ - hostname: "*"
+   service: https://localhost:8000
+extraKey: extraValue
+`},
+			want: Ingress{Rules: []Rule{
+				{
+					Hostname: "*",
+					Service:  localhost8000,
+				},
+			}},
+		},
+		{
+			name: "Hostname can be omitted",
+			args: args{rawYAML: `
+ingress:
+ - service: https://localhost:8000
+`},
+			want: Ingress{Rules: []Rule{
+				{
+					Service: localhost8000,
+				},
+			}},
+		},
+		{
+			name: "Invalid service",
+			args: args{rawYAML: `
+ingress:
+ - hostname: "*"
+   service: https://local host:8000
+`},
+			wantErr: true,
+		},
+		{
+			name: "Last rule isn't catchall",
+			args: args{rawYAML: `
+ingress:
+ - hostname: example.com
+   service: https://localhost:8000
+`},
+			wantErr: true,
+		},
+		{
+			name: "First rule is catchall",
+			args: args{rawYAML: `
+ingress:
+ - service: https://localhost:8000
+ - hostname: example.com
+   service: https://localhost:8000
+`},
+			wantErr: true,
+		},
+		{
+			name: "Catch-all rule can't have a path",
+			args: args{rawYAML: `
+ingress:
+ - service: https://localhost:8001
+   path: /subpath1/(.*)/subpath2
+`},
+			wantErr: true,
+		},
+		{
+			name: "Invalid regex",
+			args: args{rawYAML: `
+ingress:
+ - hostname: example.com
+   service: https://localhost:8000
+   path: "*/subpath2"
+ - service: https://localhost:8001
+`},
+			wantErr: true,
+		},
+		{
+			name: "Service must have a scheme",
+			args: args{rawYAML: `
+ingress:
+ - service: localhost:8000
+`},
+			wantErr: true,
+		},
+		{
+			name: "Wildcard not at start",
+			args: args{rawYAML: `
+ingress:
+ - hostname: "test.*.example.com"
+   service: https://localhost:8000
+`},
+			wantErr: true,
+		},
+		{
+			name: "Service can't have a path",
+			args: args{rawYAML: `
+ingress:
+ - service: https://localhost:8000/static/
+`},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ParseIngress(MustReadIngress(tt.args.rawYAML))
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseIngress() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func MustParseURL(t *testing.T, rawURL string) *url.URL {
+	u, err := url.Parse(rawURL)
+	require.NoError(t, err)
+	return u
+}
+
+func Test_rule_matches(t *testing.T) {
+	type fields struct {
+		Hostname string
+		Path     *regexp.Regexp
+		Service  *url.URL
+	}
+	type args struct {
+		requestURL *url.URL
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		args   args
+		want   bool
+	}{
+		{
+			name: "Just hostname, pass",
+			fields: fields{
+				Hostname: "example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Entire hostname is wildcard, should match everything",
+			fields: fields{
+				Hostname: "*",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Just hostname, fail",
+			fields: fields{
+				Hostname: "example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://foo.bar"),
+			},
+			want: false,
+		},
+		{
+			name: "Just wildcard hostname, pass",
+			fields: fields{
+				Hostname: "*.example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://adam.example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Just wildcard hostname, fail",
+			fields: fields{
+				Hostname: "*.example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://tunnel.com"),
+			},
+			want: false,
+		},
+		{
+			name: "Just wildcard outside of subdomain in hostname, fail",
+			fields: fields{
+				Hostname: "*example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://www.example.com"),
+			},
+			want: false,
+		},
+		{
+			name: "Wildcard over multiple subdomains",
+			fields: fields{
+				Hostname: "*.example.com",
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://adam.chalmers.example.com"),
+			},
+			want: true,
+		},
+		{
+			name: "Hostname and path",
+			fields: fields{
+				Hostname: "*.example.com",
+				Path:     regexp.MustCompile("/static/.*\\.html"),
+			},
+			args: args{
+				requestURL: MustParseURL(t, "https://www.example.com/static/index.html"),
+			},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := Rule{
+				Hostname: tt.fields.Hostname,
+				Path:     tt.fields.Path,
+				Service:  tt.fields.Service,
+			}
+			u := tt.args.requestURL
+			if got := r.Matches(u.Hostname(), u.Path); got != tt.want {
+				t.Errorf("rule.matches() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func BenchmarkFindMatch(b *testing.B) {
+	rulesYAML := `
+ingress:
+ - hostname: tunnel1.example.com
+   service: https://localhost:8000
+ - hostname: tunnel2.example.com
+   service: https://localhost:8001
+ - hostname: "*"
+   service: https://localhost:8002
+`
+
+	ing, err := ParseIngress(MustReadIngress(rulesYAML))
+	if err != nil {
+		b.Error(err)
+	}
+	for n := 0; n < b.N; n++ {
+		ing.FindMatchingRule("tunnel1.example.com", "")
+		ing.FindMatchingRule("tunnel2.example.com", "")
+		ing.FindMatchingRule("tunnel3.example.com", "")
+	}
+}
+
+func MustReadIngress(s string) *config.Configuration {
+	var conf config.Configuration
+	err := yaml.Unmarshal([]byte(s), &conf)
+	if err != nil {
+		panic(err)
+	}
+	return &conf
+}

jet.yaml

@@ -0,0 +1,9 @@
+---
+basename: "cloudflared"
+comment: "Cloudflare Argo Tunnel"
+copyright: "Cloudflare, Inc"
+arch: "x86"
+abi: "64"
+files:
+    - source: cloudflared
+      destination: /var/db/scripts/jet/cloudflared

logger/create.go

@@ -98,31 +98,32 @@ func Parse(opts ...Option) (*Options, error) {
 
 // New setups a new logger based on the options.
 // The default behavior is to write to standard out
-func New(opts ...Option) (Service, error) {
-	config, err := Parse(opts...)
+func New(opts ...Option) (*OutputWriter, error) {
+	options, err := Parse(opts...)
 
 	if err != nil {
 		return nil, err
 	}
 
 	l := NewOutputWriter(SharedWriteManager)
-	if config.logFileDirectory != "" {
-		l.Add(NewFileRollingWriter(SanitizeLogPath(config.logFileDirectory),
+	if options.logFileDirectory != "" {
+		l.Add(NewFileRollingWriter(SanitizeLogPath(options.logFileDirectory),
 			"cloudflared",
-			int64(config.maxFileSize),
-			config.maxFileCount),
-			NewDefaultFormatter(time.RFC3339Nano), config.supportedFileLevels...)
+			int64(options.maxFileSize),
+			options.maxFileCount),
+			NewDefaultFormatter(time.RFC3339Nano), options.supportedFileLevels...)
 	}
 
-	if !config.terminalOutputDisabled {
+	if !options.terminalOutputDisabled {
 		terminalFormatter := NewTerminalFormatter(time.RFC3339)
 
-		if len(config.supportedTerminalLevels) == 0 {
+		if len(options.supportedTerminalLevels) == 0 {
 			l.Add(os.Stderr, terminalFormatter, InfoLevel, ErrorLevel, FatalLevel)
 		} else {
-			l.Add(os.Stderr, terminalFormatter, config.supportedTerminalLevels...)
+			l.Add(os.Stderr, terminalFormatter, options.supportedTerminalLevels...)
 		}
 	}
+
 	return l, nil
 }
 

logger/formatter.go

@@ -63,6 +63,12 @@ type TerminalFormatter struct {
 	supportsColor bool
 }
 
+// UIFormatter is used for streaming logs to UI
+type UIFormatter struct {
+	format        string
+	supportsColor bool
+}
+
 // NewTerminalFormatter creates a Terminal formatter for colored output
 // format is the time format to use for timestamp formatting
 func NewTerminalFormatter(format string) Formatter {
@@ -73,10 +79,39 @@ func NewTerminalFormatter(format string) Formatter {
 	}
 }
 
+func NewUIFormatter(format string) Formatter {
+	supportsColor := (runtime.GOOS != "windows")
+	return &UIFormatter{
+		format:        format,
+		supportsColor: supportsColor,
+	}
+}
+
+// Timestamp uses formatting that is tview-specific for UI
+func (f *UIFormatter) Timestamp(l Level, d time.Time) string {
+	t := ""
+	dateStr := "[" + d.Format(f.format) + "] "
+	switch l {
+	case InfoLevel:
+		t = "[#00ffff]INFO[white]"
+	case ErrorLevel:
+		t = "[red]ERROR[white]"
+	case DebugLevel:
+		t = "[yellow]DEBUG[white]"
+	case FatalLevel:
+		t = "[red]FATAL[white]"
+	}
+	return t + dateStr
+}
+
+func (f *UIFormatter) Content(l Level, c string) string {
+	return c
+}
+
 // Timestamp returns the log level with a matching color to the log type
 func (f *TerminalFormatter) Timestamp(l Level, d time.Time) string {
 	t := ""
-	dateStr := "[" + d.Format(f.format) + "] " 
+	dateStr := "[" + d.Format(f.format) + "] "
 	switch l {
 	case InfoLevel:
 		t = f.output("INFO", skittles.Cyan)

logger/output.go

@@ -31,6 +31,8 @@ type Service interface {
 	Infof(format string, args ...interface{})
 	Debugf(format string, args ...interface{})
 	Fatalf(format string, args ...interface{})
+
+	Add(writer io.Writer, formatter Formatter, levels ...Level)
 }
 
 type sourceGroup struct {
@@ -59,7 +61,7 @@ type OutputWriter struct {
 	minLevel   Level
 }
 
-// NewOutputWriter create a new logger
+// NewOutputWriter creates a new logger
 func NewOutputWriter(syncWriter OutputManager) *OutputWriter {
 	return &OutputWriter{
 		syncWriter: syncWriter,

origin/reconnect.go

@@ -7,7 +7,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/h2mux"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
@@ -164,18 +163,17 @@ func ReconnectTunnel(
 	}
 
 	config.TransportLogger.Debug("initiating RPC stream to reconnect")
-	tunnelServer, err := connection.NewRPCClient(ctx, muxer, config.TransportLogger, openStreamTimeout)
+	rpcClient, err := newTunnelRPCClient(ctx, muxer, config, reconnect)
 	if err != nil {
-		// RPC stream open error
-		return newClientRegisterTunnelError(err, config.Metrics.rpcFail, reconnect)
+		return err
 	}
-	defer tunnelServer.Close()
+	defer rpcClient.Close()
 	// Request server info without blocking tunnel registration; must use capnp library directly.
-	serverInfoPromise := tunnelrpc.TunnelServer{Client: tunnelServer.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+	serverInfoPromise := tunnelrpc.TunnelServer{Client: rpcClient.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
 		return nil
 	})
-	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger)
-	registration := tunnelServer.ReconnectTunnel(
+	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger, config.TunnelEventChan)
+	registration := rpcClient.ReconnectTunnel(
 		ctx,
 		token,
 		eventDigest,

origin/supervisor.go

@@ -323,16 +323,16 @@ func (s *Supervisor) authenticate(ctx context.Context, numPreviousAttempts int)
 		<-muxer.Shutdown()
 	}()
 
-	tunnelServer, err := connection.NewRPCClient(ctx, muxer, s.logger, openStreamTimeout)
+	rpcClient, err := newTunnelRPCClient(ctx, muxer, s.config, authenticate)
 	if err != nil {
 		return nil, err
 	}
-	defer tunnelServer.Close()
+	defer rpcClient.Close()
 
 	const arbitraryConnectionID = uint8(0)
 	registrationOptions := s.config.RegistrationOptions(arbitraryConnectionID, edgeConn.LocalAddr().String(), s.cloudflaredUUID)
 	registrationOptions.NumPreviousAttempts = uint8(numPreviousAttempts)
-	authResponse, err := tunnelServer.Authenticate(
+	authResponse, err := rpcClient.Authenticate(
 		ctx,
 		s.config.OriginCert,
 		s.config.Hostname,

origin/tunnel.go

@@ -21,8 +21,10 @@ import (
 
 	"github.com/cloudflare/cloudflared/buffer"
 	"github.com/cloudflare/cloudflared/cmd/cloudflared/buildinfo"
+	"github.com/cloudflare/cloudflared/cmd/cloudflared/ui"
 	"github.com/cloudflare/cloudflared/connection"
 	"github.com/cloudflare/cloudflared/h2mux"
+	"github.com/cloudflare/cloudflared/ingress"
 	"github.com/cloudflare/cloudflared/logger"
 	"github.com/cloudflare/cloudflared/signal"
 	"github.com/cloudflare/cloudflared/tunnelrpc"
@@ -43,11 +45,13 @@ const (
 	FeatureQuickReconnects   = "quick_reconnects"
 )
 
-type registerRPCName string
+type rpcName string
 
 const (
-	register  registerRPCName = "register"
-	reconnect registerRPCName = "reconnect"
+	register     rpcName = "register"
+	reconnect    rpcName = "reconnect"
+	unregister   rpcName = "unregister"
+	authenticate rpcName = " authenticate"
 )
 
 type TunnelConfig struct {
@@ -88,6 +92,8 @@ type TunnelConfig struct {
 
 	NamedTunnel     *NamedTunnelConfig
 	ReplaceExisting bool
+	TunnelEventChan chan<- ui.TunnelEvent
+	IngressRules    ingress.Ingress
 }
 
 type dupConnRegisterTunnelError struct{}
@@ -119,7 +125,7 @@ type clientRegisterTunnelError struct {
 	cause error
 }
 
-func newClientRegisterTunnelError(cause error, counter *prometheus.CounterVec, name registerRPCName) clientRegisterTunnelError {
+func newRPCError(cause error, counter *prometheus.CounterVec, name rpcName) clientRegisterTunnelError {
 	counter.WithLabelValues(cause.Error(), string(name)).Inc()
 	return clientRegisterTunnelError{cause: cause}
 }
@@ -184,6 +190,10 @@ func (c *TunnelConfig) SupportedFeatures() []string {
 	return features
 }
 
+func (c *TunnelConfig) IsTrialTunnel() bool {
+	return c.Hostname == ""
+}
+
 type NamedTunnelConfig struct {
 	Auth   pogs.TunnelAuth
 	ID     uuid.UUID
@@ -234,6 +244,10 @@ func ServeTunnelLoop(ctx context.Context,
 		)
 		if recoverable {
 			if duration, ok := backoff.GetBackoffDuration(ctx); ok {
+				if config.TunnelEventChan != nil {
+					config.TunnelEventChan <- ui.TunnelEvent{Index: connectionIndex, EventType: ui.Reconnecting}
+				}
+
 				config.Logger.Infof("Retrying connection %d in %s seconds", connectionIndex, duration)
 				backoff.Backoff(ctx)
 				continue
@@ -268,6 +282,13 @@ func ServeTunnel(
 		}
 	}()
 
+	// If launch-ui flag is set, send disconnect msg
+	if config.TunnelEventChan != nil {
+		defer func() {
+			config.TunnelEventChan <- ui.TunnelEvent{Index: connectionIndex, EventType: ui.Disconnected}
+		}()
+	}
+
 	connectionTag := uint8ToString(connectionIndex)
 
 	// Returns error from parsing the origin URL or handshake errors
@@ -320,7 +341,7 @@ func ServeTunnel(
 					if config.NamedTunnel != nil {
 						_ = UnregisterConnection(ctx, handler.muxer, config)
 					} else {
-						_ = UnregisterTunnel(handler.muxer, config.GracePeriod, config.TransportLogger)
+						_ = UnregisterTunnel(handler.muxer, config)
 					}
 				}
 				handler.muxer.Shutdown()
@@ -400,14 +421,13 @@ func RegisterConnection(
 	const registerConnection = "registerConnection"
 
 	config.TransportLogger.Debug("initiating RPC stream for RegisterConnection")
-	rpc, err := connection.NewRPCClient(ctx, muxer, config.TransportLogger, openStreamTimeout)
+	rpcClient, err := newTunnelRPCClient(ctx, muxer, config, registerConnection)
 	if err != nil {
-		// RPC stream open error
-		return newClientRegisterTunnelError(err, config.Metrics.rpcFail, registerConnection)
+		return err
 	}
-	defer rpc.Close()
+	defer rpcClient.Close()
 
-	conn, err := rpc.RegisterConnection(
+	conn, err := rpcClient.RegisterConnection(
 		ctx,
 		config.NamedTunnel.Auth,
 		config.NamedTunnel.ID,
@@ -426,6 +446,11 @@ func RegisterConnection(
 	config.Metrics.regSuccess.WithLabelValues(registerConnection).Inc()
 	config.Logger.Infof("Connection %d registered with %s using ID %s", connectionIndex, conn.Location, conn.UUID)
 
+	// If launch-ui flag is set, send connect msg
+	if config.TunnelEventChan != nil {
+		config.TunnelEventChan <- ui.TunnelEvent{Index: connectionIndex, EventType: ui.Connected, Location: conn.Location}
+	}
+
 	return nil
 }
 
@@ -448,14 +473,14 @@ func UnregisterConnection(
 	config *TunnelConfig,
 ) error {
 	config.TransportLogger.Debug("initiating RPC stream for UnregisterConnection")
-	rpc, err := connection.NewRPCClient(ctx, muxer, config.TransportLogger, openStreamTimeout)
+	rpcClient, err := newTunnelRPCClient(ctx, muxer, config, register)
 	if err != nil {
 		// RPC stream open error
-		return newClientRegisterTunnelError(err, config.Metrics.rpcFail, register)
+		return err
 	}
-	defer rpc.Close()
+	defer rpcClient.Close()
 
-	return rpc.UnregisterConnection(ctx)
+	return rpcClient.UnregisterConnection(ctx)
 }
 
 func RegisterTunnel(
@@ -469,18 +494,21 @@ func RegisterTunnel(
 	uuid uuid.UUID,
 ) error {
 	config.TransportLogger.Debug("initiating RPC stream to register")
-	tunnelServer, err := connection.NewRPCClient(ctx, muxer, config.TransportLogger, openStreamTimeout)
-	if err != nil {
-		// RPC stream open error
-		return newClientRegisterTunnelError(err, config.Metrics.rpcFail, register)
+	if config.TunnelEventChan != nil {
+		config.TunnelEventChan <- ui.TunnelEvent{EventType: ui.RegisteringTunnel}
 	}
-	defer tunnelServer.Close()
+
+	rpcClient, err := newTunnelRPCClient(ctx, muxer, config, register)
+	if err != nil {
+		return err
+	}
+	defer rpcClient.Close()
 	// Request server info without blocking tunnel registration; must use capnp library directly.
-	serverInfoPromise := tunnelrpc.TunnelServer{Client: tunnelServer.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
+	serverInfoPromise := tunnelrpc.TunnelServer{Client: rpcClient.Client}.GetServerInfo(ctx, func(tunnelrpc.TunnelServer_getServerInfo_Params) error {
 		return nil
 	})
-	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger)
-	registration := tunnelServer.RegisterTunnel(
+	LogServerInfo(serverInfoPromise.Result(), connectionID, config.Metrics, logger, config.TunnelEventChan)
+	registration := rpcClient.RegisterTunnel(
 		ctx,
 		config.OriginCert,
 		config.Hostname,
@@ -490,6 +518,11 @@ func RegisterTunnel(
 		// RegisterTunnel RPC failure
 		return processRegisterTunnelError(registrationErr, config.Metrics, register)
 	}
+
+	// Send free tunnel URL to UI
+	if config.TunnelEventChan != nil {
+		config.TunnelEventChan <- ui.TunnelEvent{EventType: ui.SetUrl, Url: registration.Url}
+	}
 	credentialManager.SetEventDigest(connectionID, registration.EventDigest)
 	return processRegistrationSuccess(config, logger, connectionID, registration, register, credentialManager)
 }
@@ -499,7 +532,7 @@ func processRegistrationSuccess(
 	logger logger.Service,
 	connectionID uint8,
 	registration *tunnelpogs.TunnelRegistration,
-	name registerRPCName,
+	name rpcName,
 	credentialManager *reconnectCredentialManager,
 ) error {
 	for _, logLine := range registration.LogLines {
@@ -511,15 +544,17 @@ func processRegistrationSuccess(
 		logger.Infof("Each HA connection's tunnel IDs: %v", config.Metrics.tunnelsHA.String())
 	}
 
-	// Print out the user's trial zone URL in a nice box (if they requested and got one)
-	if isTrialTunnel := config.Hostname == ""; isTrialTunnel {
-		if url, err := url.Parse(registration.Url); err == nil {
-			for _, line := range asciiBox(trialZoneMsg(url.String()), 2) {
-				logger.Info(line)
+	// Print out the user's trial zone URL in a nice box (if they requested and got one and UI flag is not set)
+	if config.TunnelEventChan == nil {
+		if config.IsTrialTunnel() {
+			if registrationURL, err := url.Parse(registration.Url); err == nil {
+				for _, line := range asciiBox(trialZoneMsg(registrationURL.String()), 2) {
+					logger.Info(line)
+				}
+			} else {
+				logger.Error("Failed to connect tunnel, please try again.")
+				return fmt.Errorf("empty URL in response from Cloudflare edge")
 			}
-		} else {
-			logger.Error("Failed to connect tunnel, please try again.")
-			return fmt.Errorf("empty URL in response from Cloudflare edge")
 		}
 	}
 
@@ -531,7 +566,7 @@ func processRegistrationSuccess(
 	return nil
 }
 
-func processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, metrics *TunnelMetrics, name registerRPCName) error {
+func processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, metrics *TunnelMetrics, name rpcName) error {
 	if err.Error() == DuplicateConnectionError {
 		metrics.regFail.WithLabelValues("dup_edge_conn", string(name)).Inc()
 		return errDuplicationConnection
@@ -543,18 +578,18 @@ func processRegisterTunnelError(err tunnelpogs.TunnelRegistrationError, metrics
 	}
 }
 
-func UnregisterTunnel(muxer *h2mux.Muxer, gracePeriod time.Duration, logger logger.Service) error {
-	logger.Debug("initiating RPC stream to unregister")
+func UnregisterTunnel(muxer *h2mux.Muxer, config *TunnelConfig) error {
+	config.TransportLogger.Debug("initiating RPC stream to unregister")
 	ctx := context.Background()
-	tunnelServer, err := connection.NewRPCClient(ctx, muxer, logger, openStreamTimeout)
+	rpcClient, err := newTunnelRPCClient(ctx, muxer, config, unregister)
 	if err != nil {
 		// RPC stream open error
 		return err
 	}
-	defer tunnelServer.Close()
+	defer rpcClient.Close()
 
 	// gracePeriod is encoded in int64 using capnproto
-	return tunnelServer.UnregisterTunnel(ctx, gracePeriod.Nanoseconds())
+	return rpcClient.UnregisterTunnel(ctx, config.GracePeriod.Nanoseconds())
 }
 
 func LogServerInfo(
@@ -562,6 +597,7 @@ func LogServerInfo(
 	connectionID uint8,
 	metrics *TunnelMetrics,
 	logger logger.Service,
+	tunnelEventChan chan<- ui.TunnelEvent,
 ) {
 	serverInfoMessage, err := promise.Struct()
 	if err != nil {
@@ -573,12 +609,17 @@ func LogServerInfo(
 		logger.Errorf("Failed to retrieve server information: %s", err)
 		return
 	}
+	// If launch-ui flag is set, send connect msg
+	if tunnelEventChan != nil {
+		tunnelEventChan <- ui.TunnelEvent{Index: connectionID, EventType: ui.Connected, Location: serverInfo.LocationName}
+	}
 	logger.Infof("Connected to %s", serverInfo.LocationName)
 	metrics.registerServerLocation(uint8ToString(connectionID), serverInfo.LocationName)
 }
 
 type TunnelHandler struct {
 	originUrl      string
+	ingressRules   ingress.Ingress
 	httpHostHeader string
 	muxer          *h2mux.Muxer
 	httpClient     http.RoundTripper
@@ -600,12 +641,20 @@ func NewTunnelHandler(ctx context.Context,
 	connectionID uint8,
 	bufferPool *buffer.Pool,
 ) (*TunnelHandler, string, error) {
-	originURL, err := validation.ValidateUrl(config.OriginUrl)
-	if err != nil {
-		return nil, "", fmt.Errorf("unable to parse origin URL %#v", originURL)
+
+	// Check single-origin config
+	var originURL string
+	var err error
+	if config.IngressRules.IsEmpty() {
+		originURL, err = validation.ValidateUrl(config.OriginUrl)
+		if err != nil {
+			return nil, "", fmt.Errorf("unable to parse origin URL %#v", originURL)
+		}
 	}
+
 	h := &TunnelHandler{
 		originUrl:         originURL,
+		ingressRules:      config.IngressRules,
 		httpHostHeader:    config.HTTPHostHeader,
 		httpClient:        config.HTTPTransport,
 		tlsConfig:         config.ClientTlsConfig,
@@ -678,6 +727,12 @@ func (h *TunnelHandler) createRequest(stream *h2mux.MuxedStream) (*http.Request,
 		return nil, errors.Wrap(err, "invalid request received")
 	}
 	h.AppendTagHeaders(req)
+	if !h.ingressRules.IsEmpty() {
+		ruleNumber := h.ingressRules.FindMatchingRule(req.Host, req.URL.Path)
+		destination := h.ingressRules.Rules[ruleNumber].Service
+		req.URL.Host = destination.Host
+		req.URL.Scheme = destination.Scheme
+	}
 	return req, nil
 }
 
@@ -872,3 +927,18 @@ func findCfRayHeader(h1 *http.Request) string {
 func isLBProbeRequest(req *http.Request) bool {
 	return strings.HasPrefix(req.UserAgent(), lbProbeUserAgentPrefix)
 }
+
+func newTunnelRPCClient(ctx context.Context, muxer *h2mux.Muxer, config *TunnelConfig, rpcName rpcName) (tunnelpogs.TunnelServer_PogsClient, error) {
+	openStreamCtx, openStreamCancel := context.WithTimeout(ctx, openStreamTimeout)
+	defer openStreamCancel()
+	stream, err := muxer.OpenRPCStream(openStreamCtx)
+	if err != nil {
+		return tunnelpogs.TunnelServer_PogsClient{}, err
+	}
+	rpcClient, err := connection.NewTunnelRPCClient(ctx, stream, config.TransportLogger)
+	if err != nil {
+		// RPC stream open error
+		return tunnelpogs.TunnelServer_PogsClient{}, newRPCError(err, config.Metrics.rpcFail, rpcName)
+	}
+	return rpcClient, nil
+}

tunneldns/metrics.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	"github.com/coredns/coredns/plugin"
+	"github.com/coredns/coredns/plugin/metrics"
 	"github.com/coredns/coredns/plugin/metrics/vars"
 	"github.com/coredns/coredns/plugin/pkg/dnstest"
 	"github.com/coredns/coredns/plugin/pkg/rcode"
@@ -27,7 +28,6 @@ func NewMetricsPlugin(next plugin.Handler) *MetricsPlugin {
 		prometheus.MustRegister(vars.RequestDuration)
 		prometheus.MustRegister(vars.RequestSize)
 		prometheus.MustRegister(vars.RequestDo)
-		prometheus.MustRegister(vars.RequestType)
 		prometheus.MustRegister(vars.ResponseSize)
 		prometheus.MustRegister(vars.ResponseRcode)
 	})
@@ -42,7 +42,8 @@ func (p MetricsPlugin) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dn
 	status, err := plugin.NextOrFailure(p.Name(), p.Next, ctx, rw, r)
 
 	// Update built-in metrics
-	vars.Report(ctx, state, ".", rcode.ToString(rw.Rcode), rw.Len, rw.Start)
+	server := metrics.WithServer(ctx)
+	vars.Report(server, state, ".", rcode.ToString(rw.Rcode), rw.Len, rw.Start)
 
 	return status, err
 }

tunneldns/tunnel.go

@@ -27,6 +27,48 @@ type Listener struct {
 	logger logger.Service
 }
 
+func Command(hidden bool) *cli.Command {
+	return &cli.Command{
+		Name:   "proxy-dns",
+		Action: cliutil.ErrorHandler(Run),
+		Usage:  "Run a DNS over HTTPS proxy server.",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:    "metrics",
+				Value:   "localhost:",
+				Usage:   "Listen address for metrics reporting.",
+				EnvVars: []string{"TUNNEL_METRICS"},
+			},
+			&cli.StringFlag{
+				Name:    "address",
+				Usage:   "Listen address for the DNS over HTTPS proxy server.",
+				Value:   "localhost",
+				EnvVars: []string{"TUNNEL_DNS_ADDRESS"},
+			},
+			&cli.IntFlag{
+				Name:    "port",
+				Usage:   "Listen on given port for the DNS over HTTPS proxy server.",
+				Value:   53,
+				EnvVars: []string{"TUNNEL_DNS_PORT"},
+			},
+			&cli.StringSliceFlag{
+				Name:    "upstream",
+				Usage:   "Upstream endpoint URL, you can specify multiple endpoints for redundancy.",
+				Value:   cli.NewStringSlice("https://1.1.1.1/dns-query", "https://1.0.0.1/dns-query"),
+				EnvVars: []string{"TUNNEL_DNS_UPSTREAM"},
+			},
+			&cli.StringSliceFlag{
+				Name:    "bootstrap",
+				Usage:   "bootstrap endpoint URL, you can specify multiple endpoints for redundancy.",
+				Value:   cli.NewStringSlice("https://162.159.36.1/dns-query", "https://162.159.46.1/dns-query", "https://[2606:4700:4700::1111]/dns-query", "https://[2606:4700:4700::1001]/dns-query"),
+				EnvVars: []string{"TUNNEL_DNS_BOOTSTRAP"},
+			},
+		},
+		ArgsUsage: " ", // can't be the empty string or we get the default output
+		Hidden:    hidden,
+	}
+}
+
 // Run implements a foreground runner
 func Run(c *cli.Context) error {
 	logDirectory, logLevel := config.FindLogSettings()

tunnelrpc/pogs/connectionrpc.go

@@ -7,7 +7,9 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	capnp "zombiezen.com/go/capnproto2"
 	"zombiezen.com/go/capnproto2/pogs"
+	"zombiezen.com/go/capnproto2/rpc"
 	"zombiezen.com/go/capnproto2/server"
 
 	"github.com/cloudflare/cloudflared/tunnelrpc"
@@ -18,6 +20,156 @@ type RegistrationServer interface {
 	UnregisterConnection(ctx context.Context)
 }
 
+type RegistrationServer_PogsImpl struct {
+	impl RegistrationServer
+}
+
+func RegistrationServer_ServerToClient(s RegistrationServer) tunnelrpc.RegistrationServer {
+	return tunnelrpc.RegistrationServer_ServerToClient(RegistrationServer_PogsImpl{s})
+}
+
+func (i RegistrationServer_PogsImpl) RegisterConnection(p tunnelrpc.RegistrationServer_registerConnection) error {
+	server.Ack(p.Options)
+
+	auth, err := p.Params.Auth()
+	if err != nil {
+		return err
+	}
+	var pogsAuth TunnelAuth
+	err = pogsAuth.UnmarshalCapnproto(auth)
+	if err != nil {
+		return err
+	}
+	uuidBytes, err := p.Params.TunnelId()
+	if err != nil {
+		return err
+	}
+	tunnelID, err := uuid.FromBytes(uuidBytes)
+	if err != nil {
+		return err
+	}
+	connIndex := p.Params.ConnIndex()
+	options, err := p.Params.Options()
+	if err != nil {
+		return err
+	}
+	var pogsOptions ConnectionOptions
+	err = pogsOptions.UnmarshalCapnproto(options)
+	if err != nil {
+		return err
+	}
+
+	connDetails, callError := i.impl.RegisterConnection(p.Ctx, pogsAuth, tunnelID, connIndex, &pogsOptions)
+
+	resp, err := p.Results.NewResult()
+	if err != nil {
+		return err
+	}
+
+	if callError != nil {
+		if connError, err := resp.Result().NewError(); err != nil {
+			return err
+		} else {
+			return MarshalError(connError, callError)
+		}
+	}
+
+	if details, err := resp.Result().NewConnectionDetails(); err != nil {
+		return err
+	} else {
+		return connDetails.MarshalCapnproto(details)
+	}
+}
+
+func (i RegistrationServer_PogsImpl) UnregisterConnection(p tunnelrpc.RegistrationServer_unregisterConnection) error {
+	server.Ack(p.Options)
+
+	i.impl.UnregisterConnection(p.Ctx)
+	return nil
+}
+
+type RegistrationServer_PogsClient struct {
+	Client capnp.Client
+	Conn   *rpc.Conn
+}
+
+func (c RegistrationServer_PogsClient) RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error) {
+	client := tunnelrpc.TunnelServer{Client: c.Client}
+	promise := client.RegisterConnection(ctx, func(p tunnelrpc.RegistrationServer_registerConnection_Params) error {
+		tunnelAuth, err := p.NewAuth()
+		if err != nil {
+			return err
+		}
+		if err = auth.MarshalCapnproto(tunnelAuth); err != nil {
+			return err
+		}
+		err = p.SetAuth(tunnelAuth)
+		if err != nil {
+			return err
+		}
+		err = p.SetTunnelId(tunnelID[:])
+		if err != nil {
+			return err
+		}
+		p.SetConnIndex(connIndex)
+		connectionOptions, err := p.NewOptions()
+		if err != nil {
+			return err
+		}
+		err = options.MarshalCapnproto(connectionOptions)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	response, err := promise.Result().Struct()
+	if err != nil {
+		return nil, wrapRPCError(err)
+	}
+	result := response.Result()
+	switch result.Which() {
+	case tunnelrpc.ConnectionResponse_result_Which_error:
+		resultError, err := result.Error()
+		if err != nil {
+			return nil, wrapRPCError(err)
+		}
+		cause, err := resultError.Cause()
+		if err != nil {
+			return nil, wrapRPCError(err)
+		}
+		err = errors.New(cause)
+		if resultError.ShouldRetry() {
+			err = RetryErrorAfter(err, time.Duration(resultError.RetryAfter()))
+		}
+		return nil, err
+
+	case tunnelrpc.ConnectionResponse_result_Which_connectionDetails:
+		connDetails, err := result.ConnectionDetails()
+		if err != nil {
+			return nil, wrapRPCError(err)
+		}
+		details := new(ConnectionDetails)
+		if err = details.UnmarshalCapnproto(connDetails); err != nil {
+			return nil, wrapRPCError(err)
+		}
+		return details, nil
+	}
+
+	return nil, newRPCError("unknown result which %d", result.Which())
+}
+
+func (c RegistrationServer_PogsClient) UnregisterConnection(ctx context.Context) error {
+	client := tunnelrpc.TunnelServer{Client: c.Client}
+	promise := client.UnregisterConnection(ctx, func(p tunnelrpc.RegistrationServer_unregisterConnection_Params) error {
+		return nil
+	})
+	_, err := promise.Struct()
+	if err != nil {
+		return wrapRPCError(err)
+	}
+	return nil
+}
+
 type ClientInfo struct {
 	ClientID []byte `capnp:"clientId"` // must be a slice for capnp compatibility
 	Features []string
@@ -98,140 +250,3 @@ func MarshalError(s tunnelrpc.ConnectionError, err error) error {
 
 	return nil
 }
-
-func (i TunnelServer_PogsImpl) RegisterConnection(p tunnelrpc.RegistrationServer_registerConnection) error {
-	server.Ack(p.Options)
-
-	auth, err := p.Params.Auth()
-	if err != nil {
-		return err
-	}
-	var pogsAuth TunnelAuth
-	err = pogsAuth.UnmarshalCapnproto(auth)
-	if err != nil {
-		return err
-	}
-	uuidBytes, err := p.Params.TunnelId()
-	if err != nil {
-		return err
-	}
-	tunnelID, err := uuid.FromBytes(uuidBytes)
-	if err != nil {
-		return err
-	}
-	connIndex := p.Params.ConnIndex()
-	options, err := p.Params.Options()
-	if err != nil {
-		return err
-	}
-	var pogsOptions ConnectionOptions
-	err = pogsOptions.UnmarshalCapnproto(options)
-	if err != nil {
-		return err
-	}
-
-	connDetails, callError := i.impl.RegisterConnection(p.Ctx, pogsAuth, tunnelID, connIndex, &pogsOptions)
-
-	resp, err := p.Results.NewResult()
-	if err != nil {
-		return err
-	}
-
-	if callError != nil {
-		if connError, err := resp.Result().NewError(); err != nil {
-			return err
-		} else {
-			return MarshalError(connError, callError)
-		}
-	}
-
-	if details, err := resp.Result().NewConnectionDetails(); err != nil {
-		return err
-	} else {
-		return connDetails.MarshalCapnproto(details)
-	}
-}
-
-func (i TunnelServer_PogsImpl) UnregisterConnection(p tunnelrpc.RegistrationServer_unregisterConnection) error {
-	server.Ack(p.Options)
-
-	i.impl.UnregisterConnection(p.Ctx)
-	return nil
-}
-
-func (c TunnelServer_PogsClient) RegisterConnection(ctx context.Context, auth TunnelAuth, tunnelID uuid.UUID, connIndex byte, options *ConnectionOptions) (*ConnectionDetails, error) {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.RegisterConnection(ctx, func(p tunnelrpc.RegistrationServer_registerConnection_Params) error {
-		tunnelAuth, err := p.NewAuth()
-		if err != nil {
-			return err
-		}
-		if err = auth.MarshalCapnproto(tunnelAuth); err != nil {
-			return err
-		}
-		err = p.SetAuth(tunnelAuth)
-		if err != nil {
-			return err
-		}
-		err = p.SetTunnelId(tunnelID[:])
-		if err != nil {
-			return err
-		}
-		p.SetConnIndex(connIndex)
-		connectionOptions, err := p.NewOptions()
-		if err != nil {
-			return err
-		}
-		err = options.MarshalCapnproto(connectionOptions)
-		if err != nil {
-			return err
-		}
-		return nil
-	})
-	response, err := promise.Result().Struct()
-	if err != nil {
-		return nil, wrapRPCError(err)
-	}
-	result := response.Result()
-	switch result.Which() {
-	case tunnelrpc.ConnectionResponse_result_Which_error:
-		resultError, err := result.Error()
-		if err != nil {
-			return nil, wrapRPCError(err)
-		}
-		cause, err := resultError.Cause()
-		if err != nil {
-			return nil, wrapRPCError(err)
-		}
-		err = errors.New(cause)
-		if resultError.ShouldRetry() {
-			err = RetryErrorAfter(err, time.Duration(resultError.RetryAfter()))
-		}
-		return nil, err
-
-	case tunnelrpc.ConnectionResponse_result_Which_connectionDetails:
-		connDetails, err := result.ConnectionDetails()
-		if err != nil {
-			return nil, wrapRPCError(err)
-		}
-		details := new(ConnectionDetails)
-		if err = details.UnmarshalCapnproto(connDetails); err != nil {
-			return nil, wrapRPCError(err)
-		}
-		return details, nil
-	}
-
-	return nil, newRPCError("unknown result which %d", result.Which())
-}
-
-func (c TunnelServer_PogsClient) UnregisterConnection(ctx context.Context) error {
-	client := tunnelrpc.TunnelServer{Client: c.Client}
-	promise := client.UnregisterConnection(ctx, func(p tunnelrpc.RegistrationServer_unregisterConnection_Params) error {
-		return nil
-	})
-	_, err := promise.Struct()
-	if err != nil {
-		return wrapRPCError(err)
-	}
-	return nil
-}

tunnelrpc/pogs/connectionrpc_test.go

@@ -62,6 +62,10 @@ func TestConnectionRegistrationRPC(t *testing.T) {
 	clientConn := rpc.NewConn(t2)
 	defer clientConn.Close()
 	client := TunnelServer_PogsClient{
+		RegistrationServer_PogsClient: RegistrationServer_PogsClient{
+			Client: clientConn.Bootstrap(ctx),
+			Conn:   clientConn,
+		},
 		Client: clientConn.Bootstrap(ctx),
 		Conn:   clientConn,
 	}

tunnelrpc/pogs/tunnelrpc.go

@@ -210,10 +210,11 @@ type TunnelServer interface {
 }
 
 func TunnelServer_ServerToClient(s TunnelServer) tunnelrpc.TunnelServer {
-	return tunnelrpc.TunnelServer_ServerToClient(TunnelServer_PogsImpl{s})
+	return tunnelrpc.TunnelServer_ServerToClient(TunnelServer_PogsImpl{RegistrationServer_PogsImpl{s}, s})
 }
 
 type TunnelServer_PogsImpl struct {
+	RegistrationServer_PogsImpl
 	impl TunnelServer
 }
 
@@ -268,6 +269,7 @@ func (i TunnelServer_PogsImpl) ObsoleteDeclarativeTunnelConnect(p tunnelrpc.Tunn
 }
 
 type TunnelServer_PogsClient struct {
+	RegistrationServer_PogsClient
 	Client capnp.Client
 	Conn   *rpc.Conn
 }

tunnelstore/client.go

@@ -27,6 +27,7 @@ var (
 	ErrUnauthorized       = errors.New("unauthorized")
 	ErrBadRequest         = errors.New("incorrect request parameters")
 	ErrNotFound           = errors.New("not found")
+	ErrAPINoSuccess       = errors.New("API call failed")
 )
 
 type Tunnel struct {
@@ -39,14 +40,26 @@ type Tunnel struct {
 
 type Connection struct {
 	ColoName           string    `json:"colo_name"`
-	ID                 uuid.UUID `json:"uuid"`
+	ID                 uuid.UUID `json:"id"`
 	IsPendingReconnect bool      `json:"is_pending_reconnect"`
 }
 
+type Change = string
+
+const (
+	ChangeNew       = "new"
+	ChangeUpdated   = "updated"
+	ChangeUnchanged = "unchanged"
+)
+
 // Route represents a record type that can route to a tunnel
 type Route interface {
 	json.Marshaler
 	RecordType() string
+	UnmarshalResult(body io.Reader) (RouteResult, error)
+}
+
+type RouteResult interface {
 	// SuccessSummary explains what will route to this tunnel when it's provisioned successfully
 	SuccessSummary() string
 }
@@ -55,6 +68,11 @@ type DNSRoute struct {
 	userHostname string
 }
 
+type DNSRouteResult struct {
+	route *DNSRoute
+	CName Change `json:"cname"`
+}
+
 func NewDNSRoute(userHostname string) Route {
 	return &DNSRoute{
 		userHostname: userHostname,
@@ -72,12 +90,28 @@ func (dr *DNSRoute) MarshalJSON() ([]byte, error) {
 	return json.Marshal(&s)
 }
 
+func (dr *DNSRoute) UnmarshalResult(body io.Reader) (RouteResult, error) {
+	var result DNSRouteResult
+	err := parseResponse(body, &result)
+	result.route = dr
+	return &result, err
+}
+
 func (dr *DNSRoute) RecordType() string {
 	return "dns"
 }
 
-func (dr *DNSRoute) SuccessSummary() string {
-	return fmt.Sprintf("%s will route to your tunnel", dr.userHostname)
+func (res *DNSRouteResult) SuccessSummary() string {
+	var msgFmt string
+	switch res.CName {
+	case ChangeNew:
+		msgFmt = "Added CNAME %s which will route to this tunnel"
+	case ChangeUpdated: // this is not currently returned by tunnelsore
+		msgFmt = "%s updated to route to your tunnel"
+	case ChangeUnchanged:
+		msgFmt = "%s is already configured to route to your tunnel"
+	}
+	return fmt.Sprintf(msgFmt, res.route.userHostname)
 }
 
 type LBRoute struct {
@@ -85,6 +119,12 @@ type LBRoute struct {
 	lbPool string
 }
 
+type LBRouteResult struct {
+	route        *LBRoute
+	LoadBalancer Change `json:"load_balancer"`
+	Pool         Change `json:"pool"`
+}
+
 func NewLBRoute(lbName, lbPool string) Route {
 	return &LBRoute{
 		lbName: lbName,
@@ -109,8 +149,40 @@ func (lr *LBRoute) RecordType() string {
 	return "lb"
 }
 
-func (lr *LBRoute) SuccessSummary() string {
-	return fmt.Sprintf("Load balancer %s will route to this tunnel through pool %s", lr.lbName, lr.lbPool)
+func (lr *LBRoute) UnmarshalResult(body io.Reader) (RouteResult, error) {
+	var result LBRouteResult
+	err := parseResponse(body, &result)
+	result.route = lr
+	return &result, err
+}
+
+func (res *LBRouteResult) SuccessSummary() string {
+	var msg string
+	switch res.LoadBalancer + "," + res.Pool {
+	case "new,new":
+		msg = "Created load balancer %s and added a new pool %s with this tunnel as an origin"
+	case "new,updated":
+		msg = "Created load balancer %s with an existing pool %s which was updated to use this tunnel as an origin"
+	case "new,unchanged":
+		msg = "Created load balancer %s with an existing pool %s which already has this tunnel as an origin"
+	case "updated,new":
+		msg = "Added new pool %[2]s with this tunnel as an origin to load balancer %[1]s"
+	case "updated,updated":
+		msg = "Updated pool %[2]s to use this tunnel as an origin and added it to load balancer %[1]s"
+	case "updated,unchanged":
+		msg = "Added pool %[2]s, which already has this tunnel as an origin, to load balancer %[1]s"
+	case "unchanged,updated":
+		msg = "Added this tunnel as an origin in pool %[2]s which is already used by load balancer %[1]s"
+	case "unchanged,unchanged":
+		msg = "Load balancer %s already uses pool %s which has this tunnel as an origin"
+	case "unchanged,new":
+		// this state is not possible
+		fallthrough
+	default:
+		msg = "Something went wrong: failed to modify load balancer %s with pool %s; please check traffic manager configuration in the dashboard"
+	}
+
+	return fmt.Sprintf(msg, res.route.lbName, res.route.lbPool)
 }
 
 type Client interface {
@@ -119,12 +191,13 @@ type Client interface {
 	DeleteTunnel(tunnelID uuid.UUID) error
 	ListTunnels(filter *Filter) ([]*Tunnel, error)
 	CleanupConnections(tunnelID uuid.UUID) error
-	RouteTunnel(tunnelID uuid.UUID, route Route) error
+	RouteTunnel(tunnelID uuid.UUID, route Route) (RouteResult, error)
 }
 
 type RESTClient struct {
 	baseEndpoints *baseEndpoints
 	authToken     string
+	userAgent     string
 	client        http.Client
 	logger        logger.Service
 }
@@ -136,7 +209,7 @@ type baseEndpoints struct {
 
 var _ Client = (*RESTClient)(nil)
 
-func NewRESTClient(baseURL string, accountTag, zoneTag string, authToken string, logger logger.Service) (*RESTClient, error) {
+func NewRESTClient(baseURL, accountTag, zoneTag, authToken, userAgent string, logger logger.Service) (*RESTClient, error) {
 	if strings.HasSuffix(baseURL, "/") {
 		baseURL = baseURL[:len(baseURL)-1]
 	}
@@ -154,6 +227,7 @@ func NewRESTClient(baseURL string, accountTag, zoneTag string, authToken string,
 			zoneLevel:    *zoneLevelEndpoint,
 		},
 		authToken: authToken,
+		userAgent: userAgent,
 		client: http.Client{
 			Transport: &http.Transport{
 				TLSHandshakeTimeout:   defaultTimeout,
@@ -236,16 +310,18 @@ func (r *RESTClient) ListTunnels(filter *Filter) ([]*Tunnel, error) {
 	defer resp.Body.Close()
 
 	if resp.StatusCode == http.StatusOK {
-		var tunnels []*Tunnel
-		if err := json.NewDecoder(resp.Body).Decode(&tunnels); err != nil {
-			return nil, errors.Wrap(err, "failed to decode response")
-		}
-		return tunnels, nil
+		return parseListTunnels(resp.Body)
 	}
 
 	return nil, r.statusCodeToError("list tunnels", resp)
 }
 
+func parseListTunnels(body io.ReadCloser) ([]*Tunnel, error) {
+	var tunnels []*Tunnel
+	err := parseResponse(body, &tunnels)
+	return tunnels, err
+}
+
 func (r *RESTClient) CleanupConnections(tunnelID uuid.UUID) error {
 	endpoint := r.baseEndpoints.accountLevel
 	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/connections", tunnelID))
@@ -258,16 +334,20 @@ func (r *RESTClient) CleanupConnections(tunnelID uuid.UUID) error {
 	return r.statusCodeToError("cleanup connections", resp)
 }
 
-func (r *RESTClient) RouteTunnel(tunnelID uuid.UUID, route Route) error {
+func (r *RESTClient) RouteTunnel(tunnelID uuid.UUID, route Route) (RouteResult, error) {
 	endpoint := r.baseEndpoints.zoneLevel
 	endpoint.Path = path.Join(endpoint.Path, fmt.Sprintf("%v/routes", tunnelID))
 	resp, err := r.sendRequest("PUT", endpoint, route)
 	if err != nil {
-		return errors.Wrap(err, "REST request failed")
+		return nil, errors.Wrap(err, "REST request failed")
 	}
 	defer resp.Body.Close()
 
-	return r.statusCodeToError("add route", resp)
+	if resp.StatusCode == http.StatusOK {
+		return route.UnmarshalResult(resp.Body)
+	}
+
+	return nil, r.statusCodeToError("add route", resp)
 }
 
 func (r *RESTClient) sendRequest(method string, url url.URL, body interface{}) (*http.Response, error) {
@@ -284,22 +364,80 @@ func (r *RESTClient) sendRequest(method string, url url.URL, body interface{}) (
 	if err != nil {
 		return nil, errors.Wrapf(err, "can't create %s request", method)
 	}
+	req.Header.Set("User-Agent", r.userAgent)
 	if bodyReader != nil {
 		req.Header.Set("Content-Type", jsonContentType)
 	}
 	req.Header.Add("X-Auth-User-Service-Key", r.authToken)
+	req.Header.Add("Accept", "application/json;version=1")
 	return r.client.Do(req)
 }
 
+func parseResponse(reader io.Reader, data interface{}) error {
+	// Schema for Tunnelstore responses in the v1 API.
+	// Roughly, it's a wrapper around a particular result that adds failures/errors/etc
+	var result response
+	// First, parse the wrapper and check the API call succeeded
+	if err := json.NewDecoder(reader).Decode(&result); err != nil {
+		return errors.Wrap(err, "failed to decode response")
+	}
+	if err := result.checkErrors(); err != nil {
+		return err
+	}
+	if !result.Success {
+		return ErrAPINoSuccess
+	}
+	// At this point we know the API call succeeded, so, parse out the inner
+	// result into the datatype provided as a parameter.
+	return json.Unmarshal(result.Result, &data)
+}
+
 func unmarshalTunnel(reader io.Reader) (*Tunnel, error) {
 	var tunnel Tunnel
-	if err := json.NewDecoder(reader).Decode(&tunnel); err != nil {
-		return nil, errors.Wrap(err, "failed to decode response")
+	err := parseResponse(reader, &tunnel)
+	return &tunnel, err
+}
+
+type response struct {
+	Success  bool            `json:"success,omitempty"`
+	Errors   []apiErr        `json:"errors,omitempty"`
+	Messages []string        `json:"messages,omitempty"`
+	Result   json.RawMessage `json:"result,omitempty"`
+}
+
+func (r *response) checkErrors() error {
+	if len(r.Errors) == 0 {
+		return nil
 	}
-	return &tunnel, nil
+	if len(r.Errors) == 1 {
+		return r.Errors[0]
+	}
+	var messages string
+	for _, e := range r.Errors {
+		messages += fmt.Sprintf("%s; ", e)
+	}
+	return fmt.Errorf("API errors: %s", messages)
+}
+
+type apiErr struct {
+	Code    json.Number `json:"code,omitempty"`
+	Message string      `json:"message,omitempty"`
+}
+
+func (e apiErr) Error() string {
+	return fmt.Sprintf("code: %v, reason: %s", e.Code, e.Message)
 }
 
 func (r *RESTClient) statusCodeToError(op string, resp *http.Response) error {
+	if resp.Header.Get("Content-Type") == "application/json" {
+		var errorsResp response
+		if json.NewDecoder(resp.Body).Decode(&errorsResp) == nil {
+			if err := errorsResp.checkErrors(); err != nil {
+				return errors.Errorf("Failed to %s: %s", op, err)
+			}
+		}
+	}
+
 	switch resp.StatusCode {
 	case http.StatusOK:
 		return nil

tunnelstore/client_test.go

@@ -0,0 +1,207 @@
+package tunnelstore
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestDNSRouteUnmarshalResult(t *testing.T) {
+	route := &DNSRoute{
+		userHostname: "example.com",
+	}
+
+	result, err := route.UnmarshalResult(strings.NewReader(`{"success": true, "result": {"cname": "new"}}`))
+
+	assert.NoError(t, err)
+	assert.Equal(t, &DNSRouteResult{
+		route: route,
+		CName: ChangeNew,
+	}, result)
+
+	badJSON := []string{
+		`abc`,
+		`{"success": false, "result": {"cname": "new"}}`,
+		`{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}], "result": {"cname": "new"}}`,
+		`{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}, {"code": 1004, "message":"Cannot use tunnel as origin for non-proxied load balancer"}], "result": {"cname": "new"}}`,
+		`{"result": {"cname": "new"}}`,
+		`{"result": {"cname": "new"}}`,
+	}
+
+	for _, j := range badJSON {
+		_, err = route.UnmarshalResult(strings.NewReader(j))
+		assert.NotNil(t, err)
+	}
+}
+
+func TestLBRouteUnmarshalResult(t *testing.T) {
+	route := &LBRoute{
+		lbName: "lb.example.com",
+		lbPool: "pool",
+	}
+
+	result, err := route.UnmarshalResult(strings.NewReader(`{"success": true, "result": {"pool": "unchanged", "load_balancer": "updated"}}`))
+
+	assert.NoError(t, err)
+	assert.Equal(t, &LBRouteResult{
+		route:        route,
+		LoadBalancer: ChangeUpdated,
+		Pool:         ChangeUnchanged,
+	}, result)
+
+	badJSON := []string{
+		`abc`,
+		`{"success": false, "result": {"pool": "unchanged", "load_balancer": "updated"}}`,
+		`{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}], "result": {"pool": "unchanged", "load_balancer": "updated"}}`,
+		`{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}, {"code": 1004, "message":"Cannot use tunnel as origin for non-proxied load balancer"}], "result": {"pool": "unchanged", "load_balancer": "updated"}}`,
+		`{"result": {"pool": "unchanged", "load_balancer": "updated"}}`,
+	}
+
+	for _, j := range badJSON {
+		_, err = route.UnmarshalResult(strings.NewReader(j))
+		assert.NotNil(t, err)
+	}
+}
+
+func TestLBRouteResultSuccessSummary(t *testing.T) {
+	route := &LBRoute{
+		lbName: "lb.example.com",
+		lbPool: "POOL",
+	}
+
+	tests := []struct {
+		lb       Change
+		pool     Change
+		expected string
+	}{
+		{ChangeNew, ChangeNew, "Created load balancer lb.example.com and added a new pool POOL with this tunnel as an origin"},
+		{ChangeNew, ChangeUpdated, "Created load balancer lb.example.com with an existing pool POOL which was updated to use this tunnel as an origin"},
+		{ChangeNew, ChangeUnchanged, "Created load balancer lb.example.com with an existing pool POOL which already has this tunnel as an origin"},
+		{ChangeUpdated, ChangeNew, "Added new pool POOL with this tunnel as an origin to load balancer lb.example.com"},
+		{ChangeUpdated, ChangeUpdated, "Updated pool POOL to use this tunnel as an origin and added it to load balancer lb.example.com"},
+		{ChangeUpdated, ChangeUnchanged, "Added pool POOL, which already has this tunnel as an origin, to load balancer lb.example.com"},
+		{ChangeUnchanged, ChangeNew, "Something went wrong: failed to modify load balancer lb.example.com with pool POOL; please check traffic manager configuration in the dashboard"},
+		{ChangeUnchanged, ChangeUpdated, "Added this tunnel as an origin in pool POOL which is already used by load balancer lb.example.com"},
+		{ChangeUnchanged, ChangeUnchanged, "Load balancer lb.example.com already uses pool POOL which has this tunnel as an origin"},
+		{"", "", "Something went wrong: failed to modify load balancer lb.example.com with pool POOL; please check traffic manager configuration in the dashboard"},
+		{"a", "b", "Something went wrong: failed to modify load balancer lb.example.com with pool POOL; please check traffic manager configuration in the dashboard"},
+	}
+	for i, tt := range tests {
+		res := &LBRouteResult{
+			route:        route,
+			LoadBalancer: tt.lb,
+			Pool:         tt.pool,
+		}
+		actual := res.SuccessSummary()
+		assert.Equal(t, tt.expected, actual, "case %d", i+1)
+	}
+}
+
+func Test_parseListTunnels(t *testing.T) {
+	type args struct {
+		body string
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    []*Tunnel
+		wantErr bool
+	}{
+		{
+			name: "empty list",
+			args: args{body: `{"success": true, "result": []}`},
+			want: []*Tunnel{},
+		},
+		{
+			name:    "success is false",
+			args:    args{body: `{"success": false, "result": []}`},
+			wantErr: true,
+		},
+		{
+			name:    "errors are present",
+			args:    args{body: `{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}], "result": []}`},
+			wantErr: true,
+		},
+		{
+			name:    "invalid response",
+			args:    args{body: `abc`},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			body := ioutil.NopCloser(bytes.NewReader([]byte(tt.args.body)))
+			got, err := parseListTunnels(body)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("parseListTunnels() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("parseListTunnels() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_unmarshalTunnel(t *testing.T) {
+	type args struct {
+		reader io.Reader
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    *Tunnel
+		wantErr bool
+	}{
+		// TODO: Add test cases.
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := unmarshalTunnel(tt.args.reader)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("unmarshalTunnel() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("unmarshalTunnel() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestUnmarshalTunnelOk(t *testing.T) {
+
+	jsonBody := `{"success": true, "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}`
+	expected := Tunnel{
+		ID:          uuid.Nil,
+		Name:        "test",
+		CreatedAt:   time.Time{},
+		Connections: []Connection{},
+	}
+	actual, err := unmarshalTunnel(bytes.NewReader([]byte(jsonBody)))
+	assert.NoError(t, err)
+	assert.Equal(t, &expected, actual)
+}
+
+func TestUnmarshalTunnelErr(t *testing.T) {
+
+	tests := []string{
+		`abc`,
+		`{"success": true, "result": abc}`,
+		`{"success": false, "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}}`,
+		`{"errors": [{"code": 1003, "message":"An A, AAAA or CNAME record already exists with that host"}], "result": {"id": "00000000-0000-0000-0000-000000000000","name":"test","created_at":"0001-01-01T00:00:00Z","connections":[]}}}`,
+	}
+
+	for i, test := range tests {
+		_, err := unmarshalTunnel(bytes.NewReader([]byte(test)))
+		assert.Error(t, err, fmt.Sprintf("Test #%v failed", i))
+	}
+}

vendor/github.com/alecthomas/units/bytes.go

@@ -27,6 +27,7 @@ var (
 
 // ParseBase2Bytes supports both iB and B in base-2 multipliers. That is, KB
 // and KiB are both 1024.
+// However "kB", which is the correct SI spelling of 1000 Bytes, is rejected.
 func ParseBase2Bytes(s string) (Base2Bytes, error) {
 	n, err := ParseUnit(s, bytesUnitMap)
 	if err != nil {
@@ -68,12 +69,13 @@ func ParseMetricBytes(s string) (MetricBytes, error) {
 	return MetricBytes(n), err
 }
 
+// TODO: represents 1000B as uppercase "KB", while SI standard requires "kB".
 func (m MetricBytes) String() string {
 	return ToString(int64(m), 1000, "B", "B")
 }
 
 // ParseStrictBytes supports both iB and B suffixes for base 2 and metric,
-// respectively. That is, KiB represents 1024 and KB represents 1000.
+// respectively. That is, KiB represents 1024 and kB, KB represent 1000.
 func ParseStrictBytes(s string) (int64, error) {
 	n, err := ParseUnit(s, bytesUnitMap)
 	if err != nil {

vendor/github.com/alecthomas/units/go.mod

@@ -1 +1,3 @@
 module github.com/alecthomas/units
+
+require github.com/stretchr/testify v1.4.0

vendor/github.com/alecthomas/units/go.sum

@@ -0,0 +1,11 @@
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

vendor/github.com/alecthomas/units/si.go

@@ -14,13 +14,37 @@ const (
 )
 
 func MakeUnitMap(suffix, shortSuffix string, scale int64) map[string]float64 {
-	return map[string]float64{
-		shortSuffix:  1,
-		"K" + suffix: float64(scale),
+	res := map[string]float64{
+		shortSuffix: 1,
+		// see below for "k" / "K"
 		"M" + suffix: float64(scale * scale),
 		"G" + suffix: float64(scale * scale * scale),
 		"T" + suffix: float64(scale * scale * scale * scale),
 		"P" + suffix: float64(scale * scale * scale * scale * scale),
 		"E" + suffix: float64(scale * scale * scale * scale * scale * scale),
 	}
+
+	// Standard SI prefixes use lowercase "k" for kilo = 1000.
+	// For compatibility, and to be fool-proof, we accept both "k" and "K" in metric mode.
+	//
+	// However, official binary prefixes are always capitalized - "KiB" -
+	// and we specifically never parse "kB" as 1024B because:
+	//
+	// (1) people pedantic enough to use lowercase according to SI unlikely to abuse "k" to mean 1024 :-)
+	//
+	// (2) Use of capital K for 1024 was an informal tradition predating IEC prefixes:
+	//     "The binary meaning of the kilobyte for 1024 bytes typically uses the symbol KB, with an
+	//     uppercase letter K."
+	//     -- https://en.wikipedia.org/wiki/Kilobyte#Base_2_(1024_bytes)
+	//     "Capitalization of the letter K became the de facto standard for binary notation, although this
+	//     could not be extended to higher powers, and use of the lowercase k did persist.[13][14][15]"
+	//     -- https://en.wikipedia.org/wiki/Binary_prefix#History
+	//     See also the extensive https://en.wikipedia.org/wiki/Timeline_of_binary_prefixes.
+	if scale == 1024 {
+		res["K"+suffix] = float64(scale)
+	} else {
+		res["k"+suffix] = float64(scale)
+		res["K"+suffix] = float64(scale)
+	}
+	return res
 }

vendor/github.com/aws/aws-sdk-go/aws/arn/arn.go

@@ -0,0 +1,93 @@
+// Package arn provides a parser for interacting with Amazon Resource Names.
+package arn
+
+import (
+	"errors"
+	"strings"
+)
+
+const (
+	arnDelimiter = ":"
+	arnSections  = 6
+	arnPrefix    = "arn:"
+
+	// zero-indexed
+	sectionPartition = 1
+	sectionService   = 2
+	sectionRegion    = 3
+	sectionAccountID = 4
+	sectionResource  = 5
+
+	// errors
+	invalidPrefix   = "arn: invalid prefix"
+	invalidSections = "arn: not enough sections"
+)
+
+// ARN captures the individual fields of an Amazon Resource Name.
+// See http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html for more information.
+type ARN struct {
+	// The partition that the resource is in. For standard AWS regions, the partition is "aws". If you have resources in
+	// other partitions, the partition is "aws-partitionname". For example, the partition for resources in the China
+	// (Beijing) region is "aws-cn".
+	Partition string
+
+	// The service namespace that identifies the AWS product (for example, Amazon S3, IAM, or Amazon RDS). For a list of
+	// namespaces, see
+	// http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#genref-aws-service-namespaces.
+	Service string
+
+	// The region the resource resides in. Note that the ARNs for some resources do not require a region, so this
+	// component might be omitted.
+	Region string
+
+	// The ID of the AWS account that owns the resource, without the hyphens. For example, 123456789012. Note that the
+	// ARNs for some resources don't require an account number, so this component might be omitted.
+	AccountID string
+
+	// The content of this part of the ARN varies by service. It often includes an indicator of the type of resource —
+	// for example, an IAM user or Amazon RDS database - followed by a slash (/) or a colon (:), followed by the
+	// resource name itself. Some services allows paths for resource names, as described in
+	// http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html#arns-paths.
+	Resource string
+}
+
+// Parse parses an ARN into its constituent parts.
+//
+// Some example ARNs:
+// arn:aws:elasticbeanstalk:us-east-1:123456789012:environment/My App/MyEnvironment
+// arn:aws:iam::123456789012:user/David
+// arn:aws:rds:eu-west-1:123456789012:db:mysql-db
+// arn:aws:s3:::my_corporate_bucket/exampleobject.png
+func Parse(arn string) (ARN, error) {
+	if !strings.HasPrefix(arn, arnPrefix) {
+		return ARN{}, errors.New(invalidPrefix)
+	}
+	sections := strings.SplitN(arn, arnDelimiter, arnSections)
+	if len(sections) != arnSections {
+		return ARN{}, errors.New(invalidSections)
+	}
+	return ARN{
+		Partition: sections[sectionPartition],
+		Service:   sections[sectionService],
+		Region:    sections[sectionRegion],
+		AccountID: sections[sectionAccountID],
+		Resource:  sections[sectionResource],
+	}, nil
+}
+
+// IsARN returns whether the given string is an ARN by looking for
+// whether the string starts with "arn:" and contains the correct number
+// of sections delimited by colons(:).
+func IsARN(arn string) bool {
+	return strings.HasPrefix(arn, arnPrefix) && strings.Count(arn, ":") >= arnSections-1
+}
+
+// String returns the canonical representation of the ARN
+func (arn ARN) String() string {
+	return arnPrefix +
+		arn.Partition + arnDelimiter +
+		arn.Service + arnDelimiter +
+		arn.Region + arnDelimiter +
+		arn.AccountID + arnDelimiter +
+		arn.Resource
+}

vendor/github.com/aws/aws-sdk-go/aws/awsutil/path_value.go

@@ -70,7 +70,7 @@ func rValuesAtPath(v interface{}, path string, createPath, caseSensitive, nilTer
 			value = value.FieldByNameFunc(func(name string) bool {
 				if c == name {
 					return true
-				} else if !caseSensitive && strings.ToLower(name) == strings.ToLower(c) {
+				} else if !caseSensitive && strings.EqualFold(name, c) {
 					return true
 				}
 				return false

vendor/github.com/aws/aws-sdk-go/aws/client/client.go

@@ -12,6 +12,7 @@ import (
 type Config struct {
 	Config        *aws.Config
 	Handlers      request.Handlers
+	PartitionID   string
 	Endpoint      string
 	SigningRegion string
 	SigningName   string

vendor/github.com/aws/aws-sdk-go/aws/client/default_retryer.go

@@ -16,11 +16,11 @@ import (
 type DefaultRetryer struct {
 	// Num max Retries is the number of max retries that will be performed.
 	// By default, this is zero.
-	NumMaxRetries    int
+	NumMaxRetries int
 
 	// MinRetryDelay is the minimum retry delay after which retry will be performed.
 	// If not set, the value is 0ns.
-	MinRetryDelay    time.Duration
+	MinRetryDelay time.Duration
 
 	// MinThrottleRetryDelay is the minimum retry delay when throttled.
 	// If not set, the value is 0ns.
@@ -28,7 +28,7 @@ type DefaultRetryer struct {
 
 	// MaxRetryDelay is the maximum retry delay before which retry must be performed.
 	// If not set, the value is 0ns.
-	MaxRetryDelay    time.Duration
+	MaxRetryDelay time.Duration
 
 	// MaxThrottleDelay is the maximum retry delay when throttled.
 	// If not set, the value is 0ns.

vendor/github.com/aws/aws-sdk-go/aws/client/metadata/client_info.go

@@ -5,6 +5,7 @@ type ClientInfo struct {
 	ServiceName   string
 	ServiceID     string
 	APIVersion    string
+	PartitionID   string
 	Endpoint      string
 	SigningName   string
 	SigningRegion string

vendor/github.com/aws/aws-sdk-go/aws/config.go

@@ -43,7 +43,7 @@ type Config struct {
 
 	// An optional endpoint URL (hostname only or fully qualified URI)
 	// that overrides the default generated endpoint for a client. Set this
-	// to `""` to use the default generated endpoint.
+	// to `nil` or the value to `""` to use the default generated endpoint.
 	//
 	// Note: You must still provide a `Region` value when specifying an
 	// endpoint for a client.
@@ -138,7 +138,7 @@ type Config struct {
 	// `ExpectContinueTimeout` for information on adjusting the continue wait
 	// timeout. https://golang.org/pkg/net/http/#Transport
 	//
-	// You should use this flag to disble 100-Continue if you experience issues
+	// You should use this flag to disable 100-Continue if you experience issues
 	// with proxies or third party S3 compatible services.
 	S3Disable100Continue *bool
 
@@ -161,6 +161,17 @@ type Config struct {
 	// on GetObject API calls.
 	S3DisableContentMD5Validation *bool
 
+	// Set this to `true` to have the S3 service client to use the region specified
+	// in the ARN, when an ARN is provided as an argument to a bucket parameter.
+	S3UseARNRegion *bool
+
+	// Set this to `true` to enable the SDK to unmarshal API response header maps to
+	// normalized lower case map keys.
+	//
+	// For example S3's X-Amz-Meta prefixed header will be unmarshaled to lower case
+	// Metadata member's map keys. The value of the header in the map is unaffected.
+	LowerCaseHeaderMaps *bool
+
 	// Set this to `true` to disable the EC2Metadata client from overriding the
 	// default http.Client's Timeout. This is helpful if you do not want the
 	// EC2Metadata client to create a new http.Client. This options is only
@@ -172,7 +183,7 @@ type Config struct {
 	//
 	// Example:
 	//    sess := session.Must(session.NewSession(aws.NewConfig()
-	//       .WithEC2MetadataDiableTimeoutOverride(true)))
+	//       .WithEC2MetadataDisableTimeoutOverride(true)))
 	//
 	//    svc := s3.New(sess)
 	//
@@ -183,7 +194,7 @@ type Config struct {
 	// both IPv4 and IPv6 addressing.
 	//
 	// Setting this for a service which does not support dual stack will fail
-	// to make requets. It is not recommended to set this value on the session
+	// to make requests. It is not recommended to set this value on the session
 	// as it will apply to all service clients created with the session. Even
 	// services which don't support dual stack endpoints.
 	//
@@ -227,6 +238,7 @@ type Config struct {
 
 	// EnableEndpointDiscovery will allow for endpoint discovery on operations that
 	// have the definition in its model. By default, endpoint discovery is off.
+	// To use EndpointDiscovery, Endpoint should be unset or set to an empty string.
 	//
 	// Example:
 	//    sess := session.Must(session.NewSession(&aws.Config{
@@ -246,6 +258,12 @@ type Config struct {
 	// Disabling this feature is useful when you want to use local endpoints
 	// for testing that do not support the modeled host prefix pattern.
 	DisableEndpointHostPrefix *bool
+
+	// STSRegionalEndpoint will enable regional or legacy endpoint resolving
+	STSRegionalEndpoint endpoints.STSRegionalEndpoint
+
+	// S3UsEast1RegionalEndpoint will enable regional or legacy endpoint resolving
+	S3UsEast1RegionalEndpoint endpoints.S3UsEast1RegionalEndpoint
 }
 
 // NewConfig returns a new Config pointer that can be chained with builder
@@ -379,6 +397,13 @@ func (c *Config) WithS3DisableContentMD5Validation(enable bool) *Config {
 
 }
 
+// WithS3UseARNRegion sets a config S3UseARNRegion value and
+// returning a Config pointer for chaining
+func (c *Config) WithS3UseARNRegion(enable bool) *Config {
+	c.S3UseARNRegion = &enable
+	return c
+}
+
 // WithUseDualStack sets a config UseDualStack value returning a Config
 // pointer for chaining.
 func (c *Config) WithUseDualStack(enable bool) *Config {
@@ -420,6 +445,20 @@ func (c *Config) MergeIn(cfgs ...*Config) {
 	}
 }
 
+// WithSTSRegionalEndpoint will set whether or not to use regional endpoint flag
+// when resolving the endpoint for a service
+func (c *Config) WithSTSRegionalEndpoint(sre endpoints.STSRegionalEndpoint) *Config {
+	c.STSRegionalEndpoint = sre
+	return c
+}
+
+// WithS3UsEast1RegionalEndpoint will set whether or not to use regional endpoint flag
+// when resolving the endpoint for a service
+func (c *Config) WithS3UsEast1RegionalEndpoint(sre endpoints.S3UsEast1RegionalEndpoint) *Config {
+	c.S3UsEast1RegionalEndpoint = sre
+	return c
+}
+
 func mergeInConfig(dst *Config, other *Config) {
 	if other == nil {
 		return
@@ -493,6 +532,10 @@ func mergeInConfig(dst *Config, other *Config) {
 		dst.S3DisableContentMD5Validation = other.S3DisableContentMD5Validation
 	}
 
+	if other.S3UseARNRegion != nil {
+		dst.S3UseARNRegion = other.S3UseARNRegion
+	}
+
 	if other.UseDualStack != nil {
 		dst.UseDualStack = other.UseDualStack
 	}
@@ -520,6 +563,14 @@ func mergeInConfig(dst *Config, other *Config) {
 	if other.DisableEndpointHostPrefix != nil {
 		dst.DisableEndpointHostPrefix = other.DisableEndpointHostPrefix
 	}
+
+	if other.STSRegionalEndpoint != endpoints.UnsetSTSEndpoint {
+		dst.STSRegionalEndpoint = other.STSRegionalEndpoint
+	}
+
+	if other.S3UsEast1RegionalEndpoint != endpoints.UnsetS3UsEast1Endpoint {
+		dst.S3UsEast1RegionalEndpoint = other.S3UsEast1RegionalEndpoint
+	}
 }
 
 // Copy will return a shallow copy of the Config object. If any additional

vendor/github.com/aws/aws-sdk-go/aws/context_background_1_5.go

@@ -2,42 +2,8 @@
 
 package aws
 
-import "time"
-
-// An emptyCtx is a copy of the Go 1.7 context.emptyCtx type. This is copied to
-// provide a 1.6 and 1.5 safe version of context that is compatible with Go
-// 1.7's Context.
-//
-// An emptyCtx is never canceled, has no values, and has no deadline. It is not
-// struct{}, since vars of this type must have distinct addresses.
-type emptyCtx int
-
-func (*emptyCtx) Deadline() (deadline time.Time, ok bool) {
-	return
-}
-
-func (*emptyCtx) Done() <-chan struct{} {
-	return nil
-}
-
-func (*emptyCtx) Err() error {
-	return nil
-}
-
-func (*emptyCtx) Value(key interface{}) interface{} {
-	return nil
-}
-
-func (e *emptyCtx) String() string {
-	switch e {
-	case backgroundCtx:
-		return "aws.BackgroundContext"
-	}
-	return "unknown empty Context"
-}
-
-var (
-	backgroundCtx = new(emptyCtx)
+import (
+	"github.com/aws/aws-sdk-go/internal/context"
 )
 
 // BackgroundContext returns a context that will never be canceled, has no
@@ -52,5 +18,5 @@ var (
 //
 // See https://golang.org/pkg/context for more information on Contexts.
 func BackgroundContext() Context {
-	return backgroundCtx
+	return context.BackgroundCtx
 }

vendor/github.com/aws/aws-sdk-go/aws/corehandlers/handlers.go

@@ -161,7 +161,7 @@ func handleSendError(r *request.Request, err error) {
 	}
 	// Catch all request errors, and let the default retrier determine
 	// if the error is retryable.
-	r.Error = awserr.New("RequestError", "send request failed", err)
+	r.Error = awserr.New(request.ErrCodeRequestError, "send request failed", err)
 
 	// Override the error with a context canceled error, if that was canceled.
 	ctx := r.Context()
@@ -225,6 +225,8 @@ var ValidateEndpointHandler = request.NamedHandler{Name: "core.ValidateEndpointH
 	if r.ClientInfo.SigningRegion == "" && aws.StringValue(r.Config.Region) == "" {
 		r.Error = aws.ErrMissingRegion
 	} else if r.ClientInfo.Endpoint == "" {
+		// Was any endpoint provided by the user, or one was derived by the
+		// SDK's endpoint resolver?
 		r.Error = aws.ErrMissingEndpoint
 	}
 }}

vendor/github.com/aws/aws-sdk-go/aws/credentials/context_background_go1.5.go

@@ -0,0 +1,22 @@
+// +build !go1.7
+
+package credentials
+
+import (
+	"github.com/aws/aws-sdk-go/internal/context"
+)
+
+// backgroundContext returns a context that will never be canceled, has no
+// values, and no deadline. This context is used by the SDK to provide
+// backwards compatibility with non-context API operations and functionality.
+//
+// Go 1.6 and before:
+// This context function is equivalent to context.Background in the Go stdlib.
+//
+// Go 1.7 and later:
+// The context returned will be the value returned by context.Background()
+//
+// See https://golang.org/pkg/context for more information on Contexts.
+func backgroundContext() Context {
+	return context.BackgroundCtx
+}

vendor/github.com/aws/aws-sdk-go/aws/credentials/context_background_go1.7.go

@@ -0,0 +1,20 @@
+// +build go1.7
+
+package credentials
+
+import "context"
+
+// backgroundContext returns a context that will never be canceled, has no
+// values, and no deadline. This context is used by the SDK to provide
+// backwards compatibility with non-context API operations and functionality.
+//
+// Go 1.6 and before:
+// This context function is equivalent to context.Background in the Go stdlib.
+//
+// Go 1.7 and later:
+// The context returned will be the value returned by context.Background()
+//
+// See https://golang.org/pkg/context for more information on Contexts.
+func backgroundContext() Context {
+	return context.Background()
+}

vendor/github.com/aws/aws-sdk-go/aws/credentials/context_go1.5.go

@@ -0,0 +1,39 @@
+// +build !go1.9
+
+package credentials
+
+import "time"
+
+// Context is an copy of the Go v1.7 stdlib's context.Context interface.
+// It is represented as a SDK interface to enable you to use the "WithContext"
+// API methods with Go v1.6 and a Context type such as golang.org/x/net/context.
+//
+// This type, aws.Context, and context.Context are equivalent.
+//
+// See https://golang.org/pkg/context on how to use contexts.
+type Context interface {
+	// Deadline returns the time when work done on behalf of this context
+	// should be canceled. Deadline returns ok==false when no deadline is
+	// set. Successive calls to Deadline return the same results.
+	Deadline() (deadline time.Time, ok bool)
+
+	// Done returns a channel that's closed when work done on behalf of this
+	// context should be canceled. Done may return nil if this context can
+	// never be canceled. Successive calls to Done return the same value.
+	Done() <-chan struct{}
+
+	// Err returns a non-nil error value after Done is closed. Err returns
+	// Canceled if the context was canceled or DeadlineExceeded if the
+	// context's deadline passed. No other values for Err are defined.
+	// After Done is closed, successive calls to Err return the same value.
+	Err() error
+
+	// Value returns the value associated with this context for key, or nil
+	// if no value is associated with key. Successive calls to Value with
+	// the same key returns the same result.
+	//
+	// Use context values only for request-scoped data that transits
+	// processes and API boundaries, not for passing optional parameters to
+	// functions.
+	Value(key interface{}) interface{}
+}

vendor/github.com/aws/aws-sdk-go/aws/credentials/context_go1.9.go

@@ -0,0 +1,13 @@
+// +build go1.9
+
+package credentials
+
+import "context"
+
+// Context is an alias of the Go stdlib's context.Context interface.
+// It can be used within the SDK's API operation "WithContext" methods.
+//
+// This type, aws.Context, and context.Context are equivalent.
+//
+// See https://golang.org/pkg/context on how to use contexts.
+type Context = context.Context

vendor/github.com/aws/aws-sdk-go/aws/credentials/credentials.go

@@ -50,10 +50,11 @@ package credentials
 
 import (
 	"fmt"
-	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/internal/sync/singleflight"
 )
 
 // AnonymousCredentials is an empty Credential object that can be used as
@@ -106,6 +107,13 @@ type Provider interface {
 	IsExpired() bool
 }
 
+// ProviderWithContext is a Provider that can retrieve credentials with a Context
+type ProviderWithContext interface {
+	Provider
+
+	RetrieveWithContext(Context) (Value, error)
+}
+
 // An Expirer is an interface that Providers can implement to expose the expiration
 // time, if known.  If the Provider cannot accurately provide this info,
 // it should not implement this interface.
@@ -197,20 +205,68 @@ func (e *Expiry) ExpiresAt() time.Time {
 // first instance of the credentials Value. All calls to Get() after that
 // will return the cached credentials Value until IsExpired() returns true.
 type Credentials struct {
-	creds        Value
-	forceRefresh bool
-
-	m sync.RWMutex
+	creds atomic.Value
+	sf    singleflight.Group
 
 	provider Provider
 }
 
 // NewCredentials returns a pointer to a new Credentials with the provider set.
 func NewCredentials(provider Provider) *Credentials {
-	return &Credentials{
-		provider:     provider,
-		forceRefresh: true,
+	c := &Credentials{
+		provider: provider,
 	}
+	c.creds.Store(Value{})
+	return c
+}
+
+// GetWithContext returns the credentials value, or error if the credentials
+// Value failed to be retrieved. Will return early if the passed in context is
+// canceled.
+//
+// Will return the cached credentials Value if it has not expired. If the
+// credentials Value has expired the Provider's Retrieve() will be called
+// to refresh the credentials.
+//
+// If Credentials.Expire() was called the credentials Value will be force
+// expired, and the next call to Get() will cause them to be refreshed.
+//
+// Passed in Context is equivalent to aws.Context, and context.Context.
+func (c *Credentials) GetWithContext(ctx Context) (Value, error) {
+	if curCreds := c.creds.Load(); !c.isExpired(curCreds) {
+		return curCreds.(Value), nil
+	}
+
+	// Cannot pass context down to the actual retrieve, because the first
+	// context would cancel the whole group when there is not direct
+	// association of items in the group.
+	resCh := c.sf.DoChan("", func() (interface{}, error) {
+		return c.singleRetrieve(&suppressedContext{ctx})
+	})
+	select {
+	case res := <-resCh:
+		return res.Val.(Value), res.Err
+	case <-ctx.Done():
+		return Value{}, awserr.New("RequestCanceled",
+			"request context canceled", ctx.Err())
+	}
+}
+
+func (c *Credentials) singleRetrieve(ctx Context) (creds interface{}, err error) {
+	if curCreds := c.creds.Load(); !c.isExpired(curCreds) {
+		return curCreds.(Value), nil
+	}
+
+	if p, ok := c.provider.(ProviderWithContext); ok {
+		creds, err = p.RetrieveWithContext(ctx)
+	} else {
+		creds, err = c.provider.Retrieve()
+	}
+	if err == nil {
+		c.creds.Store(creds)
+	}
+
+	return creds, err
 }
 
 // Get returns the credentials value, or error if the credentials Value failed
@@ -223,30 +279,7 @@ func NewCredentials(provider Provider) *Credentials {
 // If Credentials.Expire() was called the credentials Value will be force
 // expired, and the next call to Get() will cause them to be refreshed.
 func (c *Credentials) Get() (Value, error) {
-	// Check the cached credentials first with just the read lock.
-	c.m.RLock()
-	if !c.isExpired() {
-		creds := c.creds
-		c.m.RUnlock()
-		return creds, nil
-	}
-	c.m.RUnlock()
-
-	// Credentials are expired need to retrieve the credentials taking the full
-	// lock.
-	c.m.Lock()
-	defer c.m.Unlock()
-
-	if c.isExpired() {
-		creds, err := c.provider.Retrieve()
-		if err != nil {
-			return Value{}, err
-		}
-		c.creds = creds
-		c.forceRefresh = false
-	}
-
-	return c.creds, nil
+	return c.GetWithContext(backgroundContext())
 }
 
 // Expire expires the credentials and forces them to be retrieved on the
@@ -255,10 +288,7 @@ func (c *Credentials) Get() (Value, error) {
 // This will override the Provider's expired state, and force Credentials
 // to call the Provider's Retrieve().
 func (c *Credentials) Expire() {
-	c.m.Lock()
-	defer c.m.Unlock()
-
-	c.forceRefresh = true
+	c.creds.Store(Value{})
 }
 
 // IsExpired returns if the credentials are no longer valid, and need
@@ -267,33 +297,43 @@ func (c *Credentials) Expire() {
 // If the Credentials were forced to be expired with Expire() this will
 // reflect that override.
 func (c *Credentials) IsExpired() bool {
-	c.m.RLock()
-	defer c.m.RUnlock()
-
-	return c.isExpired()
+	return c.isExpired(c.creds.Load())
 }
 
 // isExpired helper method wrapping the definition of expired credentials.
-func (c *Credentials) isExpired() bool {
-	return c.forceRefresh || c.provider.IsExpired()
+func (c *Credentials) isExpired(creds interface{}) bool {
+	return creds == nil || creds.(Value) == Value{} || c.provider.IsExpired()
 }
 
 // ExpiresAt provides access to the functionality of the Expirer interface of
 // the underlying Provider, if it supports that interface.  Otherwise, it returns
 // an error.
 func (c *Credentials) ExpiresAt() (time.Time, error) {
-	c.m.RLock()
-	defer c.m.RUnlock()
-
 	expirer, ok := c.provider.(Expirer)
 	if !ok {
 		return time.Time{}, awserr.New("ProviderNotExpirer",
-			fmt.Sprintf("provider %s does not support ExpiresAt()", c.creds.ProviderName),
+			fmt.Sprintf("provider %s does not support ExpiresAt()", c.creds.Load().(Value).ProviderName),
 			nil)
 	}
-	if c.forceRefresh {
+	if c.creds.Load().(Value) == (Value{}) {
 		// set expiration time to the distant past
 		return time.Time{}, nil
 	}
 	return expirer.ExpiresAt(), nil
 }
+
+type suppressedContext struct {
+	Context
+}
+
+func (s *suppressedContext) Deadline() (deadline time.Time, ok bool) {
+	return time.Time{}, false
+}
+
+func (s *suppressedContext) Done() <-chan struct{} {
+	return nil
+}
+
+func (s *suppressedContext) Err() error {
+	return nil
+}

vendor/github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds/ec2_role_provider.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/credentials"
@@ -87,7 +88,14 @@ func NewCredentialsWithClient(client *ec2metadata.EC2Metadata, options ...func(*
 // Error will be returned if the request fails, or unable to extract
 // the desired credentials.
 func (m *EC2RoleProvider) Retrieve() (credentials.Value, error) {
-	credsList, err := requestCredList(m.Client)
+	return m.RetrieveWithContext(aws.BackgroundContext())
+}
+
+// RetrieveWithContext retrieves credentials from the EC2 service.
+// Error will be returned if the request fails, or unable to extract
+// the desired credentials.
+func (m *EC2RoleProvider) RetrieveWithContext(ctx credentials.Context) (credentials.Value, error) {
+	credsList, err := requestCredList(ctx, m.Client)
 	if err != nil {
 		return credentials.Value{ProviderName: ProviderName}, err
 	}
@@ -97,7 +105,7 @@ func (m *EC2RoleProvider) Retrieve() (credentials.Value, error) {
 	}
 	credsName := credsList[0]
 
-	roleCreds, err := requestCred(m.Client, credsName)
+	roleCreds, err := requestCred(ctx, m.Client, credsName)
 	if err != nil {
 		return credentials.Value{ProviderName: ProviderName}, err
 	}
@@ -130,8 +138,8 @@ const iamSecurityCredsPath = "iam/security-credentials/"
 
 // requestCredList requests a list of credentials from the EC2 service.
 // If there are no credentials, or there is an error making or receiving the request
-func requestCredList(client *ec2metadata.EC2Metadata) ([]string, error) {
-	resp, err := client.GetMetadata(iamSecurityCredsPath)
+func requestCredList(ctx aws.Context, client *ec2metadata.EC2Metadata) ([]string, error) {
+	resp, err := client.GetMetadataWithContext(ctx, iamSecurityCredsPath)
 	if err != nil {
 		return nil, awserr.New("EC2RoleRequestError", "no EC2 instance role found", err)
 	}
@@ -154,8 +162,8 @@ func requestCredList(client *ec2metadata.EC2Metadata) ([]string, error) {
 //
 // If the credentials cannot be found, or there is an error reading the response
 // and error will be returned.
-func requestCred(client *ec2metadata.EC2Metadata, credsName string) (ec2RoleCredRespBody, error) {
-	resp, err := client.GetMetadata(sdkuri.PathJoin(iamSecurityCredsPath, credsName))
+func requestCred(ctx aws.Context, client *ec2metadata.EC2Metadata, credsName string) (ec2RoleCredRespBody, error) {
+	resp, err := client.GetMetadataWithContext(ctx, sdkuri.PathJoin(iamSecurityCredsPath, credsName))
 	if err != nil {
 		return ec2RoleCredRespBody{},
 			awserr.New("EC2RoleRequestError",

vendor/github.com/aws/aws-sdk-go/aws/credentials/endpointcreds/provider.go

@@ -116,7 +116,13 @@ func (p *Provider) IsExpired() bool {
 // Retrieve will attempt to request the credentials from the endpoint the Provider
 // was configured for. And error will be returned if the retrieval fails.
 func (p *Provider) Retrieve() (credentials.Value, error) {
-	resp, err := p.getCredentials()
+	return p.RetrieveWithContext(aws.BackgroundContext())
+}
+
+// RetrieveWithContext will attempt to request the credentials from the endpoint the Provider
+// was configured for. And error will be returned if the retrieval fails.
+func (p *Provider) RetrieveWithContext(ctx credentials.Context) (credentials.Value, error) {
+	resp, err := p.getCredentials(ctx)
 	if err != nil {
 		return credentials.Value{ProviderName: ProviderName},
 			awserr.New("CredentialsEndpointError", "failed to load credentials", err)
@@ -148,7 +154,7 @@ type errorOutput struct {
 	Message string `json:"message"`
 }
 
-func (p *Provider) getCredentials() (*getCredentialsOutput, error) {
+func (p *Provider) getCredentials(ctx aws.Context) (*getCredentialsOutput, error) {
 	op := &request.Operation{
 		Name:       "GetCredentials",
 		HTTPMethod: "GET",
@@ -156,6 +162,7 @@ func (p *Provider) getCredentials() (*getCredentialsOutput, error) {
 
 	out := &getCredentialsOutput{}
 	req := p.Client.NewRequest(op, nil, out)
+	req.SetContext(ctx)
 	req.HTTPRequest.Header.Set("Accept", "application/json")
 	if authToken := p.AuthorizationToken; len(authToken) != 0 {
 		req.HTTPRequest.Header.Set("Authorization", authToken)

vendor/github.com/aws/aws-sdk-go/aws/credentials/processcreds/provider.go

@@ -90,6 +90,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/internal/sdkio"
 )
 
 const (
@@ -142,7 +143,7 @@ const (
 
 	// DefaultBufSize limits buffer size from growing to an enormous
 	// amount due to a faulty process.
-	DefaultBufSize = 1024
+	DefaultBufSize = int(8 * sdkio.KibiByte)
 
 	// DefaultTimeout default limit on time a process can run.
 	DefaultTimeout = time.Duration(1) * time.Minute

vendor/github.com/aws/aws-sdk-go/aws/credentials/shared_credentials_provider.go

@@ -17,8 +17,9 @@ var (
 	ErrSharedCredentialsHomeNotFound = awserr.New("UserHomeNotFound", "user home directory not found.", nil)
 )
 
-// A SharedCredentialsProvider retrieves credentials from the current user's home
-// directory, and keeps track if those credentials are expired.
+// A SharedCredentialsProvider retrieves access key pair (access key ID,
+// secret access key, and session token if present) credentials from the current
+// user's home directory, and keeps track if those credentials are expired.
 //
 // Profile ini file example: $HOME/.aws/credentials
 type SharedCredentialsProvider struct {

vendor/github.com/aws/aws-sdk-go/aws/credentials/static_provider.go

@@ -19,7 +19,9 @@ type StaticProvider struct {
 }
 
 // NewStaticCredentials returns a pointer to a new Credentials object
-// wrapping a static credentials value provider.
+// wrapping a static credentials value provider. Token is only required
+// for temporary security credentials retrieved via STS, otherwise an empty
+// string can be passed for this parameter.
 func NewStaticCredentials(id, secret, token string) *Credentials {
 	return NewCredentials(&StaticProvider{Value: Value{
 		AccessKeyID:     id,

vendor/github.com/aws/aws-sdk-go/aws/credentials/stscreds/assume_role_provider.go

@@ -87,6 +87,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/internal/sdkrand"
 	"github.com/aws/aws-sdk-go/service/sts"
 )
@@ -118,6 +119,10 @@ type AssumeRoler interface {
 	AssumeRole(input *sts.AssumeRoleInput) (*sts.AssumeRoleOutput, error)
 }
 
+type assumeRolerWithContext interface {
+	AssumeRoleWithContext(aws.Context, *sts.AssumeRoleInput, ...request.Option) (*sts.AssumeRoleOutput, error)
+}
+
 // DefaultDuration is the default amount of time in minutes that the credentials
 // will be valid for.
 var DefaultDuration = time.Duration(15) * time.Minute
@@ -144,6 +149,13 @@ type AssumeRoleProvider struct {
 	// Session name, if you wish to reuse the credentials elsewhere.
 	RoleSessionName string
 
+	// Optional, you can pass tag key-value pairs to your session. These tags are called session tags.
+	Tags []*sts.Tag
+
+	// A list of keys for session tags that you want to set as transitive.
+	// If you set a tag key as transitive, the corresponding key and value passes to subsequent sessions in a role chain.
+	TransitiveTagKeys []*string
+
 	// Expiry duration of the STS credentials. Defaults to 15 minutes if not set.
 	Duration time.Duration
 
@@ -157,6 +169,29 @@ type AssumeRoleProvider struct {
 	// size.
 	Policy *string
 
+	// The ARNs of IAM managed policies you want to use as managed session policies.
+	// The policies must exist in the same account as the role.
+	//
+	// This parameter is optional. You can provide up to 10 managed policy ARNs.
+	// However, the plain text that you use for both inline and managed session
+	// policies can't exceed 2,048 characters.
+	//
+	// An AWS conversion compresses the passed session policies and session tags
+	// into a packed binary format that has a separate limit. Your request can fail
+	// for this limit even if your plain text meets the other requirements. The
+	// PackedPolicySize response element indicates by percentage how close the policies
+	// and tags for your request are to the upper size limit.
+	//
+	// Passing policies to this operation returns new temporary credentials. The
+	// resulting session's permissions are the intersection of the role's identity-based
+	// policy and the session policies. You can use the role's temporary credentials
+	// in subsequent AWS API calls to access resources in the account that owns
+	// the role. You cannot use session policies to grant more permissions than
+	// those allowed by the identity-based policy of the role that is being assumed.
+	// For more information, see Session Policies (https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies.html#policies_session)
+	// in the IAM User Guide.
+	PolicyArns []*sts.PolicyDescriptorType
+
 	// The identification number of the MFA device that is associated with the user
 	// who is making the AssumeRole call. Specify this value if the trust policy
 	// of the role being assumed includes a condition that requires MFA authentication.
@@ -258,6 +293,11 @@ func NewCredentialsWithClient(svc AssumeRoler, roleARN string, options ...func(*
 
 // Retrieve generates a new set of temporary credentials using STS.
 func (p *AssumeRoleProvider) Retrieve() (credentials.Value, error) {
+	return p.RetrieveWithContext(aws.BackgroundContext())
+}
+
+// RetrieveWithContext generates a new set of temporary credentials using STS.
+func (p *AssumeRoleProvider) RetrieveWithContext(ctx credentials.Context) (credentials.Value, error) {
 	// Apply defaults where parameters are not set.
 	if p.RoleSessionName == "" {
 		// Try to work out a role name that will hopefully end up unique.
@@ -269,10 +309,13 @@ func (p *AssumeRoleProvider) Retrieve() (credentials.Value, error) {
 	}
 	jitter := time.Duration(sdkrand.SeededRand.Float64() * p.MaxJitterFrac * float64(p.Duration))
 	input := &sts.AssumeRoleInput{
-		DurationSeconds: aws.Int64(int64((p.Duration - jitter) / time.Second)),
-		RoleArn:         aws.String(p.RoleARN),
-		RoleSessionName: aws.String(p.RoleSessionName),
-		ExternalId:      p.ExternalID,
+		DurationSeconds:   aws.Int64(int64((p.Duration - jitter) / time.Second)),
+		RoleArn:           aws.String(p.RoleARN),
+		RoleSessionName:   aws.String(p.RoleSessionName),
+		ExternalId:        p.ExternalID,
+		Tags:              p.Tags,
+		PolicyArns:        p.PolicyArns,
+		TransitiveTagKeys: p.TransitiveTagKeys,
 	}
 	if p.Policy != nil {
 		input.Policy = p.Policy
@@ -295,7 +338,15 @@ func (p *AssumeRoleProvider) Retrieve() (credentials.Value, error) {
 		}
 	}
 
-	roleOutput, err := p.Client.AssumeRole(input)
+	var roleOutput *sts.AssumeRoleOutput
+	var err error
+
+	if c, ok := p.Client.(assumeRolerWithContext); ok {
+		roleOutput, err = c.AssumeRoleWithContext(ctx, input)
+	} else {
+		roleOutput, err = p.Client.AssumeRole(input)
+	}
+
 	if err != nil {
 		return credentials.Value{ProviderName: ProviderName}, err
 	}

vendor/github.com/aws/aws-sdk-go/aws/credentials/stscreds/web_identity_provider.go

@@ -28,15 +28,46 @@ const (
 // compare test values.
 var now = time.Now
 
+// TokenFetcher shuold return WebIdentity token bytes or an error
+type TokenFetcher interface {
+	FetchToken(credentials.Context) ([]byte, error)
+}
+
+// FetchTokenPath is a path to a WebIdentity token file
+type FetchTokenPath string
+
+// FetchToken returns a token by reading from the filesystem
+func (f FetchTokenPath) FetchToken(ctx credentials.Context) ([]byte, error) {
+	data, err := ioutil.ReadFile(string(f))
+	if err != nil {
+		errMsg := fmt.Sprintf("unable to read file at %s", f)
+		return nil, awserr.New(ErrCodeWebIdentity, errMsg, err)
+	}
+	return data, nil
+}
+
 // WebIdentityRoleProvider is used to retrieve credentials using
 // an OIDC token.
 type WebIdentityRoleProvider struct {
 	credentials.Expiry
+	PolicyArns []*sts.PolicyDescriptorType
 
-	client       stsiface.STSAPI
+	// Duration the STS credentials will be valid for. Truncated to seconds.
+	// If unset, the assumed role will use AssumeRoleWithWebIdentity's default
+	// expiry duration. See
+	// https://docs.aws.amazon.com/sdk-for-go/api/service/sts/#STS.AssumeRoleWithWebIdentity
+	// for more information.
+	Duration time.Duration
+
+	// The amount of time the credentials will be refreshed before they expire.
+	// This is useful refresh credentials before they expire to reduce risk of
+	// using credentials as they expire. If unset, will default to no expiry
+	// window.
 	ExpiryWindow time.Duration
 
-	tokenFilePath   string
+	client stsiface.STSAPI
+
+	tokenFetcher    TokenFetcher
 	roleARN         string
 	roleSessionName string
 }
@@ -52,9 +83,15 @@ func NewWebIdentityCredentials(c client.ConfigProvider, roleARN, roleSessionName
 // NewWebIdentityRoleProvider will return a new WebIdentityRoleProvider with the
 // provided stsiface.STSAPI
 func NewWebIdentityRoleProvider(svc stsiface.STSAPI, roleARN, roleSessionName, path string) *WebIdentityRoleProvider {
+	return NewWebIdentityRoleProviderWithToken(svc, roleARN, roleSessionName, FetchTokenPath(path))
+}
+
+// NewWebIdentityRoleProviderWithToken will return a new WebIdentityRoleProvider with the
+// provided stsiface.STSAPI and a TokenFetcher
+func NewWebIdentityRoleProviderWithToken(svc stsiface.STSAPI, roleARN, roleSessionName string, tokenFetcher TokenFetcher) *WebIdentityRoleProvider {
 	return &WebIdentityRoleProvider{
 		client:          svc,
-		tokenFilePath:   path,
+		tokenFetcher:    tokenFetcher,
 		roleARN:         roleARN,
 		roleSessionName: roleSessionName,
 	}
@@ -64,10 +101,16 @@ func NewWebIdentityRoleProvider(svc stsiface.STSAPI, roleARN, roleSessionName, p
 // 'WebIdentityTokenFilePath' specified destination and if that is empty an
 // error will be returned.
 func (p *WebIdentityRoleProvider) Retrieve() (credentials.Value, error) {
-	b, err := ioutil.ReadFile(p.tokenFilePath)
+	return p.RetrieveWithContext(aws.BackgroundContext())
+}
+
+// RetrieveWithContext will attempt to assume a role from a token which is located at
+// 'WebIdentityTokenFilePath' specified destination and if that is empty an
+// error will be returned.
+func (p *WebIdentityRoleProvider) RetrieveWithContext(ctx credentials.Context) (credentials.Value, error) {
+	b, err := p.tokenFetcher.FetchToken(ctx)
 	if err != nil {
-		errMsg := fmt.Sprintf("unable to read file at %s", p.tokenFilePath)
-		return credentials.Value{}, awserr.New(ErrCodeWebIdentity, errMsg, err)
+		return credentials.Value{}, awserr.New(ErrCodeWebIdentity, "failed fetching WebIdentity token: ", err)
 	}
 
 	sessionName := p.roleSessionName
@@ -76,11 +119,22 @@ func (p *WebIdentityRoleProvider) Retrieve() (credentials.Value, error) {
 		// uses unix time in nanoseconds to uniquely identify sessions.
 		sessionName = strconv.FormatInt(now().UnixNano(), 10)
 	}
+
+	var duration *int64
+	if p.Duration != 0 {
+		duration = aws.Int64(int64(p.Duration / time.Second))
+	}
+
 	req, resp := p.client.AssumeRoleWithWebIdentityRequest(&sts.AssumeRoleWithWebIdentityInput{
+		PolicyArns:       p.PolicyArns,
 		RoleArn:          &p.roleARN,
 		RoleSessionName:  &sessionName,
 		WebIdentityToken: aws.String(string(b)),
+		DurationSeconds:  duration,
 	})
+
+	req.SetContext(ctx)
+
 	// InvalidIdentityToken error is a temporary error that can occur
 	// when assuming an Role with a JWT web identity token.
 	req.RetryErrorCodes = append(req.RetryErrorCodes, sts.ErrCodeInvalidIdentityTokenException)

vendor/github.com/aws/aws-sdk-go/aws/csm/reporter.go

@@ -66,7 +66,6 @@ func (rep *Reporter) sendAPICallAttemptMetric(r *request.Request) {
 
 		XAmzRequestID: aws.String(r.RequestID),
 
-		AttemptCount:   aws.Int(r.RetryCount + 1),
 		AttemptLatency: aws.Int(int(now.Sub(r.AttemptTime).Nanoseconds() / int64(time.Millisecond))),
 		AccessKey:      aws.String(creds.AccessKeyID),
 	}
@@ -90,7 +89,7 @@ func getMetricException(err awserr.Error) metricException {
 	code := err.Code()
 
 	switch code {
-	case "RequestError",
+	case request.ErrCodeRequestError,
 		request.ErrCodeSerialization,
 		request.CanceledErrorCode:
 		return sdkException{

vendor/github.com/aws/aws-sdk-go/aws/ec2metadata/api.go

@@ -4,28 +4,73 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/internal/sdkuri"
 )
 
+// getToken uses the duration to return a token for EC2 metadata service,
+// or an error if the request failed.
+func (c *EC2Metadata) getToken(ctx aws.Context, duration time.Duration) (tokenOutput, error) {
+	op := &request.Operation{
+		Name:       "GetToken",
+		HTTPMethod: "PUT",
+		HTTPPath:   "/latest/api/token",
+	}
+
+	var output tokenOutput
+	req := c.NewRequest(op, nil, &output)
+	req.SetContext(ctx)
+
+	// remove the fetch token handler from the request handlers to avoid infinite recursion
+	req.Handlers.Sign.RemoveByName(fetchTokenHandlerName)
+
+	// Swap the unmarshalMetadataHandler with unmarshalTokenHandler on this request.
+	req.Handlers.Unmarshal.Swap(unmarshalMetadataHandlerName, unmarshalTokenHandler)
+
+	ttl := strconv.FormatInt(int64(duration/time.Second), 10)
+	req.HTTPRequest.Header.Set(ttlHeader, ttl)
+
+	err := req.Send()
+
+	// Errors with bad request status should be returned.
+	if err != nil {
+		err = awserr.NewRequestFailure(
+			awserr.New(req.HTTPResponse.Status, http.StatusText(req.HTTPResponse.StatusCode), err),
+			req.HTTPResponse.StatusCode, req.RequestID)
+	}
+
+	return output, err
+}
+
 // GetMetadata uses the path provided to request information from the EC2
-// instance metdata service. The content will be returned as a string, or
+// instance metadata service. The content will be returned as a string, or
 // error if the request failed.
 func (c *EC2Metadata) GetMetadata(p string) (string, error) {
+	return c.GetMetadataWithContext(aws.BackgroundContext(), p)
+}
+
+// GetMetadataWithContext uses the path provided to request information from the EC2
+// instance metadata service. The content will be returned as a string, or
+// error if the request failed.
+func (c *EC2Metadata) GetMetadataWithContext(ctx aws.Context, p string) (string, error) {
 	op := &request.Operation{
 		Name:       "GetMetadata",
 		HTTPMethod: "GET",
-		HTTPPath:   sdkuri.PathJoin("/meta-data", p),
+		HTTPPath:   sdkuri.PathJoin("/latest/meta-data", p),
 	}
-
 	output := &metadataOutput{}
-	req := c.NewRequest(op, nil, output)
-	err := req.Send()
 
+	req := c.NewRequest(op, nil, output)
+
+	req.SetContext(ctx)
+
+	err := req.Send()
 	return output.Content, err
 }
 
@@ -33,21 +78,24 @@ func (c *EC2Metadata) GetMetadata(p string) (string, error) {
 // there is no user-data setup for the EC2 instance a "NotFoundError" error
 // code will be returned.
 func (c *EC2Metadata) GetUserData() (string, error) {
+	return c.GetUserDataWithContext(aws.BackgroundContext())
+}
+
+// GetUserDataWithContext returns the userdata that was configured for the service. If
+// there is no user-data setup for the EC2 instance a "NotFoundError" error
+// code will be returned.
+func (c *EC2Metadata) GetUserDataWithContext(ctx aws.Context) (string, error) {
 	op := &request.Operation{
 		Name:       "GetUserData",
 		HTTPMethod: "GET",
-		HTTPPath:   "/user-data",
+		HTTPPath:   "/latest/user-data",
 	}
 
 	output := &metadataOutput{}
 	req := c.NewRequest(op, nil, output)
-	req.Handlers.UnmarshalError.PushBack(func(r *request.Request) {
-		if r.HTTPResponse.StatusCode == http.StatusNotFound {
-			r.Error = awserr.New("NotFoundError", "user-data not found", r.Error)
-		}
-	})
-	err := req.Send()
+	req.SetContext(ctx)
 
+	err := req.Send()
 	return output.Content, err
 }
 
@@ -55,16 +103,24 @@ func (c *EC2Metadata) GetUserData() (string, error) {
 // instance metadata service for dynamic data. The content will be returned
 // as a string, or error if the request failed.
 func (c *EC2Metadata) GetDynamicData(p string) (string, error) {
+	return c.GetDynamicDataWithContext(aws.BackgroundContext(), p)
+}
+
+// GetDynamicDataWithContext uses the path provided to request information from the EC2
+// instance metadata service for dynamic data. The content will be returned
+// as a string, or error if the request failed.
+func (c *EC2Metadata) GetDynamicDataWithContext(ctx aws.Context, p string) (string, error) {
 	op := &request.Operation{
 		Name:       "GetDynamicData",
 		HTTPMethod: "GET",
-		HTTPPath:   sdkuri.PathJoin("/dynamic", p),
+		HTTPPath:   sdkuri.PathJoin("/latest/dynamic", p),
 	}
 
 	output := &metadataOutput{}
 	req := c.NewRequest(op, nil, output)
-	err := req.Send()
+	req.SetContext(ctx)
 
+	err := req.Send()
 	return output.Content, err
 }
 
@@ -72,7 +128,14 @@ func (c *EC2Metadata) GetDynamicData(p string) (string, error) {
 // instance. Error is returned if the request fails or is unable to parse
 // the response.
 func (c *EC2Metadata) GetInstanceIdentityDocument() (EC2InstanceIdentityDocument, error) {
-	resp, err := c.GetDynamicData("instance-identity/document")
+	return c.GetInstanceIdentityDocumentWithContext(aws.BackgroundContext())
+}
+
+// GetInstanceIdentityDocumentWithContext retrieves an identity document describing an
+// instance. Error is returned if the request fails or is unable to parse
+// the response.
+func (c *EC2Metadata) GetInstanceIdentityDocumentWithContext(ctx aws.Context) (EC2InstanceIdentityDocument, error) {
+	resp, err := c.GetDynamicDataWithContext(ctx, "instance-identity/document")
 	if err != nil {
 		return EC2InstanceIdentityDocument{},
 			awserr.New("EC2MetadataRequestError",
@@ -91,7 +154,12 @@ func (c *EC2Metadata) GetInstanceIdentityDocument() (EC2InstanceIdentityDocument
 
 // IAMInfo retrieves IAM info from the metadata API
 func (c *EC2Metadata) IAMInfo() (EC2IAMInfo, error) {
-	resp, err := c.GetMetadata("iam/info")
+	return c.IAMInfoWithContext(aws.BackgroundContext())
+}
+
+// IAMInfoWithContext retrieves IAM info from the metadata API
+func (c *EC2Metadata) IAMInfoWithContext(ctx aws.Context) (EC2IAMInfo, error) {
+	resp, err := c.GetMetadataWithContext(ctx, "iam/info")
 	if err != nil {
 		return EC2IAMInfo{},
 			awserr.New("EC2MetadataRequestError",
@@ -116,24 +184,36 @@ func (c *EC2Metadata) IAMInfo() (EC2IAMInfo, error) {
 
 // Region returns the region the instance is running in.
 func (c *EC2Metadata) Region() (string, error) {
-	resp, err := c.GetMetadata("placement/availability-zone")
+	return c.RegionWithContext(aws.BackgroundContext())
+}
+
+// RegionWithContext returns the region the instance is running in.
+func (c *EC2Metadata) RegionWithContext(ctx aws.Context) (string, error) {
+	ec2InstanceIdentityDocument, err := c.GetInstanceIdentityDocumentWithContext(ctx)
 	if err != nil {
 		return "", err
 	}
-
-	if len(resp) == 0 {
-		return "", awserr.New("EC2MetadataError", "invalid Region response", nil)
+	// extract region from the ec2InstanceIdentityDocument
+	region := ec2InstanceIdentityDocument.Region
+	if len(region) == 0 {
+		return "", awserr.New("EC2MetadataError", "invalid region received for ec2metadata instance", nil)
 	}
-
-	// returns region without the suffix. Eg: us-west-2a becomes us-west-2
-	return resp[:len(resp)-1], nil
+	// returns region
+	return region, nil
 }
 
 // Available returns if the application has access to the EC2 Metadata service.
 // Can be used to determine if application is running within an EC2 Instance and
 // the metadata service is available.
 func (c *EC2Metadata) Available() bool {
-	if _, err := c.GetMetadata("instance-id"); err != nil {
+	return c.AvailableWithContext(aws.BackgroundContext())
+}
+
+// AvailableWithContext returns if the application has access to the EC2 Metadata service.
+// Can be used to determine if application is running within an EC2 Instance and
+// the metadata service is available.
+func (c *EC2Metadata) AvailableWithContext(ctx aws.Context) bool {
+	if _, err := c.GetMetadataWithContext(ctx, "instance-id"); err != nil {
 		return false
 	}
 

vendor/github.com/aws/aws-sdk-go/aws/ec2metadata/service.go

@@ -5,6 +5,10 @@
 // variable "AWS_EC2_METADATA_DISABLED=true". This environment variable set to
 // true instructs the SDK to disable the EC2 Metadata client. The client cannot
 // be used while the environment variable is set to true, (case insensitive).
+//
+// The endpoint of the EC2 IMDS client can be configured via the environment
+// variable, AWS_EC2_METADATA_SERVICE_ENDPOINT when creating the client with a
+// Session. See aws/session#Options.EC2IMDSEndpoint for more details.
 package ec2metadata
 
 import (
@@ -12,7 +16,9 @@ import (
 	"errors"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -24,9 +30,25 @@ import (
 	"github.com/aws/aws-sdk-go/aws/request"
 )
 
-// ServiceName is the name of the service.
-const ServiceName = "ec2metadata"
-const disableServiceEnvVar = "AWS_EC2_METADATA_DISABLED"
+const (
+	// ServiceName is the name of the service.
+	ServiceName          = "ec2metadata"
+	disableServiceEnvVar = "AWS_EC2_METADATA_DISABLED"
+
+	// Headers for Token and TTL
+	ttlHeader   = "x-aws-ec2-metadata-token-ttl-seconds"
+	tokenHeader = "x-aws-ec2-metadata-token"
+
+	// Named Handler constants
+	fetchTokenHandlerName          = "FetchTokenHandler"
+	unmarshalMetadataHandlerName   = "unmarshalMetadataHandler"
+	unmarshalTokenHandlerName      = "unmarshalTokenHandler"
+	enableTokenProviderHandlerName = "enableTokenProviderHandler"
+
+	// TTL constants
+	defaultTTL          = 21600 * time.Second
+	ttlExpirationWindow = 30 * time.Second
+)
 
 // A EC2Metadata is an EC2 Metadata service Client.
 type EC2Metadata struct {
@@ -52,6 +74,9 @@ func New(p client.ConfigProvider, cfgs ...*aws.Config) *EC2Metadata {
 // a client when not using a session. Generally using just New with a session
 // is preferred.
 //
+// Will remove the URL path from the endpoint provided to ensure the EC2 IMDS
+// client is able to communicate with the EC2 IMDS API.
+//
 // If an unmodified HTTP client is provided from the stdlib default, or no client
 // the EC2RoleProvider's EC2Metadata HTTP client's timeout will be shortened.
 // To disable this set Config.EC2MetadataDisableTimeoutOverride to false. Enabled by default.
@@ -63,8 +88,19 @@ func NewClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio
 			// use a shorter timeout than default because the metadata
 			// service is local if it is running, and to fail faster
 			// if not running on an ec2 instance.
-			Timeout: 5 * time.Second,
+			Timeout: 1 * time.Second,
 		}
+		// max number of retries on the client operation
+		cfg.MaxRetries = aws.Int(2)
+	}
+
+	if u, err := url.Parse(endpoint); err == nil {
+		// Remove path from the endpoint since it will be added by requests.
+		// This is an artifact of the SDK adding `/latest` to the endpoint for
+		// EC2 IMDS, but this is now moved to the operation definition.
+		u.Path = ""
+		u.RawPath = ""
+		endpoint = u.String()
 	}
 
 	svc := &EC2Metadata{
@@ -80,13 +116,27 @@ func NewClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio
 		),
 	}
 
-	svc.Handlers.Unmarshal.PushBack(unmarshalHandler)
+	// token provider instance
+	tp := newTokenProvider(svc, defaultTTL)
+
+	// NamedHandler for fetching token
+	svc.Handlers.Sign.PushBackNamed(request.NamedHandler{
+		Name: fetchTokenHandlerName,
+		Fn:   tp.fetchTokenHandler,
+	})
+	// NamedHandler for enabling token provider
+	svc.Handlers.Complete.PushBackNamed(request.NamedHandler{
+		Name: enableTokenProviderHandlerName,
+		Fn:   tp.enableTokenProviderHandler,
+	})
+
+	svc.Handlers.Unmarshal.PushBackNamed(unmarshalHandler)
 	svc.Handlers.UnmarshalError.PushBack(unmarshalError)
 	svc.Handlers.Validate.Clear()
 	svc.Handlers.Validate.PushBack(validateEndpointHandler)
 
 	// Disable the EC2 Metadata service if the environment variable is set.
-	// This shortcirctes the service's functionality to always fail to send
+	// This short-circuits the service's functionality to always fail to send
 	// requests.
 	if strings.ToLower(os.Getenv(disableServiceEnvVar)) == "true" {
 		svc.Handlers.Send.SwapNamed(request.NamedHandler{
@@ -107,7 +157,6 @@ func NewClient(cfg aws.Config, handlers request.Handlers, endpoint, signingRegio
 	for _, option := range opts {
 		option(svc.Client)
 	}
-
 	return svc
 }
 
@@ -119,30 +168,74 @@ type metadataOutput struct {
 	Content string
 }
 
-func unmarshalHandler(r *request.Request) {
-	defer r.HTTPResponse.Body.Close()
-	b := &bytes.Buffer{}
-	if _, err := io.Copy(b, r.HTTPResponse.Body); err != nil {
-		r.Error = awserr.New(request.ErrCodeSerialization, "unable to unmarshal EC2 metadata response", err)
-		return
-	}
+type tokenOutput struct {
+	Token string
+	TTL   time.Duration
+}
 
-	if data, ok := r.Data.(*metadataOutput); ok {
-		data.Content = b.String()
-	}
+// unmarshal token handler is used to parse the response of a getToken operation
+var unmarshalTokenHandler = request.NamedHandler{
+	Name: unmarshalTokenHandlerName,
+	Fn: func(r *request.Request) {
+		defer r.HTTPResponse.Body.Close()
+		var b bytes.Buffer
+		if _, err := io.Copy(&b, r.HTTPResponse.Body); err != nil {
+			r.Error = awserr.NewRequestFailure(awserr.New(request.ErrCodeSerialization,
+				"unable to unmarshal EC2 metadata response", err), r.HTTPResponse.StatusCode, r.RequestID)
+			return
+		}
+
+		v := r.HTTPResponse.Header.Get(ttlHeader)
+		data, ok := r.Data.(*tokenOutput)
+		if !ok {
+			return
+		}
+
+		data.Token = b.String()
+		// TTL is in seconds
+		i, err := strconv.ParseInt(v, 10, 64)
+		if err != nil {
+			r.Error = awserr.NewRequestFailure(awserr.New(request.ParamFormatErrCode,
+				"unable to parse EC2 token TTL response", err), r.HTTPResponse.StatusCode, r.RequestID)
+			return
+		}
+		t := time.Duration(i) * time.Second
+		data.TTL = t
+	},
+}
+
+var unmarshalHandler = request.NamedHandler{
+	Name: unmarshalMetadataHandlerName,
+	Fn: func(r *request.Request) {
+		defer r.HTTPResponse.Body.Close()
+		var b bytes.Buffer
+		if _, err := io.Copy(&b, r.HTTPResponse.Body); err != nil {
+			r.Error = awserr.NewRequestFailure(awserr.New(request.ErrCodeSerialization,
+				"unable to unmarshal EC2 metadata response", err), r.HTTPResponse.StatusCode, r.RequestID)
+			return
+		}
+
+		if data, ok := r.Data.(*metadataOutput); ok {
+			data.Content = b.String()
+		}
+	},
 }
 
 func unmarshalError(r *request.Request) {
 	defer r.HTTPResponse.Body.Close()
-	b := &bytes.Buffer{}
-	if _, err := io.Copy(b, r.HTTPResponse.Body); err != nil {
-		r.Error = awserr.New(request.ErrCodeSerialization, "unable to unmarshal EC2 metadata error response", err)
+	var b bytes.Buffer
+
+	if _, err := io.Copy(&b, r.HTTPResponse.Body); err != nil {
+		r.Error = awserr.NewRequestFailure(
+			awserr.New(request.ErrCodeSerialization, "unable to unmarshal EC2 metadata error response", err),
+			r.HTTPResponse.StatusCode, r.RequestID)
 		return
 	}
 
 	// Response body format is not consistent between metadata endpoints.
 	// Grab the error message as a string and include that as the source error
-	r.Error = awserr.New("EC2MetadataError", "failed to make EC2Metadata request", errors.New(b.String()))
+	r.Error = awserr.NewRequestFailure(awserr.New("EC2MetadataError", "failed to make EC2Metadata request", errors.New(b.String())),
+		r.HTTPResponse.StatusCode, r.RequestID)
 }
 
 func validateEndpointHandler(r *request.Request) {

vendor/github.com/aws/aws-sdk-go/aws/ec2metadata/token_provider.go

@@ -0,0 +1,92 @@
+package ec2metadata
+
+import (
+	"net/http"
+	"sync/atomic"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+)
+
+// A tokenProvider struct provides access to EC2Metadata client
+// and atomic instance of a token, along with configuredTTL for it.
+// tokenProvider also provides an atomic flag to disable the
+// fetch token operation.
+// The disabled member will use 0 as false, and 1 as true.
+type tokenProvider struct {
+	client        *EC2Metadata
+	token         atomic.Value
+	configuredTTL time.Duration
+	disabled      uint32
+}
+
+// A ec2Token struct helps use of token in EC2 Metadata service ops
+type ec2Token struct {
+	token string
+	credentials.Expiry
+}
+
+// newTokenProvider provides a pointer to a tokenProvider instance
+func newTokenProvider(c *EC2Metadata, duration time.Duration) *tokenProvider {
+	return &tokenProvider{client: c, configuredTTL: duration}
+}
+
+// fetchTokenHandler fetches token for EC2Metadata service client by default.
+func (t *tokenProvider) fetchTokenHandler(r *request.Request) {
+
+	// short-circuits to insecure data flow if tokenProvider is disabled.
+	if v := atomic.LoadUint32(&t.disabled); v == 1 {
+		return
+	}
+
+	if ec2Token, ok := t.token.Load().(ec2Token); ok && !ec2Token.IsExpired() {
+		r.HTTPRequest.Header.Set(tokenHeader, ec2Token.token)
+		return
+	}
+
+	output, err := t.client.getToken(r.Context(), t.configuredTTL)
+
+	if err != nil {
+
+		// change the disabled flag on token provider to true,
+		// when error is request timeout error.
+		if requestFailureError, ok := err.(awserr.RequestFailure); ok {
+			switch requestFailureError.StatusCode() {
+			case http.StatusForbidden, http.StatusNotFound, http.StatusMethodNotAllowed:
+				atomic.StoreUint32(&t.disabled, 1)
+			case http.StatusBadRequest:
+				r.Error = requestFailureError
+			}
+
+			// Check if request timed out while waiting for response
+			if e, ok := requestFailureError.OrigErr().(awserr.Error); ok {
+				if e.Code() == request.ErrCodeRequestError {
+					atomic.StoreUint32(&t.disabled, 1)
+				}
+			}
+		}
+		return
+	}
+
+	newToken := ec2Token{
+		token: output.Token,
+	}
+	newToken.SetExpiration(time.Now().Add(output.TTL), ttlExpirationWindow)
+	t.token.Store(newToken)
+
+	// Inject token header to the request.
+	if ec2Token, ok := t.token.Load().(ec2Token); ok {
+		r.HTTPRequest.Header.Set(tokenHeader, ec2Token.token)
+	}
+}
+
+// enableTokenProviderHandler enables the token provider
+func (t *tokenProvider) enableTokenProviderHandler(r *request.Request) {
+	// If the error code status is 401, we enable the token provider
+	if e, ok := r.Error.(awserr.RequestFailure); ok && e != nil &&
+		e.StatusCode() == http.StatusUnauthorized {
+		atomic.StoreUint32(&t.disabled, 0)
+	}
+}

vendor/github.com/aws/aws-sdk-go/aws/endpoints/decode.go

@@ -83,6 +83,7 @@ func decodeV3Endpoints(modelDef modelDefinition, opts DecodeModelOptions) (Resol
 		p := &ps[i]
 		custAddEC2Metadata(p)
 		custAddS3DualStack(p)
+		custRegionalS3(p)
 		custRmIotDataService(p)
 		custFixAppAutoscalingChina(p)
 		custFixAppAutoscalingUsGov(p)
@@ -92,7 +93,7 @@ func decodeV3Endpoints(modelDef modelDefinition, opts DecodeModelOptions) (Resol
 }
 
 func custAddS3DualStack(p *partition) {
-	if p.ID != "aws" {
+	if !(p.ID == "aws" || p.ID == "aws-cn" || p.ID == "aws-us-gov") {
 		return
 	}
 
@@ -100,6 +101,33 @@ func custAddS3DualStack(p *partition) {
 	custAddDualstack(p, "s3-control")
 }
 
+func custRegionalS3(p *partition) {
+	if p.ID != "aws" {
+		return
+	}
+
+	service, ok := p.Services["s3"]
+	if !ok {
+		return
+	}
+
+	// If global endpoint already exists no customization needed.
+	if _, ok := service.Endpoints["aws-global"]; ok {
+		return
+	}
+
+	service.PartitionEndpoint = "aws-global"
+	service.Endpoints["us-east-1"] = endpoint{}
+	service.Endpoints["aws-global"] = endpoint{
+		Hostname: "s3.amazonaws.com",
+		CredentialScope: credentialScope{
+			Region: "us-east-1",
+		},
+	}
+
+	p.Services["s3"] = service
+}
+
 func custAddDualstack(p *partition, svcName string) {
 	s, ok := p.Services[svcName]
 	if !ok {

vendor/github.com/aws/aws-sdk-go/aws/endpoints/endpoints.go

@@ -3,6 +3,7 @@ package endpoints
 import (
 	"fmt"
 	"regexp"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws/awserr"
 )
@@ -46,6 +47,108 @@ type Options struct {
 	//
 	// This option is ignored if StrictMatching is enabled.
 	ResolveUnknownService bool
+
+	// STS Regional Endpoint flag helps with resolving the STS endpoint
+	STSRegionalEndpoint STSRegionalEndpoint
+
+	// S3 Regional Endpoint flag helps with resolving the S3 endpoint
+	S3UsEast1RegionalEndpoint S3UsEast1RegionalEndpoint
+}
+
+// STSRegionalEndpoint is an enum for the states of the STS Regional Endpoint
+// options.
+type STSRegionalEndpoint int
+
+func (e STSRegionalEndpoint) String() string {
+	switch e {
+	case LegacySTSEndpoint:
+		return "legacy"
+	case RegionalSTSEndpoint:
+		return "regional"
+	case UnsetSTSEndpoint:
+		return ""
+	default:
+		return "unknown"
+	}
+}
+
+const (
+
+	// UnsetSTSEndpoint represents that STS Regional Endpoint flag is not specified.
+	UnsetSTSEndpoint STSRegionalEndpoint = iota
+
+	// LegacySTSEndpoint represents when STS Regional Endpoint flag is specified
+	// to use legacy endpoints.
+	LegacySTSEndpoint
+
+	// RegionalSTSEndpoint represents when STS Regional Endpoint flag is specified
+	// to use regional endpoints.
+	RegionalSTSEndpoint
+)
+
+// GetSTSRegionalEndpoint function returns the STSRegionalEndpointFlag based
+// on the input string provided in env config or shared config by the user.
+//
+// `legacy`, `regional` are the only case-insensitive valid strings for
+// resolving the STS regional Endpoint flag.
+func GetSTSRegionalEndpoint(s string) (STSRegionalEndpoint, error) {
+	switch {
+	case strings.EqualFold(s, "legacy"):
+		return LegacySTSEndpoint, nil
+	case strings.EqualFold(s, "regional"):
+		return RegionalSTSEndpoint, nil
+	default:
+		return UnsetSTSEndpoint, fmt.Errorf("unable to resolve the value of STSRegionalEndpoint for %v", s)
+	}
+}
+
+// S3UsEast1RegionalEndpoint is an enum for the states of the S3 us-east-1
+// Regional Endpoint options.
+type S3UsEast1RegionalEndpoint int
+
+func (e S3UsEast1RegionalEndpoint) String() string {
+	switch e {
+	case LegacyS3UsEast1Endpoint:
+		return "legacy"
+	case RegionalS3UsEast1Endpoint:
+		return "regional"
+	case UnsetS3UsEast1Endpoint:
+		return ""
+	default:
+		return "unknown"
+	}
+}
+
+const (
+
+	// UnsetS3UsEast1Endpoint represents that S3 Regional Endpoint flag is not
+	// specified.
+	UnsetS3UsEast1Endpoint S3UsEast1RegionalEndpoint = iota
+
+	// LegacyS3UsEast1Endpoint represents when S3 Regional Endpoint flag is
+	// specified to use legacy endpoints.
+	LegacyS3UsEast1Endpoint
+
+	// RegionalS3UsEast1Endpoint represents when S3 Regional Endpoint flag is
+	// specified to use regional endpoints.
+	RegionalS3UsEast1Endpoint
+)
+
+// GetS3UsEast1RegionalEndpoint function returns the S3UsEast1RegionalEndpointFlag based
+// on the input string provided in env config or shared config by the user.
+//
+// `legacy`, `regional` are the only case-insensitive valid strings for
+// resolving the S3 regional Endpoint flag.
+func GetS3UsEast1RegionalEndpoint(s string) (S3UsEast1RegionalEndpoint, error) {
+	switch {
+	case strings.EqualFold(s, "legacy"):
+		return LegacyS3UsEast1Endpoint, nil
+	case strings.EqualFold(s, "regional"):
+		return RegionalS3UsEast1Endpoint, nil
+	default:
+		return UnsetS3UsEast1Endpoint,
+			fmt.Errorf("unable to resolve the value of S3UsEast1RegionalEndpoint for %v", s)
+	}
 }
 
 // Set combines all of the option functions together.
@@ -79,6 +182,12 @@ func ResolveUnknownServiceOption(o *Options) {
 	o.ResolveUnknownService = true
 }
 
+// STSRegionalEndpointOption enables the STS endpoint resolver behavior to resolve
+// STS endpoint to their regional endpoint, instead of the global endpoint.
+func STSRegionalEndpointOption(o *Options) {
+	o.STSRegionalEndpoint = RegionalSTSEndpoint
+}
+
 // A Resolver provides the interface for functionality to resolve endpoints.
 // The build in Partition and DefaultResolver return value satisfy this interface.
 type Resolver interface {
@@ -194,7 +303,7 @@ func (p Partition) ID() string { return p.id }
 // require the provided service and region to be known by the partition.
 // If the endpoint cannot be strictly resolved an error will be returned. This
 // mode is useful to ensure the endpoint resolved is valid. Without
-// StrictMatching enabled the endpoint returned my look valid but may not work.
+// StrictMatching enabled the endpoint returned may look valid but may not work.
 // StrictMatching requires the SDK to be updated if you want to take advantage
 // of new regions and services expansions.
 //
@@ -208,7 +317,7 @@ func (p Partition) EndpointFor(service, region string, opts ...func(*Options)) (
 // Regions returns a map of Regions indexed by their ID. This is useful for
 // enumerating over the regions in a partition.
 func (p Partition) Regions() map[string]Region {
-	rs := map[string]Region{}
+	rs := make(map[string]Region, len(p.p.Regions))
 	for id, r := range p.p.Regions {
 		rs[id] = Region{
 			id:   id,
@@ -223,7 +332,7 @@ func (p Partition) Regions() map[string]Region {
 // Services returns a map of Service indexed by their ID. This is useful for
 // enumerating over the services in a partition.
 func (p Partition) Services() map[string]Service {
-	ss := map[string]Service{}
+	ss := make(map[string]Service, len(p.p.Services))
 	for id := range p.p.Services {
 		ss[id] = Service{
 			id: id,
@@ -310,7 +419,7 @@ func (s Service) Regions() map[string]Region {
 // A region is the AWS region the service exists in. Whereas a Endpoint is
 // an URL that can be resolved to a instance of a service.
 func (s Service) Endpoints() map[string]Endpoint {
-	es := map[string]Endpoint{}
+	es := make(map[string]Endpoint, len(s.p.Services[s.id].Endpoints))
 	for id := range s.p.Services[s.id].Endpoints {
 		es[id] = Endpoint{
 			id:        id,
@@ -350,6 +459,9 @@ type ResolvedEndpoint struct {
 	// The endpoint URL
 	URL string
 
+	// The endpoint partition
+	PartitionID string
+
 	// The region that should be used for signing requests.
 	SigningRegion string
 

vendor/github.com/aws/aws-sdk-go/aws/endpoints/legacy_regions.go

@@ -0,0 +1,24 @@
+package endpoints
+
+var legacyGlobalRegions = map[string]map[string]struct{}{
+	"sts": {
+		"ap-northeast-1": {},
+		"ap-south-1":     {},
+		"ap-southeast-1": {},
+		"ap-southeast-2": {},
+		"ca-central-1":   {},
+		"eu-central-1":   {},
+		"eu-north-1":     {},
+		"eu-west-1":      {},
+		"eu-west-2":      {},
+		"eu-west-3":      {},
+		"sa-east-1":      {},
+		"us-east-1":      {},
+		"us-east-2":      {},
+		"us-west-1":      {},
+		"us-west-2":      {},
+	},
+	"s3": {
+		"us-east-1": {},
+	},
+}

vendor/github.com/aws/aws-sdk-go/aws/endpoints/v3model.go

@@ -7,6 +7,8 @@ import (
 	"strings"
 )
 
+var regionValidationRegex = regexp.MustCompile(`^[[:alnum:]]([[:alnum:]\-]*[[:alnum:]])?$`)
+
 type partitions []partition
 
 func (ps partitions) EndpointFor(service, region string, opts ...func(*Options)) (ResolvedEndpoint, error) {
@@ -75,24 +77,56 @@ func (p partition) canResolveEndpoint(service, region string, strictMatch bool)
 	return p.RegionRegex.MatchString(region)
 }
 
+func allowLegacyEmptyRegion(service string) bool {
+	legacy := map[string]struct{}{
+		"budgets":       {},
+		"ce":            {},
+		"chime":         {},
+		"cloudfront":    {},
+		"ec2metadata":   {},
+		"iam":           {},
+		"importexport":  {},
+		"organizations": {},
+		"route53":       {},
+		"sts":           {},
+		"support":       {},
+		"waf":           {},
+	}
+
+	_, allowed := legacy[service]
+	return allowed
+}
+
 func (p partition) EndpointFor(service, region string, opts ...func(*Options)) (resolved ResolvedEndpoint, err error) {
 	var opt Options
 	opt.Set(opts...)
 
 	s, hasService := p.Services[service]
-	if !(hasService || opt.ResolveUnknownService) {
+	if len(service) == 0 || !(hasService || opt.ResolveUnknownService) {
 		// Only return error if the resolver will not fallback to creating
 		// endpoint based on service endpoint ID passed in.
 		return resolved, NewUnknownServiceError(p.ID, service, serviceList(p.Services))
 	}
 
+	if len(region) == 0 && allowLegacyEmptyRegion(service) && len(s.PartitionEndpoint) != 0 {
+		region = s.PartitionEndpoint
+	}
+
+	if (service == "sts" && opt.STSRegionalEndpoint != RegionalSTSEndpoint) ||
+		(service == "s3" && opt.S3UsEast1RegionalEndpoint != RegionalS3UsEast1Endpoint) {
+		if _, ok := legacyGlobalRegions[service][region]; ok {
+			region = "aws-global"
+		}
+	}
+
 	e, hasEndpoint := s.endpointForRegion(region)
-	if !hasEndpoint && opt.StrictMatching {
+	if len(region) == 0 || (!hasEndpoint && opt.StrictMatching) {
 		return resolved, NewUnknownEndpointError(p.ID, service, region, endpointList(s.Endpoints))
 	}
 
 	defs := []endpoint{p.Defaults, s.Defaults}
-	return e.resolve(service, region, p.DNSSuffix, defs, opt), nil
+
+	return e.resolve(service, p.ID, region, p.DNSSuffix, defs, opt)
 }
 
 func serviceList(ss services) []string {
@@ -201,7 +235,7 @@ func getByPriority(s []string, p []string, def string) string {
 	return s[0]
 }
 
-func (e endpoint) resolve(service, region, dnsSuffix string, defs []endpoint, opts Options) ResolvedEndpoint {
+func (e endpoint) resolve(service, partitionID, region, dnsSuffix string, defs []endpoint, opts Options) (ResolvedEndpoint, error) {
 	var merged endpoint
 	for _, def := range defs {
 		merged.mergeIn(def)
@@ -209,20 +243,6 @@ func (e endpoint) resolve(service, region, dnsSuffix string, defs []endpoint, op
 	merged.mergeIn(e)
 	e = merged
 
-	hostname := e.Hostname
-
-	// Offset the hostname for dualstack if enabled
-	if opts.UseDualStack && e.HasDualStack == boxedTrue {
-		hostname = e.DualStackHostname
-	}
-
-	u := strings.Replace(hostname, "{service}", service, 1)
-	u = strings.Replace(u, "{region}", region, 1)
-	u = strings.Replace(u, "{dnsSuffix}", dnsSuffix, 1)
-
-	scheme := getEndpointScheme(e.Protocols, opts.DisableSSL)
-	u = fmt.Sprintf("%s://%s", scheme, u)
-
 	signingRegion := e.CredentialScope.Region
 	if len(signingRegion) == 0 {
 		signingRegion = region
@@ -235,13 +255,32 @@ func (e endpoint) resolve(service, region, dnsSuffix string, defs []endpoint, op
 		signingNameDerived = true
 	}
 
+	hostname := e.Hostname
+	// Offset the hostname for dualstack if enabled
+	if opts.UseDualStack && e.HasDualStack == boxedTrue {
+		hostname = e.DualStackHostname
+		region = signingRegion
+	}
+
+	if !validateInputRegion(region) {
+		return ResolvedEndpoint{}, fmt.Errorf("invalid region identifier format provided")
+	}
+
+	u := strings.Replace(hostname, "{service}", service, 1)
+	u = strings.Replace(u, "{region}", region, 1)
+	u = strings.Replace(u, "{dnsSuffix}", dnsSuffix, 1)
+
+	scheme := getEndpointScheme(e.Protocols, opts.DisableSSL)
+	u = fmt.Sprintf("%s://%s", scheme, u)
+
 	return ResolvedEndpoint{
 		URL:                u,
+		PartitionID:        partitionID,
 		SigningRegion:      signingRegion,
 		SigningName:        signingName,
 		SigningNameDerived: signingNameDerived,
 		SigningMethod:      getByPriority(e.SignatureVersions, signerPriority, defaultSigner),
-	}
+	}, nil
 }
 
 func getEndpointScheme(protocols []string, disableSSL bool) string {
@@ -306,3 +345,7 @@ const (
 	boxedFalse
 	boxedTrue
 )
+
+func validateInputRegion(region string) bool {
+	return regionValidationRegex.MatchString(region)
+}

vendor/github.com/aws/aws-sdk-go/aws/request/connection_reset_error.go

@@ -9,7 +9,8 @@ func isErrConnectionReset(err error) bool {
 		return false
 	}
 
-	if strings.Contains(err.Error(), "connection reset") ||
+	if strings.Contains(err.Error(), "use of closed network connection") ||
+		strings.Contains(err.Error(), "connection reset") ||
 		strings.Contains(err.Error(), "broken pipe") {
 		return true
 	}

vendor/github.com/aws/aws-sdk-go/aws/request/handlers.go

@@ -10,6 +10,7 @@ import (
 type Handlers struct {
 	Validate         HandlerList
 	Build            HandlerList
+	BuildStream      HandlerList
 	Sign             HandlerList
 	Send             HandlerList
 	ValidateResponse HandlerList
@@ -28,6 +29,7 @@ func (h *Handlers) Copy() Handlers {
 	return Handlers{
 		Validate:         h.Validate.copy(),
 		Build:            h.Build.copy(),
+		BuildStream:      h.BuildStream.copy(),
 		Sign:             h.Sign.copy(),
 		Send:             h.Send.copy(),
 		ValidateResponse: h.ValidateResponse.copy(),
@@ -46,6 +48,7 @@ func (h *Handlers) Copy() Handlers {
 func (h *Handlers) Clear() {
 	h.Validate.Clear()
 	h.Build.Clear()
+	h.BuildStream.Clear()
 	h.Send.Clear()
 	h.Sign.Clear()
 	h.Unmarshal.Clear()
@@ -67,6 +70,9 @@ func (h *Handlers) IsEmpty() bool {
 	if h.Build.Len() != 0 {
 		return false
 	}
+	if h.BuildStream.Len() != 0 {
+		return false
+	}
 	if h.Send.Len() != 0 {
 		return false
 	}
@@ -320,3 +326,18 @@ func MakeAddToUserAgentFreeFormHandler(s string) func(*Request) {
 		AddToUserAgent(r, s)
 	}
 }
+
+// WithSetRequestHeaders updates the operation request's HTTP header to contain
+// the header key value pairs provided. If the header key already exists in the
+// request's HTTP header set, the existing value(s) will be replaced.
+func WithSetRequestHeaders(h map[string]string) Option {
+	return withRequestHeader(h).SetRequestHeaders
+}
+
+type withRequestHeader map[string]string
+
+func (h withRequestHeader) SetRequestHeaders(r *Request) {
+	for k, v := range h {
+		r.HTTPRequest.Header[k] = []string{v}
+	}
+}

vendor/github.com/aws/aws-sdk-go/aws/request/request.go

@@ -36,6 +36,10 @@ const (
 	// API request that was canceled. Requests given a aws.Context may
 	// return this error when canceled.
 	CanceledErrorCode = "RequestCanceled"
+
+	// ErrCodeRequestError is an error preventing the SDK from continuing to
+	// process the request.
+	ErrCodeRequestError = "RequestError"
 )
 
 // A Request is the service request to be made.
@@ -51,6 +55,7 @@ type Request struct {
 	HTTPRequest            *http.Request
 	HTTPResponse           *http.Response
 	Body                   io.ReadSeeker
+	streamingBody          io.ReadCloser
 	BodyStart              int64 // offset from beginning of Body that the request body starts
 	Params                 interface{}
 	Error                  error
@@ -99,8 +104,12 @@ type Operation struct {
 	BeforePresignFn func(r *Request) error
 }
 
-// New returns a new Request pointer for the service API
-// operation and parameters.
+// New returns a new Request pointer for the service API operation and
+// parameters.
+//
+// A Retryer should be provided to direct how the request is retried. If
+// Retryer is nil, a default no retry value will be used. You can use
+// NoOpRetryer in the Client package to disable retry behavior directly.
 //
 // Params is any value of input parameters to be the request payload.
 // Data is pointer value to an object which the request's response
@@ -108,6 +117,10 @@ type Operation struct {
 func New(cfg aws.Config, clientInfo metadata.ClientInfo, handlers Handlers,
 	retryer Retryer, operation *Operation, params interface{}, data interface{}) *Request {
 
+	if retryer == nil {
+		retryer = noOpRetryer{}
+	}
+
 	method := operation.HTTPMethod
 	if method == "" {
 		method = "POST"
@@ -122,8 +135,6 @@ func New(cfg aws.Config, clientInfo metadata.ClientInfo, handlers Handlers,
 		err = awserr.New("InvalidEndpointURL", "invalid endpoint uri", err)
 	}
 
-	SanitizeHostForHeader(httpReq)
-
 	r := &Request{
 		Config:     cfg,
 		ClientInfo: clientInfo,
@@ -287,6 +298,13 @@ func (r *Request) SetReaderBody(reader io.ReadSeeker) {
 	r.ResetBody()
 }
 
+// SetStreamingBody set the reader to be used for the request that will stream
+// bytes to the server. Request's Body must not be set to any reader.
+func (r *Request) SetStreamingBody(reader io.ReadCloser) {
+	r.streamingBody = reader
+	r.SetReaderBody(aws.ReadSeekCloser(reader))
+}
+
 // Presign returns the request's signed URL. Error will be returned
 // if the signing fails. The expire parameter is only used for presigned Amazon
 // S3 API requests. All other AWS services will use a fixed expiration
@@ -406,11 +424,17 @@ func (r *Request) Sign() error {
 		return r.Error
 	}
 
+	SanitizeHostForHeader(r.HTTPRequest)
+
 	r.Handlers.Sign.Run(r)
 	return r.Error
 }
 
 func (r *Request) getNextRequestBody() (body io.ReadCloser, err error) {
+	if r.streamingBody != nil {
+		return r.streamingBody, nil
+	}
+
 	if r.safeBody != nil {
 		r.safeBody.Close()
 	}
@@ -615,6 +639,10 @@ func getHost(r *http.Request) string {
 		return r.Host
 	}
 
+	if r.URL == nil {
+		return ""
+	}
+
 	return r.URL.Host
 }
 

vendor/github.com/aws/aws-sdk-go/aws/request/request_pagination.go

@@ -17,11 +17,13 @@ import (
 // does the pagination between API operations, and Paginator defines the
 // configuration that will be used per page request.
 //
-//     cont := true
-//     for p.Next() && cont {
+//     for p.Next() {
 //         data := p.Page().(*s3.ListObjectsOutput)
 //         // process the page's data
+//         // ...
+//         // break out of loop to stop fetching additional pages
 //     }
+//
 //     return p.Err()
 //
 // See service client API operation Pages methods for examples how the SDK will

vendor/github.com/aws/aws-sdk-go/aws/request/retryer.go

@@ -35,16 +35,47 @@ type Retryer interface {
 }
 
 // WithRetryer sets a Retryer value to the given Config returning the Config
-// value for chaining.
+// value for chaining. The value must not be nil.
 func WithRetryer(cfg *aws.Config, retryer Retryer) *aws.Config {
+	if retryer == nil {
+		if cfg.Logger != nil {
+			cfg.Logger.Log("ERROR: Request.WithRetryer called with nil retryer. Replacing with retry disabled Retryer.")
+		}
+		retryer = noOpRetryer{}
+	}
 	cfg.Retryer = retryer
 	return cfg
+
+}
+
+// noOpRetryer is a internal no op retryer used when a request is created
+// without a retryer.
+//
+// Provides a retryer that performs no retries.
+// It should be used when we do not want retries to be performed.
+type noOpRetryer struct{}
+
+// MaxRetries returns the number of maximum returns the service will use to make
+// an individual API; For NoOpRetryer the MaxRetries will always be zero.
+func (d noOpRetryer) MaxRetries() int {
+	return 0
+}
+
+// ShouldRetry will always return false for NoOpRetryer, as it should never retry.
+func (d noOpRetryer) ShouldRetry(_ *Request) bool {
+	return false
+}
+
+// RetryRules returns the delay duration before retrying this request again;
+// since NoOpRetryer does not retry, RetryRules always returns 0.
+func (d noOpRetryer) RetryRules(_ *Request) time.Duration {
+	return 0
 }
 
 // retryableCodes is a collection of service response codes which are retry-able
 // without any further action.
 var retryableCodes = map[string]struct{}{
-	"RequestError":            {},
+	ErrCodeRequestError:       {},
 	"RequestTimeout":          {},
 	ErrCodeResponseTimeout:    {},
 	"RequestTimeoutException": {}, // Glacier's flavor of RequestTimeout
@@ -52,6 +83,7 @@ var retryableCodes = map[string]struct{}{
 
 var throttleCodes = map[string]struct{}{
 	"ProvisionedThroughputExceededException": {},
+	"ThrottledException":                     {}, // SNS, XRay, ResourceGroupsTagging API
 	"Throttling":                             {},
 	"ThrottlingException":                    {},
 	"RequestLimitExceeded":                   {},
@@ -60,6 +92,7 @@ var throttleCodes = map[string]struct{}{
 	"TooManyRequestsException":               {}, // Lambda functions
 	"PriorRequestNotComplete":                {}, // Route53
 	"TransactionInProgressException":         {},
+	"EC2ThrottledException":                  {}, // EC2
 }
 
 // credsExpiredCodes is a collection of error codes which signify the credentials
@@ -145,8 +178,8 @@ func shouldRetryError(origErr error) bool {
 		origErr := err.OrigErr()
 		var shouldRetry bool
 		if origErr != nil {
-			shouldRetry := shouldRetryError(origErr)
-			if err.Code() == "RequestError" && !shouldRetry {
+			shouldRetry = shouldRetryError(origErr)
+			if err.Code() == ErrCodeRequestError && !shouldRetry {
 				return false
 			}
 		}

vendor/github.com/aws/aws-sdk-go/aws/session/credentials.go

@@ -3,6 +3,7 @@ package session
 import (
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
@@ -47,10 +48,10 @@ func resolveCredentials(cfg *aws.Config,
 }
 
 // WebIdentityEmptyRoleARNErr will occur if 'AWS_WEB_IDENTITY_TOKEN_FILE' was set but
-// 'AWS_IAM_ROLE_ARN' was not set.
+// 'AWS_ROLE_ARN' was not set.
 var WebIdentityEmptyRoleARNErr = awserr.New(stscreds.ErrCodeWebIdentity, "role ARN is not set", nil)
 
-// WebIdentityEmptyTokenFilePathErr will occur if 'AWS_IAM_ROLE_ARN' was set but
+// WebIdentityEmptyTokenFilePathErr will occur if 'AWS_ROLE_ARN' was set but
 // 'AWS_WEB_IDENTITY_TOKEN_FILE' was not set.
 var WebIdentityEmptyTokenFilePathErr = awserr.New(stscreds.ErrCodeWebIdentity, "token file path is not set", nil)
 
@@ -206,7 +207,14 @@ func credsFromAssumeRole(cfg aws.Config,
 		sharedCfg.RoleARN,
 		func(opt *stscreds.AssumeRoleProvider) {
 			opt.RoleSessionName = sharedCfg.RoleSessionName
-			opt.Duration = sessOpts.AssumeRoleDuration
+
+			if sessOpts.AssumeRoleDuration == 0 &&
+				sharedCfg.AssumeRoleDuration != nil &&
+				*sharedCfg.AssumeRoleDuration/time.Minute > 15 {
+				opt.Duration = *sharedCfg.AssumeRoleDuration
+			} else if sessOpts.AssumeRoleDuration != 0 {
+				opt.Duration = sessOpts.AssumeRoleDuration
+			}
 
 			// Assume role with external ID
 			if len(sharedCfg.ExternalID) > 0 {

vendor/github.com/aws/aws-sdk-go/aws/session/doc.go

@@ -241,5 +241,22 @@ over the AWS_CA_BUNDLE environment variable, and will be used if both are set.
 Setting a custom HTTPClient in the aws.Config options will override this setting.
 To use this option and custom HTTP client, the HTTP client needs to be provided
 when creating the session. Not the service client.
+
+The endpoint of the EC2 IMDS client can be configured via the environment
+variable, AWS_EC2_METADATA_SERVICE_ENDPOINT when creating the client with a
+Session. See Options.EC2IMDSEndpoint for more details.
+
+  AWS_EC2_METADATA_SERVICE_ENDPOINT=http://169.254.169.254
+
+If using an URL with an IPv6 address literal, the IPv6 address
+component must be enclosed in square brackets.
+
+  AWS_EC2_METADATA_SERVICE_ENDPOINT=http://[::1]
+
+The custom EC2 IMDS endpoint can also be specified via the Session options.
+
+  sess, err := session.NewSessionWithOptions(session.Options{
+      EC2IMDSEndpoint: "http://[::1]",
+  })
 */
 package session

vendor/github.com/aws/aws-sdk-go/aws/session/env_config.go

@@ -1,12 +1,15 @@
 package session
 
 import (
+	"fmt"
 	"os"
 	"strconv"
+	"strings"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/defaults"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
 )
 
 // EnvProviderName provides a name of the provider when config is loaded from environment.
@@ -125,6 +128,31 @@ type envConfig struct {
 	//
 	//  AWS_ROLE_SESSION_NAME=session_name
 	RoleSessionName string
+
+	// Specifies the STS Regional Endpoint flag for the SDK to resolve the endpoint
+	// for a service.
+	//
+	// AWS_STS_REGIONAL_ENDPOINTS=regional
+	// This can take value as `regional` or `legacy`
+	STSRegionalEndpoint endpoints.STSRegionalEndpoint
+
+	// Specifies the S3 Regional Endpoint flag for the SDK to resolve the
+	// endpoint for a service.
+	//
+	// AWS_S3_US_EAST_1_REGIONAL_ENDPOINT=regional
+	// This can take value as `regional` or `legacy`
+	S3UsEast1RegionalEndpoint endpoints.S3UsEast1RegionalEndpoint
+
+	// Specifies if the S3 service should allow ARNs to direct the region
+	// the client's requests are sent to.
+	//
+	// AWS_S3_USE_ARN_REGION=true
+	S3UseARNRegion bool
+
+	// Specifies the alternative endpoint to use for EC2 IMDS.
+	//
+	// AWS_EC2_METADATA_SERVICE_ENDPOINT=http://[::1]
+	EC2IMDSEndpoint string
 }
 
 var (
@@ -179,6 +207,18 @@ var (
 	roleSessionNameEnvKey = []string{
 		"AWS_ROLE_SESSION_NAME",
 	}
+	stsRegionalEndpointKey = []string{
+		"AWS_STS_REGIONAL_ENDPOINTS",
+	}
+	s3UsEast1RegionalEndpoint = []string{
+		"AWS_S3_US_EAST_1_REGIONAL_ENDPOINT",
+	}
+	s3UseARNRegionEnvKey = []string{
+		"AWS_S3_USE_ARN_REGION",
+	}
+	ec2IMDSEndpointEnvKey = []string{
+		"AWS_EC2_METADATA_SERVICE_ENDPOINT",
+	}
 )
 
 // loadEnvConfig retrieves the SDK's environment configuration.
@@ -187,7 +227,7 @@ var (
 // If the environment variable `AWS_SDK_LOAD_CONFIG` is set to a truthy value
 // the shared SDK config will be loaded in addition to the SDK's specific
 // configuration values.
-func loadEnvConfig() envConfig {
+func loadEnvConfig() (envConfig, error) {
 	enableSharedConfig, _ := strconv.ParseBool(os.Getenv("AWS_SDK_LOAD_CONFIG"))
 	return envConfigLoad(enableSharedConfig)
 }
@@ -198,11 +238,11 @@ func loadEnvConfig() envConfig {
 // Loads the shared configuration in addition to the SDK's specific configuration.
 // This will load the same values as `loadEnvConfig` if the `AWS_SDK_LOAD_CONFIG`
 // environment variable is set.
-func loadSharedEnvConfig() envConfig {
+func loadSharedEnvConfig() (envConfig, error) {
 	return envConfigLoad(true)
 }
 
-func envConfigLoad(enableSharedConfig bool) envConfig {
+func envConfigLoad(enableSharedConfig bool) (envConfig, error) {
 	cfg := envConfig{}
 
 	cfg.EnableSharedConfig = enableSharedConfig
@@ -264,12 +304,50 @@ func envConfigLoad(enableSharedConfig bool) envConfig {
 
 	cfg.CustomCABundle = os.Getenv("AWS_CA_BUNDLE")
 
-	return cfg
+	var err error
+	// STS Regional Endpoint variable
+	for _, k := range stsRegionalEndpointKey {
+		if v := os.Getenv(k); len(v) != 0 {
+			cfg.STSRegionalEndpoint, err = endpoints.GetSTSRegionalEndpoint(v)
+			if err != nil {
+				return cfg, fmt.Errorf("failed to load, %v from env config, %v", k, err)
+			}
+		}
+	}
+
+	// S3 Regional Endpoint variable
+	for _, k := range s3UsEast1RegionalEndpoint {
+		if v := os.Getenv(k); len(v) != 0 {
+			cfg.S3UsEast1RegionalEndpoint, err = endpoints.GetS3UsEast1RegionalEndpoint(v)
+			if err != nil {
+				return cfg, fmt.Errorf("failed to load, %v from env config, %v", k, err)
+			}
+		}
+	}
+
+	var s3UseARNRegion string
+	setFromEnvVal(&s3UseARNRegion, s3UseARNRegionEnvKey)
+	if len(s3UseARNRegion) != 0 {
+		switch {
+		case strings.EqualFold(s3UseARNRegion, "false"):
+			cfg.S3UseARNRegion = false
+		case strings.EqualFold(s3UseARNRegion, "true"):
+			cfg.S3UseARNRegion = true
+		default:
+			return envConfig{}, fmt.Errorf(
+				"invalid value for environment variable, %s=%s, need true or false",
+				s3UseARNRegionEnvKey[0], s3UseARNRegion)
+		}
+	}
+
+	setFromEnvVal(&cfg.EC2IMDSEndpoint, ec2IMDSEndpointEnvKey)
+
+	return cfg, nil
 }
 
 func setFromEnvVal(dst *string, keys []string) {
 	for _, k := range keys {
-		if v := os.Getenv(k); len(v) > 0 {
+		if v := os.Getenv(k); len(v) != 0 {
 			*dst = v
 			break
 		}

vendor/github.com/aws/aws-sdk-go/aws/session/session.go

@@ -48,6 +48,8 @@ var ErrSharedConfigInvalidCredSource = awserr.New(ErrCodeSharedConfig, "credenti
 type Session struct {
 	Config   *aws.Config
 	Handlers request.Handlers
+
+	options Options
 }
 
 // New creates a new instance of the handlers merging in the provided configs
@@ -73,7 +75,7 @@ type Session struct {
 // func is called instead of waiting to receive an error until a request is made.
 func New(cfgs ...*aws.Config) *Session {
 	// load initial config from environment
-	envCfg := loadEnvConfig()
+	envCfg, envErr := loadEnvConfig()
 
 	if envCfg.EnableSharedConfig {
 		var cfg aws.Config
@@ -93,17 +95,17 @@ func New(cfgs ...*aws.Config) *Session {
 			// Session creation failed, need to report the error and prevent
 			// any requests from succeeding.
 			s = &Session{Config: defaults.Config()}
-			s.Config.MergeIn(cfgs...)
-			s.Config.Logger.Log("ERROR:", msg, "Error:", err)
-			s.Handlers.Validate.PushBack(func(r *request.Request) {
-				r.Error = err
-			})
+			s.logDeprecatedNewSessionError(msg, err, cfgs)
 		}
 
 		return s
 	}
 
-	s := deprecatedNewSession(cfgs...)
+	s := deprecatedNewSession(envCfg, cfgs...)
+	if envErr != nil {
+		msg := "failed to load env config"
+		s.logDeprecatedNewSessionError(msg, envErr, cfgs)
+	}
 
 	if csmCfg, err := loadCSMConfig(envCfg, []string{}); err != nil {
 		if l := s.Config.Logger; l != nil {
@@ -112,11 +114,8 @@ func New(cfgs ...*aws.Config) *Session {
 	} else if csmCfg.Enabled {
 		err := enableCSM(&s.Handlers, csmCfg, s.Config.Logger)
 		if err != nil {
-			err = fmt.Errorf("failed to enable CSM, %v", err)
-			s.Config.Logger.Log("ERROR:", err.Error())
-			s.Handlers.Validate.PushBack(func(r *request.Request) {
-				r.Error = err
-			})
+			msg := "failed to enable CSM"
+			s.logDeprecatedNewSessionError(msg, err, cfgs)
 		}
 	}
 
@@ -246,6 +245,23 @@ type Options struct {
 	// function to initialize this value before changing the handlers to be
 	// used by the SDK.
 	Handlers request.Handlers
+
+	// Allows specifying a custom endpoint to be used by the EC2 IMDS client
+	// when making requests to the EC2 IMDS API. The must endpoint value must
+	// include protocol prefix.
+	//
+	// If unset, will the EC2 IMDS client will use its default endpoint.
+	//
+	// Can also be specified via the environment variable,
+	// AWS_EC2_METADATA_SERVICE_ENDPOINT.
+	//
+	//   AWS_EC2_METADATA_SERVICE_ENDPOINT=http://169.254.169.254
+	//
+	// If using an URL with an IPv6 address literal, the IPv6 address
+	// component must be enclosed in square brackets.
+	//
+	//   AWS_EC2_METADATA_SERVICE_ENDPOINT=http://[::1]
+	EC2IMDSEndpoint string
 }
 
 // NewSessionWithOptions returns a new Session created from SDK defaults, config files,
@@ -279,10 +295,17 @@ type Options struct {
 //     }))
 func NewSessionWithOptions(opts Options) (*Session, error) {
 	var envCfg envConfig
+	var err error
 	if opts.SharedConfigState == SharedConfigEnable {
-		envCfg = loadSharedEnvConfig()
+		envCfg, err = loadSharedEnvConfig()
+		if err != nil {
+			return nil, fmt.Errorf("failed to load shared config, %v", err)
+		}
 	} else {
-		envCfg = loadEnvConfig()
+		envCfg, err = loadEnvConfig()
+		if err != nil {
+			return nil, fmt.Errorf("failed to load environment config, %v", err)
+		}
 	}
 
 	if len(opts.Profile) != 0 {
@@ -325,7 +348,25 @@ func Must(sess *Session, err error) *Session {
 	return sess
 }
 
-func deprecatedNewSession(cfgs ...*aws.Config) *Session {
+// Wraps the endpoint resolver with a resolver that will return a custom
+// endpoint for EC2 IMDS.
+func wrapEC2IMDSEndpoint(resolver endpoints.Resolver, endpoint string) endpoints.Resolver {
+	return endpoints.ResolverFunc(
+		func(service, region string, opts ...func(*endpoints.Options)) (
+			endpoints.ResolvedEndpoint, error,
+		) {
+			if service == ec2MetadataServiceID {
+				return endpoints.ResolvedEndpoint{
+					URL:           endpoint,
+					SigningName:   ec2MetadataServiceID,
+					SigningRegion: region,
+				}, nil
+			}
+			return resolver.EndpointFor(service, region)
+		})
+}
+
+func deprecatedNewSession(envCfg envConfig, cfgs ...*aws.Config) *Session {
 	cfg := defaults.Config()
 	handlers := defaults.Handlers()
 
@@ -337,6 +378,11 @@ func deprecatedNewSession(cfgs ...*aws.Config) *Session {
 		// endpoints for service client configurations.
 		cfg.EndpointResolver = endpoints.DefaultResolver()
 	}
+
+	if len(envCfg.EC2IMDSEndpoint) != 0 {
+		cfg.EndpointResolver = wrapEC2IMDSEndpoint(cfg.EndpointResolver, envCfg.EC2IMDSEndpoint)
+	}
+
 	cfg.Credentials = defaults.CredChain(cfg, handlers)
 
 	// Reapply any passed in configs to override credentials if set
@@ -345,6 +391,9 @@ func deprecatedNewSession(cfgs ...*aws.Config) *Session {
 	s := &Session{
 		Config:   cfg,
 		Handlers: handlers,
+		options: Options{
+			EC2IMDSEndpoint: envCfg.EC2IMDSEndpoint,
+		},
 	}
 
 	initHandlers(s)
@@ -414,6 +463,7 @@ func newSession(opts Options, envCfg envConfig, cfgs ...*aws.Config) (*Session,
 	s := &Session{
 		Config:   cfg,
 		Handlers: handlers,
+		options:  opts,
 	}
 
 	initHandlers(s)
@@ -550,6 +600,30 @@ func mergeConfigSrcs(cfg, userCfg *aws.Config,
 		}
 	}
 
+	// Regional Endpoint flag for STS endpoint resolving
+	mergeSTSRegionalEndpointConfig(cfg, []endpoints.STSRegionalEndpoint{
+		userCfg.STSRegionalEndpoint,
+		envCfg.STSRegionalEndpoint,
+		sharedCfg.STSRegionalEndpoint,
+		endpoints.LegacySTSEndpoint,
+	})
+
+	// Regional Endpoint flag for S3 endpoint resolving
+	mergeS3UsEast1RegionalEndpointConfig(cfg, []endpoints.S3UsEast1RegionalEndpoint{
+		userCfg.S3UsEast1RegionalEndpoint,
+		envCfg.S3UsEast1RegionalEndpoint,
+		sharedCfg.S3UsEast1RegionalEndpoint,
+		endpoints.LegacyS3UsEast1Endpoint,
+	})
+
+	ec2IMDSEndpoint := sessOpts.EC2IMDSEndpoint
+	if len(ec2IMDSEndpoint) == 0 {
+		ec2IMDSEndpoint = envCfg.EC2IMDSEndpoint
+	}
+	if len(ec2IMDSEndpoint) != 0 {
+		cfg.EndpointResolver = wrapEC2IMDSEndpoint(cfg.EndpointResolver, ec2IMDSEndpoint)
+	}
+
 	// Configure credentials if not already set by the user when creating the
 	// Session.
 	if cfg.Credentials == credentials.AnonymousCredentials && userCfg.Credentials == nil {
@@ -560,9 +634,35 @@ func mergeConfigSrcs(cfg, userCfg *aws.Config,
 		cfg.Credentials = creds
 	}
 
+	cfg.S3UseARNRegion = userCfg.S3UseARNRegion
+	if cfg.S3UseARNRegion == nil {
+		cfg.S3UseARNRegion = &envCfg.S3UseARNRegion
+	}
+	if cfg.S3UseARNRegion == nil {
+		cfg.S3UseARNRegion = &sharedCfg.S3UseARNRegion
+	}
+
 	return nil
 }
 
+func mergeSTSRegionalEndpointConfig(cfg *aws.Config, values []endpoints.STSRegionalEndpoint) {
+	for _, v := range values {
+		if v != endpoints.UnsetSTSEndpoint {
+			cfg.STSRegionalEndpoint = v
+			break
+		}
+	}
+}
+
+func mergeS3UsEast1RegionalEndpointConfig(cfg *aws.Config, values []endpoints.S3UsEast1RegionalEndpoint) {
+	for _, v := range values {
+		if v != endpoints.UnsetS3UsEast1Endpoint {
+			cfg.S3UsEast1RegionalEndpoint = v
+			break
+		}
+	}
+}
+
 func initHandlers(s *Session) {
 	// Add the Validate parameter handler if it is not disabled.
 	s.Handlers.Validate.Remove(corehandlers.ValidateParametersHandler)
@@ -581,6 +681,7 @@ func (s *Session) Copy(cfgs ...*aws.Config) *Session {
 	newSession := &Session{
 		Config:   s.Config.Copy(cfgs...),
 		Handlers: s.Handlers.Copy(),
+		options:  s.options,
 	}
 
 	initHandlers(newSession)
@@ -591,47 +692,69 @@ func (s *Session) Copy(cfgs ...*aws.Config) *Session {
 // ClientConfig satisfies the client.ConfigProvider interface and is used to
 // configure the service client instances. Passing the Session to the service
 // client's constructor (New) will use this method to configure the client.
-func (s *Session) ClientConfig(serviceName string, cfgs ...*aws.Config) client.Config {
-	// Backwards compatibility, the error will be eaten if user calls ClientConfig
-	// directly. All SDK services will use ClientconfigWithError.
-	cfg, _ := s.clientConfigWithErr(serviceName, cfgs...)
-
-	return cfg
-}
-
-func (s *Session) clientConfigWithErr(serviceName string, cfgs ...*aws.Config) (client.Config, error) {
+func (s *Session) ClientConfig(service string, cfgs ...*aws.Config) client.Config {
 	s = s.Copy(cfgs...)
 
-	var resolved endpoints.ResolvedEndpoint
-	var err error
-
 	region := aws.StringValue(s.Config.Region)
-
-	if endpoint := aws.StringValue(s.Config.Endpoint); len(endpoint) != 0 {
-		resolved.URL = endpoints.AddScheme(endpoint, aws.BoolValue(s.Config.DisableSSL))
-		resolved.SigningRegion = region
-	} else {
-		resolved, err = s.Config.EndpointResolver.EndpointFor(
-			serviceName, region,
-			func(opt *endpoints.Options) {
-				opt.DisableSSL = aws.BoolValue(s.Config.DisableSSL)
-				opt.UseDualStack = aws.BoolValue(s.Config.UseDualStack)
-
-				// Support the condition where the service is modeled but its
-				// endpoint metadata is not available.
-				opt.ResolveUnknownService = true
-			},
-		)
+	resolved, err := s.resolveEndpoint(service, region, s.Config)
+	if err != nil {
+		s.Handlers.Validate.PushBack(func(r *request.Request) {
+			if len(r.ClientInfo.Endpoint) != 0 {
+				// Error occurred while resolving endpoint, but the request
+				// being invoked has had an endpoint specified after the client
+				// was created.
+				return
+			}
+			r.Error = err
+		})
 	}
 
 	return client.Config{
 		Config:             s.Config,
 		Handlers:           s.Handlers,
+		PartitionID:        resolved.PartitionID,
 		Endpoint:           resolved.URL,
 		SigningRegion:      resolved.SigningRegion,
 		SigningNameDerived: resolved.SigningNameDerived,
 		SigningName:        resolved.SigningName,
-	}, err
+	}
+}
+
+const ec2MetadataServiceID = "ec2metadata"
+
+func (s *Session) resolveEndpoint(service, region string, cfg *aws.Config) (endpoints.ResolvedEndpoint, error) {
+
+	if ep := aws.StringValue(cfg.Endpoint); len(ep) != 0 {
+		return endpoints.ResolvedEndpoint{
+			URL:           endpoints.AddScheme(ep, aws.BoolValue(cfg.DisableSSL)),
+			SigningRegion: region,
+		}, nil
+	}
+
+	resolved, err := cfg.EndpointResolver.EndpointFor(service, region,
+		func(opt *endpoints.Options) {
+			opt.DisableSSL = aws.BoolValue(cfg.DisableSSL)
+			opt.UseDualStack = aws.BoolValue(cfg.UseDualStack)
+			// Support for STSRegionalEndpoint where the STSRegionalEndpoint is
+			// provided in envConfig or sharedConfig with envConfig getting
+			// precedence.
+			opt.STSRegionalEndpoint = cfg.STSRegionalEndpoint
+
+			// Support for S3UsEast1RegionalEndpoint where the S3UsEast1RegionalEndpoint is
+			// provided in envConfig or sharedConfig with envConfig getting
+			// precedence.
+			opt.S3UsEast1RegionalEndpoint = cfg.S3UsEast1RegionalEndpoint
+
+			// Support the condition where the service is modeled but its
+			// endpoint metadata is not available.
+			opt.ResolveUnknownService = true
+		},
+	)
+	if err != nil {
+		return endpoints.ResolvedEndpoint{}, err
+	}
+
+	return resolved, nil
 }
 
 // ClientConfigNoResolveEndpoint is the same as ClientConfig with the exception
@@ -641,12 +764,9 @@ func (s *Session) ClientConfigNoResolveEndpoint(cfgs ...*aws.Config) client.Conf
 	s = s.Copy(cfgs...)
 
 	var resolved endpoints.ResolvedEndpoint
-
-	region := aws.StringValue(s.Config.Region)
-
 	if ep := aws.StringValue(s.Config.Endpoint); len(ep) > 0 {
 		resolved.URL = endpoints.AddScheme(ep, aws.BoolValue(s.Config.DisableSSL))
-		resolved.SigningRegion = region
+		resolved.SigningRegion = aws.StringValue(s.Config.Region)
 	}
 
 	return client.Config{
@@ -658,3 +778,14 @@ func (s *Session) ClientConfigNoResolveEndpoint(cfgs ...*aws.Config) client.Conf
 		SigningName:        resolved.SigningName,
 	}
 }
+
+// logDeprecatedNewSessionError function enables error handling for session
+func (s *Session) logDeprecatedNewSessionError(msg string, err error, cfgs []*aws.Config) {
+	// Session creation failed, need to report the error and prevent
+	// any requests from succeeding.
+	s.Config.MergeIn(cfgs...)
+	s.Config.Logger.Log("ERROR:", msg, "Error:", err)
+	s.Handlers.Validate.PushBack(func(r *request.Request) {
+		r.Error = err
+	})
+}

vendor/github.com/aws/aws-sdk-go/aws/session/shared_config.go

@@ -2,9 +2,11 @@ package session
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/internal/ini"
 )
 
@@ -15,12 +17,13 @@ const (
 	sessionTokenKey = `aws_session_token`     // optional
 
 	// Assume Role Credentials group
-	roleArnKey          = `role_arn`          // group required
-	sourceProfileKey    = `source_profile`    // group required (or credential_source)
-	credentialSourceKey = `credential_source` // group required (or source_profile)
-	externalIDKey       = `external_id`       // optional
-	mfaSerialKey        = `mfa_serial`        // optional
-	roleSessionNameKey  = `role_session_name` // optional
+	roleArnKey             = `role_arn`          // group required
+	sourceProfileKey       = `source_profile`    // group required (or credential_source)
+	credentialSourceKey    = `credential_source` // group required (or source_profile)
+	externalIDKey          = `external_id`       // optional
+	mfaSerialKey           = `mfa_serial`        // optional
+	roleSessionNameKey     = `role_session_name` // optional
+	roleDurationSecondsKey = "duration_seconds"  // optional
 
 	// CSM options
 	csmEnabledKey  = `csm_enabled`
@@ -40,10 +43,19 @@ const (
 	// Web Identity Token File
 	webIdentityTokenFileKey = `web_identity_token_file` // optional
 
+	// Additional config fields for regional or legacy endpoints
+	stsRegionalEndpointSharedKey = `sts_regional_endpoints`
+
+	// Additional config fields for regional or legacy endpoints
+	s3UsEast1RegionalSharedKey = `s3_us_east_1_regional_endpoint`
+
 	// DefaultSharedConfigProfile is the default profile to be used when
 	// loading configuration from the config files if another profile name
 	// is not provided.
 	DefaultSharedConfigProfile = `default`
+
+	// S3 ARN Region Usage
+	s3UseARNRegionKey = "s3_use_arn_region"
 )
 
 // sharedConfig represents the configuration fields of the SDK config files.
@@ -63,10 +75,11 @@ type sharedConfig struct {
 	CredentialProcess    string
 	WebIdentityTokenFile string
 
-	RoleARN         string
-	RoleSessionName string
-	ExternalID      string
-	MFASerial       string
+	RoleARN            string
+	RoleSessionName    string
+	ExternalID         string
+	MFASerial          string
+	AssumeRoleDuration *time.Duration
 
 	SourceProfileName string
 	SourceProfile     *sharedConfig
@@ -88,6 +101,24 @@ type sharedConfig struct {
 	CSMHost     string
 	CSMPort     string
 	CSMClientID string
+
+	// Specifies the Regional Endpoint flag for the SDK to resolve the endpoint for a service
+	//
+	// sts_regional_endpoints = regional
+	// This can take value as `LegacySTSEndpoint` or `RegionalSTSEndpoint`
+	STSRegionalEndpoint endpoints.STSRegionalEndpoint
+
+	// Specifies the Regional Endpoint flag for the SDK to resolve the endpoint for a service
+	//
+	// s3_us_east_1_regional_endpoint = regional
+	// This can take value as `LegacyS3UsEast1Endpoint` or `RegionalS3UsEast1Endpoint`
+	S3UsEast1RegionalEndpoint endpoints.S3UsEast1RegionalEndpoint
+
+	// Specifies if the S3 service should allow ARNs to direct the region
+	// the client's requests are sent to.
+	//
+	// s3_use_arn_region=true
+	S3UseARNRegion bool
 }
 
 type sharedConfigFile struct {
@@ -244,8 +275,30 @@ func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile, e
 		updateString(&cfg.RoleSessionName, section, roleSessionNameKey)
 		updateString(&cfg.SourceProfileName, section, sourceProfileKey)
 		updateString(&cfg.CredentialSource, section, credentialSourceKey)
-
 		updateString(&cfg.Region, section, regionKey)
+
+		if section.Has(roleDurationSecondsKey) {
+			d := time.Duration(section.Int(roleDurationSecondsKey)) * time.Second
+			cfg.AssumeRoleDuration = &d
+		}
+
+		if v := section.String(stsRegionalEndpointSharedKey); len(v) != 0 {
+			sre, err := endpoints.GetSTSRegionalEndpoint(v)
+			if err != nil {
+				return fmt.Errorf("failed to load %s from shared config, %s, %v",
+					stsRegionalEndpointSharedKey, file.Filename, err)
+			}
+			cfg.STSRegionalEndpoint = sre
+		}
+
+		if v := section.String(s3UsEast1RegionalSharedKey); len(v) != 0 {
+			sre, err := endpoints.GetS3UsEast1RegionalEndpoint(v)
+			if err != nil {
+				return fmt.Errorf("failed to load %s from shared config, %s, %v",
+					s3UsEast1RegionalSharedKey, file.Filename, err)
+			}
+			cfg.S3UsEast1RegionalEndpoint = sre
+		}
 	}
 
 	updateString(&cfg.CredentialProcess, section, credentialProcessKey)
@@ -271,6 +324,8 @@ func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile, e
 	updateString(&cfg.CSMPort, section, csmPortKey)
 	updateString(&cfg.CSMClientID, section, csmClientIDKey)
 
+	updateBool(&cfg.S3UseARNRegion, section, s3UseARNRegionKey)
+
 	return nil
 }
 
@@ -363,6 +418,15 @@ func updateString(dst *string, section ini.Section, key string) {
 	*dst = section.String(key)
 }
 
+// updateBool will only update the dst with the value in the section key, key
+// is present in the section.
+func updateBool(dst *bool, section ini.Section, key string) {
+	if !section.Has(key) {
+		return
+	}
+	*dst = section.Bool(key)
+}
+
 // updateBoolPtr will only update the dst with the value in the section key,
 // key is present in the section.
 func updateBoolPtr(dst **bool, section ini.Section, key string) {

vendor/github.com/aws/aws-sdk-go/aws/signer/v4/header_rules.go

@@ -1,8 +1,7 @@
 package v4
 
 import (
-	"net/http"
-	"strings"
+	"github.com/aws/aws-sdk-go/internal/strings"
 )
 
 // validator houses a set of rule needed for validation of a
@@ -61,7 +60,7 @@ type patterns []string
 // been found
 func (p patterns) IsValid(value string) bool {
 	for _, pattern := range p {
-		if strings.HasPrefix(http.CanonicalHeaderKey(value), pattern) {
+		if strings.HasPrefixFold(value, pattern) {
 			return true
 		}
 	}

vendor/github.com/aws/aws-sdk-go/aws/signer/v4/request_context_go1.5.go

@@ -0,0 +1,13 @@
+// +build !go1.7
+
+package v4
+
+import (
+	"net/http"
+
+	"github.com/aws/aws-sdk-go/aws"
+)
+
+func requestContext(r *http.Request) aws.Context {
+	return aws.BackgroundContext()
+}

vendor/github.com/aws/aws-sdk-go/aws/signer/v4/request_context_go1.7.go

@@ -0,0 +1,13 @@
+// +build go1.7
+
+package v4
+
+import (
+	"net/http"
+
+	"github.com/aws/aws-sdk-go/aws"
+)
+
+func requestContext(r *http.Request) aws.Context {
+	return r.Context()
+}

vendor/github.com/aws/aws-sdk-go/aws/signer/v4/stream.go

@@ -0,0 +1,63 @@
+package v4
+
+import (
+	"encoding/hex"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws/credentials"
+)
+
+type credentialValueProvider interface {
+	Get() (credentials.Value, error)
+}
+
+// StreamSigner implements signing of event stream encoded payloads
+type StreamSigner struct {
+	region  string
+	service string
+
+	credentials credentialValueProvider
+
+	prevSig []byte
+}
+
+// NewStreamSigner creates a SigV4 signer used to sign Event Stream encoded messages
+func NewStreamSigner(region, service string, seedSignature []byte, credentials *credentials.Credentials) *StreamSigner {
+	return &StreamSigner{
+		region:      region,
+		service:     service,
+		credentials: credentials,
+		prevSig:     seedSignature,
+	}
+}
+
+// GetSignature takes an event stream encoded headers and payload and returns a signature
+func (s *StreamSigner) GetSignature(headers, payload []byte, date time.Time) ([]byte, error) {
+	credValue, err := s.credentials.Get()
+	if err != nil {
+		return nil, err
+	}
+
+	sigKey := deriveSigningKey(s.region, s.service, credValue.SecretAccessKey, date)
+
+	keyPath := buildSigningScope(s.region, s.service, date)
+
+	stringToSign := buildEventStreamStringToSign(headers, payload, s.prevSig, keyPath, date)
+
+	signature := hmacSHA256(sigKey, []byte(stringToSign))
+	s.prevSig = signature
+
+	return signature, nil
+}
+
+func buildEventStreamStringToSign(headers, payload, prevSig []byte, scope string, date time.Time) string {
+	return strings.Join([]string{
+		"AWS4-HMAC-SHA256-PAYLOAD",
+		formatTime(date),
+		scope,
+		hex.EncodeToString(prevSig),
+		hex.EncodeToString(hashSHA256(headers)),
+		hex.EncodeToString(hashSHA256(payload)),
+	}, "\n")
+}

vendor/github.com/aws/aws-sdk-go/aws/signer/v4/v4.go

@@ -76,9 +76,14 @@ import (
 )
 
 const (
+	authorizationHeader     = "Authorization"
+	authHeaderSignatureElem = "Signature="
+	signatureQueryKey       = "X-Amz-Signature"
+
 	authHeaderPrefix = "AWS4-HMAC-SHA256"
 	timeFormat       = "20060102T150405Z"
 	shortTimeFormat  = "20060102"
+	awsV4Request     = "aws4_request"
 
 	// emptyStringSHA256 is a SHA256 of an empty string
 	emptyStringSHA256 = `e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855`
@@ -87,9 +92,9 @@ const (
 var ignoredHeaders = rules{
 	blacklist{
 		mapRule{
-			"Authorization":   struct{}{},
-			"User-Agent":      struct{}{},
-			"X-Amzn-Trace-Id": struct{}{},
+			authorizationHeader: struct{}{},
+			"User-Agent":        struct{}{},
+			"X-Amzn-Trace-Id":   struct{}{},
 		},
 	},
 }
@@ -229,11 +234,9 @@ type signingCtx struct {
 
 	DisableURIPathEscaping bool
 
-	credValues         credentials.Value
-	isPresign          bool
-	formattedTime      string
-	formattedShortTime string
-	unsignedPayload    bool
+	credValues      credentials.Value
+	isPresign       bool
+	unsignedPayload bool
 
 	bodyDigest       string
 	signedHeaders    string
@@ -337,7 +340,7 @@ func (v4 Signer) signWithBody(r *http.Request, body io.ReadSeeker, service, regi
 	}
 
 	var err error
-	ctx.credValues, err = v4.Credentials.Get()
+	ctx.credValues, err = v4.Credentials.GetWithContext(requestContext(r))
 	if err != nil {
 		return http.Header{}, err
 	}
@@ -532,39 +535,56 @@ func (ctx *signingCtx) build(disableHeaderHoisting bool) error {
 	ctx.buildSignature()       // depends on string to sign
 
 	if ctx.isPresign {
-		ctx.Request.URL.RawQuery += "&X-Amz-Signature=" + ctx.signature
+		ctx.Request.URL.RawQuery += "&" + signatureQueryKey + "=" + ctx.signature
 	} else {
 		parts := []string{
 			authHeaderPrefix + " Credential=" + ctx.credValues.AccessKeyID + "/" + ctx.credentialString,
 			"SignedHeaders=" + ctx.signedHeaders,
-			"Signature=" + ctx.signature,
+			authHeaderSignatureElem + ctx.signature,
 		}
-		ctx.Request.Header.Set("Authorization", strings.Join(parts, ", "))
+		ctx.Request.Header.Set(authorizationHeader, strings.Join(parts, ", "))
 	}
 
 	return nil
 }
 
-func (ctx *signingCtx) buildTime() {
-	ctx.formattedTime = ctx.Time.UTC().Format(timeFormat)
-	ctx.formattedShortTime = ctx.Time.UTC().Format(shortTimeFormat)
+// GetSignedRequestSignature attempts to extract the signature of the request.
+// Returning an error if the request is unsigned, or unable to extract the
+// signature.
+func GetSignedRequestSignature(r *http.Request) ([]byte, error) {
 
+	if auth := r.Header.Get(authorizationHeader); len(auth) != 0 {
+		ps := strings.Split(auth, ", ")
+		for _, p := range ps {
+			if idx := strings.Index(p, authHeaderSignatureElem); idx >= 0 {
+				sig := p[len(authHeaderSignatureElem):]
+				if len(sig) == 0 {
+					return nil, fmt.Errorf("invalid request signature authorization header")
+				}
+				return hex.DecodeString(sig)
+			}
+		}
+	}
+
+	if sig := r.URL.Query().Get("X-Amz-Signature"); len(sig) != 0 {
+		return hex.DecodeString(sig)
+	}
+
+	return nil, fmt.Errorf("request not signed")
+}
+
+func (ctx *signingCtx) buildTime() {
 	if ctx.isPresign {
 		duration := int64(ctx.ExpireTime / time.Second)
-		ctx.Query.Set("X-Amz-Date", ctx.formattedTime)
+		ctx.Query.Set("X-Amz-Date", formatTime(ctx.Time))
 		ctx.Query.Set("X-Amz-Expires", strconv.FormatInt(duration, 10))
 	} else {
-		ctx.Request.Header.Set("X-Amz-Date", ctx.formattedTime)
+		ctx.Request.Header.Set("X-Amz-Date", formatTime(ctx.Time))
 	}
 }
 
 func (ctx *signingCtx) buildCredentialString() {
-	ctx.credentialString = strings.Join([]string{
-		ctx.formattedShortTime,
-		ctx.Region,
-		ctx.ServiceName,
-		"aws4_request",
-	}, "/")
+	ctx.credentialString = buildSigningScope(ctx.Region, ctx.ServiceName, ctx.Time)
 
 	if ctx.isPresign {
 		ctx.Query.Set("X-Amz-Credential", ctx.credValues.AccessKeyID+"/"+ctx.credentialString)
@@ -588,8 +608,7 @@ func (ctx *signingCtx) buildCanonicalHeaders(r rule, header http.Header) {
 	var headers []string
 	headers = append(headers, "host")
 	for k, v := range header {
-		canonicalKey := http.CanonicalHeaderKey(k)
-		if !r.IsValid(canonicalKey) {
+		if !r.IsValid(k) {
 			continue // ignored header
 		}
 		if ctx.SignedHeaderVals == nil {
@@ -653,19 +672,15 @@ func (ctx *signingCtx) buildCanonicalString() {
 func (ctx *signingCtx) buildStringToSign() {
 	ctx.stringToSign = strings.Join([]string{
 		authHeaderPrefix,
-		ctx.formattedTime,
+		formatTime(ctx.Time),
 		ctx.credentialString,
-		hex.EncodeToString(makeSha256([]byte(ctx.canonicalString))),
+		hex.EncodeToString(hashSHA256([]byte(ctx.canonicalString))),
 	}, "\n")
 }
 
 func (ctx *signingCtx) buildSignature() {
-	secret := ctx.credValues.SecretAccessKey
-	date := makeHmac([]byte("AWS4"+secret), []byte(ctx.formattedShortTime))
-	region := makeHmac(date, []byte(ctx.Region))
-	service := makeHmac(region, []byte(ctx.ServiceName))
-	credentials := makeHmac(service, []byte("aws4_request"))
-	signature := makeHmac(credentials, []byte(ctx.stringToSign))
+	creds := deriveSigningKey(ctx.Region, ctx.ServiceName, ctx.credValues.SecretAccessKey, ctx.Time)
+	signature := hmacSHA256(creds, []byte(ctx.stringToSign))
 	ctx.signature = hex.EncodeToString(signature)
 }
 
@@ -726,13 +741,13 @@ func (ctx *signingCtx) removePresign() {
 	ctx.Query.Del("X-Amz-SignedHeaders")
 }
 
-func makeHmac(key []byte, data []byte) []byte {
+func hmacSHA256(key []byte, data []byte) []byte {
 	hash := hmac.New(sha256.New, key)
 	hash.Write(data)
 	return hash.Sum(nil)
 }
 
-func makeSha256(data []byte) []byte {
+func hashSHA256(data []byte) []byte {
 	hash := sha256.New()
 	hash.Write(data)
 	return hash.Sum(nil)
@@ -804,3 +819,28 @@ func stripExcessSpaces(vals []string) {
 		vals[i] = string(buf[:m])
 	}
 }
+
+func buildSigningScope(region, service string, dt time.Time) string {
+	return strings.Join([]string{
+		formatShortTime(dt),
+		region,
+		service,
+		awsV4Request,
+	}, "/")
+}
+
+func deriveSigningKey(region, service, secretKey string, dt time.Time) []byte {
+	kDate := hmacSHA256([]byte("AWS4"+secretKey), []byte(formatShortTime(dt)))
+	kRegion := hmacSHA256(kDate, []byte(region))
+	kService := hmacSHA256(kRegion, []byte(service))
+	signingKey := hmacSHA256(kService, []byte(awsV4Request))
+	return signingKey
+}
+
+func formatShortTime(dt time.Time) string {
+	return dt.UTC().Format(shortTimeFormat)
+}
+
+func formatTime(dt time.Time) string {
+	return dt.UTC().Format(timeFormat)
+}

vendor/github.com/aws/aws-sdk-go/aws/types.go

@@ -2,6 +2,7 @@ package aws
 
 import (
 	"io"
+	"strings"
 	"sync"
 
 	"github.com/aws/aws-sdk-go/internal/sdkio"
@@ -205,3 +206,59 @@ func (b *WriteAtBuffer) Bytes() []byte {
 	defer b.m.Unlock()
 	return b.buf
 }
+
+// MultiCloser is a utility to close multiple io.Closers within a single
+// statement.
+type MultiCloser []io.Closer
+
+// Close closes all of the io.Closers making up the MultiClosers. Any
+// errors that occur while closing will be returned in the order they
+// occur.
+func (m MultiCloser) Close() error {
+	var errs errors
+	for _, c := range m {
+		err := c.Close()
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+	if len(errs) != 0 {
+		return errs
+	}
+
+	return nil
+}
+
+type errors []error
+
+func (es errors) Error() string {
+	var parts []string
+	for _, e := range es {
+		parts = append(parts, e.Error())
+	}
+
+	return strings.Join(parts, "\n")
+}
+
+// CopySeekableBody copies the seekable body to an io.Writer
+func CopySeekableBody(dst io.Writer, src io.ReadSeeker) (int64, error) {
+	curPos, err := src.Seek(0, sdkio.SeekCurrent)
+	if err != nil {
+		return 0, err
+	}
+
+	// copy errors may be assumed to be from the body.
+	n, err := io.Copy(dst, src)
+	if err != nil {
+		return n, err
+	}
+
+	// seek back to the first position after reading to reset
+	// the body for transmission.
+	_, err = src.Seek(curPos, sdkio.SeekStart)
+	if err != nil {
+		return n, err
+	}
+
+	return n, nil
+}

vendor/github.com/aws/aws-sdk-go/aws/version.go

@@ -5,4 +5,4 @@ package aws
 const SDKName = "aws-sdk-go"
 
 // SDKVersion is the version of this SDK
-const SDKVersion = "1.25.8"
+const SDKVersion = "1.34.19"