opensource/magistrala
1
0
Fork 0
mirror of https://github.com/absmach/magistrala.git synced 2026-06-23 04:10:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

NOIISUE - Update Vault setup scripts to support Vault CLI (#2091)

Browse Source 
Signed-off-by: Arvindh <arvindh91@gmail.com>
This commit is contained in:
Arvindh
2024-02-22 14:39:50 +05:30
committed by GitHub
parent 0ed79371a4
commit 5d0cb70df8
10 changed files with 165 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
certs/README.md
+7 -7
View File
@@ -9,17 +9,17 @@ When `MG_CERTS_VAULT_HOST` is set it is presumed that `Vault` is installed and `
First you'll need to set up `Vault`.
To setup `Vault` follow steps in [Build Your Own Certificate Authority (CA)](https://learn.hashicorp.com/tutorials/vault/pki-engine).
To setup certs service with `Vault` following environment variables must be set:
For lab purposes you can use docker-compose and script for setting up PKI in [https://github.com/absmach/magistrala/blob/master/docker/addons/vault/README.md](https://github.com/absmach/magistrala/blob/master/docker/addons/vault/README.md)
```bash
MG_CERTS_VAULT_HOST=vault-domain.com
MG_CERTS_VAULT_PKI_PATH=<vault_pki_path>
MG_CERTS_VAULT_ROLE=<vault_role>
MG_CERTS_VAULT_TOKEN=<vault_acces_token>
MG_CERTS_VAULT_HOST=<https://vault-domain:8200>
MG_CERTS_VAULT_NAMESPACE=<vault_namespace>
MG_CERTS_VAULT_APPROLE_ROLEID=<vault_approle_roleid>
MG_CERTS_VAULT_APPROLE_SECRET=<vault_approle_sceret>
MG_CERTS_VAULT_THINGS_CERTS_PKI_PATH=<vault_things_certs_pki_path>
MG_CERTS_VAULT_THINGS_CERTS_PKI_ROLE_NAME=<vault_things_certs_issue_role_name>
```
For lab purposes you can use docker-compose and script for setting up PKI in [https://github.com/mteodor/vault](https://github.com/mteodor/vault)
The certificates can also be revoked using `certs` service. To revoke a certificate you need to provide `thing_id` of the thing for which the certificate was issued.
```bash
docker/.env
+2 -3
View File
@@ -271,7 +271,7 @@ MG_UI_DB_SSL_ROOT_CERT=
## Addons Services
### Bootstrap
MG_BOOTSTRAP_LOG_LEVEL=debug
MG_BOOTSTRAP_ENCRYPT_KEY=v7aT0HGxJxt2gULzr3RHwf4WIf6DusPphG5Ftm2bNCWD8mTpyr
MG_BOOTSTRAP_ENCRYPT_KEY=v7aT0HGxJxt2gULzr3RHwf4WIf6DusPp
MG_BOOTSTRAP_EVENT_CONSUMER=bootstrap
MG_BOOTSTRAP_HTTP_HOST=bootstrap
MG_BOOTSTRAP_HTTP_PORT=9013
@@ -302,8 +302,7 @@ MG_PROVISION_PASS=
MG_PROVISION_API_KEY=
MG_PROVISION_CERTS_SVC_URL=http://certs:9019
MG_PROVISION_X509_PROVISIONING=false
MG_PROVISION_BS_SVC_URL=http://bootstrap:9013/things
MG_PROVISION_BS_SVC_WHITELIST_URL=http://bootstrap:9013/things/state
MG_PROVISION_BS_SVC_URL=http://bootstrap:9013
MG_PROVISION_BS_CONFIG_PROVISIONING=true
MG_PROVISION_BS_AUTO_WHITELIST=true
MG_PROVISION_BS_CONTENT=
docker/addons/vault/README.md
+19 -10
View File
@@ -6,11 +6,8 @@ When the Vault service is started, some initialization steps need to be done to
## Configuration
| Variable | Description | Default |
| :---------------------------------------- | ------------------------------------------------------------------------------- | --------------------------------------- |
| MG_VAULT_HOST | Vault service address | vault |
| MG_VAULT_PORT | Vault service port | 8200 |
| :-------------------------------------- | ----------------------------------------------------------------------------- | ------------------------------------- |
| MG_VAULT_ADDR | Vault Address | http://vault:8200 |
| MG_VAULT_UNSEAL_KEY_1 | Vault unseal key | "" |
| MG_VAULT_UNSEAL_KEY_2 | Vault unseal key | "" |
@@ -106,24 +103,34 @@ The parameters required for generating certificate are obtained from the environ
Environmental variables starting with `MG_VAULT_PKI` in `docker/.env` file are used by `vault_set_pki.sh` to generate root CA.
Environmental variables starting with`MG_VAULT_PKI_INT` in `docker/.env` file are used by `vault_set_pki.sh` to generate intermediate CA.
Passing command line args `--skip-server-cert` to `vault_set_pki.sh` will skip server certificate role & process of generation of server certificate & key.
### 5. `vault_create_approle.sh`
This script is used to enable app role authorization in Vault. Certs service used the approle credentials to issue, revoke things certificate from vault intermedate CA.
`vault_create_approle.sh` script by default tries to enable auth approle.
If approle is already enabled in vault, then use args `skip_enable_app_role` to skip enable auth approle step.
To skip enable auth approle step use the following `vault_create_approle.sh skip_enable_app_role`
If approle is already enabled in vault, then use args `--skip-enable-approle` to skip enable auth approle step.
To skip enable auth approle step use the following `vault_create_approle.sh --skip-enable-approle`
### 6. `vault_copy_certs.sh`
This scripts copies the necessary certificates and keys from `docker/addons/vault/data` to the `docker/ssl/certs` folder.
## Hashicorp Cloud Platform (HCP) Vault
To have the same PKI setup can done in Hashicorp Cloud Platform (HCP) Vault follow the below steps:
Requirement: [VAULT CLI](https://developer.hashicorp.com/vault/tutorials/getting-started/getting-started-install)
- Replace the environmental variable `MG_VAULT_ADDR` in `docker/.env` with HCP Vault address.
- Replace the environmental variable `MG_VAULT_TOKEN` in `docker/.env` with HCP Vault Admin token.
- Run script `vault_set_pki.sh` and `vault_create_approle.sh`.
- Optional step, run script `vault_copy_certs.sh` to copy certificates to magistrala default path.
## Vault CLI
It can also be useful to run the Vault CLI for inspection and administration work.
This can be done directly using the Vault image in Docker: `docker run -it magistrala/vault:latest vault`
```bash
Usage: vault <command> [args]
@@ -156,6 +163,8 @@ Other commands:
token Interact with tokens
```
### Vault Web UI
If the Vault is setup through `docker/addons/vault`, then Vault CLI can be run directly using the Vault image in Docker: `docker run -it magistrala/vault:latest vault`
The Vault Web UI is accessible by default on `http://localhost:8200/ui`.
## Vault Web UI
If the Vault is setup through `docker/addons/vault`, Then Vault Web UI is accessible by default on `http://localhost:8200/ui`.
docker/addons/vault/vault_cmd.sh
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/bash
# Copyright (c) Abstract Machines
# SPDX-License-Identifier: Apache-2.0
vault() {
if is_container_running "magistrala-vault"; then
docker exec -it magistrala-vault vault "$@"
else
if which vault &> /dev/null; then
$(which vault) "$@"
else
echo "magistrala-vault container or vault command not found. Please refer to the documentation: https://github.com/absmach/magistrala/blob/main/docker/addons/vault/README.md"
fi
fi
}
is_container_running() {
local container_name="$1"
if [ "$(docker inspect --format '{{.State.Running}}' "$container_name" 2>/dev/null)" = "true" ]; then
return 0
else
return 1
fi
}
docker/addons/vault/vault_copy_certs.sh
+24 -4
View File
@@ -25,9 +25,29 @@ if [ -n "${MG_NGINX_SERVER_NAME:-}" ]; then
fi
echo "Copying certificate files"
cp -v data/${server_name}.crt ${MAGISTRALA_DIR}/docker/ssl/certs/magistrala-server.crt
cp -v data/${server_name}.key ${MAGISTRALA_DIR}/docker/ssl/certs/magistrala-server.key
cp -v data/${MG_VAULT_PKI_INT_FILE_NAME}.key ${MAGISTRALA_DIR}/docker/ssl/certs/ca.key
cp -v data/${MG_VAULT_PKI_INT_FILE_NAME}_bundle.crt ${MAGISTRALA_DIR}/docker/ssl/certs/ca.crt
if [ -e "data/${server_name}.crt" ]; then
cp -v data/${server_name}.crt ${MAGISTRALA_DIR}/docker/ssl/certs/magistrala-server.crt
else
echo "${server_name}.crt file not available"
fi
if [ -e "data/${server_name}.key" ]; then
cp -v data/${server_name}.key ${MAGISTRALA_DIR}/docker/ssl/certs/magistrala-server.key
else
echo "${server_name}.key file not available"
fi
if [ -e "data/${MG_VAULT_PKI_INT_FILE_NAME}.key" ]; then
cp -v data/${MG_VAULT_PKI_INT_FILE_NAME}.key ${MAGISTRALA_DIR}/docker/ssl/certs/ca.key
else
echo "data/${MG_VAULT_PKI_INT_FILE_NAME}.key file not available"
fi
if [ -e "data/${MG_VAULT_PKI_INT_FILE_NAME}_bundle.crt" ]; then
cp -v data/${MG_VAULT_PKI_INT_FILE_NAME}_bundle.crt ${MAGISTRALA_DIR}/docker/ssl/certs/ca.crt
else
echo "data/${MG_VAULT_PKI_INT_FILE_NAME}_bundle.crt file not available"
fi
exit 0
docker/addons/vault/vault_copy_env.sh
+9 -4
View File
@@ -10,10 +10,15 @@ export MAGISTRALA_DIR=$scriptdir/../../../
cd $scriptdir
write_env() {
sed -i "s,MG_VAULT_UNSEAL_KEY_1=.*,MG_VAULT_UNSEAL_KEY_1=$(awk -F ": " '$1 == "Unseal Key 1" {print $2}' data/secrets)," $MAGISTRALA_DIR/docker/.env
sed -i "s,MG_VAULT_UNSEAL_KEY_2=.*,MG_VAULT_UNSEAL_KEY_2=$(awk -F ": " '$1 == "Unseal Key 2" {print $2}' data/secrets)," $MAGISTRALA_DIR/docker/.env
sed -i "s,MG_VAULT_UNSEAL_KEY_3=.*,MG_VAULT_UNSEAL_KEY_3=$(awk -F ": " '$1 == "Unseal Key 3" {print $2}' data/secrets)," $MAGISTRALA_DIR/docker/.env
sed -i "s,MG_VAULT_TOKEN=.*,MG_VAULT_TOKEN=$(awk -F ": " '$1 == "Initial Root Token" {print $2}' data/secrets)," $MAGISTRALA_DIR/docker/.env
if [ -e "data/secrets" ]; then
sed -i "s,MG_VAULT_UNSEAL_KEY_1=.*,MG_VAULT_UNSEAL_KEY_1=$(awk -F ": " '$1 == "Unseal Key 1" {print $2}' data/secrets)," $MAGISTRALA_DIR/docker/.env
sed -i "s,MG_VAULT_UNSEAL_KEY_2=.*,MG_VAULT_UNSEAL_KEY_2=$(awk -F ": " '$1 == "Unseal Key 2" {print $2}' data/secrets)," $MAGISTRALA_DIR/docker/.env
sed -i "s,MG_VAULT_UNSEAL_KEY_3=.*,MG_VAULT_UNSEAL_KEY_3=$(awk -F ": " '$1 == "Unseal Key 3" {print $2}' data/secrets)," $MAGISTRALA_DIR/docker/.env
sed -i "s,MG_VAULT_TOKEN=.*,MG_VAULT_TOKEN=$(awk -F ": " '$1 == "Initial Root Token" {print $2}' data/secrets)," $MAGISTRALA_DIR/docker/.env
echo "Vault environment varaibles are set successfully in docker/.env"
else
echo "Error: Source file 'data/secrets' not found."
fi
}
write_env
docker/addons/vault/vault_create_approle.sh
+8 -6
View File
@@ -17,9 +17,7 @@ readDotEnv() {
set +o allexport
}
vault() {
docker exec -it magistrala-vault vault "$@"
}
source vault_cmd.sh
vaultCreatePolicyFile() {
envsubst '
@@ -29,12 +27,16 @@ vaultCreatePolicyFile() {
}
vaultCreatePolicy() {
echo "Creating new policy for AppRole"
docker cp magistrala_things_certs_issue.hcl magistrala-vault:/vault/magistrala_things_certs_issue.hcl
vault policy write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} magistrala_things_certs_issue /vault/magistrala_things_certs_issue.hcl
if is_container_running "magistrala-vault"; then
docker cp magistrala_things_certs_issue.hcl magistrala-vault:/vault/magistrala_things_certs_issue.hcl
vault policy write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} magistrala_things_certs_issue /vault/magistrala_things_certs_issue.hcl
else
vault policy write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} magistrala_things_certs_issue magistrala_things_certs_issue.hcl
fi
}
vaultEnableAppRole() {
if [ "$SKIP_ENABLE_APP_ROLE" == "skip_enable_app_role" ]; then
if [ "$SKIP_ENABLE_APP_ROLE" == "--skip-enable-approle" ]; then
echo "Skipping Enable AppRole"
else
echo "Enabling AppRole"
docker/addons/vault/vault_init.sh
+9 -3
View File
@@ -9,10 +9,16 @@ export MAGISTRALA_DIR=$scriptdir/../../../
cd $scriptdir
vault() {
docker exec -it magistrala-vault vault "$@"
readDotEnv() {
set -o allexport
source $MAGISTRALA_DIR/docker/.env
set +o allexport
}
source vault_cmd.sh
readDotEnv
mkdir -p data
vault operator init 2>&1 | tee >(sed -r 's/\x1b\[[0-9;]*m//g' > data/secrets)
vault operator init -address=$MG_VAULT_ADDR 2>&1 | tee >(sed -r 's/\x1b\[[0-9;]*m//g' > data/secrets)
docker/addons/vault/vault_set_pki.sh
+57 -27
View File
@@ -7,6 +7,8 @@ set -euo pipefail
scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
export MAGISTRALA_DIR=$scriptdir/../../../
SKIP_SERVER_CERT=${1:-}
cd $scriptdir
readDotEnv() {
@@ -22,9 +24,7 @@ if [ -n "${MG_NGINX_SERVER_NAME:-}" ]; then
server_name="$MG_NGINX_SERVER_NAME"
fi
vault() {
docker exec -it magistrala-vault vault "$@"
}
source vault_cmd.sh
vaultEnablePKI() {
vault secrets enable -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -path ${MG_VAULT_PKI_PATH} pki
@@ -103,24 +103,43 @@ vaultGenerateIntermediateCSR() {
vaultSignIntermediateCSR() {
echo "Sign intermediate CSR"
docker cp data/${MG_VAULT_PKI_INT_FILE_NAME}.csr magistrala-vault:/vault/${MG_VAULT_PKI_INT_FILE_NAME}.csr
vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -format=json ${MG_VAULT_PKI_PATH}/root/sign-intermediate \
csr=@/vault/${MG_VAULT_PKI_INT_FILE_NAME}.csr ttl="8760h" \
ou="\"$MG_VAULT_PKI_INT_CA_OU\""\
organization="\"$MG_VAULT_PKI_INT_CA_O\"" \
country="\"$MG_VAULT_PKI_INT_CA_C\"" \
locality="\"$MG_VAULT_PKI_INT_CA_L\"" \
province="\"$MG_VAULT_PKI_INT_CA_ST\"" \
street_address="\"$MG_VAULT_PKI_INT_CA_ADDR\"" \
postal_code="\"$MG_VAULT_PKI_INT_CA_PO\"" \
| tee >(jq -r .data.certificate >data/${MG_VAULT_PKI_INT_FILE_NAME}.crt) \
>(jq -r .data.issuing_ca >data/${MG_VAULT_PKI_INT_FILE_NAME}_issuing_ca.crt)
if is_container_running "magistrala-vault"; then
docker cp data/${MG_VAULT_PKI_INT_FILE_NAME}.csr magistrala-vault:/vault/${MG_VAULT_PKI_INT_FILE_NAME}.csr
vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -format=json ${MG_VAULT_PKI_PATH}/root/sign-intermediate \
csr=@/vault/${MG_VAULT_PKI_INT_FILE_NAME}.csr ttl="8760h" \
ou="\"$MG_VAULT_PKI_INT_CA_OU\""\
organization="\"$MG_VAULT_PKI_INT_CA_O\"" \
country="\"$MG_VAULT_PKI_INT_CA_C\"" \
locality="\"$MG_VAULT_PKI_INT_CA_L\"" \
province="\"$MG_VAULT_PKI_INT_CA_ST\"" \
street_address="\"$MG_VAULT_PKI_INT_CA_ADDR\"" \
postal_code="\"$MG_VAULT_PKI_INT_CA_PO\"" \
| tee >(jq -r .data.certificate >data/${MG_VAULT_PKI_INT_FILE_NAME}.crt) \
>(jq -r .data.issuing_ca >data/${MG_VAULT_PKI_INT_FILE_NAME}_issuing_ca.crt)
else
vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -format=json ${MG_VAULT_PKI_PATH}/root/sign-intermediate \
csr=@data/${MG_VAULT_PKI_INT_FILE_NAME}.csr ttl="8760h" \
ou="\"$MG_VAULT_PKI_INT_CA_OU\""\
organization="\"$MG_VAULT_PKI_INT_CA_O\"" \
country="\"$MG_VAULT_PKI_INT_CA_C\"" \
locality="\"$MG_VAULT_PKI_INT_CA_L\"" \
province="\"$MG_VAULT_PKI_INT_CA_ST\"" \
street_address="\"$MG_VAULT_PKI_INT_CA_ADDR\"" \
postal_code="\"$MG_VAULT_PKI_INT_CA_PO\"" \
| tee >(jq -r .data.certificate >data/${MG_VAULT_PKI_INT_FILE_NAME}.crt) \
>(jq -r .data.issuing_ca >data/${MG_VAULT_PKI_INT_FILE_NAME}_issuing_ca.crt)
fi
}
vaultInjectIntermediateCertificate() {
echo "Inject Intermediate Certificate"
docker cp data/${MG_VAULT_PKI_INT_FILE_NAME}.crt magistrala-vault:/vault/${MG_VAULT_PKI_INT_FILE_NAME}.crt
vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_INT_PATH}/intermediate/set-signed certificate=@/vault/${MG_VAULT_PKI_INT_FILE_NAME}.crt
if is_container_running "magistrala-vault"; then
docker cp data/${MG_VAULT_PKI_INT_FILE_NAME}.crt magistrala-vault:/vault/${MG_VAULT_PKI_INT_FILE_NAME}.crt
vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_INT_PATH}/intermediate/set-signed certificate=@/vault/${MG_VAULT_PKI_INT_FILE_NAME}.crt
else
vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_INT_PATH}/intermediate/set-signed certificate=@data/${MG_VAULT_PKI_INT_FILE_NAME}.crt
fi
}
vaultGenerateIntermediateCertificateBundle() {
@@ -139,18 +158,27 @@ vaultSetupIntermediateIssuingURLs() {
}
vaultSetupServerCertsRole() {
echo "Setup Server Certs role"
vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_INT_PATH}/roles/${MG_VAULT_PKI_INT_SERVER_CERTS_ROLE_NAME} \
allow_subdomains=true \
max_ttl="4320h"
if [ "$SKIP_SERVER_CERT" == "--skip-server-cert" ]; then
echo "Skipping server certificate role"
else
echo "Setup Server certificate role"
vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} ${MG_VAULT_PKI_INT_PATH}/roles/${MG_VAULT_PKI_INT_SERVER_CERTS_ROLE_NAME} \
allow_subdomains=true \
max_ttl="4320h"
fi
}
vaultGenerateServerCertificate() {
echo "Generate server certificate"
vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -format=json ${MG_VAULT_PKI_INT_PATH}/issue/${MG_VAULT_PKI_INT_SERVER_CERTS_ROLE_NAME} \
common_name="$server_name" ttl="4320h" \
| tee >(jq -r .data.certificate >data/${server_name}.crt) \
>(jq -r .data.private_key >data/${server_name}.key)
if [ "$SKIP_SERVER_CERT" == "--skip-server-cert" ]; then
echo "Skipping generate server certificate"
else
echo "Generate server certificate"
vault write -namespace=${MG_VAULT_NAMESPACE} -address=${MG_VAULT_ADDR} -format=json ${MG_VAULT_PKI_INT_PATH}/issue/${MG_VAULT_PKI_INT_SERVER_CERTS_ROLE_NAME} \
common_name="$server_name" ttl="4320h" \
| tee >(jq -r .data.certificate >data/${server_name}.crt) \
>(jq -r .data.private_key >data/${server_name}.key)
fi
}
vaultSetupThingCertsRole() {
@@ -162,7 +190,9 @@ vaultSetupThingCertsRole() {
}
vaultCleanupFiles() {
docker exec magistrala-vault sh -c 'rm -rf /vault/*.{crt,csr}'
if is_container_running "magistrala-vault"; then
docker exec magistrala-vault sh -c 'rm -rf /vault/*.{crt,csr}'
fi
}
if ! command -v jq &> /dev/null
docker/addons/vault/vault_unseal.sh
+6 -6
View File
@@ -7,18 +7,18 @@ set -euo pipefail
scriptdir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
export MAGISTRALA_DIR=$scriptdir/../../../
cd $scriptdir
readDotEnv() {
set -o allexport
source $MAGISTRALA_DIR/docker/.env
set +o allexport
}
vault() {
docker exec -it magistrala-vault vault "$@"
}
source vault_cmd.sh
readDotEnv
vault operator unseal ${MG_VAULT_UNSEAL_KEY_1}
vault operator unseal ${MG_VAULT_UNSEAL_KEY_2}
vault operator unseal ${MG_VAULT_UNSEAL_KEY_3}
vault operator unseal -address=${MG_VAULT_ADDR} ${MG_VAULT_UNSEAL_KEY_1}
vault operator unseal -address=${MG_VAULT_ADDR} ${MG_VAULT_UNSEAL_KEY_2}
vault operator unseal -address=${MG_VAULT_ADDR} ${MG_VAULT_UNSEAL_KEY_3}