NOISSUE - Make CI scripts run in parallel (#3269)
Property Based Tests / api-test (push) Has been cancelled
Continuous Delivery / Test (push) Has been cancelled
Continuous Delivery / Compile Check (map[env:SMQ_ES_TYPE=es_redis name:redis target:mqtt]) (push) Has been cancelled
Continuous Delivery / Compile Check (map[env:SMQ_MESSAGE_BROKER_TYPE=msg_rabbitmq name:rabbitmq target:mqtt]) (push) Has been cancelled
Check the consistency of generated files / check-generated-files (push) Has been cancelled
Check License Header / check-license (push) Has been cancelled
Deploy GitHub Pages / swagger-ui (push) Has been cancelled
CI Pipeline / Lint (push) Has been cancelled
CI Pipeline / Detect Changes (push) Has been cancelled
Continuous Delivery / Build and Push (push) Has been cancelled
CI Pipeline / Build (push) Has been cancelled
CI Pipeline / Test ${{ matrix.module }} (push) Has been cancelled
CI Pipeline / Upload Coverage (push) Has been cancelled

Signed-off-by: dusan <borovcanindusan1@gmail.com>
This commit is contained in:
Dušan Borovčanin
2025-12-02 22:58:23 +01:00
committed by GitHub
parent e77e3a3bb6
commit 0df8e8451c
3 changed files with 271 additions and 174 deletions
+72 -54
View File
@@ -4,7 +4,7 @@
name: Property Based Tests
on:
pull_request:
push:
branches:
- main
paths:
@@ -18,6 +18,11 @@ on:
- "http/api/**"
- "journal/api/**"
- "users/api/**"
- "apidocs/openapi/**"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
TOKENS_URL: http://localhost:9002/users/tokens/issue
@@ -40,6 +45,8 @@ jobs:
steps:
- name: Checkout Code
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Install Go
uses: actions/setup-go@v6
@@ -47,12 +54,68 @@ jobs:
go-version: 1.25.x
cache-dependency-path: "go.sum"
- name: Check for changes in specific paths
uses: dorny/paths-filter@v3
id: changes
with:
base: main
filters: |
workflow:
- ".github/workflows/api-tests.yaml"
journal:
- "apidocs/openapi/journal.yaml"
- "journal/api/**"
auth:
- "apidocs/openapi/auth.yaml"
- "auth/api/http/**"
domains:
- "apidocs/openapi/domains.yaml"
- "domains/api/http/**"
http:
- "apidocs/openapi/http.yaml"
- "http/api/**"
clients:
- "apidocs/openapi/clients.yaml"
- "clients/api/http/**"
channels:
- "apidocs/openapi/channels.yaml"
- "channels/api/http/**"
groups:
- "apidocs/openapi/groups.yaml"
- "groups/api/http/**"
users:
- "apidocs/openapi/users.yaml"
- "users/api/**"
- name: Build images
run: make all -j $(nproc) && make dockers_dev -j $(nproc)
- name: Start containers
run: make run up args="-d" && make run_addons up args="-d"
- name: Wait for services to be ready
run: |
echo "Waiting for services to start..."
sleep 15
# Check if services are responding
for i in {1..30}; do
if curl -f -s http://localhost:9002/health > /dev/null 2>&1; then
echo "Services are ready!"
break
fi
echo "Waiting for services... ($i/30)"
sleep 2
done
- name: Set access token
run: |
export USER_TOKEN=$(curl -sSX POST $TOKENS_URL -H "Content-Type: application/json" -d "{\"identity\": \"$USER_IDENTITY\",\"secret\": \"$USER_SECRET\"}" | jq -r .access_token)
@@ -61,53 +124,8 @@ jobs:
export CLIENT_SECRET=$(supermq-cli provision test | /usr/bin/grep -Eo '"secret": "[^"]+"' | awk 'NR % 2 == 0' | sed 's/"secret": "\(.*\)"/\1/')
echo "CLIENT_SECRET=$CLIENT_SECRET" >> $GITHUB_ENV
- name: Check for changes in specific paths
uses: dorny/paths-filter@v3
id: changes
with:
filters: |
journal:
- ".github/workflows/api-tests.yaml"
- "apidocs/openapi/journal.yaml"
- "journal/api/**"
auth:
- ".github/workflows/api-tests.yaml"
- "apidocs/openapi/auth.yaml"
- "auth/api/http/**"
domains:
- ".github/workflows/api-tests.yaml"
- "apidocs/openapi/domains.yaml"
- "domains/api/http/**"
http:
- ".github/workflows/api-tests.yaml"
- "apidocs/openapi/http.yaml"
- "http/api/**"
clients:
- ".github/workflows/api-tests.yaml"
- "apidocs/openapi/clients.yaml"
- "clients/api/http/**"
channels:
- ".github/workflows/api-tests.yaml"
- "apidocs/openapi/channels.yaml"
- "channels/api/http/**"
groups:
- ".github/workflows/api-tests.yaml"
- "apidocs/openapi/groups.yaml"
- "groups/api/http/**"
users:
- ".github/workflows/api-tests.yaml"
- "apidocs/openapi/users.yaml"
- "users/api/**"
- name: Run Users API tests
if: steps.changes.outputs.users == 'true'
if: steps.changes.outputs.users == 'true' || steps.changes.outputs.workflow == 'true'
uses: schemathesis/action@v2.1.0
with:
schema: apidocs/openapi/users.yaml
@@ -116,7 +134,7 @@ jobs:
args: '--header "Authorization: Bearer ${{ env.USER_TOKEN }}" --suppress-health-check=filter_too_much --exclude-checks=positive_data_acceptance --exclude-operation-id=requestPasswordReset --phases=examples,stateful'
- name: Run Groups API tests
if: steps.changes.outputs.groups == 'true'
if: steps.changes.outputs.groups == 'true' || steps.changes.outputs.workflow == 'true'
uses: schemathesis/action@v2.1.0
with:
schema: apidocs/openapi/groups.yaml
@@ -125,7 +143,7 @@ jobs:
args: '--header "Authorization: Bearer ${{ env.USER_TOKEN }}" --suppress-health-check=filter_too_much --exclude-checks=positive_data_acceptance --phases=examples'
- name: Run Clients API tests
if: steps.changes.outputs.clients == 'true'
if: steps.changes.outputs.clients == 'true' || steps.changes.outputs.workflow == 'true'
uses: schemathesis/action@v2.1.0
with:
schema: apidocs/openapi/clients.yaml
@@ -134,7 +152,7 @@ jobs:
args: '--header "Authorization: Bearer ${{ env.USER_TOKEN }}" --suppress-health-check=filter_too_much --exclude-checks=positive_data_acceptance --phases=examples'
- name: Run Channels API tests
if: steps.changes.outputs.channels == 'true'
if: steps.changes.outputs.channels == 'true' || steps.changes.outputs.workflow == 'true'
uses: schemathesis/action@v2.1.0
with:
schema: apidocs/openapi/channels.yaml
@@ -143,7 +161,7 @@ jobs:
args: '--header "Authorization: Bearer ${{ env.USER_TOKEN }}" --suppress-health-check=filter_too_much --exclude-checks=positive_data_acceptance --phases=examples'
- name: Run HTTP Adapter API tests
if: steps.changes.outputs.http == 'true'
if: steps.changes.outputs.http == 'true' || steps.changes.outputs.workflow == 'true'
uses: schemathesis/action@v2.1.0
with:
schema: apidocs/openapi/http.yaml
@@ -152,7 +170,7 @@ jobs:
args: '--header "Authorization: Client ${{ env.CLIENT_SECRET }}" --suppress-health-check=filter_too_much --exclude-checks=positive_data_acceptance --phases=examples'
- name: Run Auth API tests
if: steps.changes.outputs.auth == 'true'
if: steps.changes.outputs.auth == 'true' || steps.changes.outputs.workflow == 'true'
uses: schemathesis/action@v2.1.0
with:
schema: apidocs/openapi/auth.yaml
@@ -161,7 +179,7 @@ jobs:
args: '--header "Authorization: Bearer ${{ env.USER_TOKEN }}" --suppress-health-check=filter_too_much --exclude-checks=positive_data_acceptance --phases=examples'
- name: Run Domains API tests
if: steps.changes.outputs.domains == 'true'
if: steps.changes.outputs.domains == 'true' || steps.changes.outputs.workflow == 'true'
uses: schemathesis/action@v2.1.0
with:
schema: apidocs/openapi/domains.yaml
@@ -170,7 +188,7 @@ jobs:
args: '--header "Authorization: Bearer ${{ env.USER_TOKEN }}" --suppress-health-check=filter_too_much --exclude-checks=positive_data_acceptance --phases=examples'
- name: Run Journal API tests
if: steps.changes.outputs.journal == 'true'
if: steps.changes.outputs.journal == 'true' || steps.changes.outputs.workflow == 'true'
uses: schemathesis/action@v2.1.0
with:
schema: apidocs/openapi/journal.yaml
+54 -10
View File
@@ -8,9 +8,13 @@ on:
branches:
- main
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: false
jobs:
build-and-push:
name: Build and Push
test:
name: Test
runs-on: ubuntu-latest
permissions:
contents: read
@@ -45,6 +49,54 @@ jobs:
files: ./coverage/*.out
verbose: true
compile-check:
name: Compile Check
runs-on: ubuntu-latest
strategy:
fail-fast: true
matrix:
variant:
- name: rabbitmq
env: SMQ_MESSAGE_BROKER_TYPE=msg_rabbitmq
target: mqtt
- name: redis
env: SMQ_ES_TYPE=es_redis
target: mqtt
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: 1.25.x
cache-dependency-path: "go.sum"
- name: Compile check for ${{ matrix.variant.name }}
run: |
${{ matrix.variant.env }} make ${{ matrix.variant.target }}
build-and-push:
name: Build and Push
runs-on: ubuntu-latest
needs: [test, compile-check]
permissions:
contents: read
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0
fetch-tags: true
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: 1.25.x
cache-dependency-path: "go.sum"
- name: Set up Docker Build
uses: docker/setup-buildx-action@v3
@@ -55,14 +107,6 @@ jobs:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}
- name: Compile check for rabbitmq
run: |
SMQ_MESSAGE_BROKER_TYPE=msg_rabbitmq make mqtt
- name: Compile check for redis
run: |
SMQ_ES_TYPE=es_redis make mqtt
- name: Build and push Docker images
run: |
make latest -j $(nproc)
+145 -110
View File
@@ -4,13 +4,20 @@
name: CI Pipeline
on:
push:
branches:
- main
pull_request:
branches:
- main
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint-and-build: # Linting and building are combined to save time for setting up Go
name: Lint and Build
lint:
name: Lint
runs-on: ubuntu-latest
steps:
@@ -48,6 +55,25 @@ jobs:
version: v2.4.0
args: --config ./tools/config/.golangci.yaml
build:
name: Build
runs-on: ubuntu-latest
needs: lint
steps:
- name: Checkout Code
uses: actions/checkout@v6
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: 1.25.x
cache-dependency-path: "go.sum"
- name: Download all dependencies
run: |
go mod download
- name: Build all Binaries
run: |
make all -j $(nproc)
@@ -60,23 +86,18 @@ jobs:
run: |
SMQ_ES_TYPE=es_redis make mqtt
run-tests:
name: Run tests
detect-changes:
name: Detect Changes
runs-on: ubuntu-latest
needs: lint-and-build
outputs:
modules: ${{ steps.set-matrix.outputs.modules }}
workflow_changed: ${{ steps.changes.outputs.workflow }}
steps:
- name: Checkout
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: 1.25.x
cache-dependency-path: "go.sum"
- name: Check for changes in specific paths
uses: dorny/paths-filter@v3
id: changes
@@ -243,123 +264,137 @@ jobs:
- "clients/**"
- "pkg/messaging/**"
- name: Create coverage directory
run: |
mkdir coverage
notifications:
- "notifications/**"
- "cmd/notifications/**"
- "auth.pb.go"
- "auth_grpc.pb.go"
- "consumers/notifier.go"
- "pkg/events/**"
- name: Run journal tests
if: steps.changes.outputs.journal == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/journal.out ./journal/...
api:
- "api/**"
- name: Run auth tests
if: steps.changes.outputs.auth == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/auth.out ./auth/...
consumers:
- "consumers/**"
- name: Run domains tests
if: steps.changes.outputs.domains == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/domains.out ./domains/...
readers:
- "readers/**"
- name: Run cli tests
if: steps.changes.outputs.cli == 'true' || steps.changes.outputs.workflow == 'true'
- name: Set matrix for changed modules
id: set-matrix
run: |
go test --race -v -count=1 -coverprofile=coverage/cli.out ./cli/...
modules=()
- name: Run CoAP tests
if: steps.changes.outputs.coap == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/coap.out ./coap/...
if [[ "${{ steps.changes.outputs.workflow }}" == "true" ]]; then
# If workflow changed, test everything
modules=("auth" "channels" "cli" "clients" "coap" "domains" "groups" "http" "internal" "journal" "logger" "mqtt" "pkg-errors" "pkg-events" "pkg-grpcclient" "pkg-messaging" "pkg-sdk" "pkg-transformers" "pkg-ulid" "pkg-uuid" "users" "ws" "notifications" "api" "consumers" "readers")
else
# Add only changed modules
[[ "${{ steps.changes.outputs.auth }}" == "true" ]] && modules+=("auth")
[[ "${{ steps.changes.outputs.channels }}" == "true" ]] && modules+=("channels")
[[ "${{ steps.changes.outputs.cli }}" == "true" ]] && modules+=("cli")
[[ "${{ steps.changes.outputs.clients }}" == "true" ]] && modules+=("clients")
[[ "${{ steps.changes.outputs.coap }}" == "true" ]] && modules+=("coap")
[[ "${{ steps.changes.outputs.domains }}" == "true" ]] && modules+=("domains")
[[ "${{ steps.changes.outputs.groups }}" == "true" ]] && modules+=("groups")
[[ "${{ steps.changes.outputs.http }}" == "true" ]] && modules+=("http")
[[ "${{ steps.changes.outputs.internal }}" == "true" ]] && modules+=("internal")
[[ "${{ steps.changes.outputs.journal }}" == "true" ]] && modules+=("journal")
[[ "${{ steps.changes.outputs.logger }}" == "true" ]] && modules+=("logger")
[[ "${{ steps.changes.outputs.mqtt }}" == "true" ]] && modules+=("mqtt")
[[ "${{ steps.changes.outputs.pkg-errors }}" == "true" ]] && modules+=("pkg-errors")
[[ "${{ steps.changes.outputs.pkg-events }}" == "true" ]] && modules+=("pkg-events")
[[ "${{ steps.changes.outputs.pkg-grpcclient }}" == "true" ]] && modules+=("pkg-grpcclient")
[[ "${{ steps.changes.outputs.pkg-messaging }}" == "true" ]] && modules+=("pkg-messaging")
[[ "${{ steps.changes.outputs.pkg-sdk }}" == "true" ]] && modules+=("pkg-sdk")
[[ "${{ steps.changes.outputs.pkg-transformers }}" == "true" ]] && modules+=("pkg-transformers")
[[ "${{ steps.changes.outputs.pkg-ulid }}" == "true" ]] && modules+=("pkg-ulid")
[[ "${{ steps.changes.outputs.pkg-uuid }}" == "true" ]] && modules+=("pkg-uuid")
[[ "${{ steps.changes.outputs.users }}" == "true" ]] && modules+=("users")
[[ "${{ steps.changes.outputs.ws }}" == "true" ]] && modules+=("ws")
[[ "${{ steps.changes.outputs.notifications }}" == "true" ]] && modules+=("notifications")
[[ "${{ steps.changes.outputs.api }}" == "true" ]] && modules+=("api")
[[ "${{ steps.changes.outputs.consumers }}" == "true" ]] && modules+=("consumers")
[[ "${{ steps.changes.outputs.readers }}" == "true" ]] && modules+=("readers")
fi
- name: Run HTTP tests
if: steps.changes.outputs.http == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/http.out ./http/...
# Convert to JSON array
json_modules=$(printf '%s\n' "${modules[@]}" | jq -R . | jq -s -c .)
echo "modules=$json_modules" >> $GITHUB_OUTPUT
echo "Testing modules: $json_modules"
- name: Run internal tests
if: steps.changes.outputs.internal == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/internal.out ./internal/...
run-tests:
name: Test ${{ matrix.module }}
runs-on: ubuntu-latest
needs: [lint, detect-changes, build]
if: needs.detect-changes.outputs.modules != '[]'
strategy:
fail-fast: true
max-parallel: 20
matrix:
module: ${{ fromJSON(needs.detect-changes.outputs.modules) }}
- name: Run logger tests
if: steps.changes.outputs.logger == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/logger.out ./logger/...
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Run MQTT tests
if: steps.changes.outputs.mqtt == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/mqtt.out ./mqtt/...
- name: Setup Go
uses: actions/setup-go@v6
with:
go-version: 1.25.x
cache-dependency-path: "go.sum"
- name: Run pkg errors tests
if: steps.changes.outputs.pkg-errors == 'true' || steps.changes.outputs.workflow == 'true'
- name: Verify dependencies
run: |
go test --race -v -count=1 -coverprofile=coverage/pkg-errors.out ./pkg/errors/...
go mod download
go mod verify
- name: Run pkg events tests
if: steps.changes.outputs.pkg-events == 'true' || steps.changes.outputs.workflow == 'true'
- name: Run ${{ matrix.module }} tests
run: |
go test --race -v -count=1 -coverprofile=coverage/pkg-events.out ./pkg/events/...
# Map module names to directories
case "${{ matrix.module }}" in
pkg-errors) dir="pkg/errors" ;;
pkg-events) dir="pkg/events" ;;
pkg-grpcclient) dir="pkg/grpcclient" ;;
pkg-messaging) dir="pkg/messaging" ;;
pkg-sdk) dir="pkg/sdk" ;;
pkg-transformers) dir="pkg/transformers" ;;
pkg-ulid) dir="pkg/ulid" ;;
pkg-uuid) dir="pkg/uuid" ;;
*) dir="${{ matrix.module }}" ;;
esac
- name: Run pkg grpcclient tests
if: steps.changes.outputs.pkg-grpcclient == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/pkg-grpcclient.out ./pkg/grpcclient/...
- name: Run pkg messaging tests
if: steps.changes.outputs.pkg-messaging == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/pkg-messaging.out ./pkg/messaging/...
- name: Run pkg sdk tests
if: steps.changes.outputs.pkg-sdk == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/pkg-sdk.out ./pkg/sdk/...
- name: Run pkg transformers tests
if: steps.changes.outputs.pkg-transformers == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/pkg-transformers.out ./pkg/transformers/...
- name: Run pkg ulid tests
if: steps.changes.outputs.pkg-ulid == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/pkg-ulid.out ./pkg/ulid/...
- name: Run pkg uuid tests
if: steps.changes.outputs.pkg-uuid == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/pkg-uuid.out ./pkg/uuid/...
- name: Run clients tests
if: steps.changes.outputs.clients == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/clients.out ./clients/...
- name: Run channels tests
if: steps.changes.outputs.channels == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/channels.out ./channels/...
- name: Run users tests
if: steps.changes.outputs.users == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/users.out ./users/...
- name: Run groups tests
if: steps.changes.outputs.groups == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/groups.out ./groups/...
- name: Run WebSocket tests
if: steps.changes.outputs.ws == 'true' || steps.changes.outputs.workflow == 'true'
run: |
go test --race -v -count=1 -coverprofile=coverage/ws.out ./ws/...
go test -mod=readonly --race -v -count=1 -failfast -coverprofile=coverage-${{ matrix.module }}.out ./$dir/...
- name: Upload coverage
uses: actions/upload-artifact@v4
with:
name: coverage-${{ matrix.module }}
path: coverage-${{ matrix.module }}.out
retention-days: 1
upload-coverage:
name: Upload Coverage
runs-on: ubuntu-latest
needs: run-tests
if: always() && needs.run-tests.result != 'cancelled'
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Download all coverage artifacts
uses: actions/download-artifact@v4
with:
pattern: coverage-*
path: coverage
merge-multiple: true
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v5
with:
token: ${{ secrets.CODECOV }}
directory: ./coverage
files: ./coverage/*.out
fail_ci_if_error: false
verbose: true