diff --git a/.github/scripts/ansible/aws_ec2.sh b/.github/scripts/ansible/aws_ec2.sh index d6ab3c03213..82e72f96594 100755 --- a/.github/scripts/ansible/aws_ec2.sh +++ b/.github/scripts/ansible/aws_ec2.sh @@ -8,6 +8,7 @@ fi OPERATION=$1 REGION=$2 +CLUSTER_NAME=$3 case $OPERATION in requirements) @@ -16,8 +17,7 @@ case $OPERATION in ;; create|delete|start|stop) if [ -f "env.yml" ]; then ANSIBLE_CUSTOM_VARS_ARG="-e @env.yml"; fi - CLUSTER_NAME=${CLUSTER_NAME:-"keycloak_$(whoami)"} - ansible-playbook aws_ec2.yml -v -e "region=$REGION" -e "operation=$OPERATION" -e "cluster_name=$CLUSTER_NAME" $ANSIBLE_CUSTOM_VARS_ARG "${@:3}" + ansible-playbook aws_ec2.yml -v -e "region=$REGION" -e "operation=$OPERATION" -e "cluster_name=$CLUSTER_NAME" $ANSIBLE_CUSTOM_VARS_ARG "${@:4}" ;; *) echo "Invalid option!" diff --git a/.github/scripts/ansible/keycloak_ec2_installer.sh b/.github/scripts/ansible/keycloak_ec2_installer.sh index 682ff159d81..f91237d8ce7 100755 --- a/.github/scripts/ansible/keycloak_ec2_installer.sh +++ b/.github/scripts/ansible/keycloak_ec2_installer.sh @@ -7,9 +7,8 @@ if [[ "$RUNNER_DEBUG" == "1" ]]; then fi REGION=$1 -KEYCLOAK_SRC=$2 - -CLUSTER_NAME=${CLUSTER_NAME:-"keycloak_$(whoami)"} +CLUSTER_NAME=$2 +KEYCLOAK_SRC=$3 ansible-playbook -i ${CLUSTER_NAME}_${REGION}_inventory.yml keycloak.yml \ -e "keycloak_src=\"${KEYCLOAK_SRC}\"" diff --git a/.github/scripts/ansible/mvn_ec2_runner.sh b/.github/scripts/ansible/mvn_ec2_runner.sh index bbbdee12a46..d5a608aea4b 100755 --- a/.github/scripts/ansible/mvn_ec2_runner.sh +++ b/.github/scripts/ansible/mvn_ec2_runner.sh @@ -7,9 +7,10 @@ if [[ "$RUNNER_DEBUG" == "1" ]]; then fi REGION=$1 -MVN_PARAMS=${@:2} +CLUSTER_NAME=$2 +MVN_PARAMS=${@:3} -CLUSTER_NAME=${CLUSTER_NAME:-"keycloak_$(whoami)"} +echo "mvn_params=\"${MVN_PARAMS}\"" ansible-playbook -i ${CLUSTER_NAME}_${REGION}_inventory.yml mvn.yml \ -e "mvn_params=\"${MVN_PARAMS}\"" diff --git a/.github/scripts/ansible/roles/aws_ec2/tasks/create-resources.yml b/.github/scripts/ansible/roles/aws_ec2/tasks/create-resources.yml index 65c029c487f..453ff2fef58 100644 --- a/.github/scripts/ansible/roles/aws_ec2/tasks/create-resources.yml +++ b/.github/scripts/ansible/roles/aws_ec2/tasks/create-resources.yml @@ -19,6 +19,13 @@ register: group no_log: "{{ no_log_sensitive }}" +- name: Delete existing key pair if it exists + amazon.aws.ec2_key: + region: '{{ region }}' + name: '{{ cluster_name }}' + state: absent + ignore_errors: true + - name: Create Key amazon.aws.ec2_key: state: present diff --git a/.github/scripts/ansible/roles/mvn_ec2_runner/tasks/run.yml b/.github/scripts/ansible/roles/mvn_ec2_runner/tasks/run.yml index 44b2a77c99b..514fa65a3cd 100644 --- a/.github/scripts/ansible/roles/mvn_ec2_runner/tasks/run.yml +++ b/.github/scripts/ansible/roles/mvn_ec2_runner/tasks/run.yml @@ -16,6 +16,7 @@ ansible.builtin.shell: | set -o pipefail cd {{ kc_home }} + echo "{{ mvn_params }}" ./mvnw {{ mvn_params }} args: executable: /usr/bin/bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5bebfe341e..6a32790e7da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,8 +33,9 @@ jobs: ci-store: ${{ steps.conditional.outputs.ci-store }} ci-sssd: ${{ steps.conditional.outputs.ci-sssd }} ci-webauthn: ${{ steps.conditional.outputs.ci-webauthn }} - ci-store-matrix: ${{ steps.conditional-stores.outputs.matrix }} ci-test-poc: ${{ steps.conditional.outputs.ci-test-poc }} + ci-aurora: ${{ steps.auroradb-tests.outputs.run-aurora-tests }} + steps: - uses: actions/checkout@v4 @@ -43,13 +44,14 @@ jobs: with: token: ${{ secrets.GITHUB_TOKEN }} - - id: conditional-stores + - name: AuroraDB conditional check + id: auroradb-tests run: | - STORES="postgres, mysql, oracle, mssql, mariadb" + RUN_AURORADB_TESTS=false if [[ $GITHUB_EVENT_NAME != "pull_request" && -n "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]]; then - STORES+=", aurora-postgres" + RUN_AURORADB_TESTS=true fi - echo "matrix=$(echo $STORES | jq -Rc 'split(", ")')" >> $GITHUB_OUTPUT + echo "run-aurora-tests=$RUN_AURORADB_TESTS" >> $GITHUB_OUTPUT build: name: Build @@ -420,85 +422,184 @@ jobs: with: job-id: remote-infinispan-integration-tests - store-integration-tests: - name: Store IT - needs: [build, conditional] - if: needs.conditional.outputs.ci-store == 'true' + auroradb-integration-tests: + name: AuroraDB IT + needs: conditional + if: needs.conditional.outputs.ci-aurora == 'true' runs-on: ubuntu-latest timeout-minutes: 150 - strategy: - matrix: - db: ${{ fromJson(needs.conditional.outputs.ci-store-matrix) }} - fail-fast: false steps: - uses: actions/checkout@v4 - id: aurora-init name: Initialize Aurora environment - if: ${{ matrix.db == 'aurora-postgres' }} run: | AWS_REGION=us-east-1 - echo "Region: ${AWS_REGION}" - + echo "AWS Region: ${AWS_REGION}" + aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }} aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws configure set region ${AWS_REGION} + + AURORA_CLUSTER_NAME="gh-action-$(git rev-parse --short HEAD)-${{ github.run_id }}-${{ github.run_attempt }}" PASS=$(tr -dc A-Za-z0-9 > $GITHUB_OUTPUT - echo "password=${PASS}" >> $GITHUB_OUTPUT + + echo "aurora-cluster-name=${AURORA_CLUSTER_NAME}" >> $GITHUB_OUTPUT + echo "aurora-cluster-password=${PASS}" >> $GITHUB_OUTPUT echo "region=${AWS_REGION}" >> $GITHUB_OUTPUT + curl --fail-with-body https://truststore.pki.rds.amazonaws.com/${AWS_REGION}/${AWS_REGION}-bundle.pem -o aws.pem + PROPS+=' -Dkeycloak.connectionsJpa.jdbcParameters=\"?ssl=true&sslmode=verify-ca&sslrootcert=/opt/keycloak/aws.pem\"' + + echo "maven_properties=${PROPS}" >> $GITHUB_OUTPUT - id: aurora-create name: Create Aurora DB - if: ${{ matrix.db == 'aurora-postgres' }} uses: ./.github/actions/aurora-create-database with: - name: ${{ steps.aurora-init.outputs.name }} - password: ${{ steps.aurora-init.outputs.password }} + name: ${{ steps.aurora-init.outputs.aurora-cluster-name }} + password: ${{ steps.aurora-init.outputs.aurora-cluster-password }} region: ${{ steps.aurora-init.outputs.region }} - - id: integration-test-setup - name: Integration test setup - if: ${{ matrix.db != 'aurora-postgres' }} - uses: ./.github/actions/integration-test-setup - - - name: Run Aurora tests on EC2 - id: aurora-tests - if: ${{ matrix.db == 'aurora-postgres' }} + - id: ec2-create + name: Create EC2 runner instance run: | - PROPS="-Dauth.server.db.host=${{ steps.aurora-create.outputs.endpoint }}" - PROPS+=" -Dkeycloak.connectionsJpa.password=${{ steps.aurora-init.outputs.password }}" - - REGION=${{ steps.aurora-init.outputs.region }} - - curl --fail-with-body https://truststore.pki.rds.amazonaws.com/${REGION}/${REGION}-bundle.pem -o aws.pem - PROPS+=" -Dkeycloak.connectionsJpa.jdbcParameters=\"?ssl=true&sslmode=verify-ca&sslrootcert=/opt/keycloak/aws.pem\"" - - TESTS=`testsuite/integration-arquillian/tests/base/testsuites/suite.sh database` - echo "Tests: $TESTS" - + AWS_REGION=${{ steps.aurora-init.outputs.region }} + EC2_CLUSTER_NAME=keycloak_$(git rev-parse --short HEAD) + echo "ec2_cluster=${EC2_CLUSTER_NAME}" >> $GITHUB_OUTPUT + git archive --format=zip --output /tmp/keycloak.zip $GITHUB_REF zip -u /tmp/keycloak.zip aws.pem + + cd .github/scripts/ansible + ./aws_ec2.sh requirements + ./aws_ec2.sh create ${AWS_REGION} ${EC2_CLUSTER_NAME} + ./keycloak_ec2_installer.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} /tmp/keycloak.zip + ./mvn_ec2_runner.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} "clean install -B -DskipTests -Pdistribution" + ./mvn_ec2_runner.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} "clean install -B -DskipTests -pl testsuite/integration-arquillian/servers/auth-server/quarkus -Pauth-server-quarkus -Pdb-aurora-postgres -Dmaven.build.cache.enabled=true" + + - name: Run Aurora migration tests on EC2 + id: aurora-migration-tests + env: + old-version: 24.0.4 + run: | + EC2_CLUSTER_NAME=${{ steps.ec2-create.outputs.ec2_cluster }} + AWS_REGION=${{ steps.aurora-init.outputs.region }} + PROPS='${{ steps.aurora-init.outputs.maven_properties }}' + + PROPS+=" -Dauth.server.db.host=${{ steps.aurora-create.outputs.endpoint }} -Dkeycloak.connectionsJpa.password=${{ steps.aurora-init.outputs.aurora-cluster-password }}" + PROPS+=" -Djdbc.mvn.groupId=software.amazon.jdbc -Djdbc.mvn.artifactId=aws-advanced-jdbc-wrapper -Djdbc.mvn.version=2.3.1 -Djdbc.driver.tmp.dir=target/unpacked/keycloak-${{ env.old-version }}/providers" cd .github/scripts/ansible - export CLUSTER_NAME=keycloak_$(git rev-parse --short HEAD) - echo "ec2_cluster=${CLUSTER_NAME}" >> $GITHUB_OUTPUT - ./aws_ec2.sh requirements - ./aws_ec2.sh create ${REGION} - ./keycloak_ec2_installer.sh ${REGION} /tmp/keycloak.zip - ./mvn_ec2_runner.sh ${REGION} "clean install -B -DskipTests -Pdistribution" - ./mvn_ec2_runner.sh ${REGION} "clean install -B -DskipTests -pl testsuite/integration-arquillian/servers/auth-server/quarkus -Pauth-server-quarkus -Pdb-aurora-postgres -Dmaven.build.cache.enabled=true" - ./mvn_ec2_runner.sh ${REGION} "test -B ${{ env.SUREFIRE_RETRY }} -Pauth-server-quarkus -Pdb-${{ matrix.db }} $PROPS -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base 2>&1 | misc/log/trimmer.sh" + ./mvn_ec2_runner.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} "clean install -B ${{ env.SUREFIRE_RETRY }} -Pauth-server-quarkus -Pdb-aurora-postgres -Pauth-server-migration $PROPS -Dtest=MigrationTest -Dmigration.mode=auto -Dmigrated.auth.server.version=${{ env.old-version }} -Dmigration.import.file.name=migration-realm-${{ env.old-version }}.json -Dauth.server.ssl.required=false -f testsuite/integration-arquillian/pom.xml 2>&1 | misc/log/trimmer.sh" + # Copy returned surefire-report directories to workspace root to ensure they're discovered + results=(files/keycloak/results/*) + rsync -a $results/* ../../../ + + rm -rf $results + + - name: Upload JVM Heapdumps + if: always() + uses: ./.github/actions/upload-heapdumps + + - uses: ./.github/actions/upload-flaky-tests + name: Upload flaky tests + env: + GH_TOKEN: ${{ github.token }} + with: + job-name: AuroraDB IT + + - name: Surefire reports + if: always() + uses: ./.github/actions/archive-surefire-reports + with: + job-id: migration-tests-${{ env.old-version }}-aurora-postgres + + - name: EC2 Maven Logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: auroraDB-migration-tests-mvn-logs + path: .github/scripts/ansible/files + + - name: Run Aurora integration tests on EC2 + id: aurora-integration-tests + run: | + EC2_CLUSTER_NAME=${{ steps.ec2-create.outputs.ec2_cluster }} + AWS_REGION=${{ steps.aurora-init.outputs.region }} + PROPS='${{ steps.aurora-init.outputs.maven_properties }}' + PROPS+=" -Dauth.server.db.host=${{ steps.aurora-create.outputs.endpoint }} -Dkeycloak.connectionsJpa.password=${{ steps.aurora-init.outputs.aurora-cluster-password }}" + + TESTS=`testsuite/integration-arquillian/tests/base/testsuites/suite.sh database` + echo "Tests: $TESTS" + + cd .github/scripts/ansible + ./mvn_ec2_runner.sh ${AWS_REGION} ${EC2_CLUSTER_NAME} "test -B ${{ env.SUREFIRE_RETRY }} -Pauth-server-quarkus -Pdb-aurora-postgres $PROPS -Dtest=$TESTS -pl testsuite/integration-arquillian/tests/base 2>&1 | misc/log/trimmer.sh" + # Copy returned surefire-report directories to workspace root to ensure they're discovered results=(files/keycloak/results/*) rsync -a $results/* ../../../ rm -rf $results + - name: Upload JVM Heapdumps + if: always() + uses: ./.github/actions/upload-heapdumps + + - uses: ./.github/actions/upload-flaky-tests + name: Upload flaky tests + env: + GH_TOKEN: ${{ github.token }} + with: + job-name: AuroraDB IT + + - name: Surefire reports + if: always() + uses: ./.github/actions/archive-surefire-reports + with: + job-id: aurora-integration-tests + + - name: EC2 Maven Logs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: aurora-integration-tests-mvn-logs + path: .github/scripts/ansible/files + + - name: Delete EC2 Instance + if: always() + working-directory: .github/scripts/ansible + run: | + ./aws_ec2.sh delete ${{ steps.aurora-init.outputs.region }} ${{ steps.ec2-create.outputs.ec2_cluster }} + + - name: Delete Aurora DB + if: always() + run: | + gh workflow run aurora-delete.yml \ + -f name=${{ steps.aurora-init.outputs.aurora-cluster-name }} \ + -f region=${{ steps.aurora-init.outputs.region }} \ + --repo ${{ github.repository }} \ + --ref ${{ github.ref_name }} + env: + GH_TOKEN: ${{ github.token }} + + store-integration-tests: + name: Store IT + needs: build + runs-on: ubuntu-latest + timeout-minutes: 75 + strategy: + matrix: + db: [postgres, mysql, oracle, mssql, mariadb] + fail-fast: false + steps: + - uses: actions/checkout@v4 + + - id: integration-test-setup + name: Integration test setup + uses: ./.github/actions/integration-test-setup + - name: Run base tests - if: ${{ matrix.db != 'aurora-postgres' }} run: | TESTS=`testsuite/integration-arquillian/tests/base/testsuites/suite.sh database` echo "Tests: $TESTS" @@ -521,31 +622,6 @@ jobs: with: job-id: store-integration-tests-${{ matrix.db }} - - name: EC2 Maven Logs - if: failure() - uses: actions/upload-artifact@v4 - with: - name: store-it-mvn-logs - path: .github/scripts/ansible/files - - - name: Delete Aurora EC2 Instance - if: ${{ always() && matrix.db == 'aurora-postgres' }} - working-directory: .github/scripts/ansible - run: | - export CLUSTER_NAME=${{ steps.aurora-tests.outputs.ec2_cluster }} - ./aws_ec2.sh delete ${{ steps.aurora-init.outputs.region }} - - - name: Delete Aurora DB - if: ${{ always() && matrix.db == 'aurora-postgres' }} - run: | - gh workflow run aurora-delete.yml \ - -f name=${{ steps.aurora-init.outputs.name }} \ - -f region=${{ steps.aurora-init.outputs.region }} \ - --repo ${{ github.repository }} \ - --ref ${{ github.ref_name }} - env: - GH_TOKEN: ${{ github.token }} - store-model-tests: name: Store Model Tests runs-on: ubuntu-latest diff --git a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/KeycloakQuarkusServerDeployableContainer.java b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/KeycloakQuarkusServerDeployableContainer.java index 244af6061dd..b44dad127a3 100644 --- a/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/KeycloakQuarkusServerDeployableContainer.java +++ b/testsuite/integration-arquillian/tests/base/src/main/java/org/keycloak/testsuite/arquillian/containers/KeycloakQuarkusServerDeployableContainer.java @@ -83,8 +83,8 @@ public class KeycloakQuarkusServerDeployableContainer extends AbstractQuarkusDep ProcessBuilder pb = new ProcessBuilder(commands); Process p = pb.directory(wrkDir).inheritIO().start(); try { - if (!p.waitFor(60, TimeUnit.SECONDS)) { - throw new IOException("Command " + command + " did not finished in 60 seconds"); + if (!p.waitFor(300, TimeUnit.SECONDS)) { + throw new IOException("Command " + command + " did not finished in 300 seconds"); } if (p.exitValue() != 0) { throw new IOException("Command " + command + " was executed with exit status " + p.exitValue());