mattermost/.github/workflows/e2e-tests-ci-template.yml

---
name: E2E Tests Template
on:
  workflow_call:
    inputs:
      # NB: this does not support using branch names that belong to forks.
      #     In those cases, you should specify directly the commit SHA that you want to test, or
      #     some wrapper workflow that does it for you (e.g. the slash command for initiating a PR test)
      commit_sha:
        type: string
        required: true
      status_check_context:
        type: string
        required: true
      workers_number:
        type: string # Should ideally be a number; see https://github.com/orgs/community/discussions/67182
        required: false
        default: "1"
      testcase_failure_fatal:
        type: boolean
        required: false
        default: true
      enable_reporting:
        type: boolean
        required: false
        default: false
      SERVER:
        type: string # Valid values are: onprem, cloud
        required: false
        default: onprem
      SERVER_IMAGE:
        type: string
        required: false
      ENABLED_DOCKER_SERVICES:
        type: string
        required: false
      TEST: # Valid values are: cypress, playwright
        type: string
        required: false
        default: "cypress"
      TEST_FILTER:
        type: string
        required: false
      MM_ENV:
        type: string
        required: false
      BRANCH:
        type: string
        required: false
      BUILD_ID:
        type: string
        required: false
      REPORT_TYPE:
        type: string
        required: false
      ROLLING_RELEASE_commit_sha:
        type: string
        required: false
      ROLLING_RELEASE_SERVER_IMAGE:
        type: string
        required: false
      PR_NUMBER:
        type: string
        required: false
    secrets:
      MM_LICENSE:
        required: false
      AUTOMATION_DASHBOARD_URL:
        required: false
      AUTOMATION_DASHBOARD_TOKEN:
        required: false
      PUSH_NOTIFICATION_SERVER:
        required: false
      REPORT_WEBHOOK_URL:
        required: false
      REPORT_TM4J_API_KEY:
        required: false
      REPORT_TM4J_TEST_CYCLE_LINK_PREFIX:
        required: false
      CWS_URL:
        required: false
      CWS_EXTRA_HTTP_HEADERS:
        required: false
      AWS_ACCESS_KEY_ID:
        required: false
      AWS_SECRET_ACCESS_KEY:
        required: false
    outputs:
      passed:
        value: "${{ jobs.report.outputs.passed }}"
      failed:
        value: "${{ jobs.report.outputs.failed }}"
      failed_expected:
        value: "${{ jobs.report.outputs.failed_expected }}"
      pass_rate:
        value: "${{ jobs.report.outputs.pass_rate }}"
      playwright_report_url:
        value: ${{ jobs.report.outputs.playwright_report_url }}

jobs:
  update-initial-status:
    runs-on: ubuntu-24.04
    steps:
      - uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63
        env:
          GITHUB_TOKEN: ${{ github.token }}
        with:
          repository_full_name: ${{ github.repository }}
          commit_sha: ${{ inputs.commit_sha }}
          context: ${{ inputs.status_check_context }}
          description: E2E tests for mattermost server app
          status: pending

  generate-build-variables:
    runs-on: ubuntu-24.04
    needs:
      - update-initial-status
    defaults:
      run:
        shell: bash
    outputs:
      workers: "${{ steps.generate.outputs.workers }}"
      node-cache-dependency-path: "${{ steps.generate.outputs.node-cache-dependency-path }}"
    steps:
      - name: ci/checkout-repo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.commit_sha }}
          fetch-depth: 0
      - name: ci/generate-build-variables
        id: generate
        env:
          WORKERS: ${{ inputs.workers_number }}
          TEST: ${{ inputs.TEST }}
        run: |
          [ "$WORKERS" -gt "0" ] # Assert that the workers number is an integer greater than 0
          echo "workers="$(jq --slurp --compact-output '[range('"$WORKERS"')] | map(tostring)' /dev/null) >> $GITHUB_OUTPUT
          echo "node-cache-dependency-path=e2e-tests/${TEST}/package-lock.json" >> $GITHUB_OUTPUT          

  generate-test-cycle:
    runs-on: ubuntu-24.04
    needs:
      - generate-build-variables
    defaults:
      run:
        shell: bash
        working-directory: e2e-tests
    outputs:
      status_check_url: "${{ steps.e2e-test-gencycle.outputs.status_check_url }}"
    steps:
      - name: ci/checkout-repo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.commit_sha }}
          fetch-depth: 0
      - name: ci/setup-node
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        id: setup_node
        with:
          node-version-file: ".nvmrc"
          cache: npm
          cache-dependency-path: "e2e-tests/cypress/package-lock.json" # NB: the generate-cycle script is cypress-specific operation for now
      - name: ci/e2e-test-gencycle
        id: e2e-test-gencycle
        env:
          AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
          AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}"
          BRANCH: "${{ inputs.BRANCH }}"
          BUILD_ID: "${{ inputs.BUILD_ID }}"
          TEST: "${{ inputs.TEST }}"
          TEST_FILTER: "${{ inputs.TEST_FILTER }}"
        run: |
          set -e -o pipefail
          make generate-test-cycle | tee generate-test-cycle.out
          # Extract cycle's dashboard URL, if present
          TEST_CYCLE_ID=$(sed -nE "s/^.*id: '([^']+)'.*$/\1/p"  <generate-test-cycle.out)
          if [ -n "$TEST_CYCLE_ID" ]; then
            echo "status_check_url=https://automation-dashboard.vercel.app/cycles/${TEST_CYCLE_ID}" >> $GITHUB_OUTPUT
          else
            echo "status_check_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_OUTPUT
          fi          

  test:
    continue-on-error: true # Individual runner failures shouldn't prevent the completion of an E2E run
    strategy:
      fail-fast: false # Individual runner failures shouldn't prevent the completion of an E2E run
      matrix:
        #
        # Note that E2E tests should be run only on ubuntu, for QA purposes.
        # But it's useful to be able to run and debug the E2E tests for different OSes.
        # Notes:
        # - For MacOS: works on developer machines, but uses too many resources to be able to run on Github Actions
        # - for Windows: cannot currently run on Github Actions, since the runners do not support running linux containers, at the moment
        #
        #os: [ubuntu-24.04, windows-2022, macos-12-xl]
        os: [ubuntu-24.04]
        worker_index: ${{ fromJSON(needs.generate-build-variables.outputs.workers) }} # https://docs.github.com/en/actions/learn-github-actions/expressions#example-returning-a-json-object
    runs-on: "${{ matrix.os }}"
    timeout-minutes: 120
    needs:
      - generate-build-variables
      - generate-test-cycle
    defaults:
      run:
        shell: bash
        working-directory: e2e-tests
    env:
      AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
      AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}"
      SERVER: "${{ inputs.SERVER }}"
      SERVER_IMAGE: "${{ inputs.SERVER_IMAGE }}"
      MM_LICENSE: "${{ secrets.MM_LICENSE }}"
      ENABLED_DOCKER_SERVICES: "${{ inputs.ENABLED_DOCKER_SERVICES }}"
      TEST: "${{ inputs.TEST }}"
      TEST_FILTER: "${{ inputs.TEST_FILTER }}"
      MM_ENV: "${{ inputs.MM_ENV }}"
      BRANCH: "${{ inputs.BRANCH }}"
      BUILD_ID: "${{ inputs.BUILD_ID }}"
      CI_BASE_URL: "${{ matrix.os }}-${{ matrix.worker_index }}"
      CYPRESS_pushNotificationServer: "${{ secrets.PUSH_NOTIFICATION_SERVER }}"
      CWS_URL: "${{ secrets.CWS_URL }}"
      CWS_EXTRA_HTTP_HEADERS: "${{ secrets.CWS_EXTRA_HTTP_HEADERS }}"
      ROLLING_RELEASE_COMMIT_SHA: "${{ inputs.ROLLING_RELEASE_commit_sha }}"
      ROLLING_RELEASE_SERVER_IMAGE: "${{ inputs.ROLLING_RELEASE_SERVER_IMAGE }}"
    steps:
      - name: ci/checkout-actions
        # Sparse-checkout just .github/actions from the triggering ref (master)
        # so the composite action below is available before the full checkout
        # overwrites the workspace with inputs.commit_sha.
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          sparse-checkout: .github/actions
          sparse-checkout-cone-mode: true
      - name: ci/runner-prep-for-openldap
        uses: ./.github/actions/runner-prep-openldap
      - name: ci/checkout-repo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.commit_sha }}
          fetch-depth: 0
      - name: ci/setup-macos-docker
        if: runner.os == 'macos'
        # https://github.com/actions/runner-images/issues/17#issuecomment-1537238473
        run: |
          brew install docker docker-compose
          colima start
          mkdir -p ~/.docker/cli-plugins
          ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose
          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock          
      - name: ci/setup-node
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        id: setup_node
        with:
          node-version-file: ".nvmrc"
          cache: npm
          cache-dependency-path: ${{ needs.generate-build-variables.outputs.node-cache-dependency-path }}
      - name: ci/e2e-test
        run: |
          make cloud-init
          if [ -n "$ROLLING_RELEASE_SERVER_IMAGE" ]; then
            echo "RollingRelease: checking out E2E test cases from revision ${ROLLING_RELEASE_COMMIT_SHA}, for initial smoketest"
            git checkout "${ROLLING_RELEASE_COMMIT_SHA}" -- "${TEST}/" && git status
            (
            echo "RollingRelease: running initial smoketest against image $ROLLING_RELEASE_SERVER_IMAGE"
            export SERVER_IMAGE="$ROLLING_RELEASE_SERVER_IMAGE"
            export TEST_FILTER=""
            export AUTOMATION_DASHBOARD_URL=""
            make
            )
            echo "RollingRelease: asserting smoketest result has zero failures."
            FAILURES=$(jq -r '.failed' "${TEST}/results/summary.json")
            if [ "$FAILURES" -ne "0" ]; then
              echo "RollingRelease: initial smoketest for rolling release E2E run has nonzero ($FAILURES) failures. Aborting test run." >&2
              exit 1
            fi
            rm -rfv "${TEST}/{results,logs}"
            echo "RollingRelease: reset the E2E test cases to the revision to test"
            git reset --hard HEAD && git status
            echo "RollingRelease: smoketest completed. Starting full E2E tests."
          fi
          make          
      - name: ci/cloud-teardown
        if: always()
        run: make cloud-teardown
      - name: ci/dump-docker-state-on-failure
        # Always run a final docker-state capture so failures unrelated to
        # openldap startup (e.g. server container later crashes) still produce
        # logs we can inspect. The script's own retry loop dumps openldap
        # state per-attempt; this step is a backstop covering the whole job.
        if: failure()
        run: |
          set +e
          DIAG="e2e-tests/docker-diagnostics/job-failure"
          mkdir -p "$DIAG"
          docker ps -a >"$DIAG/docker.ps.txt" 2>&1
          docker version >"$DIAG/docker.version.txt" 2>&1
          docker info >"$DIAG/docker.info.txt" 2>&1
          for c in $(docker ps -a --format '{{.Names}}'); do
            docker inspect "$c" >"$DIAG/$c.inspect.json" 2>&1
            docker logs "$c" >"$DIAG/$c.log" 2>&1
          done
          uname -a >"$DIAG/host.uname.txt" 2>&1
          free -m >"$DIAG/host.free.txt" 2>&1
          df -h >"$DIAG/host.df.txt" 2>&1
          sudo dmesg | tail -500 >"$DIAG/host.dmesg.tail.txt" 2>&1
          sudo dmesg | grep -iE 'apparmor|denied|oom|killed|openldap|slapd' >"$DIAG/host.dmesg.relevant.txt" 2>&1          
      - name: ci/upload-docker-diagnostics
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        if: always()
        with:
          name: docker-diagnostics-${{ inputs.TEST }}-${{ matrix.os }}-${{ matrix.worker_index }}
          path: e2e-tests/docker-diagnostics/
          retention-days: 7
          if-no-files-found: ignore
      - name: ci/e2e-test-store-results
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        if: always()
        with:
          name: e2e-test-results-${{ inputs.TEST }}-${{ matrix.os }}-${{ matrix.worker_index }}
          path: |
            e2e-tests/${{ inputs.TEST }}/logs/
            e2e-tests/${{ inputs.TEST }}/results/            
          retention-days: 1

  report:
    runs-on: ubuntu-24.04
    needs:
      - test
      - generate-build-variables
    defaults:
      run:
        shell: bash
        working-directory: e2e-tests
    outputs:
      passed: "${{ steps.calculate-results.outputs.passed }}"
      failed: "${{ steps.calculate-results.outputs.failed }}"
      failed_expected: "${{ steps.calculate-results.outputs.failed_expected }}"
      pass_rate: "${{ steps.calculate-results.outputs.pass_rate }}"
      commit_status_message: "${{ steps.calculate-results.outputs.commit_status_message }}"
      playwright_report_url: "${{ steps.upload-to-s3.outputs.report_url }}"
    steps:
      - name: ci/checkout-repo
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          ref: ${{ inputs.commit_sha }}
          fetch-depth: 0
      - name: ci/download-artifacts
        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
        with:
          pattern: e2e-test-results-${{ inputs.TEST }}-*
          path: e2e-tests/${{ inputs.TEST }}/
          merge-multiple: true
      - name: ci/upload-report-global
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: e2e-test-results-${{ inputs.TEST }}
          path: |
            e2e-tests/${{ inputs.TEST }}/logs/
            e2e-tests/${{ inputs.TEST }}/results/            
      - name: ci/setup-node
        if: "${{ inputs.enable_reporting }}"
        uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
        id: setup_node
        with:
          node-version-file: ".nvmrc"
          cache: npm
          cache-dependency-path: ${{ needs.generate-build-variables.outputs.node-cache-dependency-path }}
      - name: ci/publish-report
        if: "${{ inputs.enable_reporting }}"
        env:
          TYPE: "${{ inputs.REPORT_TYPE }}"
          TEST: "${{ inputs.TEST }}"
          SERVER: "${{ inputs.SERVER }}"
          SERVER_IMAGE: "${{ inputs.SERVER_IMAGE }}"
          AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
          WEBHOOK_URL: "${{ secrets.REPORT_WEBHOOK_URL }}"
          PR_NUMBER: "${{ inputs.PR_NUMBER }}"
          BRANCH: "${{ inputs.BRANCH }}"
          BUILD_ID: "${{ inputs.BUILD_ID }}"
          MM_ENV: "${{ inputs.MM_ENV }}"
          TM4J_API_KEY: "${{ secrets.REPORT_TM4J_API_KEY }}"
          TEST_CYCLE_LINK_PREFIX: "${{ secrets.REPORT_TM4J_TEST_CYCLE_LINK_PREFIX }}"
        run: |
          echo "DEBUG: TYPE=${TYPE}, PR_NUMBER=${PR_NUMBER:-<not set>}"
          make report          
      # The results dir may have been modified as part of the reporting: re-upload
      - name: ci/upload-report-global
        if: "${{ inputs.enable_reporting }}"
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: e2e-test-results-${{ inputs.TEST }}
          path: |
            e2e-tests/${{ inputs.TEST }}/logs/
            e2e-tests/${{ inputs.TEST }}/results/            
          overwrite: true

      # Configure AWS credentials
      - name: ci/aws-configure
        if: (inputs.TEST == 'playwright')
        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
        with:
          aws-region: us-east-1
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

      # Upload the playwright reports to S3
      - name: ci/upload-results-to-s3
        if: (inputs.TEST == 'playwright')
        id: upload-to-s3
        run: |
          echo "🔍 Checking if results directory exists..."

          PR_NUMBER="${{ inputs.PR_NUMBER }}"
          LOCAL_RESULTS_PATH="${{ inputs.TEST }}/results/"
          LOCAL_LOGS_PATH="${{ inputs.TEST }}/logs/"
          RUN_ID="${{ github.run_id }}"
          S3_PATH="server-pr-${PR_NUMBER}/e2e-reports/${{ inputs.TEST }}/${RUN_ID}"

          echo "📤 Uploading to s3://${AWS_S3_BUCKET}/${S3_PATH}/"

          if [[ -d "$LOCAL_RESULTS_PATH" ]]; then
            aws s3 sync "$LOCAL_RESULTS_PATH" "s3://${AWS_S3_BUCKET}/${S3_PATH}/results/" \
              --acl public-read \
              --cache-control "no-cache"
          fi

          REPORT_URL="https://${AWS_S3_BUCKET}.s3.amazonaws.com/${S3_PATH}/results/reporter/index.html"
          echo "✅ Report uploaded to: $REPORT_URL"

          echo "report_url=$REPORT_URL" >> "$GITHUB_OUTPUT"          
        env:
          AWS_REGION: us-east-1
          AWS_S3_BUCKET: mattermost-cypress-report

      - name: ci/report-calculate-results
        id: calculate-results
        env:
          TEST: "${{ inputs.TEST }}"
        run: |
          AD_CYCLE_FILE="${TEST}/results/ad_cycle.json"
          if [ -f "$AD_CYCLE_FILE" ]; then
            # Prefer using the Automation Dashboard's results to calculate failures
            export PASSED=$(jq -r .pass "$AD_CYCLE_FILE")
            export FAILED=$(jq -r .fail "$AD_CYCLE_FILE")
            export FAILED_EXPECTED=$(jq -r ".known + .flaky + .skipped" "$AD_CYCLE_FILE")
          else
            # Otherwise, utilize summary.json to calculate the failures
            # NB: in this job, this file only makes sense if a single worker is used, as with Playwright
            export PASSED=$(jq '.passed' "${TEST}/results/summary.json")
            export FAILED=$(jq '.failed' "${TEST}/results/summary.json")
            export FAILED_EXPECTED=$(jq '.failed_expected' "${TEST}/results/summary.json")
          fi
          export TOTAL_SPECS=$(( PASSED + FAILED ))
          export PASS_RATE=$(jq -r '100 * (env.PASSED | tonumber) / (env.TOTAL_SPECS | tonumber)' <<<'{}' | xargs -l printf '%.2f')
          if [ "$FAILED" = "0" ]; then
            export COMMIT_STATUS_MESSAGE="All test cases passed"
          else
            export COMMIT_STATUS_MESSAGE="${FAILED} test cases failed. Please check the workflow logs"
          fi
          echo "passed=${PASSED:?}" >> $GITHUB_OUTPUT
          echo "failed=${FAILED:?}" >> $GITHUB_OUTPUT
          echo "failed_expected=${FAILED_EXPECTED:?}" >> $GITHUB_OUTPUT
          echo "pass_rate=${PASS_RATE:?}%" >> $GITHUB_OUTPUT
          echo "commit_status_message=${COMMIT_STATUS_MESSAGE:?}" >> $GITHUB_OUTPUT
          echo "$COMMIT_STATUS_MESSAGE"          
      - name: ci/e2e-test-assert-results
        if: "${{ inputs.testcase_failure_fatal }}"
        run: |
          # Assert that the run contained 0 failures
          [ "${{ steps.calculate-results.outputs.failed }}" = "0" ]          

  update-failure-final-status:
    runs-on: ubuntu-24.04
    if: failure() || cancelled()
    needs:
      - generate-test-cycle
      - test
      - report
    steps:
      - uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63
        env:
          GITHUB_TOKEN: ${{ github.token }}
        with:
          repository_full_name: ${{ github.repository }}
          commit_sha: ${{ inputs.commit_sha }}
          context: ${{ inputs.status_check_context }}
          description: ${{ needs.report.outputs.commit_status_message || 'Error during test execution' }}
          status: failure
          target_url: >-
            ${{ inputs.TEST == 'playwright'
              && needs.report.outputs.playwright_report_url
              || needs.generate-test-cycle.outputs.status_check_url }}            


  update-success-final-status:
    runs-on: ubuntu-24.04
    if: success()
    needs:
      - generate-test-cycle
      - test
      - report
    steps:
      - uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63
        env:
          GITHUB_TOKEN: ${{ github.token }}
        with:
          repository_full_name: ${{ github.repository }}
          commit_sha: ${{ inputs.commit_sha }}
          context: ${{ inputs.status_check_context }}
          description: ${{ needs.report.outputs.commit_status_message || 'Error during test execution' }}
          status: success
          target_url: >-
            ${{ inputs.TEST == 'playwright'
              && needs.report.outputs.playwright_report_url
              || needs.generate-test-cycle.outputs.status_check_url }}