VAULT-45737 remove all failures, add relevant test results (#15182) (#15897)

* remove all failures, add relevant test results

* adding code change for test.

* adding back in deleted code

* changing to testonly

* adding links to tests

* update to tree, not ref

* removing test

* adding script for first test manifest

Co-authored-by: JMGoldsmith <spartanaudio@gmail.com>

.github/scripts/build-relevant-tests-manifest.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+# Copyright IBM Corp. 2016, 2025
+# SPDX-License-Identifier: BUSL-1.1
+
+# Builds a manifest of relevant Go test targets based on the set of changed files
+# in a pull request. For each changed Go file, it locates the corresponding
+# *_test.go file, extracts the top-level Test functions and their line numbers,
+# and emits a JSON array describing the directory, test file, and tests.
+#
+# Inputs (environment variables):
+#   CHANGED_FILES_JSON - JSON payload describing changed files. Supported shapes:
+#                        {"files": [...]}, {"changed_files": [...]},
+#                        {"changedFiles": [...]}; entries may be strings or
+#                        objects with a "path", "filename", or "file" key.
+#   SUITE_NAME         - Identifier used to name the output manifest file.
+#
+# Outputs:
+#   Writes the manifest to "relevant-tests-${SUITE_NAME}.json".
+#   If GITHUB_OUTPUT is set, appends "manifest=<file>" to it.
+
+set -euo pipefail
+
+suite_name="${SUITE_NAME:-}"
+out_file="relevant-tests-${suite_name}.json"
+changed_files_json="${CHANGED_FILES_JSON:-}"
+if [ -z "$changed_files_json" ]; then
+  changed_files_json='{}'
+fi
+
+emit_manifest_output() {
+  if [ -n "${GITHUB_OUTPUT:-}" ]; then
+    echo "manifest=$out_file" >> "$GITHUB_OUTPUT"
+  fi
+}
+
+# Extract changed paths from known shapes of changed-files payload.
+changed_paths=$(jq -r '
+  (.files // .changed_files // .changedFiles // [])
+  | map(if type == "string" then . else (.path // .filename // .file // empty) end)
+  | .[]?
+' <<< "${changed_files_json}" | awk 'NF' | sort -u)
+
+if [ -z "${changed_paths}" ]; then
+  echo '[]' > "$out_file"
+  emit_manifest_output
+  exit 0
+fi
+
+map_file="$(mktemp)"
+while IFS= read -r path; do
+  case "$path" in
+    *.go)
+      dir="$(dirname "$path")"
+      base="$(basename "$path")"
+      if [[ "$base" == *_test.go ]]; then
+        test_file="$path"
+      else
+        test_file="${path%.go}_test.go"
+      fi
+      printf '%s\t%s\n' "$dir" "$test_file" >> "$map_file"
+      ;;
+  esac
+done <<< "$changed_paths"
+
+if [ ! -s "$map_file" ]; then
+  echo '[]' > "$out_file"
+  emit_manifest_output
+  exit 0
+fi
+
+jq -n '[]' > "$out_file"
+while IFS=$'\t' read -r dir test_file; do
+  if [ ! -f "$test_file" ]; then
+    continue
+  fi
+
+  # Capture top-level Go test functions and their line numbers in the target test file.
+  tests=$(awk '
+    /^func[[:space:]]+Test[[:alnum:]_]+\(t[[:space:]]+\*testing\.T\)/ {
+      name=$2;
+      sub(/\(.*/, "", name);
+      print NR ":" name;
+    }
+  ' "$test_file" | jq -Rsc '
+    split("\n")
+    | map(select(length > 0))
+    | map(split(":") | {"line": (.[0] | tonumber), "name": .[1]})
+  ')
+
+  if [ "$tests" = "[]" ]; then
+    continue
+  fi
+
+  tmp_file="$(mktemp)"
+  jq \
+    --arg dir "$dir" \
+    --arg test_file "$test_file" \
+    --argjson tests "$tests" \
+    '. + [{"dir": $dir, "test_file": $test_file, "tests": $tests}]' \
+    "$out_file" > "$tmp_file"
+  mv "$tmp_file" "$out_file"
+done < <(sort -u "$map_file")
+
+rm -f "$map_file"
+
+emit_manifest_output

.github/workflows/ci.yml

@@ -183,6 +183,7 @@ jobs:
       # The regular Go tests use an extra runner to execute the binary-dependent tests. We isolate
       # them there so that the other tests aren't slowed down waiting for a binary build.
       binary-tests: true
+      changed-files: ${{ needs.setup.outputs.changed-files }}
       checkout-ref: ${{ needs.setup.outputs.checkout-ref }}
       go-arch: amd64
       go-tags: '${{ needs.setup.outputs.go-tags }},deadlock'
@@ -208,6 +209,7 @@ jobs:
       issues: read
       pull-requests: write
     with:
+      changed-files: ${{ needs.setup.outputs.changed-files }}
       checkout-ref: ${{ needs.setup.outputs.checkout-ref }}
       go-arch: amd64
       go-tags: '${{ needs.setup.outputs.go-tags }},deadlock,testonly'
@@ -239,6 +241,7 @@ jobs:
       issues: read
       pull-requests: write
     with:
+      changed-files: ${{ needs.setup.outputs.changed-files }}
       checkout-ref: ${{ needs.setup.outputs.checkout-ref }}
       env-vars: |
         {
@@ -273,6 +276,7 @@ jobs:
       issues: read
       pull-requests: write
     with:
+      changed-files: ${{ needs.setup.outputs.changed-files }}
       checkout-ref: ${{ needs.setup.outputs.checkout-ref }}
       env-vars: |
         {

.github/workflows/test-go.yml

@@ -84,6 +84,11 @@ on:
         required: false
         default: ${{ github.ref }}
         type: string
+      changed-files:
+        description: JSON payload containing changed file paths from setup workflow.
+        required: false
+        default: '{}'
+        type: string
     outputs:
       data-race-output:
         description: A textual output of any data race detector failures
@@ -755,6 +760,224 @@ jobs:
           ' shell {} \; > /dev/null 2>&1
 
           ls -lhR '${{ needs.test-matrix.outputs.go-test-dir }}'
+      - if: always() && github.event_name == 'pull_request' && inputs.name == 'testonly'
+        name: Check out source for relevant test discovery
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ inputs.checkout-ref }}
+      # Download test result artifacts from all matrix shards so relevant test results can be extracted.
+      - if: always() && github.event_name == 'pull_request' && inputs.name == 'testonly'
+        name: Download Go test result artifacts
+        continue-on-error: true
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          pattern: ${{ needs.test-go.outputs.go-test-results-download-pattern }}
+          path: pr-go-test-results
+          merge-multiple: true
+      - if: always() && github.event_name == 'pull_request' && inputs.name == 'testonly'
+        name: Build relevant test target manifest
+        id: relevant-tests
+        continue-on-error: true
+        env:
+          CHANGED_FILES_JSON: ${{ inputs.changed-files }}
+          SUITE_NAME: ${{ inputs.name }}
+        run: ./.github/scripts/build-relevant-tests-manifest.sh
+      # Post or update a PR comment with the aggregated test results
+      - if: always() && github.event_name == 'pull_request' && inputs.name == 'testonly'
+        name: Post test results to PR
+        continue-on-error: true
+        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
+        env:
+          RELEVANT_TESTS_MANIFEST: ${{ steps.relevant-tests.outputs.manifest }}
+          CHECKOUT_REF: ${{ inputs.checkout-ref }}
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+
+            const suiteName = '${{ inputs.name }}';
+            const overallResult = '${{ steps.status.outputs.result }}';
+            const dataRaceResult = '${{ steps.status.outputs.data-race-result }}';
+
+            function fileExists(filePath) {
+              try {
+                return fs.existsSync(filePath);
+              } catch {
+                return false;
+              }
+            }
+
+            let relevantManifest = [];
+            const manifestPath = process.env.RELEVANT_TESTS_MANIFEST;
+            if (manifestPath && fileExists(manifestPath)) {
+              try {
+                relevantManifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
+              } catch {
+                relevantManifest = [];
+              }
+            }
+
+            const relevantTestFileNames = new Set(relevantManifest.map(m => m.test_file));
+
+            // Build a map of test name -> {test_file, line} for link generation
+            const testLinkMap = new Map();
+            for (const entry of relevantManifest) {
+              for (const t of entry.tests || []) {
+                testLinkMap.set(`${entry.dir}::${t.name}`, { file: entry.test_file, line: t.line });
+              }
+            }
+
+            const treeId = context.sha;
+            const treeBase = `https://github.com/hashicorp/vault-enterprise/tree/${treeId}`;
+
+            const eventFiles = fileExists('pr-go-test-results')
+              ? fs.readdirSync('pr-go-test-results').filter(f => f.endsWith('.json')).sort()
+              : [];
+
+            // Keep final test status per package/test.
+            const statuses = new Map();
+            for (const file of eventFiles) {
+              const lines = fs.readFileSync(path.join('pr-go-test-results', file), 'utf8').split(/\r?\n/);
+              for (const line of lines) {
+                if (!line.trim()) continue;
+                let ev;
+                try {
+                  ev = JSON.parse(line);
+                } catch {
+                  continue;
+                }
+                if (!ev || !ev.Package || !ev.Test) continue;
+                if (!['pass', 'fail', 'skip'].includes(ev.Action)) continue;
+
+                const pkg = String(ev.Package);
+                const test = String(ev.Test);
+                const key = `${pkg}::${test}`;
+                const statusRank = ev.Action === 'fail' ? 3 : ev.Action === 'pass' ? 2 : 1;
+
+                const prev = statuses.get(key);
+                if (!prev || statusRank >= prev.rank) {
+                  statuses.set(key, {
+                    package: pkg,
+                    test,
+                    status: ev.Action,
+                    elapsed: typeof ev.Elapsed === 'number' ? ev.Elapsed : null,
+                    rank: statusRank,
+                  });
+                }
+              }
+            }
+
+            const relevantResults = [];
+            for (const value of statuses.values()) {
+              const pkg = value.package;
+              const test = value.test;
+              const isRelevant = relevantManifest.some(entry => {
+                if (!(entry.tests || []).some(t => t.name === test)) {
+                  return false;
+                }
+
+                if (entry.dir === '.' || entry.dir === '') {
+                  return true;
+                }
+
+                return pkg.endsWith(`/${entry.dir}`);
+              });
+
+              if (isRelevant) {
+                // Find the matching manifest entry to attach link info
+                const matchEntry = relevantManifest.find(entry =>
+                  (entry.tests || []).some(t => t.name === test) &&
+                  (entry.dir === '.' || entry.dir === '' || pkg.endsWith(`/${entry.dir}`))
+                );
+                const matchTest = matchEntry
+                  ? (matchEntry.tests || []).find(t => t.name === test)
+                  : null;
+                value.testFile = matchEntry ? matchEntry.test_file : null;
+                value.testLine = matchTest ? matchTest.line : null;
+                relevantResults.push(value);
+              }
+            }
+
+            relevantResults.sort((a, b) => {
+              if (a.status !== b.status) {
+                const order = { fail: 0, skip: 1, pass: 2 };
+                return order[a.status] - order[b.status];
+              }
+              if (a.package !== b.package) return a.package.localeCompare(b.package);
+              return a.test.localeCompare(b.test);
+            });
+
+            // Collect data race details from downloaded log files
+            let dataRaceDetails = '';
+            const raceDir = 'data-race-logs';
+            if (fs.existsSync(raceDir)) {
+              const files = fs.readdirSync(raceDir).filter(f => f.endsWith('.log'));
+              for (const file of files.sort()) {
+                dataRaceDetails += fs.readFileSync(path.join(raceDir, file), 'utf8');
+              }
+            }
+
+            const passed = overallResult === 'success' && dataRaceResult === 'success';
+            const statusIcon = passed ? ':white_check_mark:' : ':x:';
+            const statusText = passed ? 'All tests passed' : 'Some tests failed';
+
+            let body = `## Go Test Results — \`${suiteName}\`\n\n${statusIcon} **${statusText}**\n`;
+
+            if (relevantTestFileNames.size > 0) {
+              body += `\n### Relevant Test Files\n\n`;
+              for (const file of [...relevantTestFileNames].sort()) {
+                body += `- [\`${file}\`](${treeBase}/${file})\n`;
+              }
+            }
+
+            if (relevantResults.length > 0) {
+              body += `\n### Relevant Test Results\n\n`;
+              body += `| Status | Package | Test | Elapsed (s) |\n`;
+              body += `|--------|---------|------|-------------|\n`;
+              for (const row of relevantResults) {
+                const icon = row.status === 'pass' ? ':white_check_mark:' : row.status === 'fail' ? ':x:' : ':fast_forward:';
+                const elapsed = row.elapsed == null ? '-' : row.elapsed.toFixed(3);
+                const testLabel = (row.testFile && row.testLine)
+                  ? `[${row.test}](${treeBase}/${row.testFile}#L${row.testLine})`
+                  : row.test;
+                body += `| ${icon} ${row.status} | ${row.package} | ${testLabel} | ${elapsed} |\n`;
+              }
+            } else if (relevantTestFileNames.size > 0) {
+              body += `\nNo direct matching test events were found for the relevant test files in this run.\n`;
+            } else {
+              body += `\nNo directly relevant \`*_test.go\` files were inferred from changed Go files.\n`;
+            }
+
+            if (dataRaceDetails) {
+              body += `\n### Data Race Failures\n\n<details><summary>Show details</summary>\n\n\`\`\`\n${dataRaceDetails}\n\`\`\`\n</details>\n`;
+            }
+
+            // Use a hidden marker so we can update an existing comment rather than posting duplicates
+            const marker = `<!-- go-test-results-${suiteName} -->`;
+            body = `${marker}\n${body}`;
+
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+
+            const existing = comments.find(c => c.body && c.body.includes(marker));
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body,
+              });
+            }
       # Determine our overall pass/fail with our Go test results
       - if: always() && steps.status.outputs.result != 'success'
         name: Check for failed status