diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 7331295..e5ba829 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -31,6 +31,21 @@ on: required: false default: false type: boolean + horizonqa: + description: 'Run Horizon-QA in CI mode during runServer' + required: false + default: false + type: boolean + horizonqa-tests: + description: 'Optional -Dhorizonqa.tests selector' + required: false + default: '' + type: string + horizonqa-allow-no-tests: + description: 'Allow Horizon-QA CI to pass with no discovered tests' + required: false + default: false + type: boolean concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -52,7 +67,8 @@ jobs: - name: Checkout workflows repo uses: actions/checkout@v5 with: - repository: GTNewHorizons/GTNH-Actions-Workflows + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} path: .gtnh-workflows fetch-depth: 0 @@ -139,6 +155,12 @@ jobs: - name: Run server for up to ${{ inputs.timeout }} seconds if: ${{ !inputs.client-only }} + shell: bash + env: + HORIZONQA: ${{ inputs.horizonqa }} + HORIZONQA_TESTS: ${{ inputs.horizonqa-tests }} + HORIZONQA_ALLOW_NO_TESTS: ${{ inputs.horizonqa-allow-no-tests }} + DISABLE_SERVER_AUTO_STOP: ${{ inputs.disable-server-auto-stop }} run: | mkdir -p run/server echo "eula=true" > run/eula.txt @@ -147,12 +169,57 @@ jobs: echo "level-seed=-6202107849386030209\nonline-mode=true\n" > run/server.properties echo "level-seed=-6202107849386030209\nonline-mode=true\n" > run/server/server.properties + gradle_args=(--build-cache --info --stacktrace runServer) + + if [[ "${HORIZONQA}" == "true" ]]; then + horizonqa_jvm_args=( + "-Dhorizonqa.mode=ci" + "-Dhorizonqa.reportDir=${GITHUB_WORKSPACE}/build/horizonqa" + ) + + if [[ -n "${HORIZONQA_TESTS}" ]]; then + horizonqa_jvm_args+=("-Dhorizonqa.tests=${HORIZONQA_TESTS}") + fi + + if [[ "${HORIZONQA_ALLOW_NO_TESTS}" == "true" ]]; then + horizonqa_jvm_args+=("-Dhorizonqa.allowNoTests=true") + fi + + for jvm_arg in "${horizonqa_jvm_args[@]}"; do + gradle_args+=(--mcJvmArgs="${jvm_arg}") + done + fi + server_stdin=/dev/null - if [[ "${{ inputs.disable-server-auto-stop }}" != "true" ]]; then + if [[ "${DISABLE_SERVER_AUTO_STOP}" != "true" && "${HORIZONQA}" != "true" ]]; then echo "stop" > run/stop.txt server_stdin=run/stop.txt fi - timeout ${{ inputs.timeout }} ./gradlew --build-cache --info --stacktrace runServer 2>&1 < "${server_stdin}" | tee -a server.log || true + timeout ${{ inputs.timeout }} ./gradlew "${gradle_args[@]}" 2>&1 < "${server_stdin}" | tee -a server.log || true + + - name: Upload Horizon-QA reports + if: ${{ always() && !inputs.client-only && inputs.horizonqa }} + uses: actions/upload-artifact@v7 + continue-on-error: true + with: + name: ${{ github.repository_id }}-horizonqa + path: build/horizonqa/ + retention-days: 90 + + - name: Summarize Horizon-QA result + if: ${{ always() && !inputs.client-only && inputs.horizonqa }} + shell: bash + continue-on-error: true + run: | + chmod +x .gtnh-workflows/scripts/summarize_horizonqa_result + .gtnh-workflows/scripts/summarize_horizonqa_result + + - name: Test Horizon-QA result + if: ${{ always() && !inputs.client-only && inputs.horizonqa }} + shell: bash + run: | + chmod +x .gtnh-workflows/scripts/test_horizonqa_result + .gtnh-workflows/scripts/test_horizonqa_result - name: Test no errors reported during server run if: ${{ !inputs.client-only }} diff --git a/scripts/summarize_horizonqa_result b/scripts/summarize_horizonqa_result new file mode 100644 index 0000000..1e2e6ea --- /dev/null +++ b/scripts/summarize_horizonqa_result @@ -0,0 +1,97 @@ +#!/usr/bin/env bash + +set -euo pipefail + +status="${1:-${GITHUB_WORKSPACE:-$(pwd)}/build/horizonqa/horizonqa-result.json}" + +write_summary() { + if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + "$@" >> "$GITHUB_STEP_SUMMARY" + else + "$@" + fi +} + +if [[ ! -e "$status" ]]; then + write_summary bash -c ' + echo "## Horizon-QA" + echo + echo "No Horizon-QA status report was produced." + ' + exit 0 +fi + +print_summary() { + python3 - "$status" <<'PY' +import json +import sys + +def cell(value): + return str(value if value is not None else "").replace("|", "\\|").replace("\n", " ") + +with open(sys.argv[1], encoding="utf-8") as f: + result = json.load(f) + +counts = result.get("counts", {}) +tests = result.get("tests", []) +issues = result.get("issues", []) +non_passing = [test for test in tests if test.get("status") != "passed"] + +selected = counts.get("selectedTests", len(tests)) +passed = counts.get("passed", 0) +status = result.get("status", "unknown") +exit_code = result.get("exitCode", 2) + +print("## Horizon-QA") +print() +print(f"**Status:** `{status}` (`exitCode={exit_code}`)") +print() +print("| Metric | Count |") +print("|---|---:|") +print(f"| Selected | {selected} |") +print(f"| Passed | {passed} |") +print(f"| Required failures | {counts.get('requiredFailures', 0)} |") +print(f"| Optional failures | {counts.get('optionalFailures', 0)} |") +print(f"| Infrastructure issues | {counts.get('diagnosticErrors', 0)} |") +print(f"| JUnit failures | {counts.get('junitFailures', 0)} |") +print(f"| JUnit errors | {counts.get('junitErrors', 0)} |") +print(f"| JUnit skipped | {counts.get('junitSkipped', 0)} |") + +if non_passing: + print() + print("### Non-passing tests") + print() + print("| Test | Status | Required | Message |") + print("|---|---|---:|---|") + for test in non_passing[:25]: + failure = test.get("failure") or {} + print( + f"| `{cell(test.get('id'))}` | `{cell(test.get('status'))}` | " + f"{cell(test.get('required'))} | {cell(failure.get('message'))} |" + ) + if len(non_passing) > 25: + print() + print(f"_Showing 25 of {len(non_passing)} non-passing tests._") + +if issues: + print() + print("### Infrastructure issues") + print() + print("| Issue | Kind | Message |") + print("|---|---|---|") + for issue in issues[:10]: + print( + f"| `{cell(issue.get('id'))}` | `{cell(issue.get('kind'))}` | " + f"{cell(issue.get('message'))} |" + ) + if len(issues) > 10: + print() + print(f"_Showing 10 of {len(issues)} issues._") +PY +} + +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + print_summary >> "$GITHUB_STEP_SUMMARY" +else + print_summary +fi diff --git a/scripts/test_horizonqa_result b/scripts/test_horizonqa_result new file mode 100644 index 0000000..85d4a6e --- /dev/null +++ b/scripts/test_horizonqa_result @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -euo pipefail + +status="${1:-${GITHUB_WORKSPACE:-$(pwd)}/build/horizonqa/horizonqa-result.json}" + +if [[ ! -e "$status" ]]; then + echo "::error::Horizon-QA was enabled but did not write $status" + exit 2 +fi + +python3 - "$status" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as f: + result = json.load(f) + +print( + f"Horizon-QA status={result.get('status')} " + f"exitCode={result.get('exitCode')}" +) +sys.exit(int(result.get("exitCode", 2))) +PY diff --git a/templates/build-and-test.yml b/templates/build-and-test.yml index 9526a9a..044d731 100644 --- a/templates/build-and-test.yml +++ b/templates/build-and-test.yml @@ -15,3 +15,6 @@ jobs: # workspace: setupDecompWorkspace # client-only: false # disable-server-auto-stop: false +# horizonqa: true +# horizonqa-tests: '' +# horizonqa-allow-no-tests: false