Horizon qa (#68)

.github/workflows/build-and-test.yml

@@ -31,6 +31,21 @@ on:
         required: false
         default: false
         type: boolean
+      horizonqa:
+        description: 'Run Horizon-QA in CI mode during runServer'
+        required: false
+        default: false
+        type: boolean
+      horizonqa-tests:
+        description: 'Optional -Dhorizonqa.tests selector'
+        required: false
+        default: ''
+        type: string
+      horizonqa-allow-no-tests:
+        description: 'Allow Horizon-QA CI to pass with no discovered tests'
+        required: false
+        default: false
+        type: boolean
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -52,7 +67,8 @@ jobs:
     - name: Checkout workflows repo
       uses: actions/checkout@v5
       with:
-        repository: GTNewHorizons/GTNH-Actions-Workflows
+        repository: ${{ job.workflow_repository }}
+        ref: ${{ job.workflow_sha }}
         path: .gtnh-workflows
         fetch-depth: 0
 
@@ -139,6 +155,12 @@ jobs:
 
     - name: Run server for up to ${{ inputs.timeout }} seconds
       if: ${{ !inputs.client-only }}
+      shell: bash
+      env:
+        HORIZONQA: ${{ inputs.horizonqa }}
+        HORIZONQA_TESTS: ${{ inputs.horizonqa-tests }}
+        HORIZONQA_ALLOW_NO_TESTS: ${{ inputs.horizonqa-allow-no-tests }}
+        DISABLE_SERVER_AUTO_STOP: ${{ inputs.disable-server-auto-stop }}
       run: |
         mkdir -p run/server
         echo "eula=true" > run/eula.txt
@@ -147,12 +169,57 @@ jobs:
         echo "level-seed=-6202107849386030209\nonline-mode=true\n" > run/server.properties
         echo "level-seed=-6202107849386030209\nonline-mode=true\n" > run/server/server.properties
 
+        gradle_args=(--build-cache --info --stacktrace runServer)
+
+        if [[ "${HORIZONQA}" == "true" ]]; then
+          horizonqa_jvm_args=(
+            "-Dhorizonqa.mode=ci"
+            "-Dhorizonqa.reportDir=${GITHUB_WORKSPACE}/build/horizonqa"
+          )
+
+          if [[ -n "${HORIZONQA_TESTS}" ]]; then
+            horizonqa_jvm_args+=("-Dhorizonqa.tests=${HORIZONQA_TESTS}")
+          fi
+
+          if [[ "${HORIZONQA_ALLOW_NO_TESTS}" == "true" ]]; then
+            horizonqa_jvm_args+=("-Dhorizonqa.allowNoTests=true")
+          fi
+
+          for jvm_arg in "${horizonqa_jvm_args[@]}"; do
+            gradle_args+=(--mcJvmArgs="${jvm_arg}")
+          done
+        fi
+
         server_stdin=/dev/null
-        if [[ "${{ inputs.disable-server-auto-stop }}" != "true" ]]; then
+        if [[ "${DISABLE_SERVER_AUTO_STOP}" != "true" && "${HORIZONQA}" != "true" ]]; then
           echo "stop" > run/stop.txt
           server_stdin=run/stop.txt
         fi
-        timeout ${{ inputs.timeout }} ./gradlew --build-cache --info --stacktrace runServer 2>&1 < "${server_stdin}" | tee -a server.log || true
+        timeout ${{ inputs.timeout }} ./gradlew "${gradle_args[@]}" 2>&1 < "${server_stdin}" | tee -a server.log || true
+
+    - name: Upload Horizon-QA reports
+      if: ${{ always() && !inputs.client-only && inputs.horizonqa }}
+      uses: actions/upload-artifact@v7
+      continue-on-error: true
+      with:
+        name: ${{ github.repository_id }}-horizonqa
+        path: build/horizonqa/
+        retention-days: 90
+
+    - name: Summarize Horizon-QA result
+      if: ${{ always() && !inputs.client-only && inputs.horizonqa }}
+      shell: bash
+      continue-on-error: true
+      run: |
+        chmod +x .gtnh-workflows/scripts/summarize_horizonqa_result
+        .gtnh-workflows/scripts/summarize_horizonqa_result
+
+    - name: Test Horizon-QA result
+      if: ${{ always() && !inputs.client-only && inputs.horizonqa }}
+      shell: bash
+      run: |
+        chmod +x .gtnh-workflows/scripts/test_horizonqa_result
+        .gtnh-workflows/scripts/test_horizonqa_result
 
     - name: Test no errors reported during server run
       if: ${{ !inputs.client-only }}

scripts/summarize_horizonqa_result

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+status="${1:-${GITHUB_WORKSPACE:-$(pwd)}/build/horizonqa/horizonqa-result.json}"
+
+write_summary() {
+  if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+    "$@" >> "$GITHUB_STEP_SUMMARY"
+  else
+    "$@"
+  fi
+}
+
+if [[ ! -e "$status" ]]; then
+  write_summary bash -c '
+    echo "## Horizon-QA"
+    echo
+    echo "No Horizon-QA status report was produced."
+  '
+  exit 0
+fi
+
+print_summary() {
+  python3 - "$status" <<'PY'
+import json
+import sys
+
+def cell(value):
+    return str(value if value is not None else "").replace("|", "\\|").replace("\n", " ")
+
+with open(sys.argv[1], encoding="utf-8") as f:
+    result = json.load(f)
+
+counts = result.get("counts", {})
+tests = result.get("tests", [])
+issues = result.get("issues", [])
+non_passing = [test for test in tests if test.get("status") != "passed"]
+
+selected = counts.get("selectedTests", len(tests))
+passed = counts.get("passed", 0)
+status = result.get("status", "unknown")
+exit_code = result.get("exitCode", 2)
+
+print("## Horizon-QA")
+print()
+print(f"**Status:** `{status}` (`exitCode={exit_code}`)")
+print()
+print("| Metric | Count |")
+print("|---|---:|")
+print(f"| Selected | {selected} |")
+print(f"| Passed | {passed} |")
+print(f"| Required failures | {counts.get('requiredFailures', 0)} |")
+print(f"| Optional failures | {counts.get('optionalFailures', 0)} |")
+print(f"| Infrastructure issues | {counts.get('diagnosticErrors', 0)} |")
+print(f"| JUnit failures | {counts.get('junitFailures', 0)} |")
+print(f"| JUnit errors | {counts.get('junitErrors', 0)} |")
+print(f"| JUnit skipped | {counts.get('junitSkipped', 0)} |")
+
+if non_passing:
+    print()
+    print("### Non-passing tests")
+    print()
+    print("| Test | Status | Required | Message |")
+    print("|---|---|---:|---|")
+    for test in non_passing[:25]:
+        failure = test.get("failure") or {}
+        print(
+            f"| `{cell(test.get('id'))}` | `{cell(test.get('status'))}` | "
+            f"{cell(test.get('required'))} | {cell(failure.get('message'))} |"
+        )
+    if len(non_passing) > 25:
+        print()
+        print(f"_Showing 25 of {len(non_passing)} non-passing tests._")
+
+if issues:
+    print()
+    print("### Infrastructure issues")
+    print()
+    print("| Issue | Kind | Message |")
+    print("|---|---|---|")
+    for issue in issues[:10]:
+        print(
+            f"| `{cell(issue.get('id'))}` | `{cell(issue.get('kind'))}` | "
+            f"{cell(issue.get('message'))} |"
+        )
+    if len(issues) > 10:
+        print()
+        print(f"_Showing 10 of {len(issues)} issues._")
+PY
+}
+
+if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then
+  print_summary >> "$GITHUB_STEP_SUMMARY"
+else
+  print_summary
+fi

scripts/test_horizonqa_result

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+status="${1:-${GITHUB_WORKSPACE:-$(pwd)}/build/horizonqa/horizonqa-result.json}"
+
+if [[ ! -e "$status" ]]; then
+  echo "::error::Horizon-QA was enabled but did not write $status"
+  exit 2
+fi
+
+python3 - "$status" <<'PY'
+import json
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as f:
+    result = json.load(f)
+
+print(
+    f"Horizon-QA status={result.get('status')} "
+    f"exitCode={result.get('exitCode')}"
+)
+sys.exit(int(result.get("exitCode", 2)))
+PY

templates/build-and-test.yml

@@ -15,3 +15,6 @@ jobs:
 #      workspace: setupDecompWorkspace
 #      client-only: false
 #      disable-server-auto-stop: false
+#      horizonqa: true
+#      horizonqa-tests: ''
+#      horizonqa-allow-no-tests: false