From 6cd59ea624e105bd52a4398581eecdb2411ff484 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 21 Oct 2026 08:39:36 +0200 Subject: [PATCH] test: define manual M1 acceptance contract --- docs/research/m1-no-device-acceptance.md | 17 +++ docs/roadmap.md | 3 + docs/status/phase-0.md | 6 +- scripts/validate_m1_manual_acceptance.py | 133 ++++++++++++++++++ .../m1-manual-acceptance-template.json | 39 +++++ tests/test_m1_manual_acceptance.py | 93 ++++++++++++ 6 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 scripts/validate_m1_manual_acceptance.py create mode 100644 tests/fixtures/m1-manual-acceptance-template.json create mode 100644 tests/test_m1_manual_acceptance.py diff --git a/docs/research/m1-no-device-acceptance.md b/docs/research/m1-no-device-acceptance.md index fb31d8a..996b8f1 100644 --- a/docs/research/m1-no-device-acceptance.md +++ b/docs/research/m1-no-device-acceptance.md @@ -97,6 +97,23 @@ or configure a controller. Any device discovery prompt, physical-device status, transport attempt or unlabelled framing action is a stop condition and fails the checkpoint. +Record the run by copying +`tests/fixtures/m1-manual-acceptance-template.json` outside the repository, +filling it in and placing sanitized evidence below the report directory. Then +validate it without launching the application: + +```powershell +python scripts/validate_m1_manual_acceptance.py ` + .build/acceptance/manual-report.json ` + --output .build/acceptance/manual-validation.json +``` + +The validator verifies evidence hashes and relative paths. It reports +`incomplete` for any unrun or blocked scenario and `failed` for a failed step, +connected hardware, non-isolated app data, changed evidence or any stop +condition. Even a complete `passed` report still requires human review and +does not authorize hardware access. + ## Remaining external gates - A known-working private `.lbrn2` is still required before implementing diff --git a/docs/roadmap.md b/docs/roadmap.md index 5fd18f5..1747f7d 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -324,6 +324,9 @@ addon but deliberately refuses to install a missing Windows toolchain. An artifact produced elsewhere can be checked locally without execution: the verifier rejects unsafe paths or changed bytes and confirms the pin plus every packaged addon source before the manual GUI checklist may begin. +The manual checklist now also has a versioned, fail-closed report template and +validator. Missing steps remain incomplete, stop conditions fail the run, and +hashed screenshots stay outside the public repository pending sanitization. ## Decision rules diff --git a/docs/status/phase-0.md b/docs/status/phase-0.md index e88436c..f5d5382 100644 --- a/docs/status/phase-0.md +++ b/docs/status/phase-0.md @@ -75,11 +75,13 @@ evidence for: - independent layer visibility/output state and ordered serialization; - material-recipe matching plus a fail-closed PirateRay provenance and exact machine/source/lens/correction compatibility gate; -- one consolidated no-device acceptance report that reruns all 74 public +- one consolidated no-device acceptance report that reruns all 79 public tests and the leak guard, then hashes the regression evidence; - a non-executing Windows artifact verifier that checks portable paths, archive safety, sizes, hashes, the pinned Rayforge revision and every - packaged addon source while retaining the manual GUI gate. + packaged addon source while retaining the manual GUI gate; +- a versioned manual desktop-acceptance template and validator that cannot + pass missing steps, connected hardware, changed evidence or stop conditions. The remaining M1 gates are deliberately explicit: diff --git a/scripts/validate_m1_manual_acceptance.py b/scripts/validate_m1_manual_acceptance.py new file mode 100644 index 0000000..7b911ac --- /dev/null +++ b/scripts/validate_m1_manual_acceptance.py @@ -0,0 +1,133 @@ +"""Validate a manual M1 desktop acceptance report without running the app.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path, PurePosixPath + + +ROOT = Path(__file__).resolve().parents[1] +STEP_IDS = ( + "artifact_verified", + "isolated_portable_launch", + "offline_state_visible", + "synthetic_geometry_and_field", + "layer_editing_and_order", + "visibility_output_independent", + "mopa_values_and_units", + "recipe_compatibility_gate", + "preview_and_preflight", + "save_reopen_round_trip", + "simulation_cancel_recovery", + "theme_and_disclosure_stability", +) +STOP_CONDITIONS = ( + "device_discovery_prompt", + "physical_device_status", + "transport_attempt", + "unlabelled_physical_framing", +) +STEP_STATUSES = {"passed", "failed", "blocked", "not_run"} +SHA256 = re.compile(r"^[0-9a-f]{64}$") + + +def _validate_evidence(report_root: Path, records: list[dict]) -> list[str]: + errors = [] + seen = set() + for record in records: + value = record.get("path", "") + relative = PurePosixPath(value) + if not value or relative.is_absolute() or ".." in relative.parts: + errors.append(f"unsafe evidence path: {value!r}") + continue + if value in seen: + errors.append(f"duplicate evidence path: {value}") + continue + seen.add(value) + path = report_root.joinpath(*relative.parts) + if not path.is_file(): + errors.append(f"evidence file is missing: {value}") + continue + digest = record.get("sha256", "") + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if not SHA256.fullmatch(digest) or digest != actual: + errors.append(f"evidence SHA-256 mismatch: {value}") + return errors + + +def validate(report_path: Path) -> dict: + report_path = report_path.resolve() + report = json.loads(report_path.read_text(encoding="utf-8-sig")) + errors = [] + if report.get("schema_version") != 1: + errors.append("unsupported report schema") + if report.get("checkpoint") != "M1 packaged no-device desktop acceptance": + errors.append("unexpected checkpoint name") + + steps = report.get("steps", []) + step_ids = [step.get("id") for step in steps] + if step_ids != list(STEP_IDS): + errors.append("acceptance steps are missing, duplicated or out of order") + for step in steps: + if step.get("status") not in STEP_STATUSES: + errors.append(f"invalid step status: {step.get('id')}") + + stop_conditions = report.get("stop_conditions", {}) + if set(stop_conditions) != set(STOP_CONDITIONS): + errors.append("stop-condition set does not match the contract") + elif any(value not in (True, False, None) for value in stop_conditions.values()): + errors.append("stop conditions must be true, false or null") + + environment = report.get("environment", {}) + artifact = report.get("artifact_verification", {}) + errors.extend(_validate_evidence(report_path.parent, report.get("evidence", []))) + + explicit_failure = ( + any(step.get("status") == "failed" for step in steps) + or any(value is True for value in stop_conditions.values()) + or environment.get("hardware_disconnected") is False + or environment.get("fresh_app_data") is False + or artifact.get("status") == "failed" + ) + complete = ( + not errors + and step_ids == list(STEP_IDS) + and all(step.get("status") == "passed" for step in steps) + and all(value is False for value in stop_conditions.values()) + and environment.get("hardware_disconnected") is True + and environment.get("fresh_app_data") is True + and bool(environment.get("windows_version")) + and artifact.get("status") == "passed" + and SHA256.fullmatch(artifact.get("report_sha256", "")) is not None + ) + status = "failed" if explicit_failure or errors else "passed" if complete else "incomplete" + return { + "status": status, + "schema_valid": not errors, + "errors": errors, + "passed_steps": sum(step.get("status") == "passed" for step in steps), + "required_steps": len(STEP_IDS), + "hardware_access_authorized": False, + "manual_review_required": True, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("report", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + result = validate(args.report) + rendered = json.dumps(result, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered, encoding="utf-8", newline="\n") + print(rendered, end="") + return 0 if result["status"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/m1-manual-acceptance-template.json b/tests/fixtures/m1-manual-acceptance-template.json new file mode 100644 index 0000000..5c1b8d2 --- /dev/null +++ b/tests/fixtures/m1-manual-acceptance-template.json @@ -0,0 +1,39 @@ +{ + "schema_version": 1, + "checkpoint": "M1 packaged no-device desktop acceptance", + "artifact_verification": { + "status": "not_run", + "report_sha256": null + }, + "environment": { + "windows_version": null, + "hardware_disconnected": null, + "fresh_app_data": null + }, + "steps": [ + {"id": "artifact_verified", "status": "not_run", "notes": ""}, + {"id": "isolated_portable_launch", "status": "not_run", "notes": ""}, + {"id": "offline_state_visible", "status": "not_run", "notes": ""}, + {"id": "synthetic_geometry_and_field", "status": "not_run", "notes": ""}, + {"id": "layer_editing_and_order", "status": "not_run", "notes": ""}, + {"id": "visibility_output_independent", "status": "not_run", "notes": ""}, + {"id": "mopa_values_and_units", "status": "not_run", "notes": ""}, + {"id": "recipe_compatibility_gate", "status": "not_run", "notes": ""}, + {"id": "preview_and_preflight", "status": "not_run", "notes": ""}, + {"id": "save_reopen_round_trip", "status": "not_run", "notes": ""}, + {"id": "simulation_cancel_recovery", "status": "not_run", "notes": ""}, + {"id": "theme_and_disclosure_stability", "status": "not_run", "notes": ""} + ], + "stop_conditions": { + "device_discovery_prompt": null, + "physical_device_status": null, + "transport_attempt": null, + "unlabelled_physical_framing": null + }, + "evidence": [], + "review": { + "reviewer": null, + "reviewed_at": null, + "notes": "" + } +} diff --git a/tests/test_m1_manual_acceptance.py b/tests/test_m1_manual_acceptance.py new file mode 100644 index 0000000..347a5af --- /dev/null +++ b/tests/test_m1_manual_acceptance.py @@ -0,0 +1,93 @@ +import copy +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +from scripts.validate_m1_manual_acceptance import ROOT, STEP_IDS, validate + + +TEMPLATE = ROOT / "tests" / "fixtures" / "m1-manual-acceptance-template.json" + + +class M1ManualAcceptanceTests(unittest.TestCase): + def setUp(self): + self.template = json.loads(TEMPLATE.read_text(encoding="utf-8")) + + def _write(self, root: Path, report: dict) -> Path: + path = root / "acceptance.json" + path.write_text(json.dumps(report), encoding="utf-8") + return path + + def _passing(self, root: Path) -> dict: + report = copy.deepcopy(self.template) + report["artifact_verification"] = { + "status": "passed", + "report_sha256": "a" * 64, + } + report["environment"] = { + "windows_version": "synthetic-test-environment", + "hardware_disconnected": True, + "fresh_app_data": True, + } + for step in report["steps"]: + step["status"] = "passed" + report["stop_conditions"] = { + key: False for key in report["stop_conditions"] + } + screenshot = root / "evidence" / "offline-state.png" + screenshot.parent.mkdir() + screenshot.write_bytes(b"synthetic screenshot placeholder") + report["evidence"] = [{ + "path": "evidence/offline-state.png", + "sha256": hashlib.sha256(screenshot.read_bytes()).hexdigest(), + }] + return report + + def test_template_is_valid_but_incomplete(self): + with tempfile.TemporaryDirectory(dir=ROOT / ".build") as directory: + result = validate(self._write(Path(directory), self.template)) + self.assertEqual(result["status"], "incomplete") + self.assertTrue(result["schema_valid"]) + self.assertEqual(result["required_steps"], len(STEP_IDS)) + + def test_complete_no_device_report_passes(self): + with tempfile.TemporaryDirectory(dir=ROOT / ".build") as directory: + root = Path(directory) + result = validate(self._write(root, self._passing(root))) + self.assertEqual(result["status"], "passed") + self.assertFalse(result["hardware_access_authorized"]) + self.assertTrue(result["manual_review_required"]) + + def test_any_stop_condition_fails_checkpoint(self): + with tempfile.TemporaryDirectory(dir=ROOT / ".build") as directory: + root = Path(directory) + report = self._passing(root) + report["stop_conditions"]["transport_attempt"] = True + result = validate(self._write(root, report)) + self.assertEqual(result["status"], "failed") + + def test_missing_step_cannot_pass(self): + with tempfile.TemporaryDirectory(dir=ROOT / ".build") as directory: + root = Path(directory) + report = self._passing(root) + report["steps"].pop() + result = validate(self._write(root, report)) + self.assertEqual(result["status"], "failed") + self.assertFalse(result["schema_valid"]) + + def test_changed_or_external_evidence_is_rejected(self): + with tempfile.TemporaryDirectory(dir=ROOT / ".build") as directory: + root = Path(directory) + report = self._passing(root) + report["evidence"][0]["sha256"] = "0" * 64 + result = validate(self._write(root, report)) + self.assertEqual(result["status"], "failed") + report["evidence"][0]["path"] = "../private.png" + result = validate(self._write(root, report)) + self.assertEqual(result["status"], "failed") + + +if __name__ == "__main__": + unittest.main()