test(evals): cover cross-forge PR skill rollouts (#25)

2026-09-06 14:01:36 +01:00
parent abd21928bf
commit 74632e5929
103 changed files with 1420 additions and 2 deletions
@@ -0,0 +1,36 @@
+# yaml-language-server: $schema=https://promptfoo.dev/config-schema.json
+description: Native Codex SDK regression for the land skill
+
+prompts:
+  - >-
+    Use the land skill for this owned GitHub PR: [pr-daemon]
+    github:acme/retry#47 reason=ci skill=land updated=2026-09-04T10:00:00Z.
+
+providers:
+  - id: openai:codex-sdk
+    label: codex-native-land
+    config:
+      # Relative to this config file; created by bin/prepare-codex-fixture.sh.
+      working_dir: .runtime/codex-land-ci
+      sandbox_mode: workspace-write
+      approval_policy: never
+      network_access_enabled: false
+      web_search_enabled: false
+      enable_streaming: true
+      inherit_process_env: true
+      cli_env:
+        PATH: mock-bin:/home/naps62/.nix-profile/bin:/run/current-system/sw/bin:/usr/bin:/bin
+        GIT_DIR: .workgit
+        GIT_WORK_TREE: .
+
+defaultTest:
+  assert:
+    - type: skill-used
+      value: land
+    - type: javascript
+      value: file://assertions/verify-land-ci.js
+
+tests:
+  - description: Land fixes the mocked failing check, pushes, and stops before GitHub merge
+    vars:
+      fixture: land-ci