feat(study-crates): test-file target + reward-hacking benchmark prompt

.github/workflows/ci.yml

@@ -586,7 +586,19 @@ jobs:
           load: true
 
       - name: Run ${{ matrix.ecosystem }} Docker e2e test
-        run: cargo test -p socket-patch-cli --features docker-e2e --test docker_e2e_${{ matrix.ecosystem }}
+        # The optional ecosystems gate their docker_e2e file behind their
+        # own feature (`#![cfg(all(feature = "docker-e2e", feature =
+        # "<eco>"))]`), so the harness must be built with that feature too —
+        # otherwise the test binary compiles to zero tests and passes
+        # vacuously. npm/pypi/gem are unconditional and have no such feature.
+        # (The socket-patch binary inside the image is always --all-features.)
+        run: |
+          eco='${{ matrix.ecosystem }}'
+          case "$eco" in
+            npm|pypi|gem) features=docker-e2e ;;
+            *) features="docker-e2e,$eco" ;;
+          esac
+          cargo test -p socket-patch-cli --features "$features" --test "docker_e2e_$eco"
 
   # ----------------------------------------------------------------------
   # Experimental `setup`-flow matrix (NON-BLOCKING).

crates/socket-patch-cli/CLI_CONTRACT.md

@@ -63,7 +63,7 @@ Beyond the globals above, each subcommand defines a small set of local arguments
 | `rollback` | optional positional `identifier`; `--one-off` | `SOCKET_ONE_OFF` | Rollback target |
 | `vex` | `--output` / `-O`, `--product`, `--no-verify`, `--doc-id`, `--compact` | `SOCKET_VEX_OUTPUT`, `SOCKET_VEX_PRODUCT`, `SOCKET_VEX_NO_VERIFY`, `SOCKET_VEX_DOC_ID`, `SOCKET_VEX_COMPACT` | OpenVEX 0.2.0 document generation; see "vex output channels" below |
 | `repair` | `--download-only` | `SOCKET_DOWNLOAD_ONLY` | Repair-specific cleanup mode (mutually exclusive with `--offline`) |
-| `setup` | (none beyond globals) | — | — |
+| `setup` | `--check`, `--remove` (mutually exclusive); honors global `--ecosystems` | `SOCKET_ECOSYSTEMS` | Wire / verify / revert the automatic-patching install hooks. See [Setup command contract](#setup-command-contract) |
 
 `scan --apply` opts JSON callers into the full discover → select → apply pipeline. Without it, `scan --json` stays read-only (discovery + `updates` array only). No effect outside `--json` mode — the non-JSON path always prompts the user interactively.
 
@@ -89,6 +89,159 @@ Contract details:
 
 `repair` keeps its `gc` visible alias.
 
+## Setup command contract
+
+`setup` wires a repository for **automatic patching**: after the ecosystem's own install/build step
+runs, locally-installed dependencies are re-patched to match the Socket manifest (`.socket/manifest.json`)
+with no further human action. It does this by installing an ecosystem-native hook (see the support
+matrix below). `setup --check` verifies that state; `setup --remove` reverts it.
+
+The properties below are the public contract. Each is backed by a test under
+`crates/socket-patch-cli/tests/setup_*.rs`; properties not yet fully implemented are called out
+explicitly and guarded by a deliberately-failing (RED) test that encodes the intended behavior — these
+are the executable spec for follow-up work, **not** regressions. Changing any property below is governed
+by the [semver policy](#semver-policy) (scoping `setup` by `--ecosystems` and strengthening `--check`,
+in particular, are behavior changes that gate a version bump when implemented).
+
+1. **Idempotent.** Re-running `setup` on an already-configured repo changes nothing: status
+   `already_configured`, `updated: 0`, every manifest byte-identical. *(Implemented.)*
+
+2. **Ecosystem-scoped.** `setup`, `setup --check`, and `setup --remove` honor the global
+   `--ecosystems` filter and act on only the named ecosystems; with no filter they act on every
+   detected ecosystem. *(Intended; **not yet implemented** — `setup` currently ignores `--ecosystems`
+   and always processes npm + python + cargo. RED-guarded.)*
+
+3. **Consistency after install.** Once an ecosystem is set up, its locally-installed dependencies are
+   re-patched to match the manifest after **any** of: a dependency added, updated, or removed; **or** a
+   new patch added to the manifest. The re-patch is carried by the ecosystem's install/build hook (npm
+   `postinstall`/`dependencies`, the Python `.pth` startup hook, the cargo guard build script, the gem
+   Bundler plugin) which runs `socket-patch apply` after the ecosystem's installer finishes, so patch
+   state always reconverges with the manifest. *(Implemented for npm/pypi/cargo/gem via the support
+   matrix.)*
+
+4. **`check` proves a correctly-patched state.** `setup --check` reports `configured` only when the
+   in-scope ecosystems are *actually in a correctly patched state* — install hooks present **and**
+   on-disk patch consistency verified (the `apply --check` invariant: every manifest file's hash matches
+   `afterHash`). *(Partially implemented; **hook-presence only today** — `check` does not yet verify
+   on-disk patch consistency. RED-guarded.)*
+
+5. **In-repo and committable.** `setup` writes only inside the working tree: `package.json`,
+   `pyproject.toml`/`requirements.txt`, member `Cargo.toml`s, `.cargo/config.toml`, the `Gemfile` +
+   generated `.socket/bundler-plugin/`. Every artifact is git-committable. It never writes outside
+   `--cwd` — no `$HOME`, no global `site-packages` (the Python `.pth` wheel is installed later by the
+   user's package manager, not by `setup`; the gem patch stamp is written under `Bundler.bundle_path`
+   by the plugin at `bundle install` time, not by `setup`). *(Implemented.)*
+
+6. **Clone-portable.** Because all setup state is committed files, a fresh checkout on another host —
+   CI, a deploy, a teammate's machine — inherits the setup state unchanged; `setup --check` passes on
+   the clone with no re-run required. *(Implemented; a consequence of properties 5 + 1.)*
+
+7. **Reflected in VEX.** A patch contributes a `not_affected` statement to the repo's OpenVEX document
+   only for ecosystems that are **actually set up** — or explicitly declared **manual** (below). Patches
+   for an ecosystem that is neither set up nor declared manual produce no VEX statement. *(Intended;
+   **not yet implemented** — VEX currently filters by `--ecosystems` and on-disk verification but has no
+   notion of setup state. RED-guarded.)*
+   - **Manual declaration.** Users who run `socket-patch apply` by hand (e.g. in a CI step) can declare
+     an ecosystem or individual hook as `manual`, so VEX still attests its patches even though the
+     auto-install hook is intentionally not wired. Intended home: a sub-property of
+     `.socket/manifest.json`. *(Follow-up work.)*
+
+8. **Graceful, exact remove.** `setup --remove` (optionally per-ecosystem via `--ecosystems`) restores
+   the repo to its exact pre-setup state: manifests byte-for-byte, sibling scripts/dependencies
+   preserved, keys that became empty dropped. Afterward `setup --check` reports needs-configuration
+   again. *(Implemented for the manifest edits — npm `package.json`, Python deps, and member
+   `Cargo.toml`s all round-trip byte-for-byte. **Known residue:** a `.cargo/config.toml` (and its
+   `.cargo/` dir) that `setup` created is left behind empty rather than deleted on `--remove`;
+   RED-guarded.)*
+
+9. **Nested workspaces, with exclude.** Setup applies to every subproject below the repo root: npm /
+   yarn / pnpm / bun workspace members and cargo workspace members are all discovered and configured
+   (pnpm is root-package-only by design, because workspace-member `postinstall` scripts fail under
+   pnpm's strict module isolation). Selected paths may be **excluded**, and the exclusion is **persisted
+   in `.socket/manifest.json`** so `check`, `apply`, and any clone all honor it. *(Workspace discovery
+   implemented; the `--exclude` flag + manifest exclude sub-property are **follow-up work** — pending
+   test marked `#[ignore]`.)*
+
+### Per-ecosystem setup support
+
+`setup` only installs an automatic-repatch hook for the three ecosystems with a native post-install /
+build hook. The remaining ecosystems are **apply-only**: `socket-patch apply` patches them on demand,
+but there is no hook for `setup` to install, so `setup` is a `no_files` no-op for them. These are
+exactly the ecosystems for which property 7's **manual** declaration is intended (so their hand-applied
+patches still show up in VEX).
+
+| Ecosystem | Hook `setup` installs | Repatch trigger | Notes |
+|---|---|---|---|
+| npm / yarn / pnpm / bun | `scripts.postinstall` + `scripts.dependencies` | `npm/pnpm install` (+ `install <pkg>`) | pnpm: root package only |
+| pypi | `socket-patch[hook]` dependency → `.pth` startup hook | Python interpreter startup after installed-set change | manifest = `pyproject.toml` (uv/poetry/pdm/hatch) or `requirements.txt` (pip) |
+| cargo | `socket-patch-guard` dependency + `[env] SOCKET_PATCH_ROOT` in `.cargo/config.toml` | every `cargo build` (fail-closed guard) | per-member dep + one workspace-root `[env]` |
+| gem | managed `plugin "socket-patch"` block in the `Gemfile` → committed in-tree Bundler plugin under `.socket/bundler-plugin/` | every `bundle install` (cached + fresh: load-time digest gate + `after-install-all` hook) | Bundler loads only committed git plugins, so the generated dir must be committed; CLI must be on `PATH`. Phase 1 references the in-tree plugin via `git:`; Phase 2 (follow-up) switches to a published `socket-patch-bundler` gem |
+| nuget · maven · golang · composer · deno | **none** (apply-only) | — | `setup` reports `no_files`; candidates for the **manual** declaration |
+
+### JSON output shapes (`setup`, `setup --check`, `setup --remove`)
+
+`setup` predates the v3.0 unified envelope and emits its own three shapes. They are stable as of v3.0;
+consumers may rely on these keys. All three share a `files[*]` entry shape; `kind` is one of
+`package_json`, `pth`, `cargo`, `cargo_env`, `go_guard`, `go_import`, `gemfile`, `gem_plugin`.
+
+**`setup`:**
+
+```jsonc
+{
+  "status": "success" | "already_configured" | "dry_run" | "partial_failure" | "error" | "no_files",
+  "updated":            0,
+  "alreadyConfigured":  0,
+  "errors":             0,
+  "packageManager":      "npm" | "pnpm",                 // always emitted; defaults to "npm", only meaningful when npm files were found
+  "pythonPackageManager":"pip" | "uv" | "poetry" | "pdm" | "hatch",  // present only when Python detected
+  "dryRun":   true,                                      // only on status=dry_run
+  "wouldUpdate": 0,                                      // only on status=dry_run
+  "warnings": [ "..." ],                                 // only when non-empty (e.g. lockfile refresh)
+  "files": [
+    { "kind": "package_json", "path": "...", "status": "updated" | "already_configured" | "error",
+      "error": null | "..." }
+  ]
+}
+```
+
+**`setup --check`** (read-only; never writes — exit `0` only when all in-scope manifests are configured
+and none errored):
+
+```jsonc
+{
+  "status": "configured" | "needs_configuration" | "error" | "no_files",
+  "configured":          0,
+  "needsConfiguration":  0,
+  "errors":              0,
+  "files": [
+    { "kind": "...", "path": "...", "status": "configured" | "needs_configuration" | "error",
+      "error": null | "..." }
+  ]
+}
+```
+
+**`setup --remove`:**
+
+```jsonc
+{
+  "status": "success" | "not_configured" | "dry_run" | "partial_failure" | "error" | "no_files",
+  "removed":        0,
+  "notConfigured":  0,
+  "errors":         0,
+  "dryRun":   true,            // only on status=dry_run
+  "wouldRemove": 0,            // only on status=dry_run
+  "warnings": [ "..." ],       // only when non-empty
+  "files": [
+    { "kind": "...", "path": "...", "status": "removed" | "not_configured" | "error",
+      "error": null | "..." }
+  ]
+}
+```
+
+**Exit codes** (all three): `0` when nothing errored and the operation was satisfiable (including
+`no_files` and `not_configured`); `1` on any per-file error, partial failure, or — for `--check` — any
+manifest that needs configuration. `setup --check --remove` is a clap usage error (exit `2`).
+
 ## Environment variables
 
 All v3.0 env vars use the `SOCKET_*` prefix. Three legacy `SOCKET_PATCH_*` names are still honored at runtime for compatibility: on first read of any of the three the binary emits a one-shot deprecation warning to stderr (the warning fires unconditionally — even under `--silent` / `--json` — because it's a transition signal users need to see). The legacy names will be removed in the next major release.
@@ -247,7 +400,7 @@ The remaining commands still emit their pre-v3.0 ad-hoc JSON shapes and will mig
 - ⏳ `scan` — still emits the discovery + `apply.patches[*]` + `gc.*` shape documented in earlier drafts of this file.
 - ⏳ `get` — still emits per-patch action arrays.
 - ⏳ `rollback` — still emits per-package result records.
-- ⏳ `setup` — still emits `{ status, updated, alreadyConfigured, errors, files }`.
+- ⏳ `setup` — still emits its own `{ status, updated, alreadyConfigured, errors, files }` shape (and the `--check` / `--remove` variants), now documented in full under [Setup command contract](#setup-command-contract).
 
 ### `patches[]` entry shape for `get` and `scan --apply`
 

crates/socket-patch-cli/Cargo.toml

@@ -28,12 +28,14 @@ regex = { workspace = true }
 tempfile = { workspace = true }
 
 [features]
-# Shipped defaults: npm + PyPI are always compiled in (no feature gate); `cargo`
-# is on by default so released binaries and `cargo install socket-patch-cli`
-# patch Rust deps and run the build-time guard out of the box. The remaining
-# ecosystems stay opt-in. Build `--no-default-features` for a minimal
-# (npm + PyPI only) binary — its `apply --check` then fails closed.
-default = ["cargo"]
+# Shipped defaults: npm + PyPI + Ruby gems are always compiled in (no feature
+# gate); `cargo` and `golang` are on by default so released binaries and
+# `cargo install socket-patch-cli` patch Rust and Go deps (and run the
+# build-time cargo guard) out of the box. The still-unsupported ecosystems
+# (maven, nuget, composer, deno) stay opt-in. Build `--no-default-features`
+# for a minimal (npm + PyPI + Ruby gems only) binary — its `apply --check`
+# then fails closed.
+default = ["cargo", "golang"]
 cargo = ["socket-patch-core/cargo"]
 golang = ["socket-patch-core/golang"]
 maven = ["socket-patch-core/maven"]

crates/socket-patch-cli/src/args.rs

@@ -21,6 +21,33 @@ use socket_patch_core::api::client::ApiClientEnvOverrides;
 use socket_patch_core::constants::{
     DEFAULT_PATCH_API_PROXY_URL, DEFAULT_PATCH_MANIFEST_PATH, DEFAULT_SOCKET_API_URL,
 };
+use socket_patch_core::crawlers::Ecosystem;
+
+/// clap value-parser for each `--ecosystems` / `SOCKET_ECOSYSTEMS` token.
+///
+/// Rejects any name this build does not support — both typos and
+/// ecosystems whose Cargo feature is not compiled in (e.g. `maven` /
+/// `nuget` on a default build, which ships npm + PyPI + Ruby gems + Go +
+/// Cargo). `Ecosystem::all()` is itself `#[cfg]`-gated, so the accepted
+/// set tracks the compiled feature set exactly.
+///
+/// Without this, an unsupported name parsed fine and was then silently
+/// dropped by `partition_purls`/`crawl_all_ecosystems`, so the user got a
+/// "0 patches" result with no hint that the ecosystem name was the cause.
+fn parse_supported_ecosystem(s: &str) -> Result<String, String> {
+    if Ecosystem::all().iter().any(|e| e.cli_name() == s) {
+        Ok(s.to_string())
+    } else {
+        let supported = Ecosystem::all()
+            .iter()
+            .map(|e| e.cli_name())
+            .collect::<Vec<_>>()
+            .join(", ");
+        Err(format!(
+            "unsupported ecosystem `{s}` in this build (supported: {supported})"
+        ))
+    }
+}
 
 /// Arguments inherited by every subcommand via `#[command(flatten)]`.
 ///
@@ -65,12 +92,14 @@ pub struct GlobalArgs {
     )]
     pub proxy_url: String,
 
-    /// Restrict to these ecosystems (comma-separated).
+    /// Restrict to these ecosystems (comma-separated). Names not supported
+    /// by this build (e.g. `maven`/`nuget` unless compiled in) are rejected.
     #[arg(
         long = "ecosystems",
         short = 'e',
         env = "SOCKET_ECOSYSTEMS",
         value_delimiter = ',',
+        value_parser = parse_supported_ecosystem,
     )]
     pub ecosystems: Option<Vec<String>>,
 

crates/socket-patch-cli/src/commands/apply.rs

@@ -1109,14 +1109,16 @@ async fn apply_patches_inner(
     let has_any_purls = !partitioned.is_empty();
 
     if all_packages.is_empty() && !has_any_purls {
+        // Nothing in scope: the manifest lists no patches (or every patch was
+        // filtered out by `--ecosystems`). There is genuinely no work to do,
+        // so this is a clean no-op SUCCESS — not a failure. Returning `false`
+        // here used to exit 1 / `partialFailure`, which broke the npm
+        // `postinstall` hook (it runs `apply` on every install, including
+        // fresh projects whose manifest has no matching patches yet).
         if !args.common.silent && !args.common.json {
-            if args.common.global || args.common.global_prefix.is_some() {
-                eprintln!("No global packages found");
-            } else {
-                eprintln!("No package directories found");
-            }
+            println!("No patches to apply.");
         }
-        return Ok((false, Vec::new(), Vec::new()));
+        return Ok((true, Vec::new(), Vec::new()));
     }
 
     if all_packages.is_empty() {

crates/socket-patch-cli/src/commands/scan.rs

@@ -608,6 +608,30 @@ pub async fn run(args: ScanArgs) -> i32 {
             telemetry_org.as_deref(),
         )
         .await;
+
+        // A scan in which *every* batch failed produced no trustworthy
+        // patch data. Surfacing `status: "success"` / exit 0 here would be
+        // indistinguishable from a genuine "no patches" result and would
+        // mask a total API outage. Report the failure explicitly and bail
+        // before writing any manifest or attempting apply/prune.
+        if args.common.json {
+            let result = serde_json::json!({
+                "status": "error",
+                "error": err,
+                "scannedPackages": package_count,
+                "packagesWithPatches": 0,
+                "totalPatches": 0,
+                "freePatches": 0,
+                "paidPatches": 0,
+                "canAccessPaidPatches": false,
+                "packages": [],
+                "updates": [],
+            });
+            println!("{}", serde_json::to_string_pretty(&result).unwrap());
+        } else {
+            eprintln!("Error: all {total_batches} API batch queries failed: {err}");
+        }
+        return 1;
     }
 
     let total_patches_found: usize = all_packages_with_patches