Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions apps/cli/src/commands/results/remote.ts
Original file line number Diff line number Diff line change
Expand Up @@ -154,8 +154,8 @@ async function loadNormalizedResultsConfig(
const projectResults = project?.results
? {
mode: 'github' as const,
repo: project.results.repository,
path: project.results.localPath,
repo: project.results.repoUrl,
path: project.results.path,
auto_push: project.results.sync?.autoPush,
branch_prefix: project.results.branchPrefix,
}
Expand Down
24 changes: 12 additions & 12 deletions apps/cli/test/commands/results/serve.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1272,8 +1272,8 @@ describe('serve app', () => {
name: 'Project No Publish',
path: projectDir,
results: {
repository: `file://${remoteDir}`,
localPath: missingCloneDir,
repoUrl: `file://${remoteDir}`,
path: missingCloneDir,
sync: { autoPush: true },
},
addedAt: '2026-01-01T00:00:00.000Z',
Expand Down Expand Up @@ -1412,8 +1412,8 @@ describe('serve app', () => {
name: 'AgentV',
path: projectDir,
results: {
repository: 'EntityProcess/agentv-examples-eval-results',
localPath: '/home/entity/projects/EntityProcess/agentv-examples-eval-results',
repoUrl: 'EntityProcess/agentv-examples-eval-results',
path: '/home/entity/projects/EntityProcess/agentv-examples-eval-results',
sync: { autoPush: true },
},
addedAt: '2026-01-01T00:00:00.000Z',
Expand Down Expand Up @@ -1464,8 +1464,8 @@ describe('serve app', () => {
name: 'Project Sync Pull',
path: projectDir,
results: {
repository: `file://${remoteDir}`,
localPath: cloneDir,
repoUrl: `file://${remoteDir}`,
path: cloneDir,
sync: { autoPush: false },
},
addedAt: '2026-01-01T00:00:00.000Z',
Expand Down Expand Up @@ -1543,8 +1543,8 @@ describe('serve app', () => {
name: 'Project Sync Push',
path: projectDir,
results: {
repository: `file://${remoteDir}`,
localPath: cloneDir,
repoUrl: `file://${remoteDir}`,
path: cloneDir,
sync: { autoPush: true },
},
addedAt: '2026-01-01T00:00:00.000Z',
Expand Down Expand Up @@ -1610,8 +1610,8 @@ describe('serve app', () => {
name: 'Project Sync Offline',
path: projectDir,
results: {
repository: `file://${remoteDir}`,
localPath: cloneDir,
repoUrl: `file://${remoteDir}`,
path: cloneDir,
sync: { autoPush: true },
},
addedAt: '2026-01-01T00:00:00.000Z',
Expand Down Expand Up @@ -1669,8 +1669,8 @@ describe('serve app', () => {
name: 'Project Sync Conflict',
path: projectDir,
results: {
repository: `file://${remoteDir}`,
localPath: cloneDir,
repoUrl: `file://${remoteDir}`,
path: cloneDir,
sync: { autoPush: true },
},
addedAt: '2026-01-01T00:00:00.000Z',
Expand Down
26 changes: 13 additions & 13 deletions apps/web/src/content/docs/docs/tools/dashboard.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -189,13 +189,13 @@ agentv dashboard --add /path/to/other-evals

Each path must contain a `.agentv/` directory. Registered projects are stored under `projects:` in `$AGENTV_HOME/config.yaml`, or `~/.agentv/config.yaml` when `AGENTV_HOME` is unset.

To register a remote repo and keep it synced automatically, add `repository` and `ref` to the entry in `$AGENTV_HOME/config.yaml`. `repository` uses GitHub's standard owner/name form and AgentV resolves it to `https://github.com/<owner>/<name>.git` for clone and pull operations:
To register a remote repo and keep it synced automatically, add `repo_url` and `ref` to the entry in `$AGENTV_HOME/config.yaml`. `repo_url` is the Git remote URL AgentV passes to `git clone`, so it can be HTTPS or SSH:

```yaml
projects:
- id: my-evals
name: My Evals
repository: example/my-evals
repo_url: https://github.com/example/my-evals.git
path: /srv/agentv/my-evals
ref: main
```
Expand Down Expand Up @@ -255,17 +255,17 @@ For a registered project, put results repo settings on that project's entry in `
projects:
- id: agentv
name: AgentV
repository: EntityProcess/agentv
repo_url: https://github.com/EntityProcess/agentv.git
path: /home/entity/projects/EntityProcess/agentv
ref: main
results:
repository: EntityProcess/agentv-examples-eval-results
local_path: /home/entity/projects/EntityProcess/agentv-examples-eval-results
repo_url: git@github.com:EntityProcess/agentv-examples-eval-results.git
path: /home/entity/projects/EntityProcess/agentv-examples-eval-results
sync:
auto_push: true
```

`results.local_path` is the filesystem location of the local clone AgentV manages for the results repo. It is **not** a subdirectory inside the remote repo. `results.repository` uses GitHub owner/name form and resolves to `https://github.com/<owner>/<name>.git` for clone and push operations.
`results.path` is the filesystem location of the local clone AgentV manages for the results repo. It uses the same local-path field name as the source project. `results.repo_url` is the Git remote URL used for clone and push operations, so use HTTPS when credentials are HTTP-token based and SSH when the runtime has SSH keys configured.

You can also set a top-level global fallback in the same file. This is used when the current project is not registered or its registry entry has no `results` block:

Expand All @@ -279,10 +279,10 @@ results:

Project-local `.agentv/config.yaml` is for portable eval defaults such as `execution`, `eval_patterns`, and `dashboard`. Do not put `projects` in project-local config; AgentV warns and ignores it there. `results_by_project` is deprecated; use `projects[].results` in `$AGENTV_HOME/config.yaml`.

The project `repository` and the `results` block sync different repositories:
The project `repo_url` and the `results` block sync different repositories:

- `projects[].repository` is the eval source project. Dashboard startup clones or fast-forwards the project checkout so eval YAML, scripts, and project-local `.agentv/config.yaml` stay current.
- `projects[].results.repository` is the git-backed results store. **Sync Project** fetches, fast-forwards, and, when configured, pushes run artifacts and mutable metadata in that results repo clone.
- `projects[].repo_url` is the eval source project remote. Dashboard startup clones or fast-forwards the project checkout so eval YAML, scripts, and project-local `.agentv/config.yaml` stay current.
- `projects[].results.repo_url` is the git-backed results store remote. **Sync Project** fetches, fast-forwards, and, when configured, pushes run artifacts and mutable metadata in that results repo clone.

#### Migration from the legacy project schema

Expand All @@ -309,17 +309,17 @@ After:
projects:
- id: agentv
name: AgentV
repository: EntityProcess/agentv
repo_url: https://github.com/EntityProcess/agentv.git
path: /home/entity/projects/EntityProcess/agentv
ref: main
results:
repository: EntityProcess/agentv-eval-results
local_path: /home/entity/projects/EntityProcess/agentv-eval-results
repo_url: https://github.com/EntityProcess/agentv-eval-results.git
path: /home/entity/projects/EntityProcess/agentv-eval-results
sync:
auto_push: true
```

Legacy project fields (`source`, `results.mode`, `results.repo`, `results.path`, and `results.auto_push`) fail validation with migration guidance.
Legacy project fields (`source`, `repository`, `results.mode`, `results.repo`, `results.repository`, `results.local_path`, and `results.auto_push`) fail validation with migration guidance.

Use project-level **Sync Project** as the results exchange workflow. It handles pulled remote runs, locally edited metadata, dirty state, and blocked conflict feedback in one project-scoped action.

Expand Down
52 changes: 27 additions & 25 deletions packages/core/src/evaluation/validation/config-validator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -175,17 +175,21 @@ function validateProjects(errors: ValidationError[], filePath: string, projects:
severity: 'error',
filePath,
location: `${location}.source`,
message: `Field '${location}.source' was removed. Move 'source.url' to '${location}.repository' as a GitHub owner/name value (for example, 'example/repo') and move 'source.ref' to '${location}.ref'.`,
message: `Field '${location}.source' was removed. Move 'source.url' to '${location}.repo_url' and move 'source.ref' to '${location}.ref'. Use a Git remote URL such as https://github.com/example/repo.git or git@github.com:example/repo.git.`,
});
}

if (projectRecord.repository !== undefined) {
validateGitHubRepository(
errors,
errors.push({
severity: 'error',
filePath,
projectRecord.repository,
`${location}.repository`,
);
location: `${location}.repository`,
message: `Field '${location}.repository' was removed. Use '${location}.repo_url' with a Git remote URL instead.`,
});
}

if (projectRecord.repo_url !== undefined) {
validateGitRemoteUrl(errors, filePath, projectRecord.repo_url, `${location}.repo_url`);
}

if (projectRecord.ref !== undefined) {
Expand All @@ -212,7 +216,7 @@ function validateRequiredString(
}
}

function validateGitHubRepository(
function validateGitRemoteUrl(
errors: ValidationError[],
filePath: string,
value: unknown,
Expand All @@ -223,18 +227,18 @@ function validateGitHubRepository(
severity: 'error',
filePath,
location,
message: `Field '${location}' must be a non-empty GitHub owner/name repository (e.g., EntityProcess/agentv)`,
message: `Field '${location}' must be a non-empty Git remote URL (e.g., https://github.com/EntityProcess/agentv.git or git@github.com:EntityProcess/agentv.git)`,
});
return;
}

const repository = value.trim();
if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository)) {
const repoUrl = value.trim();
if (!/^(https?:\/\/|ssh:\/\/|git@|file:\/\/).+/.test(repoUrl)) {
errors.push({
severity: 'error',
filePath,
location,
message: `Field '${location}' must use GitHub owner/name format (e.g., EntityProcess/agentv), not a URL. It resolves to https://github.com/<owner>/<name>.git for git operations.`,
message: `Field '${location}' must be a Git remote URL, not an owner/name shorthand. Use https://github.com/owner/repo.git or git@github.com:owner/repo.git.`,
});
}
}
Expand Down Expand Up @@ -262,9 +266,10 @@ function validateProjectResultsConfig(
const resultsRecord = rawResults as Record<string, unknown>;

const removedFields: Record<string, string> = {
mode: `Remove '${location}.mode'; project results are GitHub-backed by '${location}.repository'.`,
repo: `Field '${location}.repo' was removed. Use '${location}.repository' with GitHub owner/name format instead.`,
path: `Field '${location}.path' was removed. Use '${location}.local_path' for the local clone path instead.`,
mode: `Remove '${location}.mode'; project results use '${location}.repo_url' as the Git remote URL.`,
repo: `Field '${location}.repo' was removed. Use '${location}.repo_url' with a Git remote URL instead.`,
repository: `Field '${location}.repository' was removed. Use '${location}.repo_url' with a Git remote URL instead.`,
local_path: `Field '${location}.local_path' was removed. Use '${location}.path' for the local clone path instead.`,
auto_push: `Field '${location}.auto_push' was removed. Use '${location}.sync.auto_push' instead.`,
};

Expand All @@ -279,25 +284,22 @@ function validateProjectResultsConfig(
}
}

validateGitHubRepository(errors, filePath, resultsRecord.repository, `${location}.repository`);
validateGitRemoteUrl(errors, filePath, resultsRecord.repo_url, `${location}.repo_url`);

if (resultsRecord.local_path !== undefined) {
if (
typeof resultsRecord.local_path !== 'string' ||
resultsRecord.local_path.trim().length === 0
) {
if (resultsRecord.path !== undefined) {
if (typeof resultsRecord.path !== 'string' || resultsRecord.path.trim().length === 0) {
errors.push({
severity: 'error',
filePath,
location: `${location}.local_path`,
message: `Field '${location}.local_path' must be a non-empty string`,
location: `${location}.path`,
message: `Field '${location}.path' must be a non-empty string`,
});
} else if (!isFilesystemPath(resultsRecord.local_path.trim())) {
} else if (!isFilesystemPath(resultsRecord.path.trim())) {
errors.push({
severity: 'error',
filePath,
location: `${location}.local_path`,
message: `'${location}.local_path' must be an absolute or home-relative filesystem path (e.g., ~/data/agentv-results).`,
location: `${location}.path`,
message: `'${location}.path' must be an absolute or home-relative filesystem path (e.g., ~/data/agentv-results).`,
});
}
}
Expand Down
2 changes: 1 addition & 1 deletion packages/core/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ export {
deriveProjectId,
getProjectsRegistryPath,
} from './projects.js';
export { syncProject, syncProjects, resolveGitHubRepositoryUrl } from './project-sync.js';
export { syncProject, syncProjects } from './project-sync.js';
export { trimBaselineResult } from './evaluation/baseline.js';
export { DEFAULT_CATEGORY, deriveCategory } from './evaluation/category.js';
export * from './observability/index.js';
Expand Down
27 changes: 11 additions & 16 deletions packages/core/src/project-sync.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
/**
* Project sync — pulls remote GitHub repos to the local path declared in the
* Project sync — pulls remote Git repos to the local path declared in the
* project registry before Dashboard/eval startup.
*
* Sync is oneshot only, triggered by the Dashboard UI "Sync" button or the
* `agentv project sync` CLI command. There is no daemon or continuous mode.
*
* First run — git clone --depth 1 --filter=blob:none --branch <ref> <url> <path>
* First run — git clone --depth 1 --filter=blob:none --branch <ref> <repoUrl> <path>
* Subsequent — git pull --ff-only (when <path>/.git already exists)
*
* Usage:
Expand All @@ -18,45 +18,40 @@ import { existsSync } from 'node:fs';

import type { ProjectEntry } from './projects.js';

export function resolveGitHubRepositoryUrl(repository: string): string {
return `https://github.com/${repository.trim().replace(/\.git$/, '')}.git`;
}

/**
* Clone or pull a single project entry from its declared repository.
* Clone or pull a single project entry from its declared repo URL.
* - No .git present: shallow clone into entry.path.
* - .git present: git pull --ff-only to update in place.
* Throws on git error or missing repository/ref.
* Throws on git error or missing repoUrl/ref.
*/
export async function syncProject(entry: ProjectEntry): Promise<void> {
if (!entry.repository) {
throw new Error(`Project '${entry.id}' has no repository defined`);
if (!entry.repoUrl) {
throw new Error(`Project '${entry.id}' has no repo_url defined`);
}
if (!entry.ref) {
throw new Error(`Project '${entry.id}' has no ref defined`);
}
const url = resolveGitHubRepositoryUrl(entry.repository);
const dest = entry.path;

if (existsSync(`${dest}/.git`)) {
childProcess.execFileSync('git', ['-C', dest, 'pull', '--ff-only'], { stdio: 'inherit' });
} else {
childProcess.execFileSync(
'git',
['clone', '--depth', '1', '--filter=blob:none', '--branch', entry.ref, url, dest],
['clone', '--depth', '1', '--filter=blob:none', '--branch', entry.ref, entry.repoUrl, dest],
{ stdio: 'inherit' },
);
}
}

/**
* Iterate project entries and sync any that have a repository declared.
* Entries without repository are skipped silently.
* Iterate project entries and sync any that have a repo URL declared.
* Entries without repoUrl are skipped silently.
*/
export async function syncProjects(entries: ProjectEntry[]): Promise<void> {
for (const entry of entries) {
if (!entry.repository) continue;
console.log(`Syncing project '${entry.id}' from ${entry.repository}...`);
if (!entry.repoUrl) continue;
console.log(`Syncing project '${entry.id}' from ${entry.repoUrl}...`);
await syncProject(entry);
console.log(`Project '${entry.id}' synced.`);
}
Expand Down
Loading
Loading