Suite

Suite configuration loader — reads suite.yaml and discovers tasks.

inspect_coco.suite

Suite configuration loader for inspect-coco.

A suite.yaml defines shared defaults for all eval scenarios within a skill. Each skill's eval directory gets its own suite.yaml.

Suite(name, description, skill, defaults, tasks, root) dataclass

Parsed suite configuration.

SuiteDefaults(epochs=3, timeout_sec=900, max_turns=None, idd_threshold=0.6, idd_strict=False, model=None, connection=None) dataclass

Shared defaults for all tasks in a suite.

TaskEntry(path, overrides=dict()) dataclass

A task within a suite (discovered or explicit).

load_suite(suite_path)

Load a suite.yaml file and discover tasks.

Parameters:

Name	Type	Description	Default
suite_path	Path	Path to a directory containing suite.yaml, or path to the suite.yaml file itself.	required

Returns:

Type	Description
Suite	Parsed Suite with discovered tasks.

Source code in src/inspect_coco/suite.py

def load_suite(suite_path: Path) -> Suite:
    """Load a suite.yaml file and discover tasks.

    Args:
        suite_path: Path to a directory containing suite.yaml,
                    or path to the suite.yaml file itself.

    Returns:
        Parsed Suite with discovered tasks.
    """
    if suite_path.is_file():
        suite_file = suite_path
        root = suite_path.parent
    else:
        suite_file = suite_path / SUITE_FILENAME
        root = suite_path

    if not suite_file.exists():
        raise FileNotFoundError(f"No {SUITE_FILENAME} found in {root}")

    with open(suite_file) as f:
        data = yaml.safe_load(f) or {}

    # Parse defaults
    defaults_data = data.get("defaults", {})
    defaults = SuiteDefaults(
        epochs=defaults_data.get("epochs", 3),
        timeout_sec=defaults_data.get("timeout_sec", 900),
        max_turns=defaults_data.get("max_turns"),
        idd_threshold=defaults_data.get("idd_threshold", 0.6),
        idd_strict=defaults_data.get("idd_strict", False),
        model=defaults_data.get("model"),
        connection=defaults_data.get("connection"),
    )

    # Discover or parse tasks
    tasks_config = data.get("tasks", "auto")
    exclude = data.get("exclude", [])

    if tasks_config == "auto":
        tasks = _auto_discover_tasks(root, exclude)
    else:
        tasks = _parse_explicit_tasks(root, tasks_config, exclude)

    return Suite(
        name=data.get("name", root.name),
        description=data.get("description", ""),
        skill=data.get("skill"),
        defaults=defaults,
        tasks=tasks,
        root=root,
    )

find_suites(search_path)

Find all suite.yaml files under a directory.

Parameters:

Name	Type	Description	Default
search_path	Path	Directory to search recursively.	required

Returns:

Type	Description
list[Path]	List of paths to directories containing suite.yaml.

Source code in src/inspect_coco/suite.py

def find_suites(search_path: Path) -> list[Path]:
    """Find all suite.yaml files under a directory.

    Args:
        search_path: Directory to search recursively.

    Returns:
        List of paths to directories containing suite.yaml.
    """
    suites = []
    if (search_path / SUITE_FILENAME).exists():
        suites.append(search_path)
    else:
        for suite_file in sorted(search_path.rglob(SUITE_FILENAME)):
            suites.append(suite_file.parent)
    return suites

merge_defaults(suite, task_path)

Merge suite defaults with task.toml to produce final config.

Priority: task.toml > suite per-task overrides > suite defaults > built-in

Parameters:

Name	Type	Description	Default
suite	Suite	The loaded suite configuration.	required
task_path	Path	Path to the task directory.	required

Returns:

Type	Description
dict	Merged configuration dict for coco_task().

Source code in src/inspect_coco/suite.py

def merge_defaults(suite: Suite, task_path: Path) -> dict:
    """Merge suite defaults with task.toml to produce final config.

    Priority: task.toml > suite per-task overrides > suite defaults > built-in

    Args:
        suite: The loaded suite configuration.
        task_path: Path to the task directory.

    Returns:
        Merged configuration dict for coco_task().
    """
    import toml as toml_lib

    # Start with suite defaults
    merged = {
        "epochs": suite.defaults.epochs,
        "timeout_sec": suite.defaults.timeout_sec,
        "idd_threshold": suite.defaults.idd_threshold,
        "idd_strict": suite.defaults.idd_strict,
    }
    if suite.defaults.max_turns:
        merged["max_turns"] = suite.defaults.max_turns
    if suite.defaults.model:
        merged["model"] = suite.defaults.model
    if suite.defaults.connection:
        merged["connection"] = suite.defaults.connection

    # Apply per-task overrides from suite.yaml tasks list
    for task_entry in suite.tasks:
        if task_entry.path == task_path:
            merged.update(task_entry.overrides)
            break

    # Apply task.toml values (highest priority)
    toml_path = task_path / "task.toml"
    if toml_path.exists():
        task_config = toml_lib.load(toml_path)
        metadata = task_config.get("metadata", {})
        agent = task_config.get("agent", {})

        if "epochs" in metadata:
            merged["epochs"] = metadata["epochs"]
        if "idd_threshold" in metadata:
            merged["idd_threshold"] = metadata["idd_threshold"]
        if "idd_strict" in metadata:
            merged["idd_strict"] = metadata["idd_strict"]
        if "timeout_sec" in agent:
            merged["timeout_sec"] = agent["timeout_sec"]
        if "max_turns" in agent:
            merged["max_turns"] = agent["max_turns"]
        if "model" in agent:
            merged["model"] = agent["model"]
        if "connection" in agent:
            merged["connection"] = agent["connection"]

    return merged