"""Harbor installed-agent adapter for Zed's eval-cli binary.""" import json import os import shlex import tomllib from pathlib import Path from harbor.agents.installed.base import BaseInstalledAgent, with_prompt_template from harbor.environments.base import BaseEnvironment from harbor.models.agent.context import AgentContext from .agent_common import ( add_anthropic_available_models_env, add_openai_compatible_provider_env, add_zed_eval_env, detect_workdir, eval_cli_with_log_command, patch_command, populate_context_from_result, provider_api_env, ) # Leave eval-cli time to exit cleanly and flush logs before Harbor's outer # asyncio.wait_for can kill the agent coroutine. EVAL_CLI_FINALIZE_BUFFER_SEC = 45 EVAL_CLI_MIN_TIMEOUT_SEC = 60 class ZedAgent(BaseInstalledAgent): def __init__( self, logs_dir: Path, binary_path: str | None = None, download_url: str | None = None, *args, **kwargs, ): super().__init__(logs_dir, *args, **kwargs) self._binary_path = binary_path self._download_url = download_url or os.environ.get("EVAL_CLI_DOWNLOAD_URL") @staticmethod def name() -> str: return "zed" async def _detect_workdir(self, environment: BaseEnvironment) -> str: return await detect_workdir( environment, self.exec_as_agent, self._extra_env.get, "Could not detect a working directory in the container. " "Set EVAL_CLI_WORKDIR explicitly via --ae EVAL_CLI_WORKDIR=/path/to/repo", ) async def install(self, environment: BaseEnvironment) -> None: # Detect the package manager and install base dependencies. # Supports Debian/Ubuntu (apt-get), Alpine (apk), and # Fedora/RHEL/CentOS (dnf/yum). await self.exec_as_root( environment, command=( "if command -v apt-get >/dev/null 2>&1; then " " apt-get update && " " apt-get install -y --no-install-recommends ca-certificates curl git; " "elif command -v apk >/dev/null 2>&1; then " " apk add --no-cache ca-certificates curl git bash coreutils gcompat libstdc++; " "elif command -v dnf >/dev/null 2>&1; then " " dnf install -y ca-certificates curl git; " "elif command -v yum >/dev/null 2>&1; then " " yum install -y ca-certificates curl git; " "else " " echo 'WARNING: No supported package manager found (apt-get, apk, dnf, yum)' >&2; " "fi" ), env={"DEBIAN_FRONTEND": "noninteractive"}, ) # Tooling setup is best-effort; benchmark images vary enough that one # unsupported runtime should not fail the trial before eval-cli starts. await self._install_node(environment) await self._install_lsps(environment) await self._install_uv_and_ruff(environment) # Modal can mount a prebuilt binary inside each sandbox, avoiding uploads. container_path = self._extra_env.get("EVAL_CLI_CONTAINER_PATH") if container_path: await self.exec_as_root( environment, command=( f"cp {shlex.quote(container_path)} /usr/local/bin/eval-cli && " "chmod +x /usr/local/bin/eval-cli && " "eval-cli --help" ), ) return if self._binary_path: binary = Path(self._binary_path) if not binary.exists(): raise FileNotFoundError( f"eval-cli binary not found at {binary}. " "Build it with: cargo build --release -p eval_cli" ) await environment.upload_file( source_path=binary, target_path="/usr/local/bin/eval-cli", ) await self.exec_as_root( environment, command="chmod +x /usr/local/bin/eval-cli && eval-cli --help", ) return if self._download_url: await self.exec_as_root( environment, command=( f"curl -fsSL {shlex.quote(self._download_url)} " "-o /usr/local/bin/eval-cli && " "chmod +x /usr/local/bin/eval-cli && " "eval-cli --help" ), ) return raise ValueError( "No eval-cli binary provided. " "Either pass binary_path=/path/to/target/release/eval-cli, " "set download_url=/EVAL_CLI_DOWNLOAD_URL, " "or set --ae EVAL_CLI_CONTAINER_PATH=/path/inside/container." ) async def _install_node(self, environment: BaseEnvironment) -> None: """Install Node.js from official binary tarballs. Uses the musl build on Alpine and the glibc build elsewhere. Skips if node is already on PATH. """ try: await self.exec_as_root( environment, command=( "if command -v node >/dev/null 2>&1; then " ' echo "Node.js already available: $(node --version)"; ' "else " " NODE_VER=v22.14.0; " " ARCH=$(uname -m); " ' case "$ARCH" in ' " x86_64) NODE_ARCH=x64 ;; " " aarch64) NODE_ARCH=arm64 ;; " ' *) echo "WARNING: unsupported arch $ARCH for Node.js" >&2; exit 0 ;; ' " esac; " " if ldd /bin/sh 2>&1 | grep -qi musl; then " ' NODE_URL="https://unofficial-builds.nodejs.org/download/release/${NODE_VER}/node-${NODE_VER}-linux-${NODE_ARCH}-musl.tar.gz"; ' " else " ' NODE_URL="https://nodejs.org/dist/${NODE_VER}/node-${NODE_VER}-linux-${NODE_ARCH}.tar.gz"; ' " fi; " ' echo "Downloading Node.js from $NODE_URL"; ' ' curl -fsSL "$NODE_URL" | tar -xz -C /usr/local --strip-components=1; ' ' echo "Installed Node.js $(node --version)"; ' "fi" ), ) except Exception as exc: self.logger.warning("Node.js installation failed (non-fatal): %s", exc) async def _install_lsps(self, environment: BaseEnvironment) -> None: """Pre-install language servers so Zed doesn't download them at runtime. Each LSP is installed independently so one failure doesn't block the rest. """ # npm-based LSPs — skip all if npm is not available. try: await self.exec_as_agent( environment, command="command -v npm >/dev/null 2>&1", ) except Exception: self.logger.warning("npm not available — skipping npm-based LSP installs") return lsp_installs = [ ( "basedpyright", 'DIR="$ZED_DATA_DIR/languages/basedpyright"; ' 'mkdir -p "$DIR" && npm install --prefix "$DIR" --save-exact basedpyright', ), ( "typescript-language-server", 'DIR="$ZED_DATA_DIR/languages/typescript-language-server"; ' 'mkdir -p "$DIR" && npm install --prefix "$DIR" --save-exact typescript typescript-language-server', ), ( "vtsls", 'DIR="$ZED_DATA_DIR/languages/vtsls"; ' 'mkdir -p "$DIR" && npm install --prefix "$DIR" --save-exact @vtsls/language-server typescript', ), ( "tailwindcss-language-server", 'DIR="$ZED_DATA_DIR/languages/tailwindcss-language-server"; ' 'mkdir -p "$DIR" && npm install --prefix "$DIR" --save-exact @tailwindcss/language-server', ), ] for name, cmd in lsp_installs: try: await self.exec_as_agent( environment, command=( 'ZED_DATA_DIR="${XDG_DATA_HOME:-$HOME/.local/share}/zed"; ' + cmd ), ) except Exception as exc: self.logger.warning( "LSP install '%s' failed (non-fatal): %s", name, exc ) # eslint — downloaded from GitHub and compiled separately. try: await self.exec_as_agent( environment, command=( "set -euo pipefail; " 'ZED_DATA_DIR="${XDG_DATA_HOME:-$HOME/.local/share}/zed"; ' 'ESLINT_DIR="$ZED_DATA_DIR/languages/eslint/vscode-eslint-2.4.4"; ' 'mkdir -p "$ESLINT_DIR"; ' 'curl -fsSL "https://github.com/zed-industries/vscode-eslint/archive/refs/tags/release/2.4.4.tar.gz" ' '| tar -xz -C "$ESLINT_DIR"; ' 'mv "$ESLINT_DIR"/vscode-eslint-release-2.4.4 "$ESLINT_DIR/vscode-eslint"; ' 'cd "$ESLINT_DIR/vscode-eslint" && npm install && npm run compile' ), ) except Exception as exc: self.logger.warning("eslint LSP install failed (non-fatal): %s", exc) # gopls — only when Go is present. Guarded by a 120s timeout so slow # compilation can never eat the full setup budget. gopls_script = ( "if command -v go >/dev/null 2>&1; then " "if go install golang.org/x/tools/gopls@latest 2>/dev/null; then " "echo 'Installed gopls@latest'; " "else " ' MY_GO=$(go env GOVERSION | sed "s/^go//"); ' " for v in $(curl -fsSL " "https://proxy.golang.org/golang.org/x/tools/gopls/@v/list 2>/dev/null" " | grep -E '^v[0-9]+\\.[0-9]+\\.[0-9]+$' | sort -rV | head -5); do " " NEED=$(curl -fsSL " '"https://proxy.golang.org/golang.org/x/tools/gopls/@v/${v}.mod"' " 2>/dev/null | awk '/^go /{print $2; exit}'); " ' if [ -n "$NEED" ] ' ' && [ "$(printf \'%s\\n%s\\n\' "$NEED" "$MY_GO" ' ' | sort -V | head -1)" = "$NEED" ]; then ' ' echo "Installing gopls $v (compatible with Go $MY_GO)"; ' ' go install "golang.org/x/tools/gopls@$v" && break; ' " fi; " " done; " "fi; " "fi" ) try: await self.exec_as_agent( environment, command=( "timeout 120 bash -c " + shlex.quote(gopls_script) + " || echo 'WARNING: gopls installation timed out or failed -- skipping'" ), ) except Exception as exc: self.logger.warning("gopls install failed (non-fatal): %s", exc) async def _install_uv_and_ruff(self, environment: BaseEnvironment) -> None: """Install uv and ruff for Python tooling.""" try: await self.exec_as_agent( environment, command=( "curl -LsSf https://astral.sh/uv/install.sh | sh && " '. "$HOME/.local/bin/env"' ), ) agent_home_result = await self.exec_as_agent( environment, command='printf %s "$HOME"', ) agent_home = agent_home_result.stdout.strip() if not agent_home: self.logger.warning( "Could not determine agent home directory — skipping uv symlinks" ) return await self.exec_as_root( environment, command=( f"ln -sf {shlex.quote(agent_home + '/.local/bin/uv')} /usr/local/bin/uv && " f"ln -sf {shlex.quote(agent_home + '/.local/bin/uvx')} /usr/local/bin/uvx" ), ) await self.exec_as_agent( environment, command='export PATH="$HOME/.local/bin:$PATH" && uv tool install ruff', ) except Exception as exc: self.logger.warning("uv/ruff installation failed (non-fatal): %s", exc) def populate_context_post_run(self, context: AgentContext) -> None: populate_context_from_result(self.logs_dir, context, self.logger) def _get_api_env(self) -> dict[str, str]: env = provider_api_env(self.model_name) add_openai_compatible_provider_env( env, self._extra_env.get("ZED_OPENAI_COMPATIBLE_PROVIDERS") ) add_anthropic_available_models_env( env, self._extra_env.get("ZED_ANTHROPIC_AVAILABLE_MODELS") ) return env def _harbor_agent_budget_sec(self) -> float | None: """Recover Harbor's hidden outer timeout so eval-cli can exit first. Harbor 0.15.0 computes this on ``Trial`` but does not pass it to custom installed agents. Replaying the calculation host-side lets us preserve trajectories instead of losing them to Harbor's coroutine kill. """ config_path = Path(self.logs_dir).parent / "config.json" try: trial_config = json.loads(config_path.read_text()) except (OSError, ValueError) as exc: print( f"[zed-eval] agent budget: could not read {config_path}: {exc!r}", flush=True, ) return None agent_cfg = trial_config.get("agent") or {} base_sec = agent_cfg.get("override_timeout_sec") if base_sec is None: base_sec = self._task_declared_agent_timeout_sec( trial_config.get("task") or {} ) if base_sec is None: return None max_sec = agent_cfg.get("max_timeout_sec") if max_sec is not None: base_sec = min(base_sec, max_sec) multiplier = trial_config.get("agent_timeout_multiplier") if multiplier is None: multiplier = trial_config.get("timeout_multiplier", 1.0) return float(base_sec) * float(multiplier) def _task_declared_agent_timeout_sec(self, task_cfg: dict) -> float | None: task_dir = self._resolve_task_dir(task_cfg) if task_dir is None: return None task_toml = task_dir / "task.toml" try: data = tomllib.loads(task_toml.read_text()) except (OSError, ValueError) as exc: print( f"[zed-eval] agent budget: could not parse {task_toml}: {exc!r}", flush=True, ) return None timeout_sec = (data.get("agent") or {}).get("timeout_sec") return float(timeout_sec) if timeout_sec is not None else None def _resolve_task_dir(self, task_cfg: dict) -> Path | None: path = task_cfg.get("path") if path: return Path(path).expanduser().resolve() # config.json omits Harbor's package content hash; use the newest cached # digest that contains a task.toml. name = task_cfg.get("name") if name and "/" in name: try: from harbor.constants import PACKAGE_CACHE_DIR except ImportError as exc: print( f"[zed-eval] agent budget: harbor PACKAGE_CACHE_DIR " f"unavailable: {exc!r}", flush=True, ) return None download_dir = task_cfg.get("download_dir") base_dir = Path(download_dir) if download_dir else PACKAGE_CACHE_DIR org, package = name.split("/", 1) package_root = base_dir / org / package candidates = [ digest_dir for digest_dir in package_root.glob("*") if (digest_dir / "task.toml").is_file() ] if not candidates: return None return max(candidates, key=lambda p: p.stat().st_mtime) return None @with_prompt_template async def run( self, instruction: str, environment: BaseEnvironment, context: AgentContext ) -> None: escaped_instruction = shlex.quote(instruction) env = self._get_api_env() add_zed_eval_env( env, self._extra_env, exclude={"ZED_EVAL_INSTRUCTION_SUFFIX_FILE"} ) workdir = await self._detect_workdir(environment) parts = [ "eval-cli", f"--workdir {shlex.quote(workdir)}", "--output-dir /logs/agent", ] if self.model_name: parts.append(f"--model {shlex.quote(self.model_name)}") instruction_suffix_file = self._extra_env.get( "ZED_EVAL_INSTRUCTION_SUFFIX_FILE" ) if instruction_suffix_file: parts.append( f"--instruction-suffix-file {shlex.quote(instruction_suffix_file)}" ) # Prefer eval-cli's own timeout over Harbor's coroutine kill so logs and # partial answers are still available for delivery. configured_timeout_raw = self._extra_env.get("EVAL_CLI_TIMEOUT") configured_timeout: int | None if configured_timeout_raw: try: configured_timeout = int(configured_timeout_raw) except ValueError: print( "[zed-eval] ignoring non-integer EVAL_CLI_TIMEOUT=" f"{configured_timeout_raw!r}", flush=True, ) configured_timeout = None else: configured_timeout = None budget_sec = self._harbor_agent_budget_sec() timeout_arg: int | None = None if budget_sec is not None: ceiling = ( configured_timeout if configured_timeout is not None else int(budget_sec) ) timeout_arg = min(ceiling, int(budget_sec - EVAL_CLI_FINALIZE_BUFFER_SEC)) timeout_arg = max(timeout_arg, EVAL_CLI_MIN_TIMEOUT_SEC) print( f"[zed-eval] eval-cli --timeout {timeout_arg}s (harbor agent " f"budget {int(budget_sec)}s - {EVAL_CLI_FINALIZE_BUFFER_SEC}s " f"finalize buffer; EVAL_CLI_TIMEOUT=" f"{configured_timeout if configured_timeout is not None else 'unset'})", flush=True, ) elif configured_timeout is not None: timeout_arg = configured_timeout print( f"[zed-eval] eval-cli --timeout {timeout_arg}s (harbor agent " "budget unavailable; using EVAL_CLI_TIMEOUT)", flush=True, ) if timeout_arg is not None: parts.append(f"--timeout {timeout_arg}") staff = self._extra_env.get("EVAL_CLI_STAFF") if staff and staff.lower() == "false": parts.append("--no-staff") reasoning_effort = self._extra_env.get("EVAL_CLI_REASONING_EFFORT") if reasoning_effort: parts.append(f"--reasoning-effort {shlex.quote(reasoning_effort)}") enable_thinking = self._extra_env.get("EVAL_CLI_ENABLE_THINKING") if enable_thinking: if enable_thinking.lower() == "true": parts.append("--thinking true") elif enable_thinking.lower() == "false": parts.append("--thinking false") parts.append(f"--instruction {escaped_instruction}") # Exit 2 is eval-cli's timeout, not a crash; keep the trajectory judgeable. await self.exec_as_agent( environment, command=eval_cli_with_log_command( parts, "/logs/agent/eval-cli.txt", timeout_message=( "[zed-eval] eval-cli timed out (exit 2); continuing to " "delivery/verification" ), line_buffered=True, ), env=env, ) # Modal-style remote runs can differ in agent-log collection behavior. await self.exec_as_agent( environment, command=( "mkdir -p /logs/artifacts && " "for f in result.json thread.json thread.md eval-cli.txt; do " ' cp "/logs/agent/$f" /logs/artifacts/ 2>/dev/null || true; ' "done; true" ), ) # Some harnesses mount an initialized repo before creating the first commit. await self.exec_as_agent( environment, command=patch_command("/logs/agent"), cwd=workdir, )