Files
tori/src/template.rs
Fam Zheng e4ba385112 refactor: worker mode — server offloads all LLM/exec to worker
- Split into `tori server` / `tori worker` subcommands (clap derive)
- Extract lib.rs for shared crate (agent, llm, exec, state, etc.)
- Introduce AgentUpdate channel to decouple agent loop from DB/broadcast
- New sink.rs: AgentUpdate enum + ServiceManager + handle_agent_updates
- New worker_runner.rs: connects to server WS, runs full agent loop
- Expand worker protocol: ServerToWorker (workflow_assign, comment)
  and WorkerToServer (register, result, update)
- Remove LLM from title generation (heuristic) and template selection
  (must be explicit)
- Remove KB tools (kb_search, kb_read) and remote worker tools
  (list_workers, execute_on_worker) from agent loop
- run_agent_loop/run_step_loop now take mpsc::Sender<AgentUpdate>
  instead of direct DB pool + broadcast sender
2026-04-06 12:54:57 +01:00

645 lines
21 KiB
Rust

use std::path::{Path, PathBuf};
use serde::Deserialize;
use crate::TemplateRepoConfig;
use crate::tools::ExternalToolManager;
#[derive(Debug, Deserialize)]
#[allow(dead_code)]
pub struct TemplateInfo {
pub name: String,
pub description: String,
pub match_hint: String,
/// If true, the agent will wait for user approval after update_plan
/// before entering the execution phase.
#[serde(default)]
pub require_plan_approval: bool,
}
#[allow(dead_code)]
pub struct LoadedTemplate {
pub id: String,
pub info: TemplateInfo,
pub instructions: String,
pub external_tools: ExternalToolManager,
pub kb_files: Vec<(String, String)>,
pub require_plan_approval: bool,
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct TemplateExample {
pub label: String,
pub text: String,
}
#[derive(Debug, Clone, serde::Serialize)]
pub struct TemplateListItem {
pub id: String,
pub name: String,
pub description: String,
pub examples: Vec<TemplateExample>,
}
// --- Template directories ---
fn builtin_dir() -> &'static str {
if Path::new("/app/templates").is_dir() {
"/app/templates"
} else {
"app-templates"
}
}
fn default_repo_dir() -> &'static str {
if Path::new("/app/oseng-templates").is_dir() {
"/app/oseng-templates"
} else {
"oseng-templates"
}
}
/// For backward compat.
pub fn templates_dir() -> &'static str {
builtin_dir()
}
// --- Scanning ---
/// Scan a local examples/ directory for .md files.
async fn scan_examples_dir(dir: &Path) -> Vec<TemplateExample> {
let mut examples = Vec::new();
let mut entries = match tokio::fs::read_dir(dir).await {
Ok(e) => e,
Err(_) => return examples,
};
while let Ok(Some(entry)) = entries.next_entry().await {
let path = entry.path();
if path.extension().and_then(|e| e.to_str()) != Some("md") {
continue;
}
let label = path
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("example")
.to_string();
if let Ok(text) = tokio::fs::read_to_string(&path).await {
examples.push(TemplateExample { label, text });
}
}
examples.sort_by(|a, b| a.label.cmp(&b.label));
examples
}
/// Scan examples from a git repo ref via `git ls-tree` + `git show`.
async fn scan_examples_git(repo: &Path, ref_name: &str, template_path: &str) -> Vec<TemplateExample> {
let examples_prefix = if template_path.is_empty() {
"examples/".to_string()
} else {
format!("{}/examples/", template_path)
};
let tree_output = match tokio::process::Command::new("git")
.args(["ls-tree", "--name-only", ref_name, &examples_prefix])
.current_dir(repo)
.output()
.await
{
Ok(o) if o.status.success() => String::from_utf8_lossy(&o.stdout).to_string(),
_ => return Vec::new(),
};
let mut examples = Vec::new();
for file_path in tree_output.lines() {
let file_path = file_path.trim();
if !file_path.ends_with(".md") {
continue;
}
let label = Path::new(file_path)
.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("example")
.to_string();
let show_ref = format!("{}:{}", ref_name, file_path);
if let Ok(o) = tokio::process::Command::new("git")
.args(["show", &show_ref])
.current_dir(repo)
.output()
.await
{
if o.status.success() {
let text = String::from_utf8_lossy(&o.stdout).to_string();
examples.push(TemplateExample { label, text });
}
}
}
examples.sort_by(|a, b| a.label.cmp(&b.label));
examples
}
/// Ensure the template repo is a git clone; clone or fetch as needed.
/// Called at startup (before serve) so readiness probe can gate on it.
pub async fn ensure_repo_ready(cfg: &TemplateRepoConfig) {
let repo = Path::new(&cfg.local_path);
if repo.join(".git").is_dir() {
tracing::info!("Template repo already cloned at {}, fetching...", repo.display());
let _ = tokio::process::Command::new("git")
.args(["fetch", "--all", "--prune", "-q"])
.current_dir(repo)
.env("GIT_SSL_NO_VERIFY", "true")
.output()
.await;
return;
}
// Not a git repo — remove stale dir (e.g. COPY'd without .git) and clone
let url = format!("{}/{}/{}.git", cfg.gitea_url, cfg.owner, cfg.repo);
tracing::info!("Cloning template repo {} → {}", url, repo.display());
let _ = tokio::fs::remove_dir_all(repo).await;
let output = tokio::process::Command::new("git")
.args(["clone", &url, &repo.to_string_lossy()])
.env("GIT_SSL_NO_VERIFY", "true")
.output()
.await;
match output {
Ok(o) if o.status.success() => {
tracing::info!("Template repo cloned successfully");
}
Ok(o) => {
tracing::error!("git clone failed: {}", String::from_utf8_lossy(&o.stderr));
}
Err(e) => {
tracing::error!("git clone error: {}", e);
}
}
}
/// List all templates from both built-in and repo (all branches).
pub async fn list_all_templates(repo_cfg: Option<&TemplateRepoConfig>) -> Vec<TemplateListItem> {
let mut items = Vec::new();
// 1. Built-in templates (flat: each top-level dir with template.json)
let builtin = Path::new(builtin_dir());
if let Ok(mut entries) = tokio::fs::read_dir(builtin).await {
while let Ok(Some(entry)) = entries.next_entry().await {
let is_dir = tokio::fs::metadata(entry.path())
.await
.map(|m| m.is_dir())
.unwrap_or(false);
if !is_dir {
continue;
}
let id = entry.file_name().to_string_lossy().to_string();
let meta_path = entry.path().join("template.json");
if let Ok(data) = tokio::fs::read_to_string(&meta_path).await {
if let Ok(info) = serde_json::from_str::<TemplateInfo>(&data) {
let examples = scan_examples_dir(&entry.path().join("examples")).await;
items.push(TemplateListItem {
id,
name: info.name,
description: info.description,
examples,
});
}
}
}
}
// 2. Repo templates (all branches, cloned at startup)
let repo_path = repo_cfg
.map(|c| c.local_path.as_str())
.unwrap_or_else(|| default_repo_dir());
let repo = Path::new(repo_path);
if repo.join(".git").is_dir() {
items.extend(scan_repo_all_branches(repo).await);
}
items.sort_by(|a, b| a.id.cmp(&b.id));
items
}
/// Scan all branches in a git repo for directories containing INSTRUCTIONS.md.
/// Template ID = "{branch}/{path}" e.g. "main/simple-npi", "fam/oncall/network-latency".
async fn scan_repo_all_branches(repo: &Path) -> Vec<TemplateListItem> {
let mut items = Vec::new();
// First: git fetch --all to get latest branches
let _ = tokio::process::Command::new("git")
.args(["fetch", "--all", "--prune", "-q"])
.current_dir(repo)
.output()
.await;
// List all remote branches: "origin/main", "origin/fam/foo" etc.
let output = match tokio::process::Command::new("git")
.args(["branch", "-r", "--format=%(refname:short)"])
.current_dir(repo)
.output()
.await
{
Ok(o) if o.status.success() => String::from_utf8_lossy(&o.stdout).to_string(),
_ => return items,
};
for line in output.lines() {
let line = line.trim();
if line.is_empty() || line.contains("HEAD") {
continue;
}
// line = "origin/main" or "origin/fam/feature"
let branch = match line.strip_prefix("origin/") {
Some(b) => b,
None => line,
};
// git ls-tree -r --name-only origin/branch
let tree_output = match tokio::process::Command::new("git")
.args(["ls-tree", "-r", "--name-only", line])
.current_dir(repo)
.output()
.await
{
Ok(o) if o.status.success() => String::from_utf8_lossy(&o.stdout).to_string(),
_ => continue,
};
for file_path in tree_output.lines() {
let file_path = file_path.trim();
if !file_path.ends_with("INSTRUCTIONS.md") {
continue;
}
// Template dir path (relative to repo root)
let template_path = if file_path == "INSTRUCTIONS.md" {
""
} else {
file_path.trim_end_matches("/INSTRUCTIONS.md")
};
let template_id = if template_path.is_empty() {
branch.to_string()
} else {
format!("{}/{}", branch, template_path)
};
// Try to read template.json via git show
let (name, description) = read_git_file_json(repo, line, template_path).await;
let examples = scan_examples_git(repo, line, template_path).await;
items.push(TemplateListItem {
id: template_id.clone(),
name: name.unwrap_or_else(|| template_id.clone()),
description: description.unwrap_or_default(),
examples,
});
}
}
items
}
/// Read template.json from a git ref via `git show`.
async fn read_git_file_json(
repo: &Path,
ref_name: &str,
template_path: &str,
) -> (Option<String>, Option<String>) {
let file = if template_path.is_empty() {
"template.json".to_string()
} else {
format!("{}/template.json", template_path)
};
let output = match tokio::process::Command::new("git")
.args(["show", &format!("{}:{}", ref_name, file)])
.current_dir(repo)
.output()
.await
{
Ok(o) if o.status.success() => String::from_utf8_lossy(&o.stdout).to_string(),
_ => return (None, None),
};
match serde_json::from_str::<TemplateInfo>(&output) {
Ok(info) => (Some(info.name), Some(info.description)),
Err(_) => (None, None),
}
}
/// Extract a template from a git branch to a local directory.
/// Uses `git archive` to extract the template subtree.
pub async fn extract_repo_template(template_id: &str, repo_cfg: Option<&TemplateRepoConfig>) -> anyhow::Result<PathBuf> {
let repo_path = repo_cfg
.map(|c| c.local_path.as_str())
.unwrap_or_else(|| default_repo_dir());
let repo = Path::new(repo_path);
let dest = PathBuf::from("/tmp/tori-repo-templates").join(template_id);
let _ = tokio::fs::remove_dir_all(&dest).await;
tokio::fs::create_dir_all(&dest).await?;
// Parse template_id: "main/simple-npi" → branch="main", path="simple-npi"
// But branch could be multi-segment: "fam/feature" → need to find split point
// Strategy: try each split, check if "origin/{prefix}" is a valid ref
let (ref_name, template_path) = resolve_branch_and_path(repo, template_id).await
.ok_or_else(|| anyhow::anyhow!("Cannot resolve template: {}", template_id))?;
// git archive <ref> <path> | tar -x -C <dest> --strip-components=N
let strip = if template_path.is_empty() {
0
} else {
template_path.matches('/').count() + 1
};
let archive_args = if template_path.is_empty() {
vec!["archive".to_string(), ref_name.clone()]
} else {
vec!["archive".to_string(), ref_name.clone(), template_path.clone()]
};
let git_archive = tokio::process::Command::new("git")
.args(&archive_args)
.current_dir(repo)
.stdout(std::process::Stdio::piped())
.spawn()?;
let child_stdout = git_archive.stdout.unwrap();
let stdio: std::process::Stdio = child_stdout.try_into()
.map_err(|e| anyhow::anyhow!("Failed to convert stdout: {:?}", e))?;
let mut tar_args = vec!["-x".to_string(), "-C".to_string(), dest.to_string_lossy().to_string()];
if strip > 0 {
tar_args.push(format!("--strip-components={}", strip));
}
let tar_output = tokio::process::Command::new("tar")
.args(&tar_args)
.stdin(stdio)
.output()
.await?;
if !tar_output.status.success() {
anyhow::bail!(
"tar extract failed: {}",
String::from_utf8_lossy(&tar_output.stderr)
);
}
// Make scripts in tools/ executable
let tools_dir = dest.join("tools");
if tools_dir.is_dir() {
if let Ok(mut entries) = tokio::fs::read_dir(&tools_dir).await {
while let Ok(Some(entry)) = entries.next_entry().await {
let path = entry.path();
if path.is_file() {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
if let Ok(meta) = tokio::fs::metadata(&path).await {
let mut perms = meta.permissions();
perms.set_mode(perms.mode() | 0o111);
let _ = tokio::fs::set_permissions(&path, perms).await;
}
}
}
}
}
}
// Make setup executable if present
let setup_file = dest.join("setup");
if setup_file.is_file() {
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
if let Ok(meta) = tokio::fs::metadata(&setup_file).await {
let mut perms = meta.permissions();
perms.set_mode(perms.mode() | 0o111);
let _ = tokio::fs::set_permissions(&setup_file, perms).await;
}
}
}
tracing::info!("Extracted repo template '{}' to {}", template_id, dest.display());
Ok(dest)
}
/// Given a template_id like "main/simple-npi" or "fam/feature/oncall",
/// figure out which part is the branch and which is the path.
async fn resolve_branch_and_path(repo: &Path, template_id: &str) -> Option<(String, String)> {
let parts: Vec<&str> = template_id.splitn(10, '/').collect();
// Try progressively longer branch names
for i in 1..parts.len() + 1 {
let candidate_branch = parts[..i].join("/");
let ref_name = format!("origin/{}", candidate_branch);
let output = tokio::process::Command::new("git")
.args(["rev-parse", "--verify", &ref_name])
.current_dir(repo)
.output()
.await;
if let Ok(o) = output {
if o.status.success() {
let path = if i < parts.len() {
parts[i..].join("/")
} else {
String::new()
};
return Some((ref_name, path));
}
}
}
None
}
/// Check if a template is from the repo (vs built-in).
pub fn is_repo_template(template_id: &str) -> bool {
// Built-in templates are flat names without '/'
// Repo templates always have branch/path format
template_id.contains('/')
}
// --- Template loading ---
/// Run the template's `setup` executable with cwd set to workdir.
/// The template directory is read-only; setup runs in the workdir to initialize
/// the workspace environment (e.g. pulling binaries, installing deps).
pub async fn run_setup(template_dir: &Path, workdir: &str) -> anyhow::Result<()> {
let setup = template_dir.join("setup");
if !setup.exists() {
return Ok(());
}
tracing::info!("Running template setup in workdir: {}", workdir);
let output = tokio::process::Command::new(&setup)
.current_dir(workdir)
.output()
.await
.map_err(|e| anyhow::anyhow!("Failed to run setup: {}", e))?;
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
if !stdout.is_empty() {
tracing::info!("setup stdout: {}", stdout.trim_end());
}
if !stderr.is_empty() {
tracing::warn!("setup stderr: {}", stderr.trim_end());
}
if !output.status.success() {
anyhow::bail!(
"Template setup failed (exit {}): {}",
output.status.code().unwrap_or(-1),
stderr.trim_end()
);
}
tracing::info!("Template setup completed successfully");
Ok(())
}
/// Copy template contents to workdir (excluding template.json, tools/, kb/, INSTRUCTIONS.md).
pub async fn apply_template(template_dir: &Path, workdir: &str) -> anyhow::Result<()> {
if !template_dir.is_dir() {
anyhow::bail!("Template directory not found: {}", template_dir.display());
}
copy_dir_recursive(template_dir, Path::new(workdir)).await
}
async fn copy_dir_recursive(src: &Path, dst: &Path) -> anyhow::Result<()> {
let mut stack: Vec<(PathBuf, PathBuf, bool)> =
vec![(src.to_path_buf(), dst.to_path_buf(), true)];
while let Some((src_dir, dst_dir, top_level)) = stack.pop() {
tokio::fs::create_dir_all(&dst_dir).await?;
let mut entries = tokio::fs::read_dir(&src_dir).await?;
while let Some(entry) = entries.next_entry().await? {
let name = entry.file_name();
let name_str = name.to_string_lossy();
if top_level
&& (name_str == "template.json"
|| name_str == "tools"
|| name_str == "kb"
|| name_str == "examples"
|| name_str == "setup"
|| name_str == "INSTRUCTIONS.md")
{
continue;
}
let src_path = entry.path();
let dst_path = dst_dir.join(&name);
if entry.file_type().await?.is_dir() {
stack.push((src_path, dst_path, false));
} else {
tokio::fs::copy(&src_path, &dst_path).await?;
}
}
}
Ok(())
}
impl LoadedTemplate {
pub async fn load_from_dir(template_id: &str, base: &Path) -> anyhow::Result<Self> {
if !base.is_dir() {
anyhow::bail!("Template directory not found: {}", base.display());
}
let meta_path = base.join("template.json");
let info = if let Ok(data) = tokio::fs::read_to_string(&meta_path).await {
serde_json::from_str::<TemplateInfo>(&data).unwrap_or_else(|_| TemplateInfo {
name: template_id.to_string(),
description: String::new(),
match_hint: String::new(),
require_plan_approval: false,
})
} else {
TemplateInfo {
name: template_id.to_string(),
description: String::new(),
match_hint: String::new(),
require_plan_approval: false,
}
};
let instructions = tokio::fs::read_to_string(base.join("INSTRUCTIONS.md"))
.await
.unwrap_or_default();
let external_tools = ExternalToolManager::discover(base).await;
tracing::info!("Template '{}': {} external tools", template_id, external_tools.len());
let kb_dir = base.join("kb");
let kb_files = scan_kb_files(&kb_dir).await;
tracing::info!("Template '{}': {} KB files", template_id, kb_files.len());
let require_plan_approval = info.require_plan_approval;
Ok(Self {
id: template_id.to_string(),
info,
instructions,
external_tools,
kb_files,
require_plan_approval,
})
}
pub async fn load(template_id: &str) -> anyhow::Result<Self> {
let base = Path::new(builtin_dir()).join(template_id);
Self::load_from_dir(template_id, &base).await
}
}
async fn scan_kb_files(kb_dir: &Path) -> Vec<(String, String)> {
let mut results = Vec::new();
let mut entries = match tokio::fs::read_dir(kb_dir).await {
Ok(e) => e,
Err(_) => return results,
};
while let Ok(Some(entry)) = entries.next_entry().await {
let path = entry.path();
let real_path = match tokio::fs::canonicalize(&path).await {
Ok(p) => p,
Err(_) => path.clone(),
};
let ext = real_path.extension().and_then(|e| e.to_str()).unwrap_or("");
if ext != "md" {
continue;
}
let content = match tokio::fs::read_to_string(&real_path).await {
Ok(c) => c,
Err(e) => {
tracing::warn!("Failed to read KB file {}: {}", real_path.display(), e);
continue;
}
};
let title = content
.lines()
.find(|l| l.starts_with("# "))
.map(|l| l.trim_start_matches("# ").trim().to_string())
.unwrap_or_else(|| {
path.file_stem()
.and_then(|s| s.to_str())
.unwrap_or("untitled")
.to_string()
});
results.push((title, content));
}
results
}