feat: multi-branch template scanning from git repo + manual template selection
- Rewrite template.rs to scan all remote branches via git commands (git fetch/branch -r/ls-tree/git show/git archive) - Add manual template picker dropdown in CreateForm UI - Remove sentence-transformers/embed.py from Dockerfile (separate container) - Clean up Gitea API approach, use local git repo instead - Add chat panel and sidebar layout improvements
This commit is contained in:
Fam Zheng
415
src/template.rs
415
src/template.rs
@@ -1,4 +1,4 @@
|
||||
use std::path::Path;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
@@ -6,6 +6,7 @@ use crate::llm::{ChatMessage, LlmClient};
|
||||
use crate::tools::ExternalToolManager;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[allow(dead_code)]
|
||||
pub struct TemplateInfo {
|
||||
pub name: String,
|
||||
pub description: String,
|
||||
@@ -21,7 +22,16 @@ pub struct LoadedTemplate {
|
||||
pub kb_files: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
pub fn templates_dir() -> &'static str {
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct TemplateListItem {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub description: String,
|
||||
}
|
||||
|
||||
// --- Template directories ---
|
||||
|
||||
fn builtin_dir() -> &'static str {
|
||||
if Path::new("/app/templates").is_dir() {
|
||||
"/app/templates"
|
||||
} else {
|
||||
@@ -29,45 +39,295 @@ pub fn templates_dir() -> &'static str {
|
||||
}
|
||||
}
|
||||
|
||||
/// Scan available templates and ask LLM to pick one (or none).
|
||||
pub async fn select_template(llm: &LlmClient, requirement: &str) -> Option<String> {
|
||||
let base = Path::new(templates_dir());
|
||||
let mut entries = match tokio::fs::read_dir(base).await {
|
||||
Ok(e) => e,
|
||||
Err(_) => return None,
|
||||
};
|
||||
fn repo_dir() -> &'static str {
|
||||
if Path::new("/app/oseng-templates").is_dir() {
|
||||
"/app/oseng-templates"
|
||||
} else {
|
||||
"oseng-templates"
|
||||
}
|
||||
}
|
||||
|
||||
let mut templates: Vec<(String, TemplateInfo)> = Vec::new();
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
// Use metadata() instead of file_type() to follow symlinks
|
||||
let is_dir = tokio::fs::metadata(entry.path())
|
||||
.await
|
||||
.map(|m| m.is_dir())
|
||||
.unwrap_or(false);
|
||||
if !is_dir {
|
||||
continue;
|
||||
}
|
||||
let id = entry.file_name().to_string_lossy().to_string();
|
||||
let meta_path = entry.path().join("template.json");
|
||||
if let Ok(data) = tokio::fs::read_to_string(&meta_path).await {
|
||||
if let Ok(info) = serde_json::from_str::<TemplateInfo>(&data) {
|
||||
templates.push((id, info));
|
||||
/// For backward compat.
|
||||
pub fn templates_dir() -> &'static str {
|
||||
builtin_dir()
|
||||
}
|
||||
|
||||
// --- Scanning ---
|
||||
|
||||
/// List all templates from both built-in and repo (all branches).
|
||||
pub async fn list_all_templates() -> Vec<TemplateListItem> {
|
||||
let mut items = Vec::new();
|
||||
|
||||
// 1. Built-in templates (flat: each top-level dir with template.json)
|
||||
let builtin = Path::new(builtin_dir());
|
||||
if let Ok(mut entries) = tokio::fs::read_dir(builtin).await {
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
let is_dir = tokio::fs::metadata(entry.path())
|
||||
.await
|
||||
.map(|m| m.is_dir())
|
||||
.unwrap_or(false);
|
||||
if !is_dir {
|
||||
continue;
|
||||
}
|
||||
let id = entry.file_name().to_string_lossy().to_string();
|
||||
let meta_path = entry.path().join("template.json");
|
||||
if let Ok(data) = tokio::fs::read_to_string(&meta_path).await {
|
||||
if let Ok(info) = serde_json::from_str::<TemplateInfo>(&data) {
|
||||
items.push(TemplateListItem {
|
||||
id,
|
||||
name: info.name,
|
||||
description: info.description,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if templates.is_empty() {
|
||||
// 2. Repo templates (all branches, find INSTRUCTIONS.md via git ls-tree)
|
||||
let repo = Path::new(repo_dir());
|
||||
if repo.join(".git").is_dir() {
|
||||
items.extend(scan_repo_all_branches(repo).await);
|
||||
}
|
||||
|
||||
items.sort_by(|a, b| a.id.cmp(&b.id));
|
||||
items
|
||||
}
|
||||
|
||||
/// Scan all branches in a git repo for directories containing INSTRUCTIONS.md.
|
||||
/// Template ID = "{branch}/{path}" e.g. "main/simple-npi", "fam/oncall/network-latency".
|
||||
async fn scan_repo_all_branches(repo: &Path) -> Vec<TemplateListItem> {
|
||||
let mut items = Vec::new();
|
||||
|
||||
// First: git fetch --all to get latest branches
|
||||
let _ = tokio::process::Command::new("git")
|
||||
.args(["fetch", "--all", "--prune", "-q"])
|
||||
.current_dir(repo)
|
||||
.output()
|
||||
.await;
|
||||
|
||||
// List all remote branches: "origin/main", "origin/fam/foo" etc.
|
||||
let output = match tokio::process::Command::new("git")
|
||||
.args(["branch", "-r", "--format=%(refname:short)"])
|
||||
.current_dir(repo)
|
||||
.output()
|
||||
.await
|
||||
{
|
||||
Ok(o) if o.status.success() => String::from_utf8_lossy(&o.stdout).to_string(),
|
||||
_ => return items,
|
||||
};
|
||||
|
||||
for line in output.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.contains("HEAD") {
|
||||
continue;
|
||||
}
|
||||
// line = "origin/main" or "origin/fam/feature"
|
||||
let branch = match line.strip_prefix("origin/") {
|
||||
Some(b) => b,
|
||||
None => line,
|
||||
};
|
||||
|
||||
// git ls-tree -r --name-only origin/branch
|
||||
let tree_output = match tokio::process::Command::new("git")
|
||||
.args(["ls-tree", "-r", "--name-only", line])
|
||||
.current_dir(repo)
|
||||
.output()
|
||||
.await
|
||||
{
|
||||
Ok(o) if o.status.success() => String::from_utf8_lossy(&o.stdout).to_string(),
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
for file_path in tree_output.lines() {
|
||||
let file_path = file_path.trim();
|
||||
if !file_path.ends_with("INSTRUCTIONS.md") {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Template dir path (relative to repo root)
|
||||
let template_path = if file_path == "INSTRUCTIONS.md" {
|
||||
""
|
||||
} else {
|
||||
file_path.trim_end_matches("/INSTRUCTIONS.md")
|
||||
};
|
||||
|
||||
let template_id = if template_path.is_empty() {
|
||||
branch.to_string()
|
||||
} else {
|
||||
format!("{}/{}", branch, template_path)
|
||||
};
|
||||
|
||||
// Try to read template.json via git show
|
||||
let (name, description) = read_git_file_json(repo, line, template_path).await;
|
||||
|
||||
items.push(TemplateListItem {
|
||||
id: template_id.clone(),
|
||||
name: name.unwrap_or_else(|| template_id.clone()),
|
||||
description: description.unwrap_or_default(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
items
|
||||
}
|
||||
|
||||
/// Read template.json from a git ref via `git show`.
|
||||
async fn read_git_file_json(
|
||||
repo: &Path,
|
||||
ref_name: &str,
|
||||
template_path: &str,
|
||||
) -> (Option<String>, Option<String>) {
|
||||
let file = if template_path.is_empty() {
|
||||
"template.json".to_string()
|
||||
} else {
|
||||
format!("{}/template.json", template_path)
|
||||
};
|
||||
|
||||
let output = match tokio::process::Command::new("git")
|
||||
.args(["show", &format!("{}:{}", ref_name, file)])
|
||||
.current_dir(repo)
|
||||
.output()
|
||||
.await
|
||||
{
|
||||
Ok(o) if o.status.success() => String::from_utf8_lossy(&o.stdout).to_string(),
|
||||
_ => return (None, None),
|
||||
};
|
||||
|
||||
match serde_json::from_str::<TemplateInfo>(&output) {
|
||||
Ok(info) => (Some(info.name), Some(info.description)),
|
||||
Err(_) => (None, None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract a template from a git branch to a local directory.
|
||||
/// Uses `git archive` to extract the template subtree.
|
||||
pub async fn extract_repo_template(template_id: &str) -> anyhow::Result<PathBuf> {
|
||||
let repo = Path::new(repo_dir());
|
||||
let dest = PathBuf::from("/tmp/tori-repo-templates").join(template_id);
|
||||
let _ = tokio::fs::remove_dir_all(&dest).await;
|
||||
tokio::fs::create_dir_all(&dest).await?;
|
||||
|
||||
// Parse template_id: "main/simple-npi" → branch="main", path="simple-npi"
|
||||
// But branch could be multi-segment: "fam/feature" → need to find split point
|
||||
// Strategy: try each split, check if "origin/{prefix}" is a valid ref
|
||||
let (ref_name, template_path) = resolve_branch_and_path(repo, template_id).await
|
||||
.ok_or_else(|| anyhow::anyhow!("Cannot resolve template: {}", template_id))?;
|
||||
|
||||
// git archive <ref> <path> | tar -x -C <dest> --strip-components=N
|
||||
let strip = if template_path.is_empty() {
|
||||
0
|
||||
} else {
|
||||
template_path.matches('/').count() + 1
|
||||
};
|
||||
|
||||
let archive_args = if template_path.is_empty() {
|
||||
vec!["archive".to_string(), ref_name.clone()]
|
||||
} else {
|
||||
vec!["archive".to_string(), ref_name.clone(), template_path.clone()]
|
||||
};
|
||||
|
||||
let git_archive = tokio::process::Command::new("git")
|
||||
.args(&archive_args)
|
||||
.current_dir(repo)
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.spawn()?;
|
||||
|
||||
let child_stdout = git_archive.stdout.unwrap();
|
||||
let stdio: std::process::Stdio = child_stdout.try_into()
|
||||
.map_err(|e| anyhow::anyhow!("Failed to convert stdout: {:?}", e))?;
|
||||
|
||||
let mut tar_args = vec!["-x".to_string(), "-C".to_string(), dest.to_string_lossy().to_string()];
|
||||
if strip > 0 {
|
||||
tar_args.push(format!("--strip-components={}", strip));
|
||||
}
|
||||
|
||||
let tar_output = tokio::process::Command::new("tar")
|
||||
.args(&tar_args)
|
||||
.stdin(stdio)
|
||||
.output()
|
||||
.await?;
|
||||
|
||||
if !tar_output.status.success() {
|
||||
anyhow::bail!(
|
||||
"tar extract failed: {}",
|
||||
String::from_utf8_lossy(&tar_output.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
// Make scripts in tools/ executable
|
||||
let tools_dir = dest.join("tools");
|
||||
if tools_dir.is_dir() {
|
||||
if let Ok(mut entries) = tokio::fs::read_dir(&tools_dir).await {
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
let path = entry.path();
|
||||
if path.is_file() {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
if let Ok(meta) = tokio::fs::metadata(&path).await {
|
||||
let mut perms = meta.permissions();
|
||||
perms.set_mode(perms.mode() | 0o111);
|
||||
let _ = tokio::fs::set_permissions(&path, perms).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!("Extracted repo template '{}' to {}", template_id, dest.display());
|
||||
Ok(dest)
|
||||
}
|
||||
|
||||
/// Given a template_id like "main/simple-npi" or "fam/feature/oncall",
|
||||
/// figure out which part is the branch and which is the path.
|
||||
async fn resolve_branch_and_path(repo: &Path, template_id: &str) -> Option<(String, String)> {
|
||||
let parts: Vec<&str> = template_id.splitn(10, '/').collect();
|
||||
|
||||
// Try progressively longer branch names
|
||||
for i in 1..parts.len() + 1 {
|
||||
let candidate_branch = parts[..i].join("/");
|
||||
let ref_name = format!("origin/{}", candidate_branch);
|
||||
|
||||
let output = tokio::process::Command::new("git")
|
||||
.args(["rev-parse", "--verify", &ref_name])
|
||||
.current_dir(repo)
|
||||
.output()
|
||||
.await;
|
||||
|
||||
if let Ok(o) = output {
|
||||
if o.status.success() {
|
||||
let path = if i < parts.len() {
|
||||
parts[i..].join("/")
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
return Some((ref_name, path));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
/// Check if a template is from the repo (vs built-in).
|
||||
pub fn is_repo_template(template_id: &str) -> bool {
|
||||
// Built-in templates are flat names without '/'
|
||||
// Repo templates always have branch/path format
|
||||
template_id.contains('/')
|
||||
}
|
||||
|
||||
// --- LLM template selection ---
|
||||
|
||||
pub async fn select_template(llm: &LlmClient, requirement: &str) -> Option<String> {
|
||||
let all = list_all_templates().await;
|
||||
if all.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let listing: String = templates
|
||||
let listing: String = all
|
||||
.iter()
|
||||
.map(|(id, info)| {
|
||||
format!(
|
||||
"- id: {}\n 名称: {}\n 描述: {}\n 适用场景: {}",
|
||||
id, info.name, info.description, info.match_hint
|
||||
)
|
||||
})
|
||||
.map(|t| format!("- id: {}\n 名称: {}\n 描述: {}", t.id, t.name, t.description))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
|
||||
@@ -86,34 +346,26 @@ pub async fn select_template(llm: &LlmClient, requirement: &str) -> Option<Strin
|
||||
|
||||
let answer = response.trim().to_lowercase();
|
||||
tracing::info!("Template selection LLM response: '{}' (available: {:?})",
|
||||
answer, templates.iter().map(|(id, _)| id.as_str()).collect::<Vec<_>>());
|
||||
answer, all.iter().map(|t| t.id.as_str()).collect::<Vec<_>>());
|
||||
if answer == "none" {
|
||||
return None;
|
||||
}
|
||||
|
||||
// Verify the answer matches an actual template ID
|
||||
let result = templates
|
||||
.iter()
|
||||
.find(|(id, _)| id == &answer)
|
||||
.map(|(id, _)| id.clone());
|
||||
if result.is_none() {
|
||||
tracing::warn!("Template selection: LLM returned '{}' which doesn't match any template ID", answer);
|
||||
}
|
||||
result
|
||||
all.iter().find(|t| t.id == answer).map(|t| t.id.clone())
|
||||
}
|
||||
|
||||
/// Copy template contents to workdir (excluding template.json, tools/, kb/).
|
||||
pub async fn apply_template(template_id: &str, workdir: &str) -> anyhow::Result<()> {
|
||||
let src = Path::new(templates_dir()).join(template_id);
|
||||
if !src.is_dir() {
|
||||
anyhow::bail!("Template directory not found: {}", template_id);
|
||||
// --- Template loading ---
|
||||
|
||||
/// Copy template contents to workdir (excluding template.json, tools/, kb/, INSTRUCTIONS.md).
|
||||
pub async fn apply_template(template_dir: &Path, workdir: &str) -> anyhow::Result<()> {
|
||||
if !template_dir.is_dir() {
|
||||
anyhow::bail!("Template directory not found: {}", template_dir.display());
|
||||
}
|
||||
copy_dir_recursive(&src, Path::new(workdir)).await
|
||||
copy_dir_recursive(template_dir, Path::new(workdir)).await
|
||||
}
|
||||
|
||||
/// Recursively copy directory contents, skipping template.json/tools/kb at the top level.
|
||||
async fn copy_dir_recursive(src: &Path, dst: &Path) -> anyhow::Result<()> {
|
||||
let mut stack: Vec<(std::path::PathBuf, std::path::PathBuf, bool)> =
|
||||
let mut stack: Vec<(PathBuf, PathBuf, bool)> =
|
||||
vec![(src.to_path_buf(), dst.to_path_buf(), true)];
|
||||
|
||||
while let Some((src_dir, dst_dir, top_level)) = stack.pop() {
|
||||
@@ -123,8 +375,11 @@ async fn copy_dir_recursive(src: &Path, dst: &Path) -> anyhow::Result<()> {
|
||||
let name = entry.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
|
||||
// Skip metadata/convention dirs at top level
|
||||
if top_level && (name_str == "template.json" || name_str == "tools" || name_str == "kb")
|
||||
if top_level
|
||||
&& (name_str == "template.json"
|
||||
|| name_str == "tools"
|
||||
|| name_str == "kb"
|
||||
|| name_str == "INSTRUCTIONS.md")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
@@ -143,41 +398,37 @@ async fn copy_dir_recursive(src: &Path, dst: &Path) -> anyhow::Result<()> {
|
||||
}
|
||||
|
||||
impl LoadedTemplate {
|
||||
/// Load a template: discover tools, read KB files, read instructions.
|
||||
pub async fn load(template_id: &str) -> anyhow::Result<Self> {
|
||||
let base = Path::new(templates_dir()).join(template_id);
|
||||
pub async fn load_from_dir(template_id: &str, base: &Path) -> anyhow::Result<Self> {
|
||||
if !base.is_dir() {
|
||||
anyhow::bail!("Template directory not found: {}", template_id);
|
||||
anyhow::bail!("Template directory not found: {}", base.display());
|
||||
}
|
||||
|
||||
// Read template.json
|
||||
let meta_path = base.join("template.json");
|
||||
let meta_data = tokio::fs::read_to_string(&meta_path).await?;
|
||||
let info: TemplateInfo = serde_json::from_str(&meta_data)?;
|
||||
let info = if let Ok(data) = tokio::fs::read_to_string(&meta_path).await {
|
||||
serde_json::from_str::<TemplateInfo>(&data).unwrap_or_else(|_| TemplateInfo {
|
||||
name: template_id.to_string(),
|
||||
description: String::new(),
|
||||
match_hint: String::new(),
|
||||
})
|
||||
} else {
|
||||
TemplateInfo {
|
||||
name: template_id.to_string(),
|
||||
description: String::new(),
|
||||
match_hint: String::new(),
|
||||
}
|
||||
};
|
||||
|
||||
// Read INSTRUCTIONS.md
|
||||
let instructions_path = base.join("INSTRUCTIONS.md");
|
||||
let instructions = tokio::fs::read_to_string(&instructions_path)
|
||||
let instructions = tokio::fs::read_to_string(base.join("INSTRUCTIONS.md"))
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
|
||||
// Discover external tools
|
||||
let tools_dir = base.join("tools");
|
||||
let external_tools = ExternalToolManager::discover(&tools_dir).await;
|
||||
tracing::info!(
|
||||
"Template '{}': discovered {} external tools",
|
||||
template_id,
|
||||
external_tools.len()
|
||||
);
|
||||
tracing::info!("Template '{}': {} external tools", template_id, external_tools.len());
|
||||
|
||||
// Scan KB files
|
||||
let kb_dir = base.join("kb");
|
||||
let kb_files = scan_kb_files(&kb_dir).await;
|
||||
tracing::info!(
|
||||
"Template '{}': found {} KB files",
|
||||
template_id,
|
||||
kb_files.len()
|
||||
);
|
||||
tracing::info!("Template '{}': {} KB files", template_id, kb_files.len());
|
||||
|
||||
Ok(Self {
|
||||
id: template_id.to_string(),
|
||||
@@ -187,10 +438,13 @@ impl LoadedTemplate {
|
||||
kb_files,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn load(template_id: &str) -> anyhow::Result<Self> {
|
||||
let base = Path::new(builtin_dir()).join(template_id);
|
||||
Self::load_from_dir(template_id, &base).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Scan kb/ directory for .md files. Returns (title, content) pairs.
|
||||
/// Title is extracted from the first `# heading` line, or falls back to the filename.
|
||||
async fn scan_kb_files(kb_dir: &Path) -> Vec<(String, String)> {
|
||||
let mut results = Vec::new();
|
||||
|
||||
@@ -201,18 +455,12 @@ async fn scan_kb_files(kb_dir: &Path) -> Vec<(String, String)> {
|
||||
|
||||
while let Ok(Some(entry)) = entries.next_entry().await {
|
||||
let path = entry.path();
|
||||
|
||||
// Resolve symlinks
|
||||
let real_path = match tokio::fs::canonicalize(&path).await {
|
||||
Ok(p) => p,
|
||||
Err(_) => path.clone(),
|
||||
};
|
||||
|
||||
// Only process .md files
|
||||
let ext = real_path
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.unwrap_or("");
|
||||
let ext = real_path.extension().and_then(|e| e.to_str()).unwrap_or("");
|
||||
if ext != "md" {
|
||||
continue;
|
||||
}
|
||||
@@ -225,7 +473,6 @@ async fn scan_kb_files(kb_dir: &Path) -> Vec<(String, String)> {
|
||||
}
|
||||
};
|
||||
|
||||
// Extract title: first `# heading` line, or filename without extension
|
||||
let title = content
|
||||
.lines()
|
||||
.find(|l| l.starts_with("# "))
|
||||
|
||||
Reference in New Issue
Block a user