euphon/tori
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

LLM call logging, plan persistence API, quote-to-feedback UX, requirement input improvements

Browse Source 
- Add llm_call_log table and per-call timing/token tracking in agent loop
- New GET /workflows/{id}/plan endpoint to restore plan from snapshots on page load
- New GET /workflows/{id}/llm-calls endpoint + WS LlmCallLog broadcast
- Parse Usage from LLM API response (prompt_tokens, completion_tokens)
- Detailed mode toggle in execution log showing LLM call cards with phase/tokens/latency
- Quote-to-feedback: hover quote buttons on plan steps and log entries, multi-quote chips in comment input
- Requirement input: larger textarea, multi-line display with pre-wrap and scroll

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Fam Zheng
2026-03-02 09:16:51 +00:00
parent 46424cfbc4
commit 0a8eee0285
14 changed files with 601 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
src/agent.rs
View File

@@ -33,6 +33,7 @@ pub enum WsMessage {
RequirementUpdate { workflow_id: String, requirement: String },
ReportReady { workflow_id: String },
ProjectUpdate { project_id: String, name: String },
LlmCallLog { workflow_id: String, entry: crate::db::LlmCallLogEntry },
Error { message: String },
}
@@ -45,7 +46,7 @@ pub struct PlanStepInfo {
pub status: Option<String>,
}
fn plan_infos_from_state(state: &AgentState) -> Vec<PlanStepInfo> {
pub fn plan_infos_from_state(state: &AgentState) -> Vec<PlanStepInfo> {
state.steps.iter().map(|s| {
let status = match s.status {
StepStatus::Pending => "pending",
@@ -832,6 +833,61 @@ async fn log_execution(
});
}
/// Log an LLM call to llm_call_log and broadcast to frontend.
#[allow(clippy::too_many_arguments)]
async fn log_llm_call(
pool: &SqlitePool,
broadcast_tx: &broadcast::Sender<WsMessage>,
workflow_id: &str,
step_order: i32,
phase: &str,
messages_count: i32,
tools_count: i32,
tool_calls_json: &str,
text_response: &str,
prompt_tokens: Option<u32>,
completion_tokens: Option<u32>,
latency_ms: i64,
) {
let id = uuid::Uuid::new_v4().to_string();
let _ = sqlx::query(
"INSERT INTO llm_call_log (id, workflow_id, step_order, phase, messages_count, tools_count, tool_calls, text_response, prompt_tokens, completion_tokens, latency_ms, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))"
)
.bind(&id)
.bind(workflow_id)
.bind(step_order)
.bind(phase)
.bind(messages_count)
.bind(tools_count)
.bind(tool_calls_json)
.bind(text_response)
.bind(prompt_tokens.map(|v| v as i32))
.bind(completion_tokens.map(|v| v as i32))
.bind(latency_ms as i32)
.execute(pool)
.await;
let entry = crate::db::LlmCallLogEntry {
id: id.clone(),
workflow_id: workflow_id.to_string(),
step_order,
phase: phase.to_string(),
messages_count,
tools_count,
tool_calls: tool_calls_json.to_string(),
text_response: text_response.to_string(),
prompt_tokens: prompt_tokens.map(|v| v as i32),
completion_tokens: completion_tokens.map(|v| v as i32),
latency_ms: latency_ms as i32,
created_at: String::new(),
};
let _ = broadcast_tx.send(WsMessage::LlmCallLog {
workflow_id: workflow_id.to_string(),
entry,
});
}
/// Process user feedback: call LLM to decide whether to revise the plan.
/// Returns the (possibly modified) AgentState ready for resumed execution.
async fn process_feedback(
@@ -952,8 +1008,16 @@ async fn run_agent_loop(
AgentPhase::Completed => break,
};
let messages = state.build_messages(&system_prompt, requirement);
let msg_count = messages.len() as i32;
let tool_count = tools.len() as i32;
let phase_label = match &state.phase {
AgentPhase::Planning => "planning".to_string(),
AgentPhase::Executing { step } => format!("executing({})", step),
AgentPhase::Completed => "completed".to_string(),
};
tracing::info!("[workflow {}] LLM call #{} phase={:?} msgs={}", workflow_id, iteration + 1, state.phase, messages.len());
let call_start = std::time::Instant::now();
let response = match llm.chat_with_tools(messages, tools).await {
Ok(r) => r,
Err(e) => {
@@ -961,6 +1025,11 @@ async fn run_agent_loop(
return Err(e);
}
};
let latency_ms = call_start.elapsed().as_millis() as i64;
let (prompt_tokens, completion_tokens) = response.usage.as_ref()
.map(|u| (Some(u.prompt_tokens), Some(u.completion_tokens)))
.unwrap_or((None, None));
let choice = response.choices.into_iter().next()
.ok_or_else(|| anyhow::anyhow!("No response from LLM"))?;
@@ -968,6 +1037,9 @@ async fn run_agent_loop(
// Add assistant message to chat history
state.current_step_chat_history.push(choice.message.clone());
// Collect text_response for logging
let llm_text_response = choice.message.content.clone().unwrap_or_default();
if let Some(tool_calls) = &choice.message.tool_calls {
tracing::info!("[workflow {}] Tool calls: {}", workflow_id,
tool_calls.iter().map(|tc| tc.function.name.as_str()).collect::<Vec<_>>().join(", "));
@@ -1184,6 +1256,22 @@ async fn run_agent_loop(
}
}
// Build tool_calls JSON for LLM call log
let tc_json: Vec<serde_json::Value> = tool_calls.iter().map(|tc| {
serde_json::json!({
"name": tc.function.name,
"arguments_preview": truncate_str(&tc.function.arguments, 200),
})
}).collect();
let tc_json_str = serde_json::to_string(&tc_json).unwrap_or_else(|_| "[]".to_string());
log_llm_call(
pool, broadcast_tx, workflow_id, state.current_step(),
&phase_label, msg_count, tool_count,
&tc_json_str, &llm_text_response,
prompt_tokens, completion_tokens, latency_ms,
).await;
if phase_transition {
continue;
}
@@ -1195,6 +1283,13 @@ async fn run_agent_loop(
// Log text response to execution_log for frontend display
log_execution(pool, broadcast_tx, workflow_id, state.current_step(), "text_response", "", content, "done").await;
log_llm_call(
pool, broadcast_tx, workflow_id, state.current_step(),
&phase_label, msg_count, tool_count,
"[]", content,
prompt_tokens, completion_tokens, latency_ms,
).await;
// Text response does NOT end the workflow. Only advance_step progresses.
// In Planning phase, LLM may be thinking before calling update_plan — just continue.
}

42
src/api/workflows.rs
View File

@@ -8,8 +8,9 @@ use axum::{
};
use serde::Deserialize;
use crate::AppState;
use crate::agent::AgentEvent;
use crate::db::{Workflow, ExecutionLogEntry, Comment};
use crate::agent::{AgentEvent, PlanStepInfo};
use crate::db::{Workflow, ExecutionLogEntry, Comment, LlmCallLogEntry};
use crate::state::AgentState;
use super::{ApiResult, db_err};
#[derive(serde::Serialize)]
@@ -33,6 +34,8 @@ pub fn router(state: Arc<AppState>) -> Router {
.route("/workflows/{id}/steps", get(list_steps))
.route("/workflows/{id}/comments", get(list_comments).post(create_comment))
.route("/workflows/{id}/report", get(get_report))
.route("/workflows/{id}/plan", get(get_plan))
.route("/workflows/{id}/llm-calls", get(list_llm_calls))
.with_state(state)
}
@@ -153,3 +156,38 @@ async fn get_report(
None => Err((StatusCode::NOT_FOUND, "Workflow not found").into_response()),
}
}
async fn get_plan(
State(state): State<Arc<AppState>>,
Path(workflow_id): Path<String>,
) -> ApiResult<Vec<PlanStepInfo>> {
let snapshot_json: Option<String> = sqlx::query_scalar(
"SELECT state_json FROM agent_state_snapshots WHERE workflow_id = ? ORDER BY created_at DESC LIMIT 1"
)
.bind(&workflow_id)
.fetch_optional(&state.db.pool)
.await
.map_err(db_err)?;
if let Some(json) = snapshot_json {
if let Ok(agent_state) = serde_json::from_str::<AgentState>(&json) {
return Ok(Json(crate::agent::plan_infos_from_state(&agent_state)));
}
}
Ok(Json(vec![]))
}
async fn list_llm_calls(
State(state): State<Arc<AppState>>,
Path(workflow_id): Path<String>,
) -> ApiResult<Vec<LlmCallLogEntry>> {
sqlx::query_as::<_, LlmCallLogEntry>(
"SELECT * FROM llm_call_log WHERE workflow_id = ? ORDER BY created_at"
)
.bind(&workflow_id)
.fetch_all(&state.db.pool)
.await
.map(Json)
.map_err(db_err)
}

42
src/db.rs
View File

@@ -158,6 +158,32 @@ impl Database {
.execute(&self.pool)
.await?;
sqlx::query(
"CREATE TABLE IF NOT EXISTS llm_call_log (
id TEXT PRIMARY KEY,
workflow_id TEXT NOT NULL REFERENCES workflows(id),
step_order INTEGER NOT NULL,
phase TEXT NOT NULL,
messages_count INTEGER NOT NULL,
tools_count INTEGER NOT NULL,
tool_calls TEXT NOT NULL DEFAULT '[]',
text_response TEXT NOT NULL DEFAULT '',
prompt_tokens INTEGER,
completion_tokens INTEGER,
latency_ms INTEGER NOT NULL,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
)"
)
.execute(&self.pool)
.await?;
// Migration: add text_response column to llm_call_log
let _ = sqlx::query(
"ALTER TABLE llm_call_log ADD COLUMN text_response TEXT NOT NULL DEFAULT ''"
)
.execute(&self.pool)
.await;
sqlx::query(
"CREATE TABLE IF NOT EXISTS timers (
id TEXT PRIMARY KEY,
@@ -237,3 +263,19 @@ pub struct Timer {
pub last_run_at: String,
pub created_at: String,
}
#[derive(Debug, Clone, Serialize, Deserialize, sqlx::FromRow)]
pub struct LlmCallLogEntry {
pub id: String,
pub workflow_id: String,
pub step_order: i32,
pub phase: String,
pub messages_count: i32,
pub tools_count: i32,
pub tool_calls: String,
pub text_response: String,
pub prompt_tokens: Option<i32>,
pub completion_tokens: Option<i32>,
pub latency_ms: i32,
pub created_at: String,
}

11
src/llm.rs
View File

@@ -67,9 +67,20 @@ pub struct ToolCallFunction {
pub arguments: String,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct Usage {
#[serde(default)]
pub prompt_tokens: u32,
#[serde(default)]
pub completion_tokens: u32,
#[serde(default)]
pub total_tokens: u32,
}
#[derive(Debug, Deserialize)]
pub struct ChatResponse {
pub choices: Vec<ChatChoice>,
pub usage: Option<Usage>,
}
#[derive(Debug, Deserialize)]

2
src/main.rs
View File

@@ -4,7 +4,7 @@ mod db;
mod kb;
mod llm;
mod exec;
mod state;
pub mod state;
mod timer;
mod ws;