LLM call logging, plan persistence API, quote-to-feedback UX, requirement input improvements

- Add llm_call_log table and per-call timing/token tracking in agent loop
- New GET /workflows/{id}/plan endpoint to restore plan from snapshots on page load
- New GET /workflows/{id}/llm-calls endpoint + WS LlmCallLog broadcast
- Parse Usage from LLM API response (prompt_tokens, completion_tokens)
- Detailed mode toggle in execution log showing LLM call cards with phase/tokens/latency
- Quote-to-feedback: hover quote buttons on plan steps and log entries, multi-quote chips in comment input
- Requirement input: larger textarea, multi-line display with pre-wrap and scroll

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-03-02 09:16:51 +00:00
parent 46424cfbc4
commit 0a8eee0285
14 changed files with 601 additions and 26 deletions

View File

@@ -33,6 +33,7 @@ pub enum WsMessage {
RequirementUpdate { workflow_id: String, requirement: String },
ReportReady { workflow_id: String },
ProjectUpdate { project_id: String, name: String },
LlmCallLog { workflow_id: String, entry: crate::db::LlmCallLogEntry },
Error { message: String },
}
@@ -45,7 +46,7 @@ pub struct PlanStepInfo {
pub status: Option<String>,
}
fn plan_infos_from_state(state: &AgentState) -> Vec<PlanStepInfo> {
pub fn plan_infos_from_state(state: &AgentState) -> Vec<PlanStepInfo> {
state.steps.iter().map(|s| {
let status = match s.status {
StepStatus::Pending => "pending",
@@ -832,6 +833,61 @@ async fn log_execution(
});
}
/// Log an LLM call to llm_call_log and broadcast to frontend.
#[allow(clippy::too_many_arguments)]
async fn log_llm_call(
pool: &SqlitePool,
broadcast_tx: &broadcast::Sender<WsMessage>,
workflow_id: &str,
step_order: i32,
phase: &str,
messages_count: i32,
tools_count: i32,
tool_calls_json: &str,
text_response: &str,
prompt_tokens: Option<u32>,
completion_tokens: Option<u32>,
latency_ms: i64,
) {
let id = uuid::Uuid::new_v4().to_string();
let _ = sqlx::query(
"INSERT INTO llm_call_log (id, workflow_id, step_order, phase, messages_count, tools_count, tool_calls, text_response, prompt_tokens, completion_tokens, latency_ms, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, datetime('now'))"
)
.bind(&id)
.bind(workflow_id)
.bind(step_order)
.bind(phase)
.bind(messages_count)
.bind(tools_count)
.bind(tool_calls_json)
.bind(text_response)
.bind(prompt_tokens.map(|v| v as i32))
.bind(completion_tokens.map(|v| v as i32))
.bind(latency_ms as i32)
.execute(pool)
.await;
let entry = crate::db::LlmCallLogEntry {
id: id.clone(),
workflow_id: workflow_id.to_string(),
step_order,
phase: phase.to_string(),
messages_count,
tools_count,
tool_calls: tool_calls_json.to_string(),
text_response: text_response.to_string(),
prompt_tokens: prompt_tokens.map(|v| v as i32),
completion_tokens: completion_tokens.map(|v| v as i32),
latency_ms: latency_ms as i32,
created_at: String::new(),
};
let _ = broadcast_tx.send(WsMessage::LlmCallLog {
workflow_id: workflow_id.to_string(),
entry,
});
}
/// Process user feedback: call LLM to decide whether to revise the plan.
/// Returns the (possibly modified) AgentState ready for resumed execution.
async fn process_feedback(
@@ -952,8 +1008,16 @@ async fn run_agent_loop(
AgentPhase::Completed => break,
};
let messages = state.build_messages(&system_prompt, requirement);
let msg_count = messages.len() as i32;
let tool_count = tools.len() as i32;
let phase_label = match &state.phase {
AgentPhase::Planning => "planning".to_string(),
AgentPhase::Executing { step } => format!("executing({})", step),
AgentPhase::Completed => "completed".to_string(),
};
tracing::info!("[workflow {}] LLM call #{} phase={:?} msgs={}", workflow_id, iteration + 1, state.phase, messages.len());
let call_start = std::time::Instant::now();
let response = match llm.chat_with_tools(messages, tools).await {
Ok(r) => r,
Err(e) => {
@@ -961,6 +1025,11 @@ async fn run_agent_loop(
return Err(e);
}
};
let latency_ms = call_start.elapsed().as_millis() as i64;
let (prompt_tokens, completion_tokens) = response.usage.as_ref()
.map(|u| (Some(u.prompt_tokens), Some(u.completion_tokens)))
.unwrap_or((None, None));
let choice = response.choices.into_iter().next()
.ok_or_else(|| anyhow::anyhow!("No response from LLM"))?;
@@ -968,6 +1037,9 @@ async fn run_agent_loop(
// Add assistant message to chat history
state.current_step_chat_history.push(choice.message.clone());
// Collect text_response for logging
let llm_text_response = choice.message.content.clone().unwrap_or_default();
if let Some(tool_calls) = &choice.message.tool_calls {
tracing::info!("[workflow {}] Tool calls: {}", workflow_id,
tool_calls.iter().map(|tc| tc.function.name.as_str()).collect::<Vec<_>>().join(", "));
@@ -1184,6 +1256,22 @@ async fn run_agent_loop(
}
}
// Build tool_calls JSON for LLM call log
let tc_json: Vec<serde_json::Value> = tool_calls.iter().map(|tc| {
serde_json::json!({
"name": tc.function.name,
"arguments_preview": truncate_str(&tc.function.arguments, 200),
})
}).collect();
let tc_json_str = serde_json::to_string(&tc_json).unwrap_or_else(|_| "[]".to_string());
log_llm_call(
pool, broadcast_tx, workflow_id, state.current_step(),
&phase_label, msg_count, tool_count,
&tc_json_str, &llm_text_response,
prompt_tokens, completion_tokens, latency_ms,
).await;
if phase_transition {
continue;
}
@@ -1195,6 +1283,13 @@ async fn run_agent_loop(
// Log text response to execution_log for frontend display
log_execution(pool, broadcast_tx, workflow_id, state.current_step(), "text_response", "", content, "done").await;
log_llm_call(
pool, broadcast_tx, workflow_id, state.current_step(),
&phase_label, msg_count, tool_count,
"[]", content,
prompt_tokens, completion_tokens, latency_ms,
).await;
// Text response does NOT end the workflow. Only advance_step progresses.
// In Planning phase, LLM may be thinking before calling update_plan — just continue.
}