smallcloudai · alashchev17 · May 7, 2025 · May 7, 2025 · May 7, 2025 · May 7, 2025
diff --git a/refact-agent/engine/src/call_validation.rs b/refact-agent/engine/src/call_validation.rs
@@ -237,8 +237,6 @@ pub struct ChatPost {
     #[serde(default)]
     pub n: Option<usize>,
     #[serde(default)]
-    pub tools: Option<Vec<serde_json::Value>>,
-    #[serde(default)]
     pub tool_choice: Option<String>,
     #[serde(default)]
     pub checkpoints_enabled: bool,

diff --git a/refact-agent/engine/src/http/routers/v1.rs b/refact-agent/engine/src/http/routers/v1.rs
@@ -1,3 +1,4 @@
+use at_tools::handle_v1_post_tools;
 use axum::Router;
 use axum::routing::{get, post, delete};
 use tower_http::cors::CorsLayer;
@@ -7,7 +8,7 @@ use crate::http::routers::v1::code_completion::{handle_v1_code_completion_web, h
 use crate::http::routers::v1::code_lens::handle_v1_code_lens;
 use crate::http::routers::v1::ast::{handle_v1_ast_file_dump, handle_v1_ast_file_symbols, handle_v1_ast_status};
 use crate::http::routers::v1::at_commands::{handle_v1_command_completion, handle_v1_command_preview, handle_v1_at_command_execute};
-use crate::http::routers::v1::at_tools::{handle_v1_tools, handle_v1_tools_check_if_confirmation_needed, handle_v1_tools_execute};
+use crate::http::routers::v1::at_tools::{handle_v1_get_tools, handle_v1_tools_check_if_confirmation_needed, handle_v1_tools_execute};
 use crate::http::routers::v1::caps::handle_v1_caps;
 use crate::http::routers::v1::caps::handle_v1_ping;
 use crate::http::routers::v1::chat::{handle_v1_chat, handle_v1_chat_completions};
@@ -93,7 +94,8 @@ pub fn make_v1_router() -> Router {
 
         .route("/caps", get(handle_v1_caps))
 
-        .route("/tools", get(handle_v1_tools))
+        .route("/tools", get(handle_v1_get_tools))
+        .route("/tools", post(handle_v1_post_tools))
         .route("/tools-check-if-confirmation-needed", post(handle_v1_tools_check_if_confirmation_needed))
         .route("/tools-execute", post(handle_v1_tools_execute)) // because it works remotely
 

diff --git a/refact-agent/engine/src/http/routers/v1/at_tools.rs b/refact-agent/engine/src/http/routers/v1/at_tools.rs
@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 use std::sync::Arc;
-use axum::Extension;
+use axum::{Extension, Json};
 use axum::http::{Response, StatusCode};
 use hyper::Body;
 use indexmap::IndexMap;
@@ -15,7 +15,8 @@ use crate::http::http_post_json;
 use crate::http::routers::v1::chat::CHAT_TOP_N;
 use crate::indexing_utils::wait_for_indexing_if_needed;
 use crate::integrations::docker::docker_container_manager::docker_container_get_host_lsp_port_to_connect;
-use crate::tools::tools_description::{tool_description_list_from_yaml, tools_merged_and_filtered, MatchConfirmDenyResult};
+use crate::tools::tools_description::{set_tool_config, MatchConfirmDenyResult, ToolConfig, ToolDesc, ToolSource};
+use crate::tools::tools_list::{get_available_tool_groups, get_available_tools, ToolGroupCategory};
 use crate::custom_error::ScratchError;
 use crate::global_context::{try_load_caps_quickly_if_not_present, GlobalContext};
 use crate::tools::tools_execute::run_tools;
@@ -65,38 +66,88 @@ pub struct ToolExecuteResponse {
     pub tools_ran: bool,
 }
 
-pub async fn handle_v1_tools(
+#[derive(Serialize, Deserialize)]
+pub struct ToolResponse {
+    pub spec: ToolDesc,
+    pub enabled: bool,
+}
+
+#[derive(Serialize, Deserialize)]
+pub struct ToolGroupResponse {
+    pub name: String,
+    pub description: String,
+    pub category: ToolGroupCategory,
+    pub allow_per_tool_toggle: bool,
+    pub tools: Vec<ToolResponse>,
+}
+
+pub async fn handle_v1_get_tools(
     Extension(gcx): Extension<Arc<ARwLock<GlobalContext>>>,
-    _: hyper::body::Bytes,
-) -> axum::response::Result<Response<Body>, ScratchError> {
-    let all_tools = match tools_merged_and_filtered(gcx.clone(), true).await {
-        Ok(tools) => tools,
-        Err(e) => {
-            let error_body = serde_json::json!({ "detail": e }).to_string();
-            return Ok(Response::builder()
-                .status(StatusCode::INTERNAL_SERVER_ERROR)
-                .header("Content-Type", "application/json")
-                .body(Body::from(error_body))
-                .unwrap());
+) -> Json<Vec<ToolGroupResponse>> {
+    let tool_groups = get_available_tool_groups(gcx.clone()).await;
+
+    let tool_groups: Vec<ToolGroupResponse> = tool_groups.into_iter().filter_map(|tool_group| {
+        if tool_group.tools.is_empty() {
+            return None;
         }
-    };
 
-    let turned_on = all_tools.keys().cloned().collect::<Vec<_>>();
-    let allow_experimental = gcx.read().await.cmdline.experimental;
+        let tools: Vec<ToolResponse> = tool_group.tools.into_iter().map(|tool| {
+            let spec = tool.tool_description();
+            ToolResponse {
+                spec,
+                enabled: tool.config().unwrap_or_default().enabled,
+            }
+        }).collect();
 
-    let tool_desclist = tool_description_list_from_yaml(all_tools, Some(&turned_on), allow_experimental).await.unwrap_or_else(|e| {
-        tracing::error!("Error loading compiled_in_tools: {:?}", e);
-        vec![]
-    });
+        Some(ToolGroupResponse {
+            name: tool_group.name,
+            description: tool_group.description,
+            category: tool_group.category,
+            allow_per_tool_toggle: tool_group.allow_per_tool_toggle,
+            tools,
+        })
+    }).collect();
 
-    let tools_openai_stype = tool_desclist.into_iter().map(|x| x.into_openai_style()).collect::<Vec<_>>();
+    Json(tool_groups)
+}
 
-    let body = serde_json::to_string_pretty(&tools_openai_stype).map_err(|e| ScratchError::new(StatusCode::UNPROCESSABLE_ENTITY, format!("JSON problem: {}", e)))?;
-    Ok(Response::builder()
-        .status(StatusCode::OK)
-        .header("Content-Type", "application/json")
-        .body(Body::from(body))
-        .unwrap())
+#[derive(Deserialize)]
+pub struct ToolPost {
+    name: String,
+    source: ToolSource,
+    enabled: bool,
+}
+
+#[derive(Deserialize)]
+pub struct ToolPostReq {
+    tools: Vec<ToolPost>,
+}
+
+#[derive(Serialize)]
+pub struct ToolPostResponse {
+    success: bool,
+}
+
+pub async fn handle_v1_post_tools(
+    body_bytes: hyper::body::Bytes,
+) -> Result<Json<ToolPostResponse>, ScratchError> {
+    let tools = serde_json::from_slice::<ToolPostReq>(&body_bytes)
+        .map_err(|e| ScratchError::new(StatusCode::UNPROCESSABLE_ENTITY, format!("JSON problem: {}", e)))?
+        .tools;
+
+    for tool in tools {
+        set_tool_config(
+            tool.source.config_path, 
+            tool.name,
+            ToolConfig {
+                enabled: tool.enabled,
+            }
+        ).await.map_err(|e| ScratchError::new(StatusCode::INTERNAL_SERVER_ERROR, format!("Error setting tool config: {}", e)))?;
+    }
+
+    Ok(Json(ToolPostResponse {
+        success: true,
+    }))
 }
 
 pub async fn handle_v1_tools_check_if_confirmation_needed(
@@ -144,17 +195,11 @@ pub async fn handle_v1_tools_check_if_confirmation_needed(
         "".to_string(),
     ).await)); // used only for should_confirm
 
-    let all_tools = match tools_merged_and_filtered(gcx.clone(), true).await {
-        Ok(tools) => tools,
-        Err(e) => {
-            let error_body = serde_json::json!({ "detail": e }).to_string();
-            return Ok(Response::builder()
-                .status(StatusCode::INTERNAL_SERVER_ERROR)
-                .header("Content-Type", "application/json")
-                .body(Body::from(error_body))
-                .unwrap());
-        }
-    };
+    let all_tools = get_available_tools(gcx.clone()).await.into_iter()
+        .map(|tool| {
+            let spec = tool.tool_description();
+            (spec.name, tool)
+        }).collect::<IndexMap<_, _>>();
 
     let mut result_messages = vec![];
     for tool_call in &post.tool_calls {
@@ -248,9 +293,12 @@ pub async fn handle_v1_tools_execute(
     ccx.postprocess_parameters = tools_execute_post.postprocess_parameters.clone();
     let ccx_arc = Arc::new(AMutex::new(ccx));
 
-    let mut at_tools = tools_merged_and_filtered(gcx.clone(), false).await.map_err(|e|{
-        ScratchError::new(StatusCode::INTERNAL_SERVER_ERROR, format!("Error getting at_tools: {}", e))
-    })?;
+    let mut at_tools = get_available_tools(gcx.clone()).await.into_iter()
+        .map(|tool| {
+            let spec = tool.tool_description();
+            (spec.name, tool)
+        }).collect::<IndexMap<_, _>>();
+
     let (messages, tools_ran) = run_tools(
         ccx_arc.clone(), &mut at_tools, tokenizer.clone(), tools_execute_post.maxgen, &tools_execute_post.messages, &tools_execute_post.style
     ).await.map_err(|e| ScratchError::new(StatusCode::INTERNAL_SERVER_ERROR, format!("Error running tools: {}", e)))?;

diff --git a/refact-agent/engine/src/http/routers/v1/chat.rs b/refact-agent/engine/src/http/routers/v1/chat.rs
@@ -5,54 +5,17 @@ use tokio::sync::RwLock as ARwLock;
 use axum::Extension;
 use axum::response::Result;
 use hyper::{Body, Response, StatusCode};
-use serde_json::Value;
 
-use crate::call_validation::{ChatContent, ChatMessage, ChatPost, ChatMode};
+use crate::call_validation::{ChatContent, ChatMessage, ChatPost};
 use crate::caps::resolve_chat_model;
 use crate::custom_error::ScratchError;
 use crate::at_commands::at_commands::AtCommandsContext;
 use crate::git::checkpoints::create_workspace_checkpoint;
 use crate::global_context::{GlobalContext, SharedGlobalContext};
 use crate::indexing_utils::wait_for_indexing_if_needed;
 use crate::integrations::docker::docker_container_manager::docker_container_check_status_or_start;
-
-
-pub fn available_tools_by_chat_mode(current_tools: Vec<Value>, chat_mode: &ChatMode) -> Vec<Value> {
-    fn filter_out_tools(current_tools: &Vec<Value>, blacklist: &Vec<&str>) -> Vec<Value> {
-        current_tools
-            .into_iter()
-            .filter(|x| {
-                x.get("function")
-                    .and_then(|x| x.get("name"))
-                    .and_then(|tool_name| tool_name.as_str())
-                    .map(|tool_name_str| !blacklist.contains(&tool_name_str))
-                    .unwrap_or(true)
-            })
-            .cloned()
-            .collect()
-    }
-    fn keep_tools(current_tools: &Vec<Value>, whitelist: &Vec<&str>) -> Vec<Value> {
-        current_tools
-            .into_iter()
-            .filter(|x| {
-                x.get("function")
-                    .and_then(|x| x.get("name"))
-                    .and_then(|tool_name| tool_name.as_str())
-                    .map(|tool_name_str| whitelist.contains(&tool_name_str))
-                    .unwrap_or(false)
-            })
-            .cloned()
-            .collect()
-    }
-    match chat_mode {
-        ChatMode::NO_TOOLS => {
-            vec![]
-        },
-        ChatMode::EXPLORE | ChatMode::AGENT => current_tools,
-        ChatMode::CONFIGURE => filter_out_tools(&current_tools, &vec!["tree", "locate", "knowledge", "search"]),
-        ChatMode::PROJECT_SUMMARY => keep_tools(&current_tools, &vec!["cat", "tree", "bash"]),
-    }
-}
+use crate::tools::tools_description::ToolDesc;
+use crate::tools::tools_list::get_available_tools_by_chat_mode;
 
 pub const CHAT_TOP_N: usize = 12;
 
@@ -120,27 +83,12 @@ async fn _chat(
 
     tracing::info!("chat_mode {:?}", chat_post.meta.chat_mode);
 
-    if chat_post.meta.chat_mode == ChatMode::NO_TOOLS {
-        chat_post.tools = None;
-    } else {
-        if let Some(tools) = &mut chat_post.tools {
-            for tool in &mut *tools {
-                if let Some(function) = tool.get_mut("function") {
-                    function.as_object_mut().unwrap().remove("agentic");
-                }
-            }
-            chat_post.tools = Some(available_tools_by_chat_mode(tools.clone(), &chat_post.meta.chat_mode));
-        } else {
-            // TODO at some point, get rid of /tools call on client, make so we can have chat_post.tools==None and just fill the tools here
-            chat_post.tools = Some(available_tools_by_chat_mode(vec![], &chat_post.meta.chat_mode));
-        }
-        tracing::info!("tools [{}]", chat_post.tools.as_ref().map_or("".to_string(), |tools| {
-            tools.iter()
-                .filter_map(|tool| tool.get("function").and_then(|f| f.get("name")).and_then(|n| n.as_str()))
-                .collect::<Vec<&str>>()
-                .join(", ")
-        }));
-    }
+    let tools: Vec<ToolDesc> = get_available_tools_by_chat_mode(gcx.clone(), chat_post.meta.chat_mode).await
+        .into_iter()
+        .map(|tool| tool.tool_description())
+        .collect();
+
+    tracing::info!("tools: {:?}", tools.iter().map(|t| &t.name).collect::<Vec<_>>());
 
     let caps = crate::global_context::try_load_caps_quickly_if_not_present(gcx.clone(), 0).await?;
     let model_rec = resolve_chat_model(caps, &chat_post.model)
@@ -228,6 +176,7 @@ async fn _chat(
     let mut scratchpad = crate::scratchpads::create_chat_scratchpad(
         gcx.clone(),
         &mut chat_post,
+        tools,
         &messages,
         true,
         &model_rec,

diff --git a/refact-agent/engine/src/http/routers/v1/links.rs b/refact-agent/engine/src/http/routers/v1/links.rs
@@ -80,8 +80,7 @@ pub async fn handle_v1_links(
     let mut uncommited_changes_warning = String::new();
 
     tracing::info!("for links, post.meta.chat_mode == {:?}", post.meta.chat_mode);
-    let experimental = gcx.read().await.cmdline.experimental;
-    let (_integrations_map, integration_yaml_errors) = crate::integrations::running_integrations::load_integrations(gcx.clone(), experimental, &["**/*".to_string()]).await;
+    let (_integrations_map, integration_yaml_errors) = crate::integrations::running_integrations::load_integrations(gcx.clone(), &["**/*".to_string()]).await;
 
     if post.meta.chat_mode == ChatMode::CONFIGURE {
         if last_message_assistant_without_tools_with_code_blocks(&post.messages) {

diff --git a/refact-agent/engine/src/integrations/docker/integr_docker.rs b/refact-agent/engine/src/integrations/docker/integr_docker.rs
@@ -10,7 +10,7 @@ use crate::at_commands::at_commands::AtCommandsContext;
 use crate::call_validation::{ChatContent, ChatMessage, ContextEnum};
 use crate::global_context::GlobalContext;
 use crate::integrations::integr_abstract::{IntegrationTrait, IntegrationCommon, IntegrationConfirmation};
-use crate::tools::tools_description::Tool;
+use crate::tools::tools_description::{Tool, ToolDesc, ToolParam, ToolSource, ToolSourceType};
 use crate::integrations::docker::docker_ssh_tunnel_utils::{SshConfig, forward_remote_docker_if_needed};
 use crate::integrations::utils::{serialize_num_to_str, deserialize_str_to_num};
 
@@ -135,6 +135,28 @@ impl ToolDocker {
 #[async_trait]
 impl Tool for ToolDocker {
     fn as_any(&self) -> &dyn std::any::Any { self }
+
+    fn tool_description(&self) -> ToolDesc {
+        ToolDesc {
+            name: "docker".to_string(),
+            display_name: "Docker CLI".to_string(),
+            source: ToolSource {
+                source_type: ToolSourceType::Integration,
+                config_path: self.config_path.clone(),
+            },
+            agentic: true,
+            experimental: true,
+            description: "Access to docker cli, in a non-interactive way, don't open a shell.".to_string(),
+            parameters: vec![
+                ToolParam {
+                    name: "command".to_string(),
+                    description: "Examples: docker images".to_string(),
+                    param_type: "string".to_string(),
+                },
+            ],
+            parameters_required: vec!["command".to_string()],
+        }
+    }
 
     async fn tool_execute(
         &mut self,

diff --git a/refact-agent/engine/src/integrations/docker/mod.rs b/refact-agent/engine/src/integrations/docker/mod.rs
@@ -15,7 +15,7 @@ pub mod docker_container_manager;
 pub async fn docker_and_isolation_load(gcx: Arc<ARwLock<GlobalContext>>) -> Result<(ToolDocker, Option<SettingsIsolation>), String>
 {
     let include_paths_matching = ["**/docker.yaml".to_string(), "**/isolation.yaml".to_string()];
-    let (integrations, _yaml_errors) = load_integrations(gcx.clone(), true, &include_paths_matching).await;
+    let (integrations, _yaml_errors) = load_integrations(gcx.clone(), &include_paths_matching).await;
 
     let docker_tools = integrations.get("docker")
         .ok_or("Docker integration not found".to_string())?

diff --git a/refact-agent/engine/src/integrations/integr_chrome.rs b/refact-agent/engine/src/integrations/integr_chrome.rs
@@ -14,7 +14,7 @@ use crate::global_context::GlobalContext;
 use crate::call_validation::{ChatContent, ChatMessage};
 use crate::scratchpads::multimodality::MultimodalElement;
 use crate::postprocessing::pp_command_output::{CmdlineOutputFilter, output_mini_postprocessing};
-use crate::tools::tools_description::{Tool, ToolDesc, ToolParam};
+use crate::tools::tools_description::{Tool, ToolDesc, ToolParam, ToolSource, ToolSourceType};
 use crate::integrations::integr_abstract::{IntegrationTrait, IntegrationCommon, IntegrationConfirmation};
 use crate::integrations::docker::docker_container_manager::get_container_name;
 
@@ -279,6 +279,11 @@ impl Tool for ToolChrome {
              Supported commands:\n{}", supported_commands.join("\n"));
         ToolDesc {
             name: "chrome".to_string(),
+            display_name: "Chrome".to_string(),
+            source: ToolSource {
+                source_type: ToolSourceType::Integration,
+                config_path: self.config_path.clone(),
+            },
             agentic: true,
             experimental: false,
             description: "A real web browser with graphical interface.".to_string(),