AI Newsletter Digest improvements: fixed QP soft line break decoding, URL extraction, and content cleaning

This commit is contained in:
Krilly
2026-03-04 13:29:22 +00:00
parent 29a98137a7
commit 57dd294675
13706 changed files with 2114953 additions and 237629 deletions
@@ -0,0 +1,233 @@
use browser_use::{BrowserSession, LaunchOptions,
tools::{HoverParams, ScrollParams, SelectParams, Tool, ToolContext, hover::HoverTool,
scroll::ScrollTool, select::SelectTool}};
use log::info;
#[test]
#[ignore] // Requires Chrome to be installed
fn test_select_tool() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Create a page with a select dropdown
let html = r#"
<!DOCTYPE html>
<html>
<body>
<select id="country">
<option value="us">United States</option>
<option value="uk">United Kingdom</option>
<option value="ca">Canada</option>
</select>
<div id="result"></div>
<script>
document.getElementById('country').addEventListener('change', function(e) {
document.getElementById('result').textContent = 'Selected: ' + e.target.value;
});
</script>
</body>
</html>
"#;
let data_url = format!("data:text/html,{}", html);
session.navigate(&data_url).expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
// Create tool and context
let tool = SelectTool::default();
let mut context = ToolContext::new(&session);
// Execute the tool to select an option
let result = tool
.execute_typed(
SelectParams { selector: Some("#country".to_string()), index: None, value: "uk".to_string() },
&mut context,
)
.expect("Failed to execute select tool");
// Verify the result
assert!(result.success, "Tool execution should succeed");
assert!(result.data.is_some());
let data = result.data.unwrap();
info!("Select result: {}", serde_json::to_string_pretty(&data).unwrap());
assert_eq!(data["value"].as_str(), Some("uk"));
assert_eq!(data["selectedText"].as_str(), Some("United Kingdom"));
}
#[test]
#[ignore]
fn test_hover_tool() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Create a page with a hoverable element
let html = r#"
<!DOCTYPE html>
<html>
<body>
<button id="hover-btn">Hover Me</button>
<div id="result"></div>
<script>
document.getElementById('hover-btn').addEventListener('mouseover', function() {
document.getElementById('result').textContent = 'Hovered!';
});
</script>
</body>
</html>
"#;
let data_url = format!("data:text/html,{}", html);
session.navigate(&data_url).expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
// Create tool and context
let tool = HoverTool::default();
let mut context = ToolContext::new(&session);
// Execute the tool
let result = tool
.execute_typed(HoverParams { selector: Some("#hover-btn".to_string()), index: None }, &mut context)
.expect("Failed to execute hover tool");
// Verify the result
assert!(result.success, "Tool execution should succeed");
assert!(result.data.is_some());
let data = result.data.unwrap();
info!("Hover result: {}", serde_json::to_string_pretty(&data).unwrap());
assert_eq!(data["selector"].as_str(), Some("#hover-btn"));
}
#[test]
#[ignore]
fn test_scroll_tool_with_amount() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Create a long page
let html = r#"
<!DOCTYPE html>
<html>
<body style="height: 3000px;">
<h1>Top of page</h1>
<div style="margin-top: 1000px;">Middle</div>
<div style="margin-top: 1000px;">Bottom</div>
</body>
</html>
"#;
let data_url = format!("data:text/html,{}", html);
session.navigate(&data_url).expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
// Create tool and context
let tool = ScrollTool::default();
let mut context = ToolContext::new(&session);
// Execute the tool to scroll down 500 pixels
let result =
tool.execute_typed(ScrollParams { amount: Some(500) }, &mut context).expect("Failed to execute scroll tool");
// Verify the result
assert!(result.success, "Tool execution should succeed");
assert!(result.data.is_some());
let data = result.data.unwrap();
info!("Scroll result: {}", serde_json::to_string_pretty(&data).unwrap());
let scrolled = data["scrolled"].as_i64();
assert!(scrolled.is_some() && scrolled.unwrap() > 0, "Should have scrolled");
}
#[test]
#[ignore]
fn test_scroll_tool_to_bottom() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Create a page
let html = r#"
<!DOCTYPE html>
<html>
<body style="height: 2000px;">
<h1>Top of page</h1>
<div style="margin-top: 1800px;">Bottom</div>
</body>
</html>
"#;
let data_url = format!("data:text/html,{}", html);
session.navigate(&data_url).expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
// Create tool and context
let tool = ScrollTool::default();
let mut context = ToolContext::new(&session);
// Execute the tool multiple times to reach bottom
for _ in 0..10 {
let result =
tool.execute_typed(ScrollParams { amount: None }, &mut context).expect("Failed to execute scroll tool");
assert!(result.success);
let data = result.data.as_ref().unwrap();
let is_at_bottom = data["isAtBottom"].as_bool().unwrap_or(false);
info!("Scroll iteration: scrolled={}, isAtBottom={}", data["scrolled"], is_at_bottom);
if is_at_bottom {
info!("Reached bottom of page");
break;
}
std::thread::sleep(std::time::Duration::from_millis(100));
}
}
#[test]
#[ignore]
fn test_select_with_index() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Create a page with a select dropdown
let html = r#"
<!DOCTYPE html>
<html>
<body>
<select id="color">
<option value="red">Red</option>
<option value="green">Green</option>
<option value="blue">Blue</option>
</select>
</body>
</html>
"#;
let data_url = format!("data:text/html,{}", html);
session.navigate(&data_url).expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
// First extract DOM to get indices
let _dom = session.extract_dom().expect("Failed to extract DOM");
// Create tool and context
let tool = SelectTool::default();
let mut context = ToolContext::new(&session);
// Try to select using index (the select element should have index 0 since it's the first interactive element)
let result =
tool.execute_typed(SelectParams { selector: None, index: Some(0), value: "green".to_string() }, &mut context);
// This might fail if DOM indexing doesn't include select elements, which is acceptable
// The test is mainly to verify the API works
if let Ok(result) = result {
info!("Select with index result: {}", serde_json::to_string_pretty(&result.data.unwrap()).unwrap());
} else {
info!("Select with index failed (may be expected if select not indexed)");
}
}
@@ -0,0 +1,112 @@
use std::process::Command;
use std::path::PathBuf;
use std::fs;
// Helper to get the path to the binary
fn bin_path() -> PathBuf {
let mut path = std::env::current_dir().unwrap();
path.push("target");
path.push("debug");
path.push("fast-browser-use");
path
}
#[test]
fn test_recipe_1_navigate_human_emulation() {
let output = Command::new(bin_path())
.arg("navigate")
.arg("--url")
.arg("https://example.com")
.arg("--human-emulation")
.arg("--wait-for-selector")
.arg("h1")
.output()
.expect("Failed to execute command");
assert!(output.status.success(), "Command failed: {}", String::from_utf8_lossy(&output.stderr));
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(stderr.contains("Engaging human emulation"));
assert!(stderr.contains("Waiting for selector: h1"));
}
#[test]
fn test_recipe_2_snapshot() {
let output_file = "test_e2e_state.json";
// Clean up if exists
if std::path::Path::new(output_file).exists() {
let _ = std::fs::remove_file(output_file);
}
let output = Command::new(bin_path())
.arg("snapshot")
.arg("--url")
.arg("https://example.com")
.arg("--include-styles")
.arg("--output")
.arg(output_file)
.output()
.expect("Failed to execute command");
assert!(output.status.success(), "Command failed: {}", String::from_utf8_lossy(&output.stderr));
assert!(std::path::Path::new(output_file).exists());
let content = fs::read_to_string(output_file).unwrap();
// Check for some expected content in the snapshot
assert!(content.contains("Example Domain") || content.contains("example.com"));
// Clean up
let _ = std::fs::remove_file(output_file);
}
#[test]
fn test_recipe_3_login_flow() {
let session_file = "test_e2e_auth.json";
// Clean up
if std::path::Path::new(session_file).exists() {
let _ = std::fs::remove_file(session_file);
}
// Step 1: Login (simulated with echo)
// We use a pipe to send newline to stdin
let mut child = Command::new(bin_path())
.arg("login")
.arg("--url")
.arg("https://example.com")
.arg("--save-session")
.arg(session_file)
.stdin(std::process::Stdio::piped())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.expect("Failed to spawn process");
{
let stdin = child.stdin.as_mut().expect("Failed to open stdin");
use std::io::Write;
stdin.write_all(b"\n").expect("Failed to write to stdin");
}
let output = child.wait_with_output().expect("Failed to read stdout");
assert!(output.status.success(), "Login command failed: {}", String::from_utf8_lossy(&output.stderr));
assert!(std::path::Path::new(session_file).exists(), "Session file was not created");
// Step 2: Reuse session
let output_nav = Command::new(bin_path())
.arg("navigate")
.arg("--url")
.arg("https://example.com")
.arg("--load-session")
.arg(session_file)
.output()
.expect("Failed to execute navigate command");
assert!(output_nav.status.success(), "Navigate with session failed: {}", String::from_utf8_lossy(&output_nav.stderr));
let stderr_nav = String::from_utf8_lossy(&output_nav.stderr);
assert!(stderr_nav.contains("Loading session from"));
// Clean up
let _ = std::fs::remove_file(session_file);
}
@@ -0,0 +1,56 @@
use browser_use::{BrowserSession, LaunchOptions,
tools::{cookies::{CookieParam, GetCookiesParams, SetCookiesParams},
Tool, ToolContext, cookies::{GetCookiesTool, SetCookiesTool}}};
use log::info;
#[test]
#[ignore]
fn test_cookies_workflow() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Navigate to a domain first
session.navigate("https://example.com").expect("Failed to navigate");
// Create tool context
let mut context = ToolContext::new(&session);
let set_cookies_tool = SetCookiesTool::default();
let get_cookies_tool = GetCookiesTool::default();
// 1. Set a cookie
let cookie = CookieParam {
name: "test_cookie".to_string(),
value: "test_value".to_string(),
domain: Some("example.com".to_string()),
path: Some("/".to_string()),
secure: Some(false),
http_only: Some(false),
same_site: None,
expires: None,
url: Some("https://example.com".to_string()),
};
let set_result = set_cookies_tool
.execute_typed(SetCookiesParams { cookies: vec![cookie] }, &mut context)
.expect("Failed to execute set_cookies");
assert!(set_result.success, "set_cookies should succeed");
// 2. Get cookies
let get_result = get_cookies_tool
.execute_typed(GetCookiesParams { urls: None }, &mut context)
.expect("Failed to execute get_cookies");
assert!(get_result.success, "get_cookies should succeed");
let cookies_json = get_result.data.unwrap();
let cookies = cookies_json.as_array().expect("Data should be an array");
// Verify the cookie was set
let found = cookies.iter().any(|c| {
c["name"].as_str() == Some("test_cookie") && c["value"].as_str() == Some("test_value")
});
assert!(found, "Should find the set cookie");
info!("Successfully set and retrieved cookies!");
}
@@ -0,0 +1,83 @@
use browser_use::{BrowserSession, LaunchOptions,
tools::{debug::{GetConsoleLogsParams, GetNetworkErrorsParams, GetConsoleLogsTool, GetNetworkErrorsTool},
Tool, ToolContext}};
use log::info;
use std::thread;
use std::time::Duration;
#[test]
#[ignore]
fn test_debug_tools() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Create a page that generates logs and network errors
// Note: We use a non-existent domain for network error
let html = r#"
<!DOCTYPE html>
<html>
<body>
<script>
console.log('Test log message');
console.warn('Test warning message');
// Trigger a network error (async)
fetch('http://this-domain-should-not-exist-at-all-12345.com/fail')
.catch(e => console.error('Fetch failed as expected'));
</script>
</body>
</html>
"#;
let data_url = format!("data:text/html,{}", html);
session.navigate(&data_url).expect("Failed to navigate");
// Wait for events to be captured
thread::sleep(Duration::from_secs(2));
// Create tool context
let mut context = ToolContext::new(&session);
let logs_tool = GetConsoleLogsTool::default();
let errors_tool = GetNetworkErrorsTool::default();
// 1. Get console logs
let logs_result = logs_tool
.execute_typed(GetConsoleLogsParams {}, &mut context)
.expect("Failed to execute get_console_logs");
assert!(logs_result.success);
let logs = logs_result.data.unwrap();
let logs_arr = logs.as_array().expect("Logs should be an array");
info!("Captured logs: {:?}", logs_arr);
// Verify we captured the logs
let has_log = logs_arr.iter().any(|l| l["text"].as_str().unwrap_or("").contains("Test log message"));
let has_warn = logs_arr.iter().any(|l| l["text"].as_str().unwrap_or("").contains("Test warning message"));
assert!(has_log, "Should capture console.log");
assert!(has_warn, "Should capture console.warn");
// 2. Get network errors
// Note: Network errors might take longer or behave differently in headless depending on environment
let errors_result = errors_tool
.execute_typed(GetNetworkErrorsParams {}, &mut context)
.expect("Failed to execute get_network_errors");
assert!(errors_result.success);
let errors = errors_result.data.unwrap();
let errors_arr = errors.as_array().expect("Errors should be an array");
info!("Captured network errors: {:?}", errors_arr);
// We expect at least one error from the failed fetch
// Note: precise error text varies by OS/Network, so we just check if we got *something* or if the logs caught the error
// In many cases, fetch failure also logs to console.error.
let has_fetch_error_log = logs_arr.iter().any(|l| l["text"].as_str().unwrap_or("").contains("Fetch failed"));
assert!(has_fetch_error_log, "Should capture fetch failure in console logs");
// Network errors might be empty if the browser handles it purely as a console error for data: URLs
// But let's see.
}
@@ -0,0 +1,170 @@
use browser_use::{BrowserSession, LaunchOptions};
use log::info;
#[test]
#[ignore] // Requires Chrome to be installed
fn test_dom_extraction() {
// Launch browser
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Navigate to a simple page
session
.navigate(
"data:text/html,<html><body><button id='test-btn'>Click me</button><a href='#'>Link</a></body></html>",
)
.expect("Failed to navigate");
// Extract DOM
let dom = session.extract_dom().expect("Failed to extract DOM");
// Verify DOM structure
assert_eq!(dom.root.role, "fragment");
assert!(dom.count_nodes() > 0);
// Note: interactive elements might be 0 due to visibility issues with data: URLs
// Just verify we got the structure
info!("DOM tree element count: {}", dom.count_nodes());
info!("Interactive elements: {}", dom.count_interactive());
// Convert to JSON
let json = dom.to_json().expect("Failed to convert to JSON");
assert!(json.contains("button"));
}
#[test]
#[ignore]
fn test_simplified_dom_extraction() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Page with script and style tags that should be removed
// Use a simple HTML page
session
.navigate("data:text/html,<html><head></head><body><p>Hello</p><button>Click</button></body></html>")
.expect("Failed to navigate");
// Small delay to let page render
std::thread::sleep(std::time::Duration::from_millis(500));
// Extract simplified DOM
let dom = session.extract_dom().expect("Failed to extract DOM");
// Verify we got content
let json = dom.to_json().expect("Failed to convert to JSON");
assert!(json.contains("button") || json.contains("body"));
info!("Simplified DOM: {}", json);
}
#[test]
#[ignore]
fn test_read_links() {
use browser_use::tools::{ReadLinksParams, Tool, ToolContext, read_links::ReadLinksTool};
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
let html = concat!(
"<html><head><title>Links Test</title></head><body>",
"<a href=\"https://example.com\">Example</a>",
"<a href=\"/path\">Relative</a>",
"<a href=\"#anchor\">Anchor</a>",
"<a href=\"https://rust-lang.org\">Rust</a>",
"<a>No Href</a>",
"<a href=\"\">Empty</a>",
"</body></html>"
);
session.navigate(&format!("data:text/html,{}", html)).expect("Failed navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
let tool = ReadLinksTool::default();
let mut context = ToolContext::new(&session);
let result = tool.execute_typed(ReadLinksParams {}, &mut context).expect("Failed execute");
assert!(result.success);
let data = result.data.unwrap();
let links = data["links"].as_array().unwrap();
let count = data["count"].as_u64().unwrap();
info!("Links found: {}", count);
for link in links {
info!(" {} -> {}", link["text"].as_str().unwrap_or(""), link["href"].as_str().unwrap_or(""));
}
// Due to data: URL limitations, we may not get all links
assert!(count >= 2, "Expected at least 2 links");
assert_eq!(links.len() as u64, count);
let texts: Vec<&str> = links.iter().filter_map(|l| l["text"].as_str()).collect();
// Verify the links we do get are correct
assert!(texts.contains(&"Example"));
assert!(texts.contains(&"Relative"));
// Verify href values
let ex_link = links.iter().find(|l| l["text"].as_str() == Some("Example")).expect("Example link not found");
assert_eq!(ex_link["href"].as_str(), Some("https://example.com"));
}
#[test]
#[ignore]
fn test_press_key_enter() {
use browser_use::tools::{PressKeyParams, Tool, ToolContext, press_key::PressKeyTool};
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Create a page with an input field that responds to Enter key
let html = r#"
<html>
<head><title>Press Key Test</title></head>
<body>
<input type="text" id="input1" value="test">
<div id="output"></div>
<script>
document.getElementById('input1').addEventListener('keydown', function(e) {
if (e.key === 'Enter') {
document.getElementById('output').textContent = 'Enter pressed!';
}
});
</script>
</body>
</html>
"#;
session.navigate(&format!("data:text/html,{}", html)).expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
// Focus the input element first
session.tab().unwrap().find_element("#input1").expect("Input not found").click().expect("Failed to click input");
std::thread::sleep(std::time::Duration::from_millis(200));
// Create tool and context
let tool = PressKeyTool::default();
let mut context = ToolContext::new(&session);
// Execute the tool to press Enter
let result = tool
.execute_typed(PressKeyParams { key: "Enter".to_string() }, &mut context)
.expect("Failed to execute press_key tool");
// Verify the result
assert!(result.success, "Tool execution should succeed");
assert!(result.data.is_some());
let data = result.data.unwrap();
assert_eq!(data["key"].as_str(), Some("Enter"));
std::thread::sleep(std::time::Duration::from_millis(200));
// Verify that the event was triggered
let output = session.tab().unwrap().wait_for_element("#output").ok().and_then(|elem| elem.get_inner_text().ok());
info!("Output after Enter key: {:?}", output);
// 校验 output 内容
assert_eq!(output.as_deref(), Some("Enter pressed!"), "Output should be 'Enter pressed!', but was: {:?}", output);
// Note: Due to limitations with data: URLs and event handling,
// we mainly verify that the tool executes without error
}
@@ -0,0 +1,75 @@
use browser_use::{BrowserSession, LaunchOptions,
tools::{local_storage::{ClearLocalStorageParams, GetLocalStorageParams, RemoveLocalStorageParams, SetLocalStorageParams},
Tool, ToolContext, local_storage::{ClearLocalStorageTool, GetLocalStorageTool, RemoveLocalStorageTool, SetLocalStorageTool}}};
use log::info;
#[test]
#[ignore]
fn test_local_storage_workflow() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Navigate to a domain to have a context for localStorage
session.navigate("https://example.com").expect("Failed to navigate");
let mut context = ToolContext::new(&session);
let get_tool = GetLocalStorageTool::default();
let set_tool = SetLocalStorageTool::default();
let remove_tool = RemoveLocalStorageTool::default();
let clear_tool = ClearLocalStorageTool::default();
// 1. Set a value
let set_result = set_tool
.execute_typed(SetLocalStorageParams { key: "test_key".to_string(), value: "test_value".to_string() }, &mut context)
.expect("Failed to set localStorage");
assert!(set_result.success);
// 2. Get the value
let get_result = get_tool
.execute_typed(GetLocalStorageParams { key: Some("test_key".to_string()) }, &mut context)
.expect("Failed to get localStorage");
assert!(get_result.success);
assert_eq!(get_result.data.unwrap().as_str(), Some("test_value"));
// 3. Set another value
set_tool
.execute_typed(SetLocalStorageParams { key: "key2".to_string(), value: "value2".to_string() }, &mut context)
.expect("Failed to set localStorage key2");
// 4. Get all values
let get_all_result = get_tool
.execute_typed(GetLocalStorageParams { key: None }, &mut context)
.expect("Failed to get all localStorage");
let all_data = get_all_result.data.unwrap();
println!("All data: {:?}", all_data);
assert_eq!(all_data["test_key"].as_str(), Some("test_value"));
assert_eq!(all_data["key2"].as_str(), Some("value2"));
// 5. Remove one value
let remove_result = remove_tool
.execute_typed(RemoveLocalStorageParams { key: "test_key".to_string() }, &mut context)
.expect("Failed to remove localStorage item");
assert!(remove_result.success);
// Verify removal
let get_removed = get_tool
.execute_typed(GetLocalStorageParams { key: Some("test_key".to_string()) }, &mut context)
.expect("Failed to get localStorage");
assert!(get_removed.data.unwrap().is_null());
// 6. Clear all
let clear_result = clear_tool
.execute_typed(ClearLocalStorageParams, &mut context)
.expect("Failed to clear localStorage");
assert!(clear_result.success);
// Verify clear
let get_all_cleared = get_tool
.execute_typed(GetLocalStorageParams { key: None }, &mut context)
.expect("Failed to get all localStorage after clear");
// The implementation returns an empty object {}
let cleared_obj = get_all_cleared.data.unwrap();
assert!(cleared_obj.as_object().unwrap().is_empty());
info!("LocalStorage workflow passed!");
}
@@ -0,0 +1,448 @@
use browser_use::{BrowserSession, LaunchOptions,
tools::{GetMarkdownParams, Tool, ToolContext, markdown::GetMarkdownTool}};
use log::info;
/// Test basic markdown extraction from a simple HTML page
#[test]
#[ignore] // Requires Chrome to be installed
fn test_basic_markdown_extraction() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Create a simple article page
let html = r#"
<!DOCTYPE html>
<html>
<head>
<title>Test Article</title>
</head>
<body>
<article>
<h1>Main Article Title</h1>
<p>This is the first paragraph of the article.</p>
<p>This is the second paragraph with <strong>bold text</strong> and <em>italic text</em>.</p>
</article>
</body>
</html>
"#;
let data_url = format!("data:text/html,{}", urlencoding::encode(html));
session.navigate(&data_url).expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
// Create tool and context
let tool = GetMarkdownTool::default();
let mut context = ToolContext::new(&session);
// Execute the tool
let result =
tool.execute_typed(GetMarkdownParams::default(), &mut context).expect("Failed to execute markdown tool");
// Verify the result
assert!(result.success, "Tool execution should succeed");
assert!(result.data.is_some());
let data = result.data.unwrap();
info!("Markdown result: {}", serde_json::to_string_pretty(&data).unwrap());
let markdown = data["markdown"].as_str().expect("Should have markdown");
// Verify content was extracted
assert!(markdown.contains("Main Article Title"), "Should contain title");
assert!(markdown.contains("first paragraph"), "Should contain first paragraph");
assert!(markdown.contains("second paragraph"), "Should contain second paragraph");
// Verify metadata
assert_eq!(data["currentPage"].as_u64(), Some(1));
assert_eq!(data["totalPages"].as_u64(), Some(1));
assert_eq!(data["hasMorePages"].as_bool(), Some(false));
}
/// Test markdown extraction with Readability filtering
#[test]
#[ignore]
fn test_readability_filtering() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Create a page with navigation, sidebar, and main content
let html = r#"
<!DOCTYPE html>
<html>
<head>
<title>Article with Navigation</title>
</head>
<body>
<nav>
<ul>
<li>Home</li>
<li>About</li>
<li>Contact</li>
</ul>
</nav>
<aside class="sidebar">
<h3>Advertisement</h3>
<p>Buy our product!</p>
</aside>
<article>
<h1>Important Article</h1>
<p>This is the main content that should be extracted by Readability.</p>
<p>It contains valuable information for the reader.</p>
<p>Navigation and ads should be filtered out.</p>
</article>
<footer>
<p>Copyright 2025</p>
</footer>
</body>
</html>
"#;
let data_url = format!("data:text/html,{}", urlencoding::encode(html));
session.navigate(&data_url).expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
let tool = GetMarkdownTool::default();
let mut context = ToolContext::new(&session);
let result =
tool.execute_typed(GetMarkdownParams::default(), &mut context).expect("Failed to execute markdown tool");
assert!(result.success);
let data = result.data.unwrap();
let markdown = data["markdown"].as_str().expect("Should have markdown");
info!("Extracted markdown:\n{}", markdown);
// Main content should be present
assert!(markdown.contains("Important Article"), "Should contain article title");
assert!(markdown.contains("main content"), "Should contain main content");
// The exact filtering depends on Readability's algorithm
// In some cases, it might include navigation/footer if the article is too short
// So we just verify the main content is present
}
/// Test pagination with large content
#[test]
#[ignore]
fn test_markdown_pagination() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Create a long article that will require multiple pages
let mut paragraphs = String::new();
for i in 1..=200 {
paragraphs.push_str(&format!(
"<p>This is paragraph number {}. It contains some text to make the content longer. Lorem ipsum dolor sit amet, consectetur adipiscing elit.</p>\n",
i
));
}
let html = format!(
r#"
<!DOCTYPE html>
<html>
<head>
<title>Long Article</title>
</head>
<body>
<article>
<h1>Very Long Article</h1>
{}
</article>
</body>
</html>
"#,
paragraphs
);
let data_url = format!("data:text/html,{}", urlencoding::encode(&html));
session.navigate(&data_url).expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(1000));
let tool = GetMarkdownTool::default();
let mut context = ToolContext::new(&session);
// Get first page with small page size
let result = tool
.execute_typed(
GetMarkdownParams {
page: 1,
page_size: 5000, // Small page size to force pagination
},
&mut context,
)
.expect("Failed to execute markdown tool");
assert!(result.success);
let data = result.data.unwrap();
info!("Pagination result: {}", serde_json::to_string_pretty(&data).unwrap());
let markdown = data["markdown"].as_str().expect("Should have markdown");
let current_page = data["currentPage"].as_u64().expect("Should have currentPage");
let total_pages = data["totalPages"].as_u64().expect("Should have totalPages");
let has_more = data["hasMorePages"].as_bool().expect("Should have hasMorePages");
// Verify pagination
assert_eq!(current_page, 1);
assert!(total_pages > 1, "Should have multiple pages, got total_pages={}", total_pages);
assert!(has_more, "Should have more pages");
// Verify title is on first page (either the original or what Readability extracted)
let title_present = markdown.contains("Very Long Article") || markdown.contains("Long Article");
assert!(title_present, "First page should have title. Markdown: {}", &markdown[..200.min(markdown.len())]);
// Verify pagination footer
assert!(markdown.contains("Page 1 of"), "Should have pagination info");
assert!(markdown.contains("more page"), "Should indicate more pages");
// Note: Testing second page in the same session sometimes fails due to
// Readability caching. In production this works fine as each call is independent.
// Uncomment below to test second page with a new session:
/*
// Test getting second page
let result2 = tool
.execute_typed(
GetMarkdownParams {
page: 2,
page_size: 5000,
},
&mut context,
)
.expect("Failed to execute markdown tool");
assert!(result2.success);
let data2 = result2.data.unwrap();
let markdown2 = data2["markdown"].as_str().expect("Should have markdown");
// Second page should not have the title
assert!(!markdown2.starts_with("# Very Long Article"), "Second page should not start with title");
// Should have different content than first page
assert_ne!(markdown, markdown2, "Pages should have different content");
*/
}
/// Test edge case: empty page
#[test]
#[ignore]
fn test_empty_page() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
let html = r#"
<!DOCTYPE html>
<html>
<head>
<title>Empty Page</title>
</head>
<body>
</body>
</html>
"#;
let data_url = format!("data:text/html,{}", urlencoding::encode(html));
session.navigate(&data_url).expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
let tool = GetMarkdownTool::default();
let mut context = ToolContext::new(&session);
let result = tool.execute_typed(GetMarkdownParams::default(), &mut context);
// Should handle empty content gracefully
// Readability might fail on empty pages, which is acceptable
match result {
Ok(res) => {
info!("Empty page result: {:?}", res);
// If it succeeds, it should have minimal content
}
Err(e) => {
info!("Empty page error (expected): {:?}", e);
// Failing on empty pages is acceptable
}
}
}
/// Test page with tables (GFM support)
#[test]
#[ignore]
fn test_table_conversion() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
let html = r#"
<!DOCTYPE html>
<html>
<head>
<title>Table Test</title>
</head>
<body>
<article>
<h1>Data Table</h1>
<table>
<thead>
<tr>
<th>Name</th>
<th>Age</th>
<th>City</th>
</tr>
</thead>
<tbody>
<tr>
<td>Alice</td>
<td>30</td>
<td>New York</td>
</tr>
<tr>
<td>Bob</td>
<td>25</td>
<td>London</td>
</tr>
</tbody>
</table>
</article>
</body>
</html>
"#;
let data_url = format!("data:text/html,{}", urlencoding::encode(html));
session.navigate(&data_url).expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
let tool = GetMarkdownTool::default();
let mut context = ToolContext::new(&session);
let result =
tool.execute_typed(GetMarkdownParams::default(), &mut context).expect("Failed to execute markdown tool");
assert!(result.success);
let data = result.data.unwrap();
let markdown = data["markdown"].as_str().expect("Should have markdown");
info!("Table markdown:\n{}", markdown);
// Verify table content is present
assert!(markdown.contains("Name"), "Should contain table header");
assert!(markdown.contains("Alice"), "Should contain table data");
assert!(markdown.contains("Bob"), "Should contain table data");
// Table should be formatted (exact format depends on html2md library)
assert!(markdown.contains("30"), "Should contain age data");
assert!(markdown.contains("London"), "Should contain city data");
}
/// Test calling get_markdown twice on the same page
/// This reproduces the bug where the second call fails with "No value returned from JavaScript"
#[test]
#[ignore]
fn test_double_execution_same_page() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Create a simple article page
let html = r#"
<!DOCTYPE html>
<html>
<head>
<title>Double Execution Test</title>
</head>
<body>
<article>
<h1>Test Article</h1>
<p>This is paragraph one with some content.</p>
<p>This is paragraph two with more content.</p>
<p>This is paragraph three with even more content.</p>
</article>
</body>
</html>
"#;
let data_url = format!("data:text/html,{}", urlencoding::encode(html));
session.navigate(&data_url).expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
let tool = GetMarkdownTool::default();
let mut context = ToolContext::new(&session);
// First execution
info!("Executing get_markdown (first call)...");
let result1 = tool
.execute_typed(GetMarkdownParams::default(), &mut context)
.expect("First call to get_markdown should succeed");
assert!(result1.success, "First execution should succeed");
let data1 = result1.data.expect("First call should return data");
let markdown1 = data1["markdown"].as_str().expect("Should have markdown");
info!("First call succeeded, markdown length: {}", markdown1.len());
assert!(markdown1.contains("Test Article"), "First call should contain title");
assert!(markdown1.contains("paragraph one"), "First call should contain content");
// Second execution on the same page - this is where the bug occurs
info!("Executing get_markdown (second call on same page)...");
let result2 = tool
.execute_typed(GetMarkdownParams::default(), &mut context)
.expect("Second call to get_markdown should also succeed");
assert!(result2.success, "Second execution should succeed");
let data2 = result2.data.expect("Second call should return data");
let markdown2 = data2["markdown"].as_str().expect("Should have markdown");
info!("Second call succeeded, markdown length: {}", markdown2.len());
assert!(markdown2.contains("Test Article"), "Second call should contain title");
assert!(markdown2.contains("paragraph one"), "Second call should contain content");
// The content should be the same (or at least very similar)
assert_eq!(markdown1, markdown2, "Both calls should return the same content");
info!("Double execution test passed!");
}
/// Test requesting page beyond available pages
#[test]
#[ignore]
fn test_page_clamping() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
let html = r#"
<!DOCTYPE html>
<html>
<head>
<title>Short Article</title>
</head>
<body>
<article>
<h1>Short Content</h1>
<p>This is a very short article.</p>
</article>
</body>
</html>
"#;
let data_url = format!("data:text/html,{}", urlencoding::encode(html));
session.navigate(&data_url).expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
let tool = GetMarkdownTool::default();
let mut context = ToolContext::new(&session);
// Request page 999 (way beyond available content)
let result = tool
.execute_typed(GetMarkdownParams { page: 999, page_size: 100_000 }, &mut context)
.expect("Failed to execute markdown tool");
assert!(result.success);
let data = result.data.unwrap();
// Should clamp to last available page (page 1)
assert_eq!(data["currentPage"].as_u64(), Some(1));
assert_eq!(data["totalPages"].as_u64(), Some(1));
assert_eq!(data["hasMorePages"].as_bool(), Some(false));
}
@@ -0,0 +1,241 @@
use browser_use::{BrowserSession, LaunchOptions,
tools::{CloseParams, GoBackParams, GoForwardParams, Tool, ToolContext, close::CloseTool,
go_back::GoBackTool, go_forward::GoForwardTool}};
use log::info;
#[test]
#[ignore] // Requires Chrome to be installed
fn test_go_back_tool() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Navigate to first page
session.navigate("data:text/html,<html><body><h1>Page 1</h1></body></html>").expect("Failed to navigate to page 1");
std::thread::sleep(std::time::Duration::from_millis(500));
// Navigate to second page
session.navigate("data:text/html,<html><body><h1>Page 2</h1></body></html>").expect("Failed to navigate to page 2");
std::thread::sleep(std::time::Duration::from_millis(500));
// Verify we're on page 2
let current_url = session.tab().unwrap().get_url();
assert!(current_url.contains("Page 2"));
// Create tool and context
let tool = GoBackTool::default();
let mut context = ToolContext::new(&session);
// Execute the tool to go back
let result = tool.execute_typed(GoBackParams {}, &mut context).expect("Failed to execute go_back tool");
// Verify the result
assert!(result.success, "Tool execution should succeed");
assert!(result.data.is_some());
let data = result.data.unwrap();
info!("Go back result: {}", serde_json::to_string_pretty(&data).unwrap());
assert_eq!(data["message"].as_str(), Some("Navigated back in history"));
std::thread::sleep(std::time::Duration::from_millis(500));
// Verify we went back to page 1
let new_url = session.tab().unwrap().get_url();
assert!(new_url.contains("Page 1"));
}
#[test]
#[ignore]
fn test_go_forward_tool() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Navigate to first page
session.navigate("data:text/html,<html><body><h1>Page 1</h1></body></html>").expect("Failed to navigate to page 1");
std::thread::sleep(std::time::Duration::from_millis(500));
// Navigate to second page
session.navigate("data:text/html,<html><body><h1>Page 2</h1></body></html>").expect("Failed to navigate to page 2");
std::thread::sleep(std::time::Duration::from_millis(500));
// Go back to page 1
session.go_back().expect("Failed to go back");
std::thread::sleep(std::time::Duration::from_millis(500));
// Verify we're on page 1
let current_url = session.tab().unwrap().get_url();
assert!(current_url.contains("Page 1"));
// Create tool and context
let tool = GoForwardTool::default();
let mut context = ToolContext::new(&session);
// Execute the tool to go forward
let result = tool.execute_typed(GoForwardParams {}, &mut context).expect("Failed to execute go_forward tool");
// Verify the result
assert!(result.success, "Tool execution should succeed");
assert!(result.data.is_some());
let data = result.data.unwrap();
info!("Go forward result: {}", serde_json::to_string_pretty(&data).unwrap());
assert_eq!(data["message"].as_str(), Some("Navigated forward in history"));
std::thread::sleep(std::time::Duration::from_millis(500));
// Verify we went forward to page 2
let new_url = session.tab().unwrap().get_url();
assert!(new_url.contains("Page 2"));
}
#[test]
#[ignore]
fn test_navigation_workflow() {
// Test a complete workflow: navigate to multiple pages, go back, go forward
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Navigate to page 1
session
.navigate("data:text/html,<html><body><h1>Page 1</h1><a id='link' href='data:text/html,<html><body><h1>Page 2</h1></body></html>'>Next</a></body></html>")
.expect("Failed to navigate to page 1");
std::thread::sleep(std::time::Duration::from_millis(500));
info!("On page 1");
// Navigate to page 2
session.navigate("data:text/html,<html><body><h1>Page 2</h1></body></html>").expect("Failed to navigate to page 2");
std::thread::sleep(std::time::Duration::from_millis(500));
info!("On page 2");
// Navigate to page 3
session.navigate("data:text/html,<html><body><h1>Page 3</h1></body></html>").expect("Failed to navigate to page 3");
std::thread::sleep(std::time::Duration::from_millis(500));
info!("On page 3");
// Create tools
let go_back_tool = GoBackTool::default();
let go_forward_tool = GoForwardTool::default();
// Go back to page 2
let mut context = ToolContext::new(&session);
let result = go_back_tool.execute_typed(GoBackParams {}, &mut context).expect("Failed to go back");
assert!(result.success);
info!("Went back to page 2");
std::thread::sleep(std::time::Duration::from_millis(500));
// Go back to page 1
let mut context = ToolContext::new(&session);
let result = go_back_tool.execute_typed(GoBackParams {}, &mut context).expect("Failed to go back");
assert!(result.success);
info!("Went back to page 1");
std::thread::sleep(std::time::Duration::from_millis(500));
// Verify we're on page 1
let current_url = session.tab().unwrap().get_url();
assert!(current_url.contains("Page 1"));
// Go forward to page 2
let mut context = ToolContext::new(&session);
let result = go_forward_tool.execute_typed(GoForwardParams {}, &mut context).expect("Failed to go forward");
assert!(result.success);
info!("Went forward to page 2");
std::thread::sleep(std::time::Duration::from_millis(500));
// Verify we're on page 2
let current_url = session.tab().unwrap().get_url();
assert!(current_url.contains("Page 2"));
}
#[test]
#[ignore]
fn test_close_tool() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Navigate to a page
session.navigate("data:text/html,<html><body><h1>Test Page</h1></body></html>").expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
// Verify browser is working
let tabs = session.get_tabs().expect("Failed to get tabs");
assert!(!tabs.is_empty(), "Should have at least one tab");
// Create tool and context
let tool = CloseTool::default();
let mut context = ToolContext::new(&session);
// Execute the tool to close the browser
let result = tool.execute_typed(CloseParams {}, &mut context).expect("Failed to execute close tool");
// Verify the result
assert!(result.success, "Tool execution should succeed");
assert!(result.data.is_some());
let data = result.data.unwrap();
info!("Close result: {}", serde_json::to_string_pretty(&data).unwrap());
assert_eq!(data["message"].as_str(), Some("Browser closed successfully"));
// Note: After closing, subsequent operations may fail
// The browser tabs should be closed
std::thread::sleep(std::time::Duration::from_millis(500));
}
#[test]
#[ignore]
fn test_go_back_on_first_page() {
// Test that going back on the first page doesn't crash
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Navigate to only one page
session.navigate("data:text/html,<html><body><h1>First Page</h1></body></html>").expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
// Create tool and context
let tool = GoBackTool::default();
let mut context = ToolContext::new(&session);
// Execute the tool - should succeed but do nothing
let result = tool.execute_typed(GoBackParams {}, &mut context).expect("Failed to execute go_back tool");
assert!(result.success, "Tool execution should succeed even if no previous page");
info!("Go back on first page result: {}", serde_json::to_string_pretty(&result.data.unwrap()).unwrap());
}
#[test]
#[ignore]
fn test_go_forward_on_last_page() {
// Test that going forward when there's no forward history doesn't crash
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Navigate to a page
session.navigate("data:text/html,<html><body><h1>Page</h1></body></html>").expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
// Create tool and context
let tool = GoForwardTool::default();
let mut context = ToolContext::new(&session);
// Execute the tool - should succeed but do nothing
let result = tool.execute_typed(GoForwardParams {}, &mut context).expect("Failed to execute go_forward tool");
assert!(result.success, "Tool execution should succeed even if no forward history");
info!("Go forward on last page result: {}", serde_json::to_string_pretty(&result.data.unwrap()).unwrap());
}
@@ -0,0 +1,326 @@
//! Integration tests for the sitemap analysis tool
use browser_use::{
BrowserSession, LaunchOptions,
tools::{sitemap::{SitemapTool, SitemapParams}, Tool, ToolContext},
};
use log::info;
#[test]
#[ignore] // Requires Chrome to be installed
fn test_sitemap_tool_basic() {
env_logger::try_init().ok();
let session = BrowserSession::launch(LaunchOptions::new().headless(true))
.expect("Failed to launch browser");
// Test with a real site that has a sitemap (example.com is simple)
let tool = SitemapTool::default();
let mut context = ToolContext::new(&session);
let params = SitemapParams {
url: "https://example.com".to_string(),
analyze_structure: false,
max_pages: 5,
};
let result = tool.execute_typed(params, &mut context).expect("Failed to execute sitemap tool");
assert!(result.success, "Tool execution should succeed");
assert!(result.data.is_some(), "Should have result data");
let data = result.data.unwrap();
info!("Sitemap result: {}", serde_json::to_string_pretty(&data).unwrap());
// Verify basic structure
assert!(data["base_url"].as_str().is_some());
assert!(data["sitemaps"].is_array());
assert!(data["pages"].is_array());
}
#[test]
#[ignore] // Requires Chrome to be installed
fn test_sitemap_tool_with_structure_analysis() {
env_logger::try_init().ok();
let session = BrowserSession::launch(LaunchOptions::new().headless(true))
.expect("Failed to launch browser");
let tool = SitemapTool::default();
let mut context = ToolContext::new(&session);
// Use a site that we know has structure
let params = SitemapParams {
url: "https://example.com".to_string(),
analyze_structure: true,
max_pages: 2,
};
let result = tool.execute_typed(params, &mut context).expect("Failed to execute sitemap tool");
assert!(result.success, "Tool execution should succeed");
assert!(result.data.is_some(), "Should have result data");
let data = result.data.unwrap();
info!("Sitemap with structure: {}", serde_json::to_string_pretty(&data).unwrap());
// Verify structure analysis was performed
let page_structures = data["page_structures"].as_array();
assert!(page_structures.is_some(), "Should have page_structures array");
// If any pages were analyzed, check the structure format
if let Some(structures) = page_structures {
if !structures.is_empty() {
let first = &structures[0];
assert!(first["url"].as_str().is_some(), "Page structure should have url");
assert!(first["title"].as_str().is_some(), "Page structure should have title");
assert!(first["headings"].is_array(), "Page structure should have headings array");
assert!(first["nav_links"].is_array(), "Page structure should have nav_links array");
assert!(first["sections"].is_array(), "Page structure should have sections array");
}
}
}
#[test]
#[ignore] // Requires Chrome to be installed
fn test_sitemap_analyze_function() {
env_logger::try_init().ok();
let session = BrowserSession::launch(LaunchOptions::new().headless(true))
.expect("Failed to launch browser");
// Test the standalone analyze_sitemap function
let result = browser_use::tools::sitemap::analyze_sitemap(
&session,
"https://example.com",
true,
2,
).expect("Failed to analyze sitemap");
info!("Analyze sitemap result: {:?}", result);
assert_eq!(result.base_url, "https://example.com");
// If structure analysis was performed on homepage
if !result.page_structures.is_empty() {
let homepage = &result.page_structures[0];
assert!(!homepage.url.is_empty(), "Should have URL");
assert!(!homepage.title.is_empty(), "Should have title");
}
}
#[test]
#[ignore] // Requires Chrome to be installed
fn test_sitemap_page_structure_extraction() {
env_logger::try_init().ok();
let session = BrowserSession::launch(LaunchOptions::new().headless(true))
.expect("Failed to launch browser");
// Create a test page with known structure using data: URL
let test_html = r#"
<!DOCTYPE html>
<html>
<head>
<title>Test Page</title>
<meta name="description" content="A test page for sitemap analysis">
<meta name="keywords" content="test, sitemap, analysis">
<link rel="canonical" href="https://example.com/test">
</head>
<body>
<header>
<nav>
<a href="/home">Home</a>
<a href="/about">About</a>
<a href="/contact">Contact</a>
</nav>
</header>
<main id="content" role="main">
<h1>Welcome to the Test Page</h1>
<section>
<h2>Section One</h2>
<p>Some content here with multiple words to count.</p>
</section>
<section>
<h2>Section Two</h2>
<p>More content in this section.</p>
</section>
</main>
<aside>
<h3>Sidebar</h3>
</aside>
<footer>
<p>Footer content</p>
</footer>
</body>
</html>
"#;
// Navigate to the test page
let data_url = format!("data:text/html,{}", urlencoding::encode(test_html));
session.navigate(&data_url).expect("Failed to navigate");
session.wait_for_navigation().expect("Failed to wait for navigation");
std::thread::sleep(std::time::Duration::from_millis(500));
// Extract structure using JavaScript evaluation directly
let tab = session.tab().expect("Failed to get tab");
let structure_js = r#"
(function() {
var structure = {
url: window.location.href,
title: document.title,
headings: [],
nav_links: [],
sections: [],
main_content: null,
meta: {}
};
var headings = document.querySelectorAll('h1, h2, h3, h4, h5, h6');
for (var i = 0; i < headings.length && i < 50; i++) {
structure.headings.push({
level: parseInt(headings[i].tagName.charAt(1)),
text: headings[i].innerText.trim().substring(0, 200)
});
}
var navs = document.querySelectorAll('nav a, header a, [role="navigation"] a');
var seenLinks = new Set();
for (var i = 0; i < navs.length && structure.nav_links.length < 30; i++) {
var href = navs[i].getAttribute('href');
var text = navs[i].innerText.trim();
if (href && text && !seenLinks.has(href)) {
seenLinks.add(href);
structure.nav_links.push({ text: text.substring(0, 100), href: href });
}
}
var sections = document.querySelectorAll('main, article, section, aside, footer');
for (var i = 0; i < sections.length && i < 20; i++) {
var el = sections[i];
structure.sections.push({
tag: el.tagName.toLowerCase(),
id: el.id || null,
class: el.className ? el.className.substring(0, 100) : null,
role: el.getAttribute('role') || null
});
}
var main = document.querySelector('main, [role="main"], #main, #content, .main-content');
if (main) {
structure.main_content = {
tag: main.tagName.toLowerCase(),
id: main.id || null,
word_count: main.innerText.split(/\s+/).length
};
}
var metaDesc = document.querySelector('meta[name="description"]');
if (metaDesc) structure.meta.description = metaDesc.getAttribute('content');
return JSON.stringify(structure);
})()
"#;
let result = tab.evaluate(structure_js, false).expect("Failed to evaluate JS");
let json_str = result.value.unwrap();
let json_str = json_str.as_str().unwrap();
let structure: serde_json::Value = serde_json::from_str(json_str).expect("Failed to parse structure");
info!("Page structure: {}", serde_json::to_string_pretty(&structure).unwrap());
// Verify structure extraction
assert_eq!(structure["title"].as_str(), Some("Test Page"));
// Check headings
let headings = structure["headings"].as_array().unwrap();
assert!(headings.len() >= 3, "Should have at least 3 headings (h1, h2, h2, h3)");
assert_eq!(headings[0]["level"].as_u64(), Some(1));
assert_eq!(headings[0]["text"].as_str(), Some("Welcome to the Test Page"));
// Check nav links
let nav_links = structure["nav_links"].as_array().unwrap();
assert_eq!(nav_links.len(), 3, "Should have 3 nav links");
// Check sections
let sections = structure["sections"].as_array().unwrap();
assert!(!sections.is_empty(), "Should have sections");
// Check main content
let main_content = &structure["main_content"];
assert!(main_content.is_object(), "Should have main_content");
assert_eq!(main_content["tag"].as_str(), Some("main"));
assert_eq!(main_content["id"].as_str(), Some("content"));
assert!(main_content["word_count"].as_u64().unwrap() > 0, "Should have word count");
// Check meta
assert_eq!(
structure["meta"]["description"].as_str(),
Some("A test page for sitemap analysis")
);
}
#[test]
#[ignore] // Requires Chrome to be installed
fn test_sitemap_robots_txt_parsing() {
env_logger::try_init().ok();
// This test verifies that we can parse sitemap references from robots.txt
// We'll test with a known site that has a proper robots.txt
let session = BrowserSession::launch(LaunchOptions::new().headless(true))
.expect("Failed to launch browser");
let tool = SitemapTool::default();
let mut context = ToolContext::new(&session);
// Test with a site known to have robots.txt with sitemap
let params = SitemapParams {
url: "https://www.google.com".to_string(),
analyze_structure: false,
max_pages: 1,
};
let result = tool.execute_typed(params, &mut context).expect("Failed to execute sitemap tool");
assert!(result.success, "Tool execution should succeed");
let data = result.data.unwrap();
info!("Google sitemap result: {}", serde_json::to_string_pretty(&data).unwrap());
// Google should have robots.txt
// Note: This test may be flaky depending on network conditions
assert_eq!(data["base_url"].as_str(), Some("https://www.google.com"));
}
#[test]
#[ignore] // Requires Chrome to be installed
fn test_sitemap_max_pages_limit() {
env_logger::try_init().ok();
let session = BrowserSession::launch(LaunchOptions::new().headless(true))
.expect("Failed to launch browser");
let tool = SitemapTool::default();
let mut context = ToolContext::new(&session);
let params = SitemapParams {
url: "https://example.com".to_string(),
analyze_structure: true,
max_pages: 1, // Limit to 1 page
};
let result = tool.execute_typed(params, &mut context).expect("Failed to execute sitemap tool");
assert!(result.success);
let data = result.data.unwrap();
let page_structures = data["page_structures"].as_array().unwrap();
// Should not exceed max_pages
assert!(
page_structures.len() <= 1,
"Should not analyze more than max_pages (1)"
);
}
@@ -0,0 +1,300 @@
use browser_use::{BrowserSession, LaunchOptions,
tools::{CloseTabParams, NewTabParams, SwitchTabParams, TabListParams, Tool, ToolContext,
close_tab::CloseTabTool, new_tab::NewTabTool, switch_tab::SwitchTabTool,
tab_list::TabListTool}};
use log::info;
#[test]
#[ignore]
fn test_new_tab() {
use browser_use::tools::{NewTabParams, Tool, ToolContext, new_tab::NewTabTool};
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Navigate to initial page
session.navigate("data:text/html,<html><body><h1>First Tab</h1></body></html>").expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
// Get initial tab count
let initial_tabs = session.get_tabs().expect("Failed to get tabs");
let initial_count = initial_tabs.len();
info!("Initial tab count: {}", initial_count);
// Create tool and context
let tool = NewTabTool::default();
let mut context = ToolContext::new(&session);
// Execute the tool to create a new tab
let result = tool
.execute_typed(
NewTabParams { url: "data:text/html,<html><body><h1>Second Tab</h1></body></html>".to_string() },
&mut context,
)
.expect("Failed to execute new_tab tool");
// Verify the result
assert!(result.success, "Tool execution should succeed");
assert!(result.data.is_some());
let data = result.data.unwrap();
assert!(data["url"].as_str().is_some(), "Result should contain url field");
assert!(data["message"].as_str().is_some(), "Result should contain message field");
info!("New tab result: {}", serde_json::to_string_pretty(&data).unwrap());
std::thread::sleep(std::time::Duration::from_millis(500));
// Verify tab count increased
let final_tabs = session.get_tabs().expect("Failed to get tabs");
let final_count = final_tabs.len();
info!("Final tab count: {}", final_count);
assert_eq!(final_count, initial_count + 1, "Tab count should increase by 1");
}
#[test]
#[ignore] // Requires Chrome to be installed
fn test_tab_list() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Navigate to a simple page
session.navigate("data:text/html,<html><body><h1>First Tab</h1></body></html>").expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
// Create tool and context
let tool = TabListTool::default();
let mut context = ToolContext::new(&session);
// Execute the tool
let result = tool.execute_typed(TabListParams {}, &mut context).expect("Failed to execute tab_list tool");
// Verify the result
assert!(result.success, "Tool execution should succeed");
assert!(result.data.is_some());
let data = result.data.unwrap();
let tab_list = data["tab_list"].as_array().expect("No tab_list field");
let count = data["count"].as_u64().expect("No count field");
info!("Tab list: {}", serde_json::to_string_pretty(&tab_list).unwrap());
// Should have at least 1 tab
assert!(count >= 1, "Expected at least 1 tab");
assert_eq!(tab_list.len() as u64, count);
// Check first tab structure
let first_tab = &tab_list[0];
assert!(first_tab["index"].is_number(), "Tab should have index");
assert!(first_tab["active"].is_boolean(), "Tab should have active flag");
assert!(first_tab["title"].is_string(), "Tab should have title");
assert!(first_tab["url"].is_string(), "Tab should have url");
}
#[test]
#[ignore]
fn test_new_tab_and_switch() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Navigate to initial page
session.navigate("data:text/html,<html><body><h1>First Tab</h1></body></html>").expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
// Create a new tab
let new_tab_tool = NewTabTool::default();
let mut context = ToolContext::new(&session);
let result = new_tab_tool
.execute_typed(
NewTabParams { url: "data:text/html,<html><body><h1>Second Tab</h1></body></html>".to_string() },
&mut context,
)
.expect("Failed to execute new_tab tool");
assert!(result.success, "New tab creation should succeed");
std::thread::sleep(std::time::Duration::from_millis(500));
// List tabs to verify count increased by 1
let tab_list_tool = TabListTool::default();
let mut context = ToolContext::new(&session);
let result = tab_list_tool.execute_typed(TabListParams {}, &mut context).expect("Failed to execute tab_list tool");
assert!(result.success);
let data = result.data.unwrap();
let count = data["count"].as_u64().expect("No count field");
info!("Tab count after creating new tab: {}", count);
assert!(count >= 2, "Should have at least 2 tabs, got {}", count);
// Switch to first tab (index 0)
let switch_tab_tool = SwitchTabTool::default();
let mut context = ToolContext::new(&session);
let result = switch_tab_tool
.execute_typed(SwitchTabParams { index: 0 }, &mut context)
.expect("Failed to execute switch_tab tool");
assert!(result.success, "Switch tab should succeed");
let data = result.data.unwrap();
assert_eq!(data["index"].as_u64(), Some(0));
info!("Switched to tab: {}", serde_json::to_string_pretty(&data).unwrap());
}
#[test]
#[ignore]
fn test_switch_tab_invalid_index() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
session.navigate("data:text/html,<html><body><h1>Tab</h1></body></html>").expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
// Try to switch to invalid index
let switch_tab_tool = SwitchTabTool::default();
let mut context = ToolContext::new(&session);
let result = switch_tab_tool
.execute_typed(SwitchTabParams { index: 999 }, &mut context)
.expect("Failed to execute switch_tab tool");
// Should fail gracefully
assert!(!result.success, "Should fail for invalid index");
assert!(result.error.is_some());
info!("Expected error: {}", result.error.unwrap());
}
#[test]
#[ignore]
fn test_close_tab() {
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Create two tabs
session.navigate("data:text/html,<html><body><h1>First Tab</h1></body></html>").expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(500));
let new_tab_tool = NewTabTool::default();
let mut context = ToolContext::new(&session);
new_tab_tool
.execute_typed(
NewTabParams { url: "data:text/html,<html><body><h1>Second Tab</h1></body></html>".to_string() },
&mut context,
)
.expect("Failed to create new tab");
std::thread::sleep(std::time::Duration::from_millis(500));
// Verify we have at least 2 tabs
let tab_list_tool = TabListTool::default();
let mut context = ToolContext::new(&session);
let result = tab_list_tool.execute_typed(TabListParams {}, &mut context).expect("Failed to execute tab_list tool");
let count_before = result.data.unwrap()["count"].as_u64().unwrap();
info!("Tab count before closing: {}", count_before);
assert!(count_before >= 2, "Should have at least 2 tabs before closing, got {}", count_before);
// Close the active tab (second tab)
let close_tab_tool = CloseTabTool::default();
let mut context = ToolContext::new(&session);
let result =
close_tab_tool.execute_typed(CloseTabParams {}, &mut context).expect("Failed to execute close_tab tool");
assert!(result.success, "Close tab should succeed");
info!("Closed tab: {}", serde_json::to_string_pretty(&result.data.unwrap()).unwrap());
std::thread::sleep(std::time::Duration::from_millis(500));
// Verify we now have one less tab
let mut context = ToolContext::new(&session);
let result = tab_list_tool.execute_typed(TabListParams {}, &mut context).expect("Failed to execute tab_list tool");
let count_after = result.data.unwrap()["count"].as_u64().unwrap();
info!("Tab count after closing: {}", count_after);
assert_eq!(count_after, count_before - 1, "Should have one less tab after closing");
}
#[test]
#[ignore]
fn test_tab_workflow() {
// Test a complete workflow: create multiple tabs, switch between them, list them, and close one
let session = BrowserSession::launch(LaunchOptions::new().headless(true)).expect("Failed to launch browser");
// Start with first tab
session.navigate("data:text/html,<html><body><h1>Tab 1</h1></body></html>").expect("Failed to navigate");
std::thread::sleep(std::time::Duration::from_millis(300));
// Create second tab
let new_tab_tool = NewTabTool::default();
let mut context = ToolContext::new(&session);
new_tab_tool
.execute_typed(
NewTabParams { url: "data:text/html,<html><body><h1>Tab 2</h1></body></html>".to_string() },
&mut context,
)
.expect("Failed to create tab 2");
std::thread::sleep(std::time::Duration::from_millis(300));
// Create third tab
let mut context = ToolContext::new(&session);
new_tab_tool
.execute_typed(
NewTabParams { url: "data:text/html,<html><body><h1>Tab 3</h1></body></html>".to_string() },
&mut context,
)
.expect("Failed to create tab 3");
std::thread::sleep(std::time::Duration::from_millis(300));
// List all tabs
let tab_list_tool = TabListTool::default();
let mut context = ToolContext::new(&session);
let result = tab_list_tool.execute_typed(TabListParams {}, &mut context).expect("Failed to list tabs");
let count = result.data.as_ref().unwrap()["count"].as_u64().unwrap();
info!("Total tabs: {}", count);
assert!(count >= 3, "Should have at least 3 tabs, got {}", count);
info!("All tabs: {}", result.data.unwrap()["summary"].as_str().unwrap());
// Switch to second tab (index 1)
let switch_tab_tool = SwitchTabTool::default();
let mut context = ToolContext::new(&session);
let result =
switch_tab_tool.execute_typed(SwitchTabParams { index: 1 }, &mut context).expect("Failed to switch to tab 1");
assert!(result.success);
assert_eq!(result.data.unwrap()["index"].as_u64(), Some(1));
std::thread::sleep(std::time::Duration::from_millis(300));
// Close the current tab (tab 2, index 1)
let close_tab_tool = CloseTabTool::default();
let mut context = ToolContext::new(&session);
let result = close_tab_tool.execute_typed(CloseTabParams {}, &mut context).expect("Failed to close tab");
assert!(result.success);
info!("Closed: {}", result.data.unwrap()["message"].as_str().unwrap());
std::thread::sleep(std::time::Duration::from_millis(300));
// List tabs again to verify we have 2 tabs left
let mut context = ToolContext::new(&session);
let result = tab_list_tool.execute_typed(TabListParams {}, &mut context).expect("Failed to list tabs");
let final_count = result.data.unwrap()["count"].as_u64().unwrap();
info!("Final tab count: {}", final_count);
assert_eq!(final_count, count - 1, "Should have one less tab after closing");
}