Update browser_tools.py

browser_tools.py

@@ -9,6 +9,12 @@ from typing import Optional
 
 logger = logging.getLogger(__name__)
 
+# Default headers to mimic a browser and reduce chance of 403 responses
+DEFAULT_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0 Safari/537.36",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+}
+
 try:
     from bs4 import BeautifulSoup
 except Exception:
@@ -24,7 +30,10 @@ def visit_page(url: str, max_chars: int = 2000) -> str:
     if not url:
         return "(no url provided)"
     try:
-        resp = requests.get(url, timeout=10)
+        # Some domains have SSL certificate issues; disable verification for those
+        verify_ssl = not any(domain in url for domain in ["npb.or.jp", "npb.jp"])
+        # include headers to reduce bot-blocking by some sites
+        resp = requests.get(url, timeout=10, headers=DEFAULT_HEADERS, verify=verify_ssl)
         resp.raise_for_status()
         text = resp.text
         if BeautifulSoup is not None:
@@ -39,6 +48,12 @@ def visit_page(url: str, max_chars: int = 2000) -> str:
         # fallback: return first chunk of raw HTML (not ideal)
         return text[:max_chars]
     except Exception as e:
+        # Improve error message for common network/DNS failures so the agent
+        # can fallback to alternative tools instead of crashing.
+        msg = str(e)
+        if "NameResolutionError" in msg or "Failed to resolve" in msg or "Temporary failure in name resolution" in msg:
+            logger.warning("visit_page DNS/network error for %s: %s", url, e)
+            return f"(visit_page error) network/DNS failure when fetching {url}"
         logger.warning("visit_page failed for %s: %s", url, e)
         return f"(visit_page error) {e}"