\u5728 AI \u5de5\u5177\u5f00\u53d1\u4e2d，\u6211\u4eec\u5e38\u5e38\u9762\u4e34\u8fd9\u6837\u7684\u56f0\u5883：\u5f53\u9700\u8981\u8c03\u7528\u5916\u90e8\u8d44\u6e90\u65f6，\u4e0d\u540c\u6846\u67b6\u7684\u534f\u8bae\u5dee\u5f02\u5bfc\u81f4\u91cd\u590d\u5f00\u53d1，\u800c\u624b\u52a8\u5904\u7406 URL \u6293\u53d6\u53c8\u5bb9\u6613\u9677\u5165\u5185\u5bb9\u622a\u65ad\u3001\u683c\u5f0f\u8f6c\u6362\u3001robots.txt \u9650\u5236\u7b49\u9677\u9631\u3002\u4eca\u5929，\u6211\u4eec\u5c06\u7ed3\u5408 MCP（Model Context Protocol）\u534f\u8bae\u7684\u6807\u51c6\u5316\u80fd\u529b，\u6df1\u5165\u5256\u6790\u5982\u4f55\u6784\u5efa\u4e00\u4e2a\u9ad8\u6548\u3001\u5408\u89c4\u7684 Fetch \u670d\u52a1\u5668，\u5f7b\u5e95\u89e3\u51b3\u8fd9\u4e9b\u75db\u70b9\u3002\u672c\u6587\u7279\u522b\u589e\u52a0 30% \u6838\u5fc3\u4ee3\u7801\u7ec6\u8282，\u5e26\u4f60\u770b\u61c2\u6bcf\u4e2a\u6280\u672f\u70b9\u7684\u5b9e\u73b0\u903b\u8f91\u3002<\/p>\n
python<\/p>\n
\u8fd0\u884c<\/p>\n
from pydantic import BaseModel, Field, AnyUrl
\nfrom typing import Annotated<\/p>\n
class Fetch(BaseModel):
\n """URL\u83b7\u53d6\u53c2\u6570\u6a21\u578b（\u5e26\u5b8c\u6574\u9a8c\u8bc1\u903b\u8f91）"""
\n url: Annotated[
\n AnyUrl,
\n Field(
\n description="\u5fc5\u586b\u53c2\u6570，\u76ee\u6807URL\u5730\u5740（\u9700\u5305\u542b\u534f\u8bae\u5934，\u5982https:\/\/）",
\n examples=["https:\/\/developer.mozilla.org\/en-US\/"] # \u793a\u4f8b\u8f93\u5165
\n )
\n ]
\n max_length: Annotated[
\n int,
\n Field(
\n default=5000,
\n ge=1, # \u6700\u5c0f1\u5b57\u7b26
\n le=1000000, # \u6700\u5927100\u4e07\u5b57\u7b26
\n description="\u8fd4\u56de\u5185\u5bb9\u7684\u6700\u5927\u5b57\u7b26\u6570，\u9632\u6b62\u5185\u5b58\u6ea2\u51fa（\u9ed8\u8ba45000）"
\n )
\n ]
\n start_index: Annotated[
\n int,
\n Field(
\n default=0,
\n ge=0, # \u8d77\u59cb\u7d22\u5f15\u4e0d\u80fd\u4e3a\u8d1f
\n description="\u5206\u9875\u83b7\u53d6\u7684\u8d77\u59cb\u5b57\u7b26\u7d22\u5f15（\u9ed8\u8ba4\u4ece0\u5f00\u59cb）"
\n )
\n ]
\n raw: Annotated[
\n bool,
\n Field(
\n default=False,
\n description="\u662f\u5426\u8fd4\u56de\u539f\u59cbHTML（\u9ed8\u8ba4\u8f6c\u6362\u4e3aMarkdown）"
\n )
\n ]<\/p>\n
# \u81ea\u5b9a\u4e49\u9a8c\u8bc1：\u68c0\u67e5URL\u662f\u5426\u4e3a\u53ef\u8bbf\u95ee\u7684HTTP\/HTTPS\u5730\u5740
\n @validator("url")
\n def validate_url_scheme(cls, value):
\n scheme = urlparse(value).scheme
\n if scheme not in ["http", "https"]:
\n raise ValueError("URL\u5fc5\u987b\u4f7f\u7528http\u6216https\u534f\u8bae")
\n return value<\/p>\n
\u5173\u952e\u7ec6\u8282\u89e3\u6790：<\/p>\n<\/p>\n
python<\/p>\n
\u8fd0\u884c<\/p>\n
from httpx import AsyncClient, HTTPError
\nfrom typing import Tuple
\nfrom mcp.shared.exceptions import McpError
\nfrom .utils import extract_content_from_html # \u5047\u8bbeutils\u6a21\u5757\u5b58\u653e\u8f85\u52a9\u51fd\u6570<\/p>\n
async def fetch_url(
\n url: str,
\n user_agent: str,
\n force_raw: bool = False,
\n proxy_url: str | None = None
\n) -> Tuple[str, str]:
\n """\u6838\u5fc3\u6293\u53d6\u51fd\u6570（\u5e26\u5b8c\u6574\u9519\u8bef\u5904\u7406）"""
\n async with AsyncClient(
\n proxies=proxy_url, # \u652f\u6301\u4ee3\u7406\u670d\u52a1\u5668\u914d\u7f6e
\n follow_redirects=True, # \u81ea\u52a8\u5904\u7406\u6700\u591a5\u6b21\u91cd\u5b9a\u5411（\u9ed8\u8ba4\u914d\u7f6e）
\n timeout=30 # 30\u79d2\u8d85\u65f6\u63a7\u5236
\n ) as client:
\n try:
\n response = await client.get(
\n url,
\n headers={"User-Agent": user_agent} # \u643a\u5e26\u81ea\u5b9a\u4e49UA
\n )
\n except HTTPError as e:
\n # \u533a\u5206\u4e0d\u540c\u7c7b\u578b\u7684\u7f51\u7edc\u9519\u8bef
\n if isinstance(e, TimeoutException):
\n raise McpError("\u8bf7\u6c42\u8d85\u65f6，\u8bf7\u68c0\u67e5\u7f51\u7edc\u8fde\u63a5")
\n elif isinstance(e, ConnectError):
\n raise McpError("\u65e0\u6cd5\u8fde\u63a5\u5230\u76ee\u6807\u670d\u52a1\u5668")
\n else:
\n raise McpError(f"\u7f51\u7edc\u9519\u8bef：{str(e)}")<\/p>\n
# \u5904\u7406HTTP\u72b6\u6001\u7801
\n if 400 <= response.status_code < 500:
\n raise McpError(f"\u5ba2\u6237\u7aef\u9519\u8bef：\u72b6\u6001\u7801{response.status_code}")
\n elif response.status_code >= 500:
\n raise McpError(f"\u670d\u52a1\u5668\u9519\u8bef：\u72b6\u6001\u7801{response.status_code}")<\/p>\n
# \u667a\u80fd\u5224\u65ad\u5185\u5bb9\u7c7b\u578b
\n page_raw = response.text
\n content_type = response.headers.get("content-type", "").lower()
\n is_html = (
\n "text\/html" in content_type
\n or (not content_type and "<html" in page_raw[:100]) # \u5904\u7406\u7f3a\u5931content-type\u7684\u60c5\u51b5
\n )<\/p>\n
# \u683c\u5f0f\u8f6c\u6362\u903b\u8f91
\n if is_html and not force_raw:
\n # \u8c03\u7528\u5185\u5bb9\u63d0\u53d6\u51fd\u6570（\u89c1\u4e0b\u6587\u8be6\u7ec6\u5b9e\u73b0）
\n return extract_content_from_html(page_raw), ""
\n else:
\n return (
\n page_raw,
\n f"[\u539f\u59cb\u5185\u5bb9] \u5185\u5bb9\u7c7b\u578b:{content_type}，\u4ee5\u4e0b\u662f\u539f\u59cb\u6570\u636e：\\\\n"
\n )<\/p>\n
\u6280\u672f\u4eae\u70b9：<\/p>\n<\/p>\n
python<\/p>\n
\u8fd0\u884c<\/p>\n
from protego import Protego
\nfrom urllib.parse import urlparse, urlunparse<\/p>\n
async def check_may_autonomously_fetch_url(
\n url: str,
\n user_agent: str,
\n proxy_url: str | None = None
\n) -> None:
\n """robots.txt\u5408\u89c4\u6027\u68c0\u67e5（\u5e26\u89e3\u6790\u7ec6\u8282）"""
\n # 1. \u751f\u6210robots.txt\u5730\u5740
\n robots_url = get_robots_txt_url(url) # \u89c1\u4e0b\u65b9\u5de5\u5177\u51fd\u6570<\/p>\n
# 2. \u83b7\u53d6\u5e76\u89e3\u6790robots.txt\u5185\u5bb9
\n async with AsyncClient(proxies=proxy_url) as client:
\n try:
\n robots_response = await client.get(robots_url, headers={"User-Agent": user_agent})
\n except Exception as e:
\n raise McpError(f"\u65e0\u6cd5\u83b7\u53d6robots.txt：{str(e)}")<\/p>\n
# 3. \u6e05\u6d17\u89c4\u5219（\u53bb\u9664\u6ce8\u91ca\u548c\u7a7a\u884c）
\n raw_rules = [
\n line.strip() for line in robots_response.text.splitlines()
\n if line.strip() and not line.strip().startswith("#") # \u53bb\u9664\u6ce8\u91ca
\n ]
\n robot_parser = Protego.parse("\\\\n".join(raw_rules)) # \u751f\u6210\u89c4\u5219\u89e3\u6790\u5668<\/p>\n
# 4. \u6838\u5fc3\u68c0\u67e5\u903b\u8f91
\n if not robot_parser.can_fetch(url, user_agent):
\n raise McpError(
\n f"\u76ee\u6807\u7f51\u7ad9\u7981\u6b62\u6293\u53d6：\\\\n"
\n f"\u2022 URL: {url}\\\\n"
\n f"\u2022 robots.txt: {robots_url}\\\\n"
\n f"\u2022 \u7981\u7528\u89c4\u5219: {robot_parser.get_disallow_rules(url, user_agent)}" # \u663e\u793a\u5177\u4f53\u7981\u7528\u89c4\u5219
\n )<\/p>\n
def get_robots_txt_url(url: str) -> str:
\n """\u751f\u6210\u6807\u51c6robots.txt\u5730\u5740（\u5e26\u534f\u8bae\u548c\u57df\u540d\u4fdd\u7559）"""
\n parsed = urlparse(url)
\n return urlunparse((
\n parsed.scheme, # \u4fdd\u7559\u539f\u534f\u8bae（http\/https）
\n parsed.netloc, # \u4fdd\u7559\u57df\u540d
\n "\/robots.txt", # \u56fa\u5b9a\u8def\u5f84
\n "", "", "" # \u6e05\u7a7a\u8def\u5f84\u3001\u53c2\u6570\u3001\u951a\u70b9
\n ))<\/p>\n
\u5173\u952e\u903b\u8f91：<\/p>\n<\/p>\n
python<\/p>\n
\u8fd0\u884c<\/p>\n
import markdownify
\nimport readabilipy.simple_json<\/p>\n
def extract_content_from_html(html: str) -> str:
\n """HTML\u51c0\u5316+Markdown\u8f6c\u6362（\u5e26\u964d\u566a\u903b\u8f91）"""
\n # 1. \u63d0\u53d6\u6838\u5fc3\u5185\u5bb9（\u53bb\u9664\u5e7f\u544a\/\u5bfc\u822a）
\n try:
\n content_data = readabilipy.simple_json.simple_json_from_html_string(
\n html,
\n use_readability=True # \u542f\u7528Readability\u7b97\u6cd5\u63d0\u53d6\u4e3b\u5185\u5bb9
\n )
\n except Exception as e:
\n return f"<\u9519\u8bef> \u5185\u5bb9\u63d0\u53d6\u5931\u8d25：{str(e)}"<\/p>\n
if not content_data.get("content"):
\n return "<\u9519\u8bef> \u672a\u68c0\u6d4b\u5230\u6709\u6548\u5185\u5bb9"<\/p>\n
# 2. \u8f6c\u6362\u4e3aMarkdown（\u5e26\u6807\u9898\u683c\u5f0f\u4f18\u5316）
\n return markdownify.markdownify(
\n content_data["content"],
\n heading_style=markdownify.ATX, # \u4f7f\u7528#\u53f7\u4f5c\u4e3a\u6807\u9898\u6807\u8bb0（\u5982# \u4e00\u7ea7\u6807\u9898）
\n linkify=False, # \u4e0d\u81ea\u52a8\u8f6c\u6362\u94fe\u63a5（\u4fdd\u6301\u539f\u59cb\u683c\u5f0f）
\n bold_tags=["strong", "b"], # \u81ea\u5b9a\u4e49\u52a0\u7c97\u6807\u7b7e
\n italic_tags=["em", "i"] # \u81ea\u5b9a\u4e49\u659c\u4f53\u6807\u7b7e
\n )<\/p>\n
\u6280\u672f\u7ec6\u8282：<\/p>\n<\/p>\n
python<\/p>\n
\u8fd0\u884c<\/p>\n
from mcp.server import Server
\nfrom mcp.types import Tool, TextContent<\/p>\n
async def serve(…):
\n server = Server("mcp-fetch")<\/p>\n
# \u6ce8\u518c\u5de5\u5177API
\n @server.call_tool()
\n async def call_tool(name, arguments):
\n try:
\n # \u53c2\u6570\u9a8c\u8bc1（\u4f7f\u7528Fetch\u7c7b\u81ea\u52a8\u6821\u9a8c）
\n args = Fetch(**arguments)
\n except ValidationError as e:
\n # \u8f6c\u6362\u4e3a\u53cb\u597d\u7684\u9519\u8bef\u4fe1\u606f
\n return [TextContent(
\n type="text",
\n text=f"\u53c2\u6570\u9519\u8bef：{e.errors()[0]['msg']}"
\n )]<\/p>\n
# robots.txt\u68c0\u67e5（\u4ec5\u81ea\u52a8\u6a21\u5f0f）
\n if not ignore_robots_txt:
\n await check_may_autonomously_fetch_url(args.url, user_agent_autonomous)<\/p>\n
# \u6267\u884c\u6293\u53d6
\n content, prefix = await fetch_url(
\n args.url,
\n user_agent_autonomous,
\n force_raw=args.raw,
\n proxy_url=proxy_url
\n )<\/p>\n
# \u5206\u9875\u5904\u7406（\u5173\u952e\u903b\u8f91）
\n original_length = len(content)
\n end_index = args.start_index + args.max_length
\n if args.start_index >= original_length:
\n content = "<\u63d0\u793a> \u5df2\u5230\u8fbe\u5185\u5bb9\u672b\u5c3e，\u6ca1\u6709\u66f4\u591a\u6570\u636e"
\n else:
\n # \u5207\u7247\u64cd\u4f5c\u5e76\u6dfb\u52a0\u622a\u65ad\u63d0\u793a
\n content = content[args.start_index:end_index]
\n if end_index < original_length:
\n content += f"\\\\n\\\\n<\u63d0\u793a> \u5185\u5bb9\u88ab\u622a\u65ad，\u5269\u4f59{original_length – end_index}\u5b57\u7b26，\u53ef\u901a\u8fc7start_index={end_index}\u7ee7\u7eed\u83b7\u53d6"<\/p>\n
return [TextContent(type="text", text=f"{prefix}{args.url} \u7684\u5185\u5bb9：\\\\n{content}")]<\/p>\n
\u5173\u952e\u6d41\u7a0b：<\/p>\n<\/p>\n
python<\/p>\n
\u8fd0\u884c<\/p>\n
# \u4ee3\u7406\u6c60\u6a21\u5757（\u4f2a\u4ee3\u7801，\u5b9e\u9645\u9700\u5b9e\u73b0\u6709\u6548\u6027\u68c0\u6d4b）
\nproxy_pool = [
\n "http:\/\/proxy1.example.com:8080",
\n "http:\/\/proxy2.example.com:8080",
\n # \u66f4\u591a\u4ee3\u7406\u5730\u5740
\n]<\/p>\n
async def get_random_proxy():
\n """\u4ece\u4ee3\u7406\u6c60\u4e2d\u83b7\u53d6\u6709\u6548\u4ee3\u7406（\u5e26\u7b80\u5355\u8d1f\u8f7d\u5747\u8861）"""
\n return random.choice(proxy_pool)<\/p>\n
# \u4f7f\u7528\u65f6\u53ea\u9700：
\nproxy_url = await get_random_proxy()
\ncontent, _ = await fetch_url(url, user_agent, proxy_url=proxy_url)<\/p>\n
python<\/p>\n
\u8fd0\u884c<\/p>\n
# \u5168\u5c40\u8fde\u63a5\u6c60（\u51cf\u5c11TCP\u63e1\u624b\u5f00\u9500）
\nclient = AsyncClient(
\n proxies=proxy_url,
\n follow_redirects=True,
\n limits=Limits(max_connections=100, max_keepalive_connections=50) # \u914d\u7f6e\u8fde\u63a5\u6c60\u53c2\u6570
\n)<\/p>\n
\u901a\u8fc7\u5b8c\u6574\u7684\u4ee3\u7801\u5b9e\u73b0，\u6211\u4eec\u770b\u5230 MCP Fetch \u670d\u52a1\u5668\u5982\u4f55\u901a\u8fc7\u53c2\u6570\u6821\u9a8c\u3001\u5f02\u6b65 IO\u3001\u5408\u89c4\u68c0\u67e5\u3001\u5185\u5bb9\u5904\u7406\u7b49\u6a21\u5757，\u6784\u5efa\u8d77\u5065\u58ee\u7684 URL \u6293\u53d6\u80fd\u529b\u3002\u8fd9\u4e9b\u4ee3\u7801\u7ec6\u8282\u4e0d\u4ec5\u89e3\u51b3\u4e86\u5b9e\u9645\u5f00\u53d1\u4e2d\u7684\u5e38\u89c1\u95ee\u9898，\u66f4\u4f53\u73b0\u4e86\u5de5\u7a0b\u5316\u8bbe\u8ba1\u7684\u6838\u5fc3\u601d\u60f3 \u2014\u2014\u5728\u590d\u6742\u573a\u666f\u4e2d\u5efa\u7acb\u6709\u5e8f\u7684\u89c4\u5219\u4f53\u7cfb\u3002<\/p>\n<\/p>\n
\u5982\u679c\u4f60\u5728\u5f00\u53d1\u4e2d\u9700\u8981\u5904\u7406\u7f51\u7edc\u5185\u5bb9\u83b7\u53d6，\u4e0d\u59a8\u76f4\u63a5\u590d\u7528\u6587\u4e2d\u7684\u53c2\u6570\u6a21\u578b\u548c\u6838\u5fc3\u51fd\u6570，\u6839\u636e\u5b9e\u9645\u9700\u6c42\u8c03\u6574\u4ee3\u7406\u914d\u7f6e\u548c\u5185\u5bb9\u8f6c\u6362\u903b\u8f91\u3002\u8bb0\u5f97\u70b9\u8d5e\u6536\u85cf，\u5173\u6ce8\u6211\u83b7\u53d6\u66f4\u591a AI \u5de5\u5177\u5f00\u53d1\u7684\u786c\u6838\u6280\u672f\u89e3\u6790！<\/p>\n","protected":false},"excerpt":{"rendered":"
\u6587\u7ae0\u6d4f\u89c8\u9605\u8bfb626\u6b21\uff0c\u70b9\u8d5e7\u6b21\uff0c\u6536\u85cf8\u6b21\u3002\u901a\u8fc7\u5b8c\u6574\u7684\u4ee3\u7801\u5b9e\u73b0\uff0c\u6211\u4eec\u770b\u5230 MCP Fetch \u670d\u52a1\u5668\u5982\u4f55\u901a\u8fc7\u53c2\u6570\u6821\u9a8c\u3001\u5f02\u6b65 IO\u3001\u5408\u89c4\u68c0\u67e5\u3001\u5185\u5bb9\u5904\u7406\u7b49\u6a21\u5757\uff0c\u6784\u5efa\u8d77\u5065\u58ee\u7684 URL \u6293\u53d6\u80fd\u529b\u3002\u8fd9\u4e9b\u4ee3\u7801\u7ec6\u8282\u4e0d\u4ec5\u89e3\u51b3\u4e86\u5b9e\u9645\u5f00\u53d1\u4e2d\u7684\u5e38\u89c1\u95ee\u9898\uff0c\u66f4\u4f53\u73b0\u4e86\u5de5\u7a0b\u5316\u8bbe\u8ba1\u7684\u6838\u5fc3\u601d\u60f3 \u2014\u2014\u5728\u590d\u6742\u573a\u666f\u4e2d\u5efa\u7acb\u6709\u5e8f\u7684\u89c4\u5219\u4f53\u7cfb\u3002\u5982\u679c\u4f60\u5728\u5f00\u53d1\u4e2d\u9700\u8981\u5904\u7406\u7f51\u7edc\u5185\u5bb9\u83b7\u53d6\uff0c\u4e0d\u59a8\u76f4\u63a5\u590d\u7528\u6587\u4e2d\u7684\u53c2\u6570\u6a21\u578b\u548c\u6838\u5fc3\u51fd\u6570\uff0c\u6839\u636e\u5b9e\u9645\u9700\u6c42\u8c03\u6574\u4ee3\u7406\u914d\u7f6e\u548c\u5185\u5bb9\u8f6c\u6362\u903b\u8f91\u3002_fetch mcp<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[73,81,2880,43],"topic":[],"class_list":["post-44597","post","type-post","status-publish","format-standard","hentry","category-server","tag-mcp","tag-python","tag-2880","tag-43"],"yoast_head":"\n