[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_128":129},"routes/modules.$creator.$moduleId",{"_14":15,"_16":17,"_18":19,"_20":21},"routes/modules.$creator.$moduleId.endpoints",{"_11":12,"_13":-5},"error","Module not found","tools","module",{"_20":22,"_23":24,"_25":26,"_27":28,"_29":30,"_31":32,"_33":34,"_35":26,"_36":37,"_38":-5,"_39":-5,"_40":-5,"_41":42,"_43":44,"_45":46,"_47":48,"_49":50,"_51":52,"_53":54,"_55":-5,"_56":48,"_57":58,"_59":60,"_61":-5,"_62":-5,"_63":-5,"_64":-5,"_65":66,"_67":68,"_69":70,"_71":72,"_73":74,"_75":-5,"_76":-5,"_77":-5,"_78":48,"_79":-5,"_80":81,"_82":83,"_84":85,"_86":87,"_88":89,"_90":91,"_92":93},"apiUrl","https://odel.app/api","creator","mldsveda","moduleId","pyscrappy","@mldsveda/pyscrappy","displayName","PyScrappy","tagline","Web-scraping toolkit with 22 tools for structured web data as JSON for AI agents.","iconKey","https://avatars.githubusercontent.com/u/86206345?v=4","category","local","status","published","visibility","public","metaDescription","deploymentType","external","workerName","bindingName","mcpConfig","pricingModel","free","creatorUserId","system:external-import","createdAt",1785126976000,"updatedAt",1788069012000,"sourceRegistry","official-mcp-registry","sourceUrl","https://registry.modelcontextprotocol.io/v0/servers?search=io.github.mldsveda%2Fpyscrappy","sourceVerified",true,"sourceMissingSince","importedAt","upstreamStars",194,"upstreamUpdatedAt",1787856683000,"detectedTransport","detectedAuthType","probeStatus","lastProbedAt","githubRepoUrl","https://github.com/mldsveda/PyScrappy","githubRepoFullName","mldsveda/PyScrappy","releaseId","ext-mldsveda-pyscrappy","version","1.5.9","descriptionMarkdown","

\n $\"PyScrappy\"$ \n

\n\n

Adaptive Python web scraping toolkit (self-healing, stealth)
+ MCP server for AI agents

\n\n[![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/)\n[![PyPI Latest Release](https://img.shields.io/pypi/v/PyScrappy.svg)](https://pypi.org/project/PyScrappy/)\n[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/mldsveda/PyScrappy/blob/main/LICENSE)\n[![Downloads](https://static.pepy.tech/badge/pyscrappy)](https://pepy.tech/project/pyscrappy)\n[![Glama quality](https://glama.ai/mcp/servers/mldsveda/PyScrappy/badges/score.svg)](https://glama.ai/mcp/servers/mldsveda/PyScrappy)\n[![Documentation](https://img.shields.io/badge/docs-pyscrappy-117866.svg)](https://pyscrappy.vercel.app)\n[![MCP Toplist](https://mcptoplist.com/badge/io.github.mldsveda%2Fpyscrappy.svg)](https://mcptoplist.com/server/io.github.mldsveda%2Fpyscrappy)\n\n\n\nPyScrappy is an AI-native web scraping toolkit that turns websites into structured, LLM-ready data. Use it as a Python library or expose it as an MCP server for AI agents.\n\n📖 **Documentation:** [pyscrappy.vercel.app](https://pyscrappy.vercel.app)\n\n## Key features\n\n- **Generic scraper** — give it any URL, get back structured text, links, images, tables, and metadata\n- **LLM-ready output** — `.to_markdown()` turns any result into clean Markdown; also `.to_json()` and `.to_dataframe()`\n- **MCP server** — expose the scrapers as tools for AI agents (Claude, Cursor, local LLMs, …)\n- **JS rendering** — optional Playwright backend for JavaScript-heavy sites\n- **Custom selectors** — pass CSS selectors to extract exactly what you need\n- **Chainable `Selector`** — navigate HTML directly with CSS/XPath, `find_all`, `find_by_text`, and `find_similar` (Scrapy/BeautifulSoup-style)\n- **Adaptive (self-healing) selectors** — remember an element and relocate it by similarity when a site changes its markup, so scrapers don't silently break\n- **Concurrent scraping** — `scrape_many` / `scrape_all` run scrapes in parallel\n- **Sitemap crawling** — enumerate and scrape a whole site from its `sitemap.xml` (index + gzip aware)\n- **Proxy & scraping-API support** — route through a proxy or ScraperAPI/ScrapeOps for blocked sites\n- **TLS-fingerprint impersonation** — `impersonate=\"chrome\"` gets past anti-bot filters that block plain clients (optional `curl_cffi` backend)\n- **Command-line extract** — `pyscrappy extract out.md` scrapes a URL straight to a file, no code\n- **Retry & rate-limiting** — built-in exponential backoff and per-domain rate limiting\n- **Type-safe** — full type hints, `py.typed` marker\n- **20+ built-in scrapers** — Wikipedia, IMDB, stocks, news, GitHub, Amazon/IKEA, YouTube, and [more](#built-in-scrapers)\n\n## Installation\n\n```sh\npip install pyscrappy\n```\n\n**Optional extras:**\n\n```sh\n# Browser support (for JS-rendered pages)\npip install 'pyscrappy[browser]'\nplaywright install chromium\n\n# DataFrame support\npip install 'pyscrappy[dataframe]'\n\n# MCP server (use PyScrappy's scrapers as AI-agent tools)\npip install 'pyscrappy[mcp]'\n\n# Stealth (TLS-fingerprint impersonation to bypass anti-bot filters)\npip install 'pyscrappy[stealth]'\n\n# Parquet / Excel export (ScrapeResult.to_parquet() / .to_excel())\npip install 'pyscrappy[parquet]'\npip install 'pyscrappy[excel]'\n\n# Everything\npip install 'pyscrappy[all]'\n```\n\n## For AI agents\n\nPyScrappy ships an [MCP server](#mcp-server-use-pyscrappy-from-an-ai-agent) that\nexposes its scrapers as tools, so an agent (Claude, Cursor, an OpenAI agent, a\nlocal LLM) can pull structured web data from any URL and hand it straight to the\nmodel:\n\n```text\nAI agent ──MCP tool call──▶ PyScrappy ──fetch + extract──▶ Any website\n ▲ │\n └────────────── clean Markdown / JSON ◀───────────────────────┘\n```\n\n```sh\npip install 'pyscrappy[mcp]'\nclaude mcp add pyscrappy pyscrappy-mcp\n```\n\nThen just ask: *\"use pyscrappy to summarize the latest headlines from bbc.com.\"*\nSee [MCP server](#mcp-server-use-pyscrappy-from-an-ai-agent) for the full setup\nand tool list.\n\n### Local models (Ollama), no MCP host needed\n\nOllama can't talk MCP on its own, so normally you'd run a host (Goose, Cline, …)\nin between. PyScrappy skips that with a built-in agent that talks to Ollama\ndirectly and lets a local model call the scrapers as tools:\n\n```sh\npip install 'pyscrappy[mcp]' # needs Python 3.10+\npyscrappy chat --model qwen2.5 \"what's the current AAPL quote?\"\n```\n\nIt exposes the same 22 tools as the MCP server. The only requirement is a model\nthat supports **tool calling** (Llama 3.1, Qwen 2.5, Mistral, …); how well it\n*picks* the right tool is up to the model. Point it at a remote Ollama with\n`--host`, and pass `-v` to see each tool call.\n\n## MCP server (use PyScrappy from an AI agent)\n\nPyScrappy ships an optional [Model Context Protocol](https://modelcontextprotocol.io)\nserver, so an AI agent (e.g. Claude) can call PyScrappy's scrapers as tools and\nget structured web data back.\n\n\n $\"PyScrappy$ \n\n\n```sh\npip install 'pyscrappy[mcp]'\n```\n\nThe MCP extra installs the standalone `fastmcp` package and requires Python 3.10\nor newer. On Python 3.9 the core scraping library still works, but the MCP server\nis unavailable.\n\nThis installs the `pyscrappy-mcp` command. It uses stdio by default for local MCP\nclients; Streamable HTTP and legacy SSE are available for remote deployments:\n\n```sh\npyscrappy-mcp # stdio (default)\npyscrappy-mcp --http # Streamable HTTP\npyscrappy-mcp --sse # legacy SSE\n```\n\nYou can also run the stdio server with `python -m pyscrappy.mcp`.\n\n### Register with Claude Code\n\n```sh\nclaude mcp add pyscrappy pyscrappy-mcp\n```\n\n### Register with Claude Desktop\n\nAdd to your `claude_desktop_config.json` and restart the app:\n\n```json\n{\n \"mcpServers\": {\n \"pyscrappy\": {\n \"command\": \"pyscrappy-mcp\"\n }\n }\n}\n```\n\n> **Tip:** Claude Desktop does not inherit your shell `PATH`. If `pyscrappy-mcp`\n> is not found, use the absolute path to the command (e.g. the one printed by\n> `which pyscrappy-mcp`).\n\n### Available tools\n\nThe server exposes **20+ tools**. The most common ones are **`scrape_url`** (any\nURL → text, links, images, tables, metadata), **`scrape_wikipedia`**,\n**`scrape_stock`**, **`scrape_news`**, and **`search_github`** — plus many more\ncovering image/YouTube/LinkedIn/Hacker News/book search, weather, crypto,\ncurrency, dictionary, Amazon/Newegg/IKEA/SoundCloud, IMDB, and Zomato/Uber Eats.\n\nTo see the full, live list, ask the agent to call the **`list_available_scrapers`**\ntool, or from a shell:\n\n```sh\npython -c \"from pyscrappy import list_scrapers; print(', '.join(sorted(list_scrapers())))\"\n```\n\nThe `lookup_movie` tool needs a free [OMDb](https://www.omdbapi.com/apikey.aspx) API\nkey. Pass it to the server through your MCP client config, e.g. for Claude Desktop:\n\n```json\n{\n \"mcpServers\": {\n \"pyscrappy\": {\n \"command\": \"pyscrappy-mcp\",\n \"env\": { \"OMDB_API_KEY\": \"your-key\" }\n }\n }\n}\n```\n\nOnce registered, just ask the agent naturally, e.g. *\"use pyscrappy to get the\nlatest headlines from bbc.co.uk and the AAPL stock quote.\"*\n\n## Built-in scrapers\n\nPyScrappy ships **24 built-in scrapers**, and every one that works without a\nproxy is also exposed as an [MCP tool](#mcp-server-use-pyscrappy-from-an-ai-agent).\n\nA few of them:\n\n- **`GenericScraper`** — scrape any URL with auto-extraction (text, links, images, tables, metadata)\n- **Data / research** — **`WikipediaScraper`**, **`StockScraper`** (Yahoo Finance), **`NewsScraper`** (RSS/Atom), **`GitHubScraper`**, **`HackerNewsScraper`**, plus weather, crypto, currency, dictionary, image, LinkedIn-jobs, and book search\n- **E-commerce** — **`AmazonScraper`**, `NeweggScraper`, `IKEAScraper`\n- **Social / media / food** — **`YouTubeScraper`**, SoundCloud, Zomato, Uber Eats (Instagram / Twitter / Spotify also ship, but are blocked and need a proxy)\n\n…and many more. To see the full, live list:\n\n```sh\npython -c \"from pyscrappy import list_scrapers; print(', '.join(sorted(list_scrapers())))\"\n```\n\n**`IMDBScraper`** (`lookup_movie`) is the one exception that needs a key — a free\n[OMDb](https://www.omdbapi.com/apikey.aspx) `OMDB_API_KEY` (see the\n[MCP config](#available-tools) above for how to pass it).\n\n## Plugins\n\nPyScrappy is extensible: you can add your own scrapers, and third parties can\nship them as standalone `pyscrappy-` packages. A registered scraper works\neverywhere a built-in does, including the MCP server and the `pyscrappy chat`\nagent, with no change to PyScrappy core.\n\n**In your own code** — register with the decorator:\n\n```python\nfrom pyscrappy import BaseScraper, register_scraper, get_scraper\nfrom pyscrappy.core.models import ScrapeResult, ScrapeMetadata\n\n@register_scraper(\"reddit\")\nclass RedditScraper(BaseScraper):\n def scrape(self, subreddit: str, **kwargs) -> ScrapeResult:\n data = self.fetch_and_parse(f\"https://old.reddit.com/r/{subreddit}/.json\")\n # ... build a list of dicts ...\n return ScrapeResult(data=[...], metadata=ScrapeMetadata(scraper=\"reddit\"))\n\nget_scraper(\"reddit\")().scrape(subreddit=\"python\")\n```\n\n**As a distributable package** — advertise an entry point in your\n`pyproject.toml`, and PyScrappy discovers it once your package is installed:\n\n```toml\n[project.entry-points.\"pyscrappy.scrapers\"]\nreddit = \"pyscrappy_reddit:RedditScraper\"\n```\n\nAfter `pip install pyscrappy-reddit`, the scraper shows up in\n`list_scrapers()`, and an AI agent can call it via the `scrape_with` MCP tool —\nno core change required.\n\n**First-class MCP tools (optional).** Add an `mcp_tools` mapping and your scraper\nbecomes a dedicated, typed MCP tool instead of only being reachable through the\ngeneric `scrape_with` — its schema is derived from the method signature, so\nagents get proper named arguments:\n\n```python\n@register_scraper(\"reddit\")\nclass RedditScraper(BaseScraper):\n mcp_tools = {\"search_reddit\": \"scrape\"} # tool name -> method\n\n def scrape(self, subreddit: str, sort: str = \"hot\") -> ScrapeResult:\n ...\n```\n\nSee the [plugin template](https://github.com/mldsveda/PyScrappy/blob/main/plugin-template/) for a complete, copyable starting\npoint, and the [plugin guide](https://pyscrappy.vercel.app/docs/plugins/) for\nthe full walkthrough.\n\n## Quick start\n\n### Scrape any URL → clean, LLM-ready Markdown\n\n```python\nfrom pyscrappy import scrape\n\nresult = scrape(\"https://en.wikipedia.org/wiki/Web_scraping\")\n\nprint(result.to_markdown()) # feed straight to an LLM\n# ...or result.to_json() / result.to_dataframe()\n\n# Write to a file — format inferred from the extension:\nresult.save(\"out.json\") # .json .csv .md .ndjson .yaml .parquet .xlsx\n# (.parquet needs pyscrappy[parquet]; .xlsx needs pyscrappy[excel])\n```\n\nPrefer raw fields? Every result is a `ScrapeResult` with `.data` (a list of\ndicts):\n\n```python\nprint(result.data[0][\"metadata\"][\"title\"])\nprint(result.data[0][\"text\"][\"word_count\"])\n```\n\n### Custom CSS selectors\n\n```python\nfrom pyscrappy import GenericScraper\n\nwith GenericScraper() as gs:\n result = gs.scrape(\n url=\"https://news.ycombinator.com\",\n selectors={\"title\": \".titleline a\", \"score\": \".score\"},\n )\n for item in result.data:\n print(item[\"title\"], item.get(\"score\", \"\"))\n```\n\n### Navigate HTML with `Selector`\n\nWhen you want to traverse markup directly (Scrapy/BeautifulSoup-style) rather than\nget back structured dicts, use `Selector`:\n\n```python\nfrom pyscrappy import Selector\n\npage = Selector(html) # or navigate any HTML string\npage.css(\".title::text\").getall() # CSS with ::text / ::attr(name)\npage.xpath(\"//a/@href\").getall() # XPath (elements, text(), @attr)\npage.find_all(\"h2\", class_=\"title\") # BeautifulSoup-style search\npage.find_by_text(\"Add to cart\", tag=\"button\") # search by text content\n\nfirst = page.css(\".product\")[0]\nfirst.css(\".price::text\").get() # chainable\nfirst.find_similar() # sibling elements shaped like this one\n```\n\n`css()` / `xpath()` return a `SelectorList` with `.get()` / `.getall()` / `.text()`.\n`find_similar()` locates elements with the same tag and overlapping classes, handy\nfor pulling every card/row once you've found one.\n\n### Adaptive (self-healing) selectors\n\nA hard-coded CSS selector silently breaks the day a site changes its markup.\nAdaptive selectors survive that: save a fingerprint of the element the first time,\nand if the selector later matches nothing, relocate it by structural and textual\nsimilarity instead of returning empty.\n\n```python\nfrom pyscrappy import Selector\n\n# First run: match normally and remember this element under an id.\npage = Selector(html_v1, url=\"https://shop.example.com\")\nprice = page.css(\".price\", auto_save=True, adaptive_id=\"price\").get()\n\n# Later, after a redesign renamed \".price\" — heal instead of breaking.\n# `expect` is an optional contract: the relocated element must satisfy it,\n# so a good structural score can't smuggle in the wrong field.\npage = Selector(html_v2, url=\"https://shop.example.com\")\nresult = page.css(\n \".price\",\n adaptive=True,\n adaptive_id=\"price\",\n expect=lambda s: s.text().startswith(\"$\"),\n)\nprint(result.get(), \"→ confidence:\", result.adaptive_confidence)\n```\n\nHow the relocation decides — and where it's stronger than a naive similarity match:\n\n- **Weighted signals, not a flat average.** A stable `id` / `data-*` hook counts\n far more than a sibling-tag list, so weak signals can't outvote strong ones.\n- **Anchor-relative.** It remembers the nearest stable ancestor (an id'd / `data-*`\n container) and depth, so it survives layout reshuffles that move absolute positions.\n- **Volatility-aware text.** Prices, dates, and counts are down-weighted, so\n healing stays reliable on exactly the fields that change most between scrapes.\n- **Confidence-scored.** `SelectorList.adaptive_confidence` (0-100) tells you how\n sure the relocation was; `threshold=` sets the minimum to accept.\n- **Contract-enforced (opt-in).** Pass `expect=` to require the healed\n element to satisfy an invariant (e.g. \"text looks like a price\"). A heal that\n clears the threshold but fails the contract is rejected, so structural\n similarity alone never redefines what a field means.\n\nA heal is a change to what a selector resolves to, so **every accepted heal is\nrecorded**. The store keeps an append-only audit log (`adaptive.heal.ndjson`\nbeside the fingerprint store) with the confidence, the runner-up gap, and the\nbefore/after fingerprint, readable via `store.heal_log()` — so drift stays\nobservable instead of being silently absorbed. For an at-a-glance summary,\n`store.heal_report()` aggregates the log into one row per selector (heal count,\nlatest/lowest/average confidence, when it last healed), sorted most-healed first\n— so the selectors that have drifted the most, and the shakiest relocations\n(lowest confidence), surface at the top for a human to review.\n\nFingerprints persist in a small JSON store (`~/.pyscrappy/adaptive.json` by\ndefault, or `$PYSCRAPPY_HOME`), namespaced by site so the same `adaptive_id` on\ntwo sites never collides. Adaptive is entirely opt-in: without `adaptive=True`, a\nbroken selector still just returns empty, exactly as before.\n\n### Site-specific scrapers\n\nEvery built-in scraper follows the same pattern — instantiate, `scrape(...)`,\nread `result.data` (or `.to_dataframe()` / `.to_markdown()`):\n\n```python\nfrom pyscrappy import WikipediaScraper\n\nwith WikipediaScraper() as ws:\n result = ws.scrape(query=\"Python (programming language)\", mode=\"summary\")\n print(result.data[0][\"text\"])\n```\n\nEach scraper has its own arguments (Wikipedia, stocks, IMDB, news, YouTube,\nAmazon/Newegg/IKEA, Uber Eats, and more — see the [full list](#built-in-scrapers)).\nFor per-scraper arguments and examples, see the\n[documentation](https://pyscrappy.vercel.app/docs/scrapers/).\n\n### From the command line\n\nScrape a URL straight to a file without writing any code — the output format is\ninferred from the file extension:\n\n```sh\npyscrappy extract https://example.com out.md # clean Markdown\npyscrappy extract https://example.com out.json # structured JSON\npyscrappy extract https://example.com out.txt # extracted page text\npyscrappy extract https://example.com out.html # raw fetched HTML\n\n# Narrow to elements matching a CSS selector, or render JS first:\npyscrappy extract https://example.com items.txt --css-selector \".product\"\npyscrappy extract https://example.com page.md --render-js\n```\n\n## Configuration\n\n```python\nfrom pyscrappy import ScraperConfig, GenericScraper\n\nconfig = ScraperConfig(\n timeout=20.0, # request timeout in seconds\n max_retries=3, # retry failed requests\n retry_jitter=True, # spread exponential retries to avoid lockstep traffic\n rate_limit=2.0, # seconds between requests per domain\n proxy=\"http://...\", # proxy URL, or a list to rotate through\n scraper_api=None, # route via a scraping-API service (see below)\n headless=True, # browser runs headless\n render_js=\"auto\", # auto-detect if JS rendering is needed\n cache_ttl=0, # response cache TTL in seconds (0 = disabled)\n cache_dir=None, # also persist the cache to disk (survives restarts)\n cache_dir_max_size=512, # max live entries kept on disk before oldest are pruned\n impersonate=None, # e.g. \"chrome\" to spoof a browser's TLS fingerprint (see below)\n)\n\nwith GenericScraper(config) as gs:\n result = gs.scrape(url=\"https://example.com\")\n```\n\n### Proxies and blocked sites\n\nSome sites (e.g. eBay, Instagram, Twitter/X, Spotify) block direct automated\nrequests. PyScrappy supports two ways to get through them.\n\n**A proxy** (or a rotating list) — applies to both the HTTP and browser backends:\n\n```python\nfrom pyscrappy import ScraperConfig, AmazonScraper\n\n# Single proxy\nconfig = ScraperConfig(proxy=\"http://user:pass@host:port\")\n\n# Rotating list (one picked per request)\nconfig = ScraperConfig(proxy=[\"http://p1:8080\", \"http://p2:8080\"])\n```\n\n**A scraping-API service** (ScraperAPI, ScrapeOps, ScrapingBee) — routes requests\nthrough the service, which handles proxies and anti-bot challenges for you:\n\n```python\nconfig = ScraperConfig(scraper_api={\n \"provider\": \"scraperapi\", # or \"scrapeops\", \"scrapingbee\"\n \"api_key\": \"YOUR_KEY\",\n \"render_js\": True, # optional\n})\n\n# Now any scraper works through the service, unchanged:\nwith AmazonScraper(config) as scraper:\n result = scraper.scrape(query=\"laptop\")\n```\n\nThis is the reliable way to use the scrapers marked \"needs proxy\" above.\n\n**TLS-fingerprint impersonation** — many anti-bot systems block a plain HTTP\nclient by its TLS/JA3 fingerprint before serving any content. Set `impersonate`\nto mimic a real browser's fingerprint and get past that class of block without a\nheadless browser:\n\n```python\nfrom pyscrappy import ScraperConfig, GenericScraper\n\n# needs the optional extra: pip install 'pyscrappy[stealth]'\nconfig = ScraperConfig(impersonate=\"chrome\") # or \"chrome124\", \"safari\", \"firefox\"\n\nwith GenericScraper(config) as gs:\n result = gs.scrape(\"https://example.com\")\n```\n\nImpersonation works on **both the sync and async paths** (async uses\n`curl_cffi`'s `AsyncSession`), so you can combine stealth with high-throughput\nasync scraping. All the usual retry, rate-limiting, caching, and robots handling\nstill apply.\n\n```python\nimport asyncio\nfrom pyscrappy import scrape_async, ScraperConfig\n\nasync def main():\n cfg = ScraperConfig(impersonate=\"chrome\")\n return await scrape_async(\"https://example.com\", config=cfg)\n\nasyncio.run(main())\n```\n\n### Concurrent scraping\n\nScraping is I/O-bound, so running several scrapes at once parallelizes the\nnetwork waits. `scrape_many` runs one scraper over many inputs; `scrape_all`\nruns a mix of scrapers together. Both preserve input order.\n\n```python\nfrom pyscrappy import scrape_many, scrape_all, AmazonScraper, WikipediaScraper, NewsScraper\n\n# One scraper, many queries, concurrently:\nresults = scrape_many(AmazonScraper, [{\"query\": \"laptop\"}, {\"query\": \"phone\"}])\n\n# Different scrapers at once:\nresults = scrape_all([\n lambda: WikipediaScraper().scrape(query=\"Python\"),\n lambda: NewsScraper().scrape(feed_url=\"https://rss.nytimes.com/services/xml/rss/nyt/World.xml\"),\n])\n```\n\n### Sitemap crawling\n\nPagination follows next-page links; a **sitemap** enumerates a whole site's URLs\ndirectly. `GenericScraper` can read `/sitemap.xml` (discovered from `robots.txt`\n`Sitemap:` directives, or the conventional path), follow a `` into\nits child sitemaps, and scrape every listed page.\n\n```python\nfrom pyscrappy import GenericScraper\n\nwith GenericScraper() as gs:\n # Just enumerate the URLs:\n urls = gs.sitemap_urls(\"https://example.com\") # -> list[str]\n\n # Or fetch + extract each, concurrently, into one result:\n result = gs.scrape_sitemap(\"https://example.com\", max_urls=100)\n print(len(result.data), \"pages scraped\")\n```\n\nHandles `` leaves and `` files (recursing one level),\ngzip-compressed sitemaps (`.xml.gz`), and de-duplicates URLs. Fetches go through\nthe usual rate-limiting, caching, proxy, and stealth machinery, and the fan-out\nreuses `scrape_all`. `max_urls` caps the crawl (a sitemap can list tens of\nthousands of URLs, so it's required for `scrape_sitemap`).\n\n### Response caching\n\nSet `cache_ttl` to a positive number of seconds to cache successful GET\nresponses. Repeated requests for the same URL (and query params) within the TTL\nare served from cache, skipping both the network and the rate limiter. Caching\nis **disabled by default** (`cache_ttl=0`).\n\n```python\nfrom pyscrappy import WikipediaScraper\nfrom pyscrappy import ScraperConfig\n\nconfig = ScraperConfig(cache_ttl=300) # cache for 5 minutes\n\nwith WikipediaScraper(config) as ws:\n ws.scrape(query=\"Python\") # fetched over the network\n ws.scrape(query=\"Python\") # served from cache\n```\n\nThe cache is in memory and shared across scraper instances in the same process\n(so it also speeds up repeated calls through the MCP server), and is cleared\nwhen the process exits. Call `HttpClient.clear_cache()` to empty it manually.\n\nIt is **LRU-bounded**: at most `cache_max_size` live entries (default `512`),\nwith the least-recently-used entry evicted once the cap is reached. So a\nlong-running process (e.g. the MCP server) that fetches many distinct URLs stays\nbounded rather than growing until restart. Raise or lower the cap as needed:\n\n```python\nconfig = ScraperConfig(cache_ttl=300, cache_max_size=2000)\n```\n\n**Persistent (on-disk) cache.** Set `cache_dir` to also persist responses to\ndisk, so cache hits survive across process restarts and separate runs — useful\nfor re-running a scrape or a CLI job without re-fetching. The in-memory cache\nstill fronts it for speed; a disk hit is promoted back into memory.\n\n```python\nconfig = ScraperConfig(cache_ttl=3600, cache_dir=\"~/.cache/pyscrappy\")\n```\n\nThe on-disk cache is bounded too: each write prunes expired entries and trims the\noldest past `cache_dir_max_size` (default `512`), so a `cache_dir` doesn't grow\none file per distinct URL forever.\n\n`clear_cache()` empties the in-memory cache; the on-disk cache persists by\ndesign — delete its `cache_dir` to clear it.\n\n### Observability hooks\n\nFor long crawls, pass lightweight callbacks to watch requests live (progress\nbars, metrics) without turning on logging:\n\n```python\nconfig = ScraperConfig(\n on_request=lambda url: print(\"GET\", url), # before a network fetch\n on_retry=lambda url, attempt, delay, err: print(\"retry\", attempt, url),\n on_cache_hit=lambda url: print(\"cached\", url), # served from cache\n)\n```\n\n- `on_request(url)` fires once before a URL is fetched (not on a cache hit).\n- `on_retry(url, attempt, delay, error)` fires before each backoff sleep.\n- `on_cache_hit(url)` fires when a request is served from cache.\n\nAll three are best-effort: a callback that raises is logged at debug and never\nbreaks the scrape. They fire on both the sync and async paths.\n\n## Dependencies\n\n**Required:** `httpx`, `beautifulsoup4`, `lxml`\n\n**Optional:** `playwright` (JS rendering), `pandas` (DataFrames), `fastmcp`\n(MCP server, Python 3.10+)\n\n## License\n\n[MIT](https://github.com/mldsveda/PyScrappy/blob/main/LICENSE)\n\n## Contributing\n\nAll contributions welcome. See [Issues](https://github.com/mldsveda/PyScrappy/issues).\n\n**This package is for educational and research purposes.**\n","changelogMarkdown","videoKey","requiredSecretsJson","publishedAt","priceCents","trialDays",0,"iconType","image","upstream",{"_110":111,"_112":113,"_114":115,"_116":117,"_118":119,"_120":121},"requiredSecrets",[],"tags",[96,97,98,99,100,101,102,103,21,104,105,106,107,108,109],"assets",[],"stats",{"_94":81,"_95":81},"ratingAvg","reviewCount","ai","ai-agents","ecommerce-scrapper","food-scrapper","hacktoberfest","hacktoberfest-accepted","image-scrapper","news-scrapper","python","social-media-scrapper","song-scrapper","stock-scrapper","web-scrapping","wikipedia-scrapper","topics",[96,97,98,99,100,101,102,126,127,103,21,104,105,106,107,108,109],"license","MIT","language","Python","registryName","io.github.mldsveda/pyscrappy","homepage","https://pyscrappy.vercel.app","packages",[122],{"_123":124,"_125":21,"_71":72},"registryType","pypi","identifier","mcp","mcp-server","ENV",{"_130":131,"_132":133,"_134":135},"ENVIRONMENT","production","TURNSTILE_SITE_KEY","0x4AAAAAAB8-rEI2gNDf_QhU","SENTRY_DSN","https://45fd1229f78089d5c3ee78be128030e1@o4510583261757440.ingest.de.sentry.io/4510584253448272"]

Adaptive Python web scraping toolkit (self-healing, stealth)+ MCP server for AI agents

Adaptive Python web scraping toolkit (self-healing, stealth)
+ MCP server for AI agents