[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_119":120},"routes/modules.$creator.$moduleId",{"_14":15,"_16":17,"_18":19,"_20":21},"routes/modules.$creator.$moduleId.endpoints",{"_11":12,"_13":-5},"error","Module not found","tools","module",{"_20":22,"_23":24,"_25":26,"_27":28,"_29":30,"_31":32,"_33":34,"_35":26,"_36":37,"_38":-5,"_39":-5,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":-5,"_57":49,"_58":59,"_60":61,"_62":63,"_64":65,"_66":67,"_68":69,"_70":71,"_72":73,"_74":75,"_76":77,"_78":79,"_80":-5,"_81":-5,"_82":-5,"_83":49,"_84":-5,"_85":86,"_87":88,"_89":90,"_91":92,"_93":94,"_95":96},"apiUrl","https://odel.app/api","creator","spronta","moduleId","crawlie","@spronta/crawlie","displayName","Crawlie","tagline","Technical SEO + GEO (AI-search) site audits: hosted crawls, prioritized fixes, report diffs.","iconKey","https://www.crawlie.co/favicon.svg","category","remote","status","published","visibility","public","metaDescription","deploymentType","external","workerName","bindingName","mcpConfig","{\"url\":\"https://crawlie.app/mcp\",\"transport\":\"streamable-http\",\"authType\":\"none\"}","pricingModel","free","creatorUserId","system:external-import","createdAt",1783743482000,"updatedAt",1788322759000,"sourceRegistry","official-mcp-registry","sourceUrl","https://registry.modelcontextprotocol.io/v0/servers?search=app.crawlie%2Fcrawlie","sourceVerified",false,"sourceMissingSince","importedAt","upstreamStars",102,"upstreamUpdatedAt",1784404969000,"detectedTransport","streamable-http","detectedAuthType","oauth","probeStatus","ok","lastProbedAt",1787631536000,"githubRepoUrl","https://github.com/spronta/crawlie","githubRepoFullName","spronta/crawlie","releaseId","ext-spronta-crawlie","version","0.1.0","descriptionMarkdown","

\n\n# crawlie\n\n**The fast, free, open-source technical SEO + GEO crawler — built for humans and agents.**\n\nCrawl any site for broken links, redirects, missing metadata, and 40+ SEO & Generative-Engine checks — with plain-English guidance on every fix. Runs locally, ships a CLI and an MCP server, and costs nothing.\n\n[![npm](https://img.shields.io/npm/v/crawlie?color=cb3837&logo=npm&label=crawlie)](https://www.npmjs.com/package/crawlie)\n[![CI](https://github.com/spronta/crawlie/actions/workflows/ci.yml/badge.svg)](https://github.com/spronta/crawlie/actions/workflows/ci.yml)\n[![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)\n\n

\n Setup ·\n CLI ·\n MCP & agents ·\n Changelog ·\n Use cases ·\n Why I built this ·\n Desktop app ·\n Checks ·\n Compare ·\n Architecture\n

\n\n*[Read the docs → crawlie.dev](https://crawlie.dev/docs)* \n\n[💡Share feedback & suggestions here](https://github.com/spronta/crawlie/discussions/2)\n\n

\n\n![Showcase of the example app demonstrating a report](https://cdn.spronta.com/spronta-8d32a2/pretty_snap_2026_5_18_23_58%20(2).png)\n\n---\n\n## Setup\n\n**The easy way — npm** (installs the `crawlie` CLI and the `crawlie-mcp` server):\n\n```bash\nnpm i -g crawlie\n```\n\n**The macOS app** — grab the signed `.dmg` from [Releases](https://github.com/spronta/crawlie/releases).\n\n**From source** — needs [Rust](https://rustup.rs) (engine/CLI/MCP) and, for the desktop app, [pnpm](https://pnpm.io) + Node:\n\n```bash\ngit clone https://github.com/spronta/crawlie\ncd crawlie\ncargo build --release\n# → target/release/crawlie and target/release/crawlie-mcp\n\n# or install onto your PATH:\ncargo install --path crates/crawlie-cli # installs `crawlie`\ncargo install --path crates/crawlie-mcp # installs `crawlie-mcp`\n```\n\n> **How it ships:** the **CLI + MCP** come *only* through npm — the right native binary installs automatically as a platform package (nothing to download or unblock). The **desktop app** is the only direct download: a signed, notarized `.dmg` on [Releases](https://github.com/spronta/crawlie/releases).\n\n---\n\n## How to use (CLI)\n\n```bash\n# Crawl a whole site (respects robots.txt, seeds from sitemap.xml)\ncrawlie crawl https://example.com --format pretty\n\n# Audit a single page, or a specific set of pages\ncrawlie audit https://example.com/pricing\ncrawlie audit https://example.com/a https://example.com/b\n\n# Save a shareable, self-contained HTML report\ncrawlie crawl https://example.com --format html -o report.html\n\n# Clean JSON on stdout (perfect for piping / scripting / agents)\ncrawlie crawl https://example.com --format json -o report.json\n\n# Learn why any finding matters and how to fix it\ncrawlie explain geo-not-answerable\n```\n\n**Output formats:** `pretty` (terminal), `json` (machine-readable, the default), `csv` (issues), `html` (shareable file).\n\n**Common flags:**\n\n| Flag | What it does |\n|---|---|\n| `--max-pages ` | Cap pages fetched (default 500) |\n| `--max-depth ` | Max click depth from the seed |\n| `--concurrency ` | Parallel requests (default 16) |\n| `--include ` / `--exclude ` | Scope the crawl by URL pattern |\n| `--no-robots` / `--no-sitemap` / `--no-external` | Turn off robots.txt, sitemap seeding, external link checks |\n| `--severity error\\|warning\\|notice` | Only output findings at/above a level |\n| `--save` | Save to local report history (`crawlie reports`, `crawlie report `) |\n| `--fail-on error\\|warning` | Non-zero exit code for CI gating |\n\nEvery crawl returns three scores: a **Health** score (technical SEO), a **GEO** score (AI-search readiness), and an **Accessibility** score (WCAG conformance) — each reported separately so one kind of problem never hides another.\n\n---\n\n## Use with agents (MCP)\n\ncrawlie ships a [Model Context Protocol](https://modelcontextprotocol.io) server so an LLM agent can run a full audit and act on it — no human in the loop. This is the part most SEO tools don't have.\n\n### Connect it\n\nAfter `npm i -g crawlie`, `crawlie-mcp` is on your `PATH`. For **Claude Desktop**, edit `claude_desktop_config.json`:\n\n```jsonc\n{\n \"mcpServers\": {\n \"crawlie\": {\n \"command\": \"crawlie-mcp\"\n }\n }\n}\n```\n\nFor **Claude Code**:\n\n```bash\nclaude mcp add crawlie crawlie-mcp\n```\n\n(If you built from source instead, use the absolute path to `target/release/crawlie-mcp`.)\n\n(Any MCP-compatible client works — Cursor, Cline, your own agent. It speaks JSON-RPC over stdio.)\n\n### Hosted: the Crawlie Cloud MCP (no install)\n\nPrefer not to install anything, or want crawls to run on our infrastructure? Point any MCP client at the hosted endpoint and authenticate with a Crawlie API key (create one in the dashboard under Settings, API keys). It speaks MCP Streamable HTTP.\n\n**Endpoint:** `https://crawlie.app/mcp`\n\nFor **Claude Code**:\n\n```bash\nclaude mcp add --transport http crawlie-cloud https://crawlie.app/mcp \\\n --header \"Authorization: Bearer crw_your_key\"\n```\n\nFor **Claude Desktop** (or any client that takes a JSON config):\n\n```jsonc\n{\n \"mcpServers\": {\n \"crawlie-cloud\": {\n \"type\": \"http\",\n \"url\": \"https://crawlie.app/mcp\",\n \"headers\": { \"Authorization\": \"Bearer crw_your_key\" }\n }\n }\n}\n```\n\nHosted crawls run on the same engine as the dashboard, are scoped to your team, and are metered against your plan. The tools mirror the local server (`crawl_site`, `audit_url`, `top_fixes`, `geo_gaps`, `affected_urls`, `diff_reports`, plus `crawl_status` to poll a long crawl and `get_report` / `list_reports` over your saved cloud reports). Every crawl returns a `reportId` you can re-slice later without re-crawling.\n\n### One-step install: the Claude Code plugin\n\nThe fastest path. The [`crawlie` plugin](https://github.com/spronta/crawlie/blob/main/.claude-plugin/plugin.json) bundles the MCP server **and** a set of skills (audit playbooks) in a single install — the MCP server auto-runs via `npx`, so you don't even pre-install the binary:\n\n```bash\n# add this repo as a marketplace, then install the plugin\nclaude plugin marketplace add spronta/crawlie\nclaude plugin install crawlie@spronta\n```\n\n### Skills (works with *any* agent, even without the MCP)\n\nThe [`skills/`](https://github.com/spronta/crawlie/blob/main/skills/) folder holds standalone [Agent Skills](https://agentskills.io) that teach an agent how to run real audits — full-site SEO + GEO, broken-link fixes, pre-launch gates, and AI-search readiness. Each is **self-contained**: it needs neither this repo nor a pre-installed crawlie. Missing the binary? The skill runs it on demand via `npx -y -p crawlie …` (the install *is* the run), and automatically uses the MCP tools when they're present. See [skills/README.md](https://github.com/spronta/crawlie/blob/main/skills/README.md).\n\n### Tools exposed\n\n| Tool | Purpose |\n|---|---|\n| `crawl_site` | Crawl + audit a whole site (SEO + GEO), returns scores, issues, per-page data |\n| `audit_url` | Audit a single page |\n| `audit_urls` | Audit an explicit list of pages |\n| `explain_issue` | Why a rule matters + how to fix it |\n| `list_rules` | The full catalogue of checks |\n| `list_reports` / `get_report` | Read saved crawl history |\n\n### Example agent prompts\n\n> *\"Crawl crawlie.dev, then give me the top 5 fixes that would most improve my GEO score, with the exact change for each.\"*\n\n> *\"Audit these three landing pages and tell me which is least ready to be cited by AI search, and why.\"*\n\n> *\"Run a crawl with `--fail-on error` semantics — are there any broken links or 5xx pages blocking launch?\"*\n\nThe agent calls `crawl_site`, reads the structured issues, and uses `explain_issue` to turn findings into a prioritized, actionable plan.\n\n---\n\n## Use cases\n\n- **Pre-launch QA** — catch broken links, redirects, 4xx/5xx, and missing metadata before you ship.\n- **GEO optimization** — make pages citable by AI search: structured data, semantic HTML, answer-ready content, authorship/E-E-A-T.\n- **Agent workflows** — let a marketing/SEO agent audit a site and propose fixes autonomously via MCP.\n- **CI/CD gating** — `crawlie crawl … --fail-on error` in a pipeline to block regressions.\n- **Client reporting** — generate a polished, shareable HTML report in one command.\n- **Auditing AI-generated sites** — verify that the site your agent just built is actually built for search.\n\n---\n\n## Why I built this\n\nI'm **Sean Ryan**. I've spent 6+ years as a Lead Marketing Engineer, and on the side I build AI tooling for marketers.\n\nWith AI, it's faster than ever to ship a marketing site — but most of what gets generated is slop that was never built to be found. And the tools meant to catch that fall short: most SEO auditors cost money, don't play nicely with your agents, or tell you *what's* wrong without telling you *how to actually rank* for SEO **and** GEO (Generative Engine Optimization — being cited by AI search like ChatGPT, Perplexity, and Google AI Overviews).\n\ncrawlie fixes that. It's free, it's local-first, it's agent-native, and every issue it finds comes with *why it matters* and *how to fix it*.\n\n**If this is useful to you, [connect with me on LinkedIn →](https://linkedin.com/in/sean-exe)** — I share what I'm learning building AI for marketers and SEO/GEO tooling, and I'd love to hear how you're using crawlie.\n\n---\n\n## Desktop app\n\nA beautiful Tauri + React app (Geist design, light/dark, seamless window chrome):\n\n```bash\ncd apps/desktop\npnpm install\npnpm tauri dev # live native crawls\npnpm dev # preview the UI in a browser (demo data, no backend)\n```\n\nWhole-site / single-page / URL-list modes, live progress, **Health**, **GEO** & **Accessibility** score rings, issues with built-in *why-it-matters* guidance, a sortable pages table, a per-page drawer (GEO signals, headers, schema, hreflang…), auto-saved report history, and one-click shareable HTML export.\n\n> First run, generate the icon set: `cd src-tauri/icons && python3 generate.py && cd .. && pnpm tauri icon icons/source.png`\n\n---\n\n## What it checks\n\n*57 rules and counting.*\n\n**Technical SEO** — broken links · 4xx/5xx · redirects & chains · titles & meta descriptions (missing / duplicate / length) · H1s · canonicals · noindex / nofollow / X-Robots-Tag · robots.txt blocking · images missing alt · thin & duplicate content · orphan & deep pages\n\n**Performance & security** — slow responses · large pages · missing compression · HTTPS · mixed content · HSTS\n\n**Accessibility (WCAG)** — links & buttons without an accessible name · form controls without a label · iframes missing a title · zoom-blocking viewport · positive `tabindex` · skipped heading levels\n\n**Mobile, international & social** — viewport · `lang` · hreflang · Open Graph · Twitter cards · structured data\n\n**Structured-data validation** — parses JSON-LD and checks each item against Google's rich-result requirements: invalid markup, missing required fields, and missing recommended fields (Product, Article, Recipe, Event, FAQ, Breadcrumb, and more)\n\n**JavaScript rendering** — crawl with `--render` to audit each page's post-JavaScript DOM via headless Chrome, so client-rendered content (React/Next/Vue) is seen, and `content-requires-js` flags pages whose content only exists after JS runs\n\n**GEO — Generative Engine Optimization** — structured data, semantic HTML, answer-readiness, authorship/E-E-A-T, dated content, question-style headings, and extractable blocks, rolled into a per-page **GEO score**.\n\nEvery finding links to plain-English guidance: **why it matters**, **how to fix it**, and **what happens if you ignore it**.\n\n---\n\n## How it compares\n\n| | **crawlie** | Screaming Frog | Sitebulb |\n|---|:---:|:---:|:---:|\n| Price | **Free & open-source** | £259/yr to unlock | from £13.50/mo |\n| Engine | **Rust, async, tiny binary** | Java (JVM) | .NET |\n| CLI with JSON output | ✅ | partial | ❌ |\n| JavaScript rendering | ✅ headless Chrome | ✅ | ✅ |\n| **MCP server (agent-native)** | ✅ | ❌ | ❌ |\n| **GEO — AI/answer-engine audit** | ✅ | ❌ | ❌ |\n| **\"Why it matters\" built in** | ✅ every issue | ❌ | partial |\n| Shareable HTML report | ✅ | paid | ✅ |\n| Source you can read & extend | ✅ | ❌ | ❌ |\n\n---\n\n## Architecture\n\n```\ncrates/\n crawlie-core # the engine — crawl, audit, score, knowledge base, reports\n crawlie-cli # `crawlie` — JSON / pretty / CSV / HTML output\n crawlie-mcp # `crawlie-mcp` — Model Context Protocol server (stdio)\napps/\n desktop # Tauri v2 + React (Geist) desktop app\n```\n\n`crawlie-core` has zero host dependencies — the same audited engine drops straight into a cloud worker (it already targets `wasm32`). One engine, every surface, identical results.\n\n---\n\n## Roadmap\n\n- Cloud workers (shared Rust core) for scheduled/remote crawls\n- JavaScript rendering for SPA-heavy sites\n- Crawl-to-crawl comparison & regression alerts\n- Internal-link graph visualization\n\n---\n\n## License & author\n\n**MIT** © **[Spronta Ltd](https://crawlie.dev)** — the crawler engine, CLI, MCP\nserver, and desktop app: everything in this repository is MIT. Crawlie Cloud (the\nhosted crawl service and the marketing site) is a separate, closed-source product\nand is not part of this repository. Pull requests to the open engine, CLI, and\ndesktop app are very welcome.\n\nBuilt by Sean Ryan — Lead Marketing Engineer at Pendo.io, building AI for marketers on the side. **[Connect on LinkedIn →](https://linkedin.com/in/sean-exe)**\n\nIf crawlie saves you time, a ⭐ on the repo and a hello on LinkedIn mean a lot.\n","changelogMarkdown","videoKey","requiredSecretsJson","publishedAt","priceCents","trialDays",0,"upstream",{"_106":107,"_108":109,"_110":111,"_112":113,"_114":115,"_116":117},"requiredSecrets",[],"tags",[99,100,101,102,103,104,105],"assets",[],"stats",{"_97":86,"_98":86},"ratingAvg","reviewCount","aeo","crawler","geo","marketing-tools","search-optimisation","search-optimization","seo","topics",[99,100,101,102,118,103,104,105],"license","NOASSERTION","language","Rust","registryName","app.crawlie/crawlie","homepage","https://crawlie.app","packages",[],"mcp","ENV",{"_121":122,"_123":124,"_125":126},"ENVIRONMENT","production","TURNSTILE_SITE_KEY","0x4AAAAAAB8-rEI2gNDf_QhU","SENTRY_DSN","https://45fd1229f78089d5c3ee78be128030e1@o4510583261757440.ingest.de.sentry.io/4510584253448272"]