[{"_1":2,"_3":-5,"_4":-5},"loaderData",{"_5":6,"_7":8,"_9":10},"actionData","errors","root",{"_137":138},"routes/modules.$creator.$moduleId",{"_14":15,"_16":17,"_18":19,"_20":21},"routes/modules.$creator.$moduleId.endpoints",{"_11":12,"_13":-5},"error","Module not found","tools","module",{"_20":22,"_23":24,"_25":26,"_27":28,"_29":30,"_31":32,"_33":34,"_35":26,"_36":37,"_38":-5,"_39":-5,"_40":41,"_42":43,"_44":45,"_46":47,"_48":49,"_50":51,"_52":53,"_54":55,"_56":-5,"_57":49,"_58":59,"_60":61,"_62":63,"_64":65,"_66":67,"_68":69,"_70":71,"_72":73,"_74":75,"_76":77,"_78":79,"_80":-5,"_81":-5,"_82":83,"_84":49,"_85":-5,"_86":87,"_88":89,"_90":91,"_92":93,"_94":95,"_96":97},"apiUrl","https://odel.app/api","creator","josejux","moduleId","rapidapi-metadata-extractor","@josejux/rapidapi-metadata-extractor","displayName","Web Metadata, OpenGraph & Contact Extractor","tagline","Turns any URL into SEO metadata, contacts, tech stack, and AI-ready Markdown, in one call.","iconKey","https://avatars.githubusercontent.com/u/248734725?v=4","category","remote","status","published","visibility","public","metaDescription","deploymentType","external","workerName","bindingName","mcpConfig","{\"url\":\"https://mcp.rapidapi.com\",\"transport\":\"streamable-http\",\"authType\":\"headers\",\"requiredSecrets\":[{\"name\":\"X_API_HOST\",\"scope\":\"per-user\",\"header\":\"x-api-host\"},{\"name\":\"X_API_KEY\",\"scope\":\"per-user\",\"header\":\"x-api-key\"}]}","pricingModel","free","creatorUserId","system:external-import","createdAt",1786508679000,"updatedAt",1788328711000,"sourceRegistry","official-mcp-registry","sourceUrl","https://registry.modelcontextprotocol.io/v0/servers?search=io.github.JosejuX%2Frapidapi-metadata-extractor","sourceVerified",true,"sourceMissingSince","importedAt","upstreamStars",5,"upstreamUpdatedAt",1787907960000,"detectedTransport","streamable-http","detectedAuthType","none","probeStatus","ok","lastProbedAt",1787718713000,"githubRepoUrl","https://github.com/JosejuX/rapidapi-metadata-extractor","githubRepoFullName","JosejuX/rapidapi-metadata-extractor","releaseId","ext-josejux-rapidapi-metadata-extractor","version","1.0.0","descriptionMarkdown","

\n\n ![Turn any URL into structured intelligence — Web Metadata, OpenGraph and Contact Extractor API](https://raw.githubusercontent.com/JosejuX/rapidapi-metadata-extractor/main/assets/banner.svg)\n\n # Turn any URL into structured intelligence.\n\n **Extract SEO, technologies, contacts, products, social profiles, security signals, and AI-ready content from any public website — with one API call.**\n\n Free to try. No signup required for the demo.\n\n [![Python](https://img.shields.io/badge/Python-3.12%2B-blue.svg)](https://www.python.org/)\n [![FastAPI](https://img.shields.io/badge/FastAPI-0.100%2B-009688.svg)](https://fastapi.tiangolo.com/)\n [![Version](https://img.shields.io/badge/Version-4.1.0-blueviolet.svg)]()\n [![Rust ORJSON](https://img.shields.io/badge/JSON%20Engine-Rust%20ORJSON-orange.svg)]()\n [![RapidAPI](https://img.shields.io/badge/RapidAPI-Available-0052CC.svg)](https://rapidapi.com/josejuanjocoding/api/web-metadata-and-contact-extractor)\n [![Response Time](https://img.shields.io/badge/Response%20Time-%3C200ms-brightgreen.svg)]()\n [![Cache Speed](https://img.shields.io/badge/Cache%20Speed-0.01ms-flash.svg)]()\n [![License](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)\n [![CI](https://github.com/JosejuX/rapidapi-metadata-extractor/actions/workflows/ci.yml/badge.svg)](https://github.com/JosejuX/rapidapi-metadata-extractor/actions)\n [![Keep Alive](https://github.com/JosejuX/rapidapi-metadata-extractor/actions/workflows/keep-alive.yml/badge.svg)](https://github.com/JosejuX/rapidapi-metadata-extractor/actions/workflows/keep-alive.yml)\n\n [🎮 **Try the Live Demo**](https://webmetadataextractor.com/) • [🔑 Get Free API Key on RapidAPI](https://rapidapi.com/josejuanjocoding/api/web-metadata-and-contact-extractor) • [📖 API Documentation](#-api-endpoint-documentation) • [⚡ Code Examples](#-quick-start)\n\n ![Terminal demo: curl request and JSON response](https://raw.githubusercontent.com/JosejuX/rapidapi-metadata-extractor/main/assets/demo-terminal.svg)\n\n

\n\n**Paste a URL, get real results in seconds** — [webmetadataextractor.com](https://webmetadataextractor.com/) runs the actual pipeline live, no API key needed. Try your own site, or one of these:\n\n[![Analyze buffer.com](https://img.shields.io/badge/Analyze-buffer.com-e0893f)](https://rapidapi-metadata-extractor.onrender.com/demo/extract?url=buffer.com) [![Analyze techcrunch.com](https://img.shields.io/badge/Analyze-techcrunch.com-4fb08a)](https://rapidapi-metadata-extractor.onrender.com/demo/extract?url=techcrunch.com) [![Analyze basecamp.com](https://img.shields.io/badge/Analyze-basecamp.com-b481d9)](https://rapidapi-metadata-extractor.onrender.com/demo/extract?url=basecamp.com)\n\n> `/demo/extract` is the same no-key route the live demo page above uses (see \"Honest Limitations\" — it's rate-limited per IP, not a documented/stable API endpoint). For integrating in code, use `/api/v1/extract` with a RapidAPI key as shown below.\n\n**Or call it directly** (swap in your key from RapidAPI for production use):\n\n```bash\ncurl \"https://web-metadata-and-contact-extractor.p.rapidapi.com/api/v1/extract?url=github.com\" \\\n -H \"X-RapidAPI-Key: YOUR_KEY\" -H \"X-RapidAPI-Host: web-metadata-and-contact-extractor.p.rapidapi.com\"\n```\n\n---\n\n## 🎯 Use Cases\n\n| | Use case | What you get |\n|:---|:---|:---|\n| 🧲 | **Lead generation** | Find companies on a given tech stack (e.g. Shopify) with a weak SEO score, then prioritize outreach. |\n| 🤖 | **AI agents** | Feed `markdown_content` and structured JSON straight into an agent or RAG pipeline as clean, LLM-ready context for any URL. |\n| 🔎 | **SEO audits** | Run the 14-point on-page score plus graded security headers across a URL list, on a schedule. |\n| 🛒 | **Ecommerce** | Pull product name/price/currency/availability/brand from Schema.org, OpenGraph, and Microdata — cross-checked for conflicts. |\n| 🕵️ | **Competitive intelligence** | Track a competitor's tech stack, metadata, and product data over time by re-checking the same URL list. |\n\n

Example: lead generation — flag Shopify sites with a weak SEO score

\n\n```python\nimport requests\n\nRAPIDAPI_KEY = \"YOUR_RAPIDAPI_KEY\"\nHEADERS = {\n \"X-RapidAPI-Key\": RAPIDAPI_KEY,\n \"X-RapidAPI-Host\": \"web-metadata-and-contact-extractor.p.rapidapi.com\",\n}\nBASE = \"https://web-metadata-and-contact-extractor.p.rapidapi.com/api/v1/extract\"\n\nfor url in [\"shop-a.com\", \"shop-b.com\", \"shop-c.com\"]:\n data = requests.get(BASE, headers=HEADERS, params={\"url\": url}).json()\n if \"Shopify\" in data[\"detected_technologies\"] and data[\"seo_score_percentage\"] < 60:\n print(f\"{url}: Shopify + {data['seo_score_percentage']}% SEO -> outreach candidate\")\n```\n

\n\n

Example: AI agent tool call — clean Markdown for any URL

\n\n```python\nimport requests\n\ndef read_url_as_markdown(url: str) -> str:\n \"\"\"Tool function an LLM agent can call to read any public webpage.\"\"\"\n resp = requests.get(\n \"https://web-metadata-and-contact-extractor.p.rapidapi.com/api/v1/markdown\",\n headers={\"X-RapidAPI-Key\": \"YOUR_KEY\", \"X-RapidAPI-Host\": \"web-metadata-and-contact-extractor.p.rapidapi.com\"},\n params={\"url\": url},\n ).json()\n return resp[\"markdown_content\"]\n```\n

\n\n---\n\n## 🌟 Key Features\n\n| Feature | Detail |\n|:---|:---|\n| ⚡ **Ultra-Fast Performance** | ~150–300ms live fetch. Powered by `selectolax` (C-Lexbor parser), Rust `ORJSON`, `uvloop`, HTTP/2 multiplexing, async DNS (non-blocking), 5-min DNS TTL cache. |\n| 🧠 **Adaptive SPA Byte Limit** | Auto-detects React, Next.js, Vue, Angular, Nuxt, Svelte, Gatsby, Remix, Astro and expands download to 256 KB for richer data extraction. Static sites stay at 64 KB. |\n| 🛡️ **IP-Pinned Anti-SSRF Shield** | DNS resolved once, IP validated against private/loopback/cloud-metadata ranges, connection pinned to IP with TLS SNI. Eliminates DNS Rebinding & Redirect SSRF. |\n| 🎯 **Rich SEO & OpenGraph Metadata** | Title, Description, OG Image, OG Type, OG URL, OG Video, Favicon, Canonical URL, Language, Author, Theme Color, Robots directive, hreflang tags, H1 headings, image count. |\n| 📧 **Contact Extractor** | Public emails and phone numbers with smart DOM cleaning to eliminate false positives. Phones are additionally normalized (`phone_details`) via libphonenumber — E.164 format, ISO country code, and possible/valid flags — for numbers that include a country code; numbers without one aren't guessed. |\n| 📲 **Social Profile Finder** | Auto-detects 20 platforms: Twitter/X, LinkedIn, Facebook, Instagram, GitHub, YouTube, Telegram, TikTok, Threads, Bluesky, Mastodon, Discord, Reddit, Pinterest, Medium, GitLab, Vimeo, Dribbble, Behance, Snapchat. |\n| 🛠️ **40+ Tech Stack Detector** | WordPress, Shopify, WooCommerce, Webflow, React, Next.js, Vue, Angular, Svelte, TailwindCSS, Stripe, GA4, and more across CMS/ecommerce, frameworks, and analytics/payment/hosting categories. |\n| 📦 **Schema.org JSON-LD + Product Parser** | Parses all structured data schemas AND auto-extracts Product price, currency, availability, brand, rating and review count. |\n| 🎯 **Multi-Source Data Quality & Conflict Detection** | Product price/currency/availability/brand are cross-checked across three distinct structured encodings — JSON-LD, OpenGraph's product extension, and schema.org Microdata. `product_field_confidence` reports a per-field confidence score (more of these encodings agreeing = higher confidence, same philosophy as tech-stack detection) and which source won. When they genuinely disagree, a `SOURCE_CONFLICT` warning names the field, every value found, and which one was chosen and why. A top-level `quality` object (`score`, `rendered: false`, `sources_used`, `warnings`) summarizes response trustworthiness for programmatic consumers. **Caveat**: these three encodings often come from the same underlying product record on a given site, so agreement means \"internally consistent,\" not \"independently verified\" — see Honest Limitations below. |\n| 📊 **On-Page SEO Health Audit Score** | 14-point automated on-page technical SEO diagnostic (0–100%) with actionable warnings list, plus a structured `seo_checks` breakdown (`check` id, `passed`, `severity`, `evidence`) for programmatic use. |\n| 🔗 **Internal vs External Link Classifier** | Categorizes up to 100 hyperlinks per page. |\n| 🤖 **AI & LLM Clean Markdown Reader** | Converts article text to clean Markdown for ChatGPT, Claude, RAG, and AI agents. Includes word count and reading time. |\n| 📡 **RSS / Atom Feed Discovery** | Auto-discovers RSS and Atom feed URLs. |\n| 🔒 **Graded Security Headers Audit** | HSTS, CSP, X-Frame-Options, X-Content-Type-Options, Referrer Policy, Permissions Policy — each graded `missing`/`weak`/`reasonable`/`strong` from its actual value (e.g. a CSP with `unsafe-inline` scores as weak, not as \"present = full credit\"), not just presence, rolled into the percentage score. `/api/v1/security` additionally returns `security_header_grades` with the per-header breakdown. |\n| 🚀 **15-Min In-Memory Cache** | Cached responses served in **< 0.01 ms** server-side processing time. |\n| ⚡ **Multi-Worker + Auto-Reconnect Redis** | `uvicorn --workers N` multi-process cluster (set via the `WORKERS` env var; `gunicorn` is a listed dependency for anyone who prefers it as the process manager, but the shipped Dockerfile launches uvicorn directly). Distributed rate limiting via Redis with startup ping validation, automatic reconnect every 30s, and immediate `degraded_fallback` status propagation to `/health/details`. |\n| 🔒 **Split `/health` + `/health/details`** | Minimal public liveness probe (`/health`). Full operational details (Redis mode, status, engine) secured via `HEALTH_DETAILS_SECRET` header on `/health/details`. |\n| 🧯 **Per-Host Circuit Breaker** | Trips after repeated timeouts/connection failures/5xx to one host, fails fast during the cooldown window instead of burning the full connect+read timeout budget on every request, and probes recovery automatically (never trips on ordinary 4xx). |\n| 🧩 **Request Single-Flight + DNS Coalescing** | Concurrent requests for the same URL (or the same hostname's DNS lookup) share one in-flight fetch instead of hammering the origin N times. |\n| 🩹 **Negative-Result Cache** | Short-TTL caching of recent upstream failures (DNS/timeout/5xx) so a broken target fails fast instead of repeating the same slow failure for every request during an outage. |\n| 🧱 **Adaptive Byte-Limit Hardening** | Streaming byte cap is enforced on the *decoded* chunk, closing a decompression-bomb gap where a small gzip/br payload could otherwise expand to megabytes in a single read. Response headers are also count- and size-bounded. |\n| 🏷️ **Structured Error Codes** | Every error response carries a machine-readable `error.code` / `error.retryable` object (e.g. `SSRF_BLOCKED`, `UPSTREAM_TIMEOUT`, `CIRCUIT_OPEN`) alongside the existing `detail` string — fully additive, v1 contract unchanged. |\n| 📈 **Prometheus Metrics + Structured JSON Logs** | `/metrics` exposes request/cache/SSRF/circuit-breaker/rate-limit counters and latency histograms. Every log line is a JSON object with `request_id` for end-to-end tracing; known-sensitive fields are redacted automatically. |\n| 🧠 **Confidence-Scored Tech Detection** | `/api/v1/tech-stack` now also returns `technology_details`: per-technology confidence score, matched evidence, and category (cms/ecommerce/framework/analytics/payment/hosting/…), alongside the original flat list. |\n| 📦 **Deeper Product & JSON-LD Parsing** | Traverses `@graph` and top-level JSON-LD arrays (not just top-level objects), extracting SKU, MPN, GTIN/ISBN, seller, condition, price range, and images. |\n| 🔎 **Unicode-Aware SEO & Keywords** | Keyword extraction now matches non-ASCII scripts correctly (accented/Cyrillic/etc. content), and the SEO audit adds `lang` attribute, viewport, noindex, multi-H1, Twitter Card, and structured-data checks. |\n| ⚡ **Lazy Extraction Per Endpoint** | Specialized endpoints only run the extractors they actually need instead of the full pipeline — `/security` never even parses the HTML tree, `/tech-stack` skips links/metadata/markdown/SEO entirely. `/api/v1/extract` runs the complete pipeline by default, but `?fields=` now maps to the same narrow extraction as the matching specialized endpoint (`fields=detected_technologies` costs exactly what `/tech-stack` costs) instead of computing everything and filtering the response afterward. Results for different endpoints (or `fields=` combinations) hitting the same URL share one upstream fetch and accumulate into the same cache entry. |\n| 🐘 **Mastodon Social Detection** | Best-effort detection of the largest public Mastodon instances (mastodon.social, fosstodon.org, hachyderm.io, ...) alongside the existing 20 platforms — decentralization means a hostname map can't cover every self-hosted instance, so this is intentionally partial rather than a false-positive risk. |\n| 🚦 **Rate-Limit Response Headers** | Every response (success or `429`) now carries `X-RateLimit-Limit`, `X-RateLimit-Remaining`, and `X-RateLimit-Reset`, so clients can back off proactively instead of learning the limit by hitting `429`. |\n| 🛡️ **Bot-Protection Detection** | Cheap heuristic signature check (no extra requests, no JS execution) flags when the fetched page is a Cloudflare/Akamai/PerimeterX/CAPTCHA challenge page rather than real content — either as `bot_protection_detected: true` on a normal response, or as a specific `BOT_PROTECTION_DETECTED` error code (instead of a generic upstream error) when the challenge is served with a 403/429/503 status. Distinguishes \"the target blocked this request\" from \"the target genuinely has no title/description\" or \"a real outage\". |\n| 🔗 **Redirect Chain & Shortened-URL Detection** | Every response — every endpoint, not just `/extract` — now includes `redirect_count` (how many HTTP redirect hops were followed to reach `final_url`) and `is_shortened_url` (whether the URL you passed in is on a known shortener domain like `bit.ly`/`tinyurl.com`/`t.co`, checked against your original input, not `final_url`). |\n| 📖 **Readability Metrics & Full Heading Structure** | `/api/v1/seo-audit` and `/api/v1/extract` now include a `readability` object: `sentence_count`, `paragraph_count`, `avg_words_per_sentence`, and a full `heading_structure` breakdown (`h1` through `h6` counts) — simple regex/DOM-count heuristics, no NLP dependency. |\n| 📦 **Batch URL Processing** | `POST /api/v1/batch` runs a lightweight link-preview extraction over up to 10 URLs in a single request. Each URL is fetched independently — one failing URL returns `{\"success\": false, \"error\": ...}` for that item without affecting the rest of the batch. Counts as N requests (one per URL) against the per-IP rate limit, not one. |\n| 🔐 **Opt-In TLS Certificate Inspection** | `/api/v1/security?include_tls_details=true` performs a live TLS handshake to the target host and returns certificate issuer, subject, validity dates, and negotiated TLS version as `tls_details`. Off by default — it's an extra network round-trip beyond the normal headers-only audit — and any handshake failure (timeout, self-signed cert, non-TLS host) degrades to `tls_details: null` rather than breaking the response. |\n| 🌐 **DNS / WHOIS Domain Intelligence** | `GET /api/v1/domain` is a standalone endpoint that never fetches the target page — only DNS (A/AAAA/MX/NS/TXT via `dnspython`) and WHOIS (registrar, creation/expiration dates via `python-whois`) lookups against the hostname. A flaky/rate-limited WHOIS server degrades to `whois_info: null` instead of failing the whole request when DNS already succeeded. |\n| 📋 **CSP Report-Only Detection & Header Explanations** | A site running only `Content-Security-Policy-Report-Only` (rollout in progress, nothing blocked yet) used to be invisible to the security audit — only the enforcing header was ever read. It now gets its own `\"report-only\"` grade, distinct from both `\"missing\"` and an enforced policy, and the raw value is exposed additively as `content_security_policy_report_only`. Every graded header (`security_headers`/`/api/v1/security`) now also ships a one-line `security_header_explanations` entry describing why it matters, not just a pass/fail grade. |\n\n---\n\n## 🎯 Performance SLA & Technical Architecture Notes\n\n> [!NOTE]\n> - **Latency & Performance SLA**: Server-side processing overhead (DOM cleaning, C-Lexbor parsing, Rust serialization) averages **< 5ms**. Live execution times depend on the target website's network latency and origin server response time. Repeating requests for the same URL hit the in-memory cache and return in **< 0.01ms**.\n> - **IPv6 & IPv4-Mapped SSRF Shield**: The Anti-SSRF validation engine enforces strict resolution checks across both IPv4 and IPv6, blocking loopback (`127.0.0.1`, `::1`), link-local (`169.254.169.254`, `fe80::/10`), and IPv4-mapped IPv6 (`::ffff:127.0.0.1`) addresses.\n> - **Redis Rate Limiter (Fixed-Window, Auto-Reconnect)**: When `REDIS_URL` is configured, rate limiting is distributed across all workers via a single atomic Redis Lua script (`INCR` + first-hit `EXPIRE` in one round-trip — no window where a dropped connection could leave a counter with no TTL). Redis is pinged at startup and re-validated every 30 seconds. On failure, the service falls back to per-process TTLCache immediately and marks `redis_status: degraded_fallback` in `/health/details`.\n> - **Split Health Endpoints**: `/health` returns a minimal public liveness payload. `/health/details` returns full operational status (Redis mode, trust_proxy, engine) and requires the `X-Health-Secret` header when `HEALTH_DETAILS_SECRET` env var is set.\n> - **Horizontal Scaling**: Single-instance → in-memory TTLCache (60 req/min/IP). Multi-worker → distributed Redis. Enterprise scale → RapidAPI Gateway or Nginx.\n> - **Zero-Trust Self-Hosting**: Full Dockerfile (uvicorn multi-worker) and test suite included for self-hosted production deployments.\n> - **Modular Codebase**: The service is organized as an `app/` package (`security/`, `fetcher/`, `cache/`, `ratelimit/`, `extraction/`, `observability/`, `api/`) rather than a single file — `main.py` is a thin backward-compatibility shim so `uvicorn main:app` / `gunicorn main:app` keep working unchanged.\n> - **Validated Configuration**: `app/config.py` is backed by `pydantic-settings` — every tunable (rate limits, cache sizes/TTLs, byte-fetch limits, header limits, ...) is validated at startup with the exact same defaults as before, and now opportunistically overridable via env vars. An invalid value (negative sizes, `STREAM_SOFT_LIMIT` above `STREAM_HARD_LIMIT`, ...) fails immediately with a clear error instead of surfacing as a runtime bug later.\n> - **Lazy Extraction**: `/api/v1/extract` with no `fields=` runs the full pipeline. Specialized endpoints — and `/api/v1/extract?fields=...` — instead run only their required extractor groups — e.g. `/api/v1/security` skips HTML parsing entirely (headers-only), `/api/v1/tech-stack` skips metadata/links/markdown/SEO. Calling two different specialized endpoints (or field selections) for the same URL shares one upstream fetch and merges into one cache entry.\n> - **Tech Signature Prefiltering**: Each of the 40+ technology signatures is gated by a cheap substring check against a once-lowercased copy of the page before its regex ever runs, instead of running every regex unconditionally — same detection output, substantially less CPU on pages with many `