Scalable Web Crawler API
For Massive AI Data Ingestion

# Launch Asynchronous Domain Crawl Job
curl -X POST https://api.flycrawl.net/v1/crawl \
  -H "Authorization: Bearer YOUR_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://docs.example.com",
    "depth": 3,
    "limit": 500,
    "workers": 16,
    "allowPaths": ["/api/*", "/guides/*"],
    "webhook": "https://your-server.com/flycrawl-hook"
  }'

# Python SDK Async Crawl & SSE Stream
from flycrawl import FlyCrawl

client = FlyCrawl(api_key="YOUR_API_KEY")

# Initiate crawl
job = client.crawl_url(
    "https://docs.example.com",
    depth=3,
    limit=200
)

# Stream pages live as they are scraped
for page in client.stream_crawl(job["id"]):
    print(f"Indexed: {page['url']} ({len(page['markdown'])} chars)")

// Node.js SSE Stream listener
import { FlyCrawl } from 'flycrawl';

const client = new FlyCrawl({ apiKey: 'YOUR_API_KEY' });
const job = await client.crawlUrl('https://docs.example.com', {
  depth: 2,
  limit: 100
});

client.onCrawlProgress(job.id, (event) => {
  console.log(`Page: ${event.url} [${event.completed}/${event.total}]`);
});