LangChain Web Scraping &
RAG Knowledge Ingestion

# Complete End-to-End LangChain RAG with Wire-Compatible FlyCrawl
from langchain_core.documents import Document
from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import Chroma
from langchain_openai import OpenAIEmbeddings
from firecrawl import FirecrawlApp

# 1. Scrape live web knowledge into token-optimized Fit-Markdown (Zero lock-in)
app = FirecrawlApp(api_key="fly_live_YOUR_API_KEY", api_url="https://api.flycrawl.net/v1")
res = app.scrape_url("https://docs.anthropic.com/en/docs/models", params={"formats": ["markdown"]})

# 2. Package as LangChain Document with rich source metadata
doc = Document(
    page_content=res["markdown"],
    metadata={"source": res.get("metadata", {}).get("sourceURL", "https://docs.anthropic.com")}
)

# 3. Chunk with zero HTML noise
splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
chunks = splitter.split_documents([doc])

# 4. Ingest into vector store & query
vectorstore = Chroma.from_documents(chunks, OpenAIEmbeddings())
retriever = vectorstore.as_retriever(search_kwargs={"k": 3})
print(f"Ingested {len(chunks)} high-density semantic chunks via FlyCrawl!")

# LangChain / LangGraph Autonomous Web Browsing Tool
from langchain_core.tools import tool
from firecrawl import FirecrawlApp

app = FirecrawlApp(api_key="fly_live_YOUR_API_KEY", api_url="https://api.flycrawl.net/v1")

@tool
def scrape_webpage(url: str) -> str:
    """Scrapes any URL and returns clean, noise-free Fit-Markdown via FlyCrawl."""
    res = app.scrape_url(url, params={"formats": ["markdown"], "onlyMainContent": True})
    return res.get("markdown", "")

# Pass tools to ChatOpenAI or LangGraph ReAct agent
tools = [scrape_webpage]

// TypeScript / Node.js LangChain Ingestion
import { Document } from '@langchain/core/documents';
import FirecrawlApp from '@mendable/firecrawl-js';

const app = new FirecrawlApp({ apiKey: process.env.FLYCRAWL_API_KEY!, apiUrl: 'https://api.flycrawl.net/v1' });
const data = await app.scrapeUrl('https://example.com', { formats: ['markdown'] });

const doc = new Document({
  pageContent: data.markdown,
  metadata: { source: data.metadata?.sourceURL || 'https://example.com' }
});