from scrapling.fetchers import Fetcher, FetcherSession

# 单次请求
page = Fetcher.get('https://quotes.toscrape.com/')
quotes = page.css('.quote .text::text').getall()

# 会话请求（保持 Cookie）
with FetcherSession(impersonate='chrome') as session:
    page = session.get('https://example.com/')
    data = page.css('.item::text').getall()

from scrapling.fetchers import StealthyFetcher, StealthySession

# 单次请求（自动打开浏览器，完成后关闭）
page = StealthyFetcher.fetch(
    'https://nopecha.com/demo/cloudflare',
    solve_cloudflare=True
)
data = page.css('#padded_content a').getall()

# 会话请求（保持浏览器会话）
with StealthySession(headless=True) as session:
    page = session.fetch('https://example.com/')
    data = page.css('.content').getall()

from scrapling.fetchers import DynamicFetcher, DynamicSession

# 单次请求
page = DynamicFetcher.fetch(
    'https://quotes.toscrape.com/',
    headless=True,
    network_idle=True
)
data = page.css('.quote .text::text').getall()

# 会话请求
with DynamicSession(headless=True, disable_resources=False) as session:
    page = session.fetch('https://example.com/')
    # 使用 XPath 或 CSS 选择器
    data = page.xpath('//span[@class="text"]/text()').getall()

from scrapling.spiders import Spider, Response

class QuotesSpider(Spider):
    name = "quotes"
    start_urls = ["https://quotes.toscrape.com/"]
    concurrent_requests = 10  # 并发数

    async def parse(self, response: Response):
        for quote in response.css('.quote'):
            yield {
                "text": quote.css('.text::text').get(),
                "author": quote.css('.author::text').get(),
            }

        # 跟进分页
        next_page = response.css('.next a')
        if next_page:
            yield response.follow(next_page[0])

# 启动爬虫
result = QuotesSpider().start()
print(f"Scraped {len(result.items)} quotes")
result.items.to_json("quotes.json")

from scrapling.spiders import Spider, Request, Response
from scrapling.fetchers import FetcherSession, AsyncStealthySession

class MultiSessionSpider(Spider):
    name = "multi"
    start_urls = ["https://example.com/"]

    def configure_sessions(self, manager):
        # 添加不同类型的会话
        manager.add("fast", FetcherSession(impersonate="chrome"))
        manager.add("stealth", AsyncStealthySession(headless=True), lazy=True)

    async def parse(self, response: Response):
        for link in response.css('a::attr(href)').getall():
            if "protected" in link:
                # 受保护页面使用 stealth 会话
                yield Request(link, sid="stealth")
            else:
                # 快速页面使用 fast 会话
                yield Request(link, sid="fast", callback=self.parse)

# 启动爬虫（带检查点）
result = QuotesSpider(crawldir="./crawl_data").start()

# 按 Ctrl+C 优雅停止 - 进度自动保存
# 之后重新运行时：
result = QuotesSpider(crawldir="./crawl_data").start()  # 自动从上次位置恢复

class StreamingSpider(Spider):
    name = "streaming"
    start_urls = ["https://example.com/"]

    async def parse(self, response: Response):
        for item in response.css('.item'):
            yield {"title": item.css('h2::text').get()}

# 流式处理（实时获取结果）
async for item in StreamingSpider().stream():
    print(item)  # 实时输出
    # 适合：UI 展示、数据管道、长时间爬取

from scrapling.fetchers import Fetcher

page = Fetcher.get('https://quotes.toscrape.com/')

# CSS 选择器
quotes = page.css('.quote')

# XPath 选择器
quotes = page.xpath('//div[@class="quote"]')

# BeautifulSoup 风格
quotes = page.find_all('div', {'class': 'quote'})

# 文本搜索
quotes = page.find_by_text('quote', tag='div')

# 链式选择
first_quote = page.css('.quote')[0]
author = first_quote.css('.author::text').get()

# 元素关系导航
parent = first_quote.parent
siblings = first_quote.sibling_elements()
children = first_quote.css('.text')

# 保存元素（网站改版后自动重新定位）
page = StealthyFetcher.fetch('https://example.com/')
products = page.css('.product', auto_save=True)

# 后续使用（网站结构可能已变化）
# Scrapling 会自动尝试找到新位置的元素
products = page.css('.product', adaptive=True)

first_item = page.css('.product')[0]

# 查找相似元素
similar_items = first_item.find_similar()

# 查找下方元素
below_items = first_item.below_elements()

from scrapling.fetchers import Fetcher, ProxyRotator

# 单次请求使用代理
page = Fetcher.get(
    'https://example.com/',
    proxies=["http://proxy1:port", "http://proxy2:port"]
)

# 使用代理轮换器
rotator = ProxyRotator(["proxy1", "proxy2", "proxy3"])
page = Fetcher.get('https://example.com/', proxy=rotator)

from scrapling.fetchers import FetcherSession

# 按域名设置不同代理
session = FetcherSession(
    proxy={"example.com": "proxy1", "api.example.com": "proxy2"}
)

# 轮换策略
rotator = ProxyRotator(
    proxies=["p1", "p2", "p3"],
    strategy="cyclic"  # 或 "random"
)

pip install "scrapling[ai]"

# 在 Claude Code / Cursor 中使用
# AI 理解自然语言指令，自动调用 Scrapling 提取数据

# 仅解析器（无请求功能）
pip install scrapling

# 含请求功能
pip install "scrapling[fetchers]"

# 安装浏览器
scrapling install        # 正常安装
scrapling install --force  # 强制重装

# 从 DockerHub 拉取
docker pull pyd4vinci/scrapling

# 或从 GitHub Registry
docker pull ghcr.io/d4vinci/scrapling:latest

# 缓存第一次请求，后续复用（不重新请求目标服务器）
class DevSpider(Spider):
    name = "dev"
    start_urls = ["https://example.com/"]

    def configure_sessions(self, manager):
        manager.dev_mode = True  # 启用开发模式

class PoliteSpider(Spider):
    name = "polite"
    robots_txt_obey = True  # 遵守 robots.txt
    crawl_delay = 1  # 爬取间隔（秒）

# 重要爬取任务使用检查点
class ImportantSpider(Spider):
    name = "important"
    concurrent_requests = 5  # 降低并发
    crawldir = "./checkpoints/"  # 检查点目录

# 按 Ctrl+C 保存进度
# 之后恢复继续

# 提取为 Markdown
scrapling extract get 'https://example.com' content.md

# 指定 CSS 选择器
scrapling extract get 'https://example.com' content.txt \
    --css-selector '#main-content'

# 使用隐身模式
scrapling extract stealthy-fetch 'https://nopecha.com/demo/cloudflare' \
    captchas.html --css-selector '#padded_content a' \
    --solve-cloudflare

# 启动交互式爬虫 Shell
scrapling shell

pip install "scrapling[fetchers]"
scrapling install