Python Web Scraping: BeautifulSoup, Scrapy, and Playwright

Web scraping extracts data from websites when no API is available. Python offers three major tools for this, each suited to different complexity levels: BeautifulSoup for quick parsing, Scrapy for production crawlers, and Playwright for pages that require a browser. This guide covers all three with working examples.

BeautifulSoup: Quick and Simple Parsing

BeautifulSoup is a parsing library. Pair it with requests to fetch and parse HTML in a few lines.

# pip install beautifulsoup4 requests

import requests
from bs4 import BeautifulSoup

url = "https://news.ycombinator.com/"
response = requests.get(url)
soup = BeautifulSoup(response.text, "html.parser")

# Extract all story titles
titles = soup.select(".titleline > a")
for i, title in enumerate(titles[:10], 1):
    print(f"{i}. {title.get_text()} -> {title['href']}")

Navigating the Parse Tree

BeautifulSoup gives you multiple ways to find elements:

from bs4 import BeautifulSoup

html = """
<div class="product" data-id="101">
    <h2 class="name">Wireless Mouse</h2>
    <span class="price">$29.99</span>
    <p class="description">Ergonomic wireless mouse with USB receiver.</p>
    <ul class="features">
        <li>2.4GHz wireless</li>
        <li>1600 DPI</li>
        <li>Battery life: 12 months</li>
    </ul>
</div>
"""

soup = BeautifulSoup(html, "html.parser")

# By CSS selector (most flexible)
name = soup.select_one(".product .name").get_text()

# By tag and attribute
price = soup.find("span", class_="price").get_text()

# Get attribute values
product_id = soup.find("div", class_="product")["data-id"]

# Find all list items
features = [li.get_text() for li in soup.select(".features li")]

print(f"Product: {name}")
print(f"Price: {price}")
print(f"ID: {product_id}")
print(f"Features: {features}")

Handling Pagination

Most sites split content across multiple pages:

import requests
from bs4 import BeautifulSoup
import time

def scrape_paginated(base_url: str, max_pages: int = 5) -> list[dict]:
    all_items = []

    for page in range(1, max_pages + 1):
        url = f"{base_url}?page={page}"
        response = requests.get(url, headers={
            "User-Agent": "Mozilla/5.0 (compatible; MyScraper/1.0)"
        })

        if response.status_code != 200:
            print(f"Failed on page {page}: {response.status_code}")
            break

        soup = BeautifulSoup(response.text, "html.parser")
        items = soup.select(".item")

        if not items:
            break  # No more content

        for item in items:
            all_items.append({
                "title": item.select_one(".title").get_text(strip=True),
                "link": item.select_one("a")["href"],
            })

        print(f"Page {page}: found {len(items)} items")
        time.sleep(1)  # Be polite: wait between requests

    return all_items

Extracting Tables

Tables are common targets for scraping:

import requests
from bs4 import BeautifulSoup
import csv

def scrape_table(url: str) -> list[dict]:
    response = requests.get(url)
    soup = BeautifulSoup(response.text, "html.parser")

    table = soup.find("table")
    if not table:
        return []

    # Extract headers
    headers = [th.get_text(strip=True) for th in table.select("thead th")]

    # Extract rows
    rows = []
    for tr in table.select("tbody tr"):
        cells = [td.get_text(strip=True) for td in tr.select("td")]
        if cells and headers:
            rows.append(dict(zip(headers, cells)))

    return rows

def save_to_csv(data: list[dict], filename: str) -> None:
    if not data:
        return
    with open(filename, "w", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=data[0].keys())
        writer.writeheader()
        writer.writerows(data)

Scrapy: Production-Grade Crawling

When you need to scrape thousands of pages, handle retries, respect robots.txt, and export structured data, Scrapy is the right tool.

# pip install scrapy

Creating a Scrapy Spider

# myspider.py
import scrapy

class BookSpider(scrapy.Spider):
    name = "books"
    start_urls = ["https://books.toscrape.com/"]

    def parse(self, response):
        # Extract book data from the current page
        for book in response.css("article.product_pod"):
            yield {
                "title": book.css("h3 a::attr(title)").get(),
                "price": book.css(".price_color::text").get(),
                "rating": book.css("p.star-rating::attr(class)").get(),
                "available": book.css(".instock.availability::text").getall()[-1].strip()
                if book.css(".instock.availability") else "Out of stock",
            }

        # Follow pagination links
        next_page = response.css("li.next a::attr(href)").get()
        if next_page:
            yield response.follow(next_page, callback=self.parse)

Run with scrapy runspider myspider.py -o books.json.

Scrapy Items and Pipelines

For structured data processing, define items and pipelines:

# items.py
import scrapy

class BookItem(scrapy.Item):
    title = scrapy.Field()
    price = scrapy.Field()
    rating = scrapy.Field()
    url = scrapy.Field()

# pipelines.py
class CleanPricePipeline:
    def process_item(self, item, spider):
        # Convert price string to float
        price_str = item.get("price", "")
        if price_str.startswith("£"):
            item["price"] = float(price_str[1:])
        return item

class DuplicateFilterPipeline:
    def __init__(self):
        self.seen_titles = set()

    def process_item(self, item, spider):
        title = item.get("title")
        if title in self.seen_titles:
            raise scrapy.exceptions.DropItem(f"Duplicate: {title}")
        self.seen_titles.add(title)
        return item

Scrapy Settings for Polite Crawling

# settings.py
ROBOTSTXT_OBEY = True
DOWNLOAD_DELAY = 1  # seconds between requests
CONCURRENT_REQUESTS_PER_DOMAIN = 4
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_TARGET_CONCURRENCY = 2.0

# Enable pipelines
ITEM_PIPELINES = {
    "myproject.pipelines.CleanPricePipeline": 100,
    "myproject.pipelines.DuplicateFilterPipeline": 200,
}

# Rotate user agents
USER_AGENT = "MyBot/1.0 (+https://example.com/bot)"

# Cache responses during development
HTTPCACHE_ENABLED = True
HTTPCACHE_DIR = "httpcache"

Scrapy with Login and Sessions

import scrapy

class AuthenticatedSpider(scrapy.Spider):
    name = "auth_spider"
    login_url = "https://example.com/login"
    start_urls = ["https://example.com/dashboard"]

    def start_requests(self):
        yield scrapy.Request(self.login_url, callback=self.login)

    def login(self, response):
        # Extract CSRF token if present
        csrf = response.css("input[name='csrf_token']::attr(value)").get()
        yield scrapy.FormRequest.from_response(
            response,
            formdata={
                "username": "myuser",
                "password": "mypassword",
                "csrf_token": csrf or "",
            },
            callback=self.after_login,
        )

    def after_login(self, response):
        if "Welcome" in response.text:
            # Successfully logged in, start crawling
            for url in self.start_urls:
                yield scrapy.Request(url, callback=self.parse)

    def parse(self, response):
        for item in response.css(".dashboard-item"):
            yield {
                "name": item.css(".item-name::text").get(),
                "value": item.css(".item-value::text").get(),
            }

Playwright: Scraping JavaScript-Rendered Pages

Many modern websites render content with JavaScript. BeautifulSoup and Scrapy only see the initial HTML. Playwright runs a real browser.

# pip install playwright
# playwright install chromium

Basic Playwright Scraping

from playwright.sync_api import sync_playwright

def scrape_spa(url: str) -> list[dict]:
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto(url)

        # Wait for dynamic content to load
        page.wait_for_selector(".product-card")

        products = []
        cards = page.query_selector_all(".product-card")
        for card in cards:
            products.append({
                "name": card.query_selector(".name").inner_text(),
                "price": card.query_selector(".price").inner_text(),
            })

        browser.close()
        return products

Handling Infinite Scroll

from playwright.sync_api import sync_playwright
import time

def scrape_infinite_scroll(url: str, max_scrolls: int = 10) -> list[str]:
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.goto(url)

        items = set()
        previous_count = 0

        for scroll in range(max_scrolls):
            # Scroll to bottom
            page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
            page.wait_for_timeout(2000)  # Wait for content to load

            # Collect items
            elements = page.query_selector_all(".item-title")
            for el in elements:
                items.add(el.inner_text())

            current_count = len(items)
            print(f"Scroll {scroll + 1}: {current_count} items")

            if current_count == previous_count:
                break  # No new content loaded
            previous_count = current_count

        browser.close()
        return list(items)

Async Playwright for Better Performance

import asyncio
from playwright.async_api import async_playwright

async def scrape_multiple(urls: list[str]) -> list[dict]:
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        context = await browser.new_context()

        results = []
        for url in urls:
            page = await context.new_page()
            await page.goto(url)
            await page.wait_for_load_state("networkidle")

            title = await page.title()
            content = await page.inner_text("body")
            results.append({
                "url": url,
                "title": title,
                "length": len(content),
            })
            await page.close()

        await browser.close()
        return results

# Run it
urls = ["https://example.com", "https://example.org"]
data = asyncio.run(scrape_multiple(urls))

Intercepting Network Requests

Playwright can capture API calls the page makes internally:

from playwright.sync_api import sync_playwright
import json

def capture_api_data(url: str) -> list[dict]:
    api_responses = []

    def handle_response(response):
        if "/api/" in response.url and response.status == 200:
            try:
                data = response.json()
                api_responses.append({
                    "url": response.url,
                    "data": data,
                })
            except Exception:
                pass

    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page()
        page.on("response", handle_response)
        page.goto(url)
        page.wait_for_timeout(5000)
        browser.close()

    return api_responses

This is often easier than parsing HTML. If a website loads data from a JSON API, you can capture that API response directly.

Choosing the Right Tool

Scenario	Tool	Why
Simple static pages	BeautifulSoup	Fast, minimal setup
Crawling thousands of pages	Scrapy	Built-in concurrency, retries, caching
JavaScript-rendered sites	Playwright	Real browser rendering
API data behind a SPA	Playwright (intercept)	Capture JSON directly
Quick one-off extraction	BeautifulSoup	Fewest lines of code

Best Practices and Ethics

Responsible scraping protects both you and the site operator.

import requests
import time
from urllib.robotparser import RobotFileParser

def check_robots(base_url: str, path: str) -> bool:
    """Check if scraping a path is allowed by robots.txt."""
    rp = RobotFileParser()
    rp.set_url(f"{base_url}/robots.txt")
    rp.read()
    return rp.can_fetch("*", f"{base_url}{path}")

def polite_request(url: str, delay: float = 1.0) -> requests.Response:
    """Make a request with rate limiting and proper headers."""
    headers = {
        "User-Agent": "MyResearchBot/1.0 (contact@example.com)",
        "Accept": "text/html",
    }
    time.sleep(delay)
    response = requests.get(url, headers=headers, timeout=10)
    response.raise_for_status()
    return response

Key guidelines to follow:

• Check robots.txt before scraping any site.
• Rate limit your requests. One request per second is a reasonable default.
• Identify your bot with a descriptive User-Agent string and contact information.
• Cache responses during development so you do not hit the server repeatedly.
• Check terms of service. Some sites explicitly prohibit scraping.
• Prefer APIs when available. Many sites offer free APIs that are faster and more reliable.

Handling Anti-Scraping Measures

Some sites actively block scrapers. Common techniques and responses:

import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

def create_session() -> requests.Session:
    """Create a session with retry logic and timeouts."""
    session = requests.Session()

    retries = Retry(
        total=3,
        backoff_factor=1,
        status_forcelist=[429, 500, 502, 503, 504],
    )
    session.mount("https://", HTTPAdapter(max_retries=retries))

    session.headers.update({
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
                      "AppleWebKit/537.36 (KHTML, like Gecko) "
                      "Chrome/120.0.0.0 Safari/537.36",
        "Accept-Language": "en-US,en;q=0.9",
    })

    return session

When you receive a 429 (Too Many Requests), back off and slow down. Respect the Retry-After header if present.

Wrapping Up

Python’s web scraping ecosystem covers every complexity level. Start with BeautifulSoup and requests for simple static pages. Move to Scrapy when you need to crawl at scale with retries, caching, and structured pipelines. Use Playwright when JavaScript rendering is required or when you want to intercept API calls directly. Regardless of the tool, always scrape responsibly by respecting robots.txt, rate limiting your requests, and checking terms of service. When a public API exists, use it instead.