PDF Link Downloaders — Chips’nCode

I wrote these because occasionally I find a page containing a large collection of useful PDFs, and clicking every one individually gets old very quickly.

Single page

Use this when the page already contains direct links to PDF files.

Downloaddownload_pdfs.py

Inspect source download_pdfs.py · 134 lines

import os
import re
import sys
from urllib.parse import urljoin, urlparse, unquote

import requests
from bs4 import BeautifulSoup


def safe_filename_from_url(url: str, fallback: str = "downloaded.pdf") -> str:
    """
    Create a safe local filename from a URL.
    """
    path = urlparse(url).path
    name = os.path.basename(path)
    name = unquote(name).strip()

    if not name:
        name = fallback

    name = re.sub(r'[<>:"/\\|?*]+', "_", name)

    if not name.lower().endswith(".pdf"):
        name += ".pdf"

    return name


def is_pdf_link(url: str) -> bool:
    """
    Check whether a URL looks like a PDF link.
    """
    path = urlparse(url).path.lower()
    return path.endswith(".pdf")


def download_file(url: str, output_folder: str, session: requests.Session) -> None:
    """
    Download a PDF file to the output folder.
    """
    filename = safe_filename_from_url(url)
    filepath = os.path.join(output_folder, filename)

    base, ext = os.path.splitext(filename)
    counter = 1
    while os.path.exists(filepath):
        filepath = os.path.join(output_folder, f"{base}_{counter}{ext}")
        counter += 1

    try:
        with session.get(url, stream=True, timeout=30) as response:
            response.raise_for_status()

            content_type = response.headers.get("Content-Type", "").lower()
            if "pdf" not in content_type and not url.lower().endswith(".pdf"):
                print(f"Skipping non-PDF response: {url}")
                return

            with open(filepath, "wb") as f:
                for chunk in response.iter_content(chunk_size=8192):
                    if chunk:
                        f.write(chunk)

        print(f"Downloaded: {url} -> {filepath}")

    except requests.RequestException as e:
        print(f"Failed to download {url}: {e}")


def find_pdf_links(webpage_url: str) -> list[str]:
    """
    Find all PDF links on a webpage.
    """
    headers = {
        "User-Agent": "Mozilla/5.0 (compatible; PDFDownloader/1.0)"
    }

    with requests.Session() as session:
        session.headers.update(headers)

        response = session.get(webpage_url, timeout=30)
        response.raise_for_status()

        soup = BeautifulSoup(response.text, "html.parser")
        pdf_links = set()

        for link in soup.find_all("a", href=True):
            href = link["href"].strip()
            absolute_url = urljoin(webpage_url, href)

            if is_pdf_link(absolute_url):
                pdf_links.add(absolute_url)

        return sorted(pdf_links)


def download_all_pdfs(webpage_url: str, output_folder: str = "downloaded_pdfs") -> None:
    """
    Find and download all PDF links from the given webpage.
    """
    os.makedirs(output_folder, exist_ok=True)

    headers = {
        "User-Agent": "Mozilla/5.0 (compatible; PDFDownloader/1.0)"
    }

    try:
        pdf_links = find_pdf_links(webpage_url)
    except requests.RequestException as e:
        print(f"Failed to read webpage: {e}")
        return

    if not pdf_links:
        print("No PDF links found on the page.")
        return

    print(f"Found {len(pdf_links)} PDF link(s).")

    with requests.Session() as session:
        session.headers.update(headers)

        for pdf_url in pdf_links:
            download_file(pdf_url, output_folder, session)


if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Usage: python download_pdfs.py <webpage_url> [output_folder]")
        sys.exit(1)

    url = sys.argv[1]
    folder = sys.argv[2] if len(sys.argv) > 2 else "downloaded_pdfs"

    download_all_pdfs(url, folder)

python -m pip install requests beautifulsoup4
python download_pdfs.py "https://example.com/page-with-pdfs"

The script finds direct .pdf links on that page and saves the files into downloaded_pdfs.

Same-site crawler

Use this when the PDFs are spread across several pages on the same website.

Downloaddownload_pdfs_crawl_site.py

Inspect source download_pdfs_crawl_site.py · 154 lines

import argparse
import os
import re
from collections import deque
from urllib.parse import urldefrag, urljoin, urlparse, unquote

import requests
from bs4 import BeautifulSoup


HEADERS = {
    "User-Agent": "Mozilla/5.0 (compatible; PDFSiteCrawler/1.0)"
}


def safe_filename_from_url(url: str, fallback: str = "downloaded.pdf") -> str:
    path = urlparse(url).path
    name = unquote(os.path.basename(path)).strip() or fallback
    name = re.sub(r'[<>:"/\\|?*]+', "_", name)

    if not name.lower().endswith(".pdf"):
        name += ".pdf"

    return name


def is_pdf_link(url: str) -> bool:
    return urlparse(url).path.lower().endswith(".pdf")


def same_site(url: str, start_url: str) -> bool:
    return urlparse(url).netloc == urlparse(start_url).netloc


def normalise_url(url: str) -> str:
    url, _fragment = urldefrag(url)
    return url


def download_file(url: str, output_folder: str, session: requests.Session) -> None:
    filename = safe_filename_from_url(url)
    filepath = os.path.join(output_folder, filename)

    base, ext = os.path.splitext(filename)
    counter = 1
    while os.path.exists(filepath):
        filepath = os.path.join(output_folder, f"{base}_{counter}{ext}")
        counter += 1

    try:
        with session.get(url, stream=True, timeout=30) as response:
            response.raise_for_status()

            content_type = response.headers.get("Content-Type", "").lower()
            if "pdf" not in content_type and not is_pdf_link(url):
                print(f"Skipping non-PDF response: {url}")
                return

            with open(filepath, "wb") as f:
                for chunk in response.iter_content(chunk_size=8192):
                    if chunk:
                        f.write(chunk)

        print(f"Downloaded: {url} -> {filepath}")

    except requests.RequestException as e:
        print(f"Failed to download {url}: {e}")


def crawl_for_pdfs(
    start_url: str,
    session: requests.Session,
    max_pages: int,
    max_depth: int,
) -> list[str]:
    seen_pages = set()
    seen_pdfs = set()
    queue = deque([(normalise_url(start_url), 0)])

    while queue and len(seen_pages) < max_pages:
        page_url, depth = queue.popleft()

        if page_url in seen_pages:
            continue

        seen_pages.add(page_url)
        print(f"Scanning: {page_url}")

        try:
            response = session.get(page_url, timeout=30)
            response.raise_for_status()
        except requests.RequestException as e:
            print(f"Failed to read page: {page_url} ({e})")
            continue

        content_type = response.headers.get("Content-Type", "").lower()
        if "html" not in content_type:
            continue

        soup = BeautifulSoup(response.text, "html.parser")

        for link in soup.find_all("a", href=True):
            href = link["href"].strip()
            absolute_url = normalise_url(urljoin(page_url, href))

            if is_pdf_link(absolute_url):
                seen_pdfs.add(absolute_url)
                continue

            if depth < max_depth and same_site(absolute_url, start_url):
                scheme = urlparse(absolute_url).scheme
                if scheme in {"http", "https"} and absolute_url not in seen_pages:
                    queue.append((absolute_url, depth + 1))

    return sorted(seen_pdfs)


def download_site_pdfs(
    start_url: str,
    output_folder: str = "downloaded_pdfs",
    max_pages: int = 50,
    max_depth: int = 2,
) -> None:
    os.makedirs(output_folder, exist_ok=True)

    with requests.Session() as session:
        session.headers.update(HEADERS)
        pdf_links = crawl_for_pdfs(start_url, session, max_pages, max_depth)

        if not pdf_links:
            print("No PDF links found.")
            return

        print(f"Found {len(pdf_links)} PDF link(s).")

        for pdf_url in pdf_links:
            download_file(pdf_url, output_folder, session)


def main() -> None:
    parser = argparse.ArgumentParser(
        description="Crawl pages on the same site and download directly linked PDFs."
    )
    parser.add_argument("url", help="Starting webpage URL")
    parser.add_argument("output_folder", nargs="?", default="downloaded_pdfs")
    parser.add_argument("--max-pages", type=int, default=50)
    parser.add_argument("--max-depth", type=int, default=2)
    args = parser.parse_args()

    download_site_pdfs(args.url, args.output_folder, args.max_pages, args.max_depth)


if __name__ == "__main__":
    main()

python -m pip install requests beautifulsoup4
python download_pdfs_crawl_site.py "https://example.com/start-page"

It follows links on the same domain and downloads the PDFs it finds. You can also stop it wandering too far:

python download_pdfs_crawl_site.py "https://example.com/start-page" "my_pdfs" --max-pages 25 --max-depth 2

JavaScript-rendered pages

If the PDF links only appear after JavaScript runs, use the Selenium version.

Downloaddownload_pdfs_selenium.py

Inspect source download_pdfs_selenium.py · 125 lines

import argparse
import os
import re
import time
from urllib.parse import urljoin, urlparse, unquote

import requests
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options


HEADERS = {
    "User-Agent": "Mozilla/5.0 (compatible; PDFSeleniumDownloader/1.0)"
}


def safe_filename_from_url(url: str, fallback: str = "downloaded.pdf") -> str:
    path = urlparse(url).path
    name = unquote(os.path.basename(path)).strip() or fallback
    name = re.sub(r'[<>:"/\\|?*]+', "_", name)

    if not name.lower().endswith(".pdf"):
        name += ".pdf"

    return name


def is_pdf_link(url: str) -> bool:
    return urlparse(url).path.lower().endswith(".pdf")


def download_file(url: str, output_folder: str, session: requests.Session) -> None:
    filename = safe_filename_from_url(url)
    filepath = os.path.join(output_folder, filename)

    base, ext = os.path.splitext(filename)
    counter = 1
    while os.path.exists(filepath):
        filepath = os.path.join(output_folder, f"{base}_{counter}{ext}")
        counter += 1

    try:
        with session.get(url, stream=True, timeout=30) as response:
            response.raise_for_status()

            content_type = response.headers.get("Content-Type", "").lower()
            if "pdf" not in content_type and not is_pdf_link(url):
                print(f"Skipping non-PDF response: {url}")
                return

            with open(filepath, "wb") as f:
                for chunk in response.iter_content(chunk_size=8192):
                    if chunk:
                        f.write(chunk)

        print(f"Downloaded: {url} -> {filepath}")

    except requests.RequestException as e:
        print(f"Failed to download {url}: {e}")


def find_pdf_links_with_selenium(webpage_url: str, wait_seconds: float = 3.0) -> list[str]:
    chrome_options = Options()
    chrome_options.add_argument("--headless=new")
    chrome_options.add_argument("--disable-gpu")
    chrome_options.add_argument("--no-sandbox")

    driver = webdriver.Chrome(options=chrome_options)

    try:
        driver.get(webpage_url)
        time.sleep(wait_seconds)

        soup = BeautifulSoup(driver.page_source, "html.parser")
        pdf_links = set()

        for link in soup.find_all("a", href=True):
            absolute_url = urljoin(webpage_url, link["href"].strip())

            if is_pdf_link(absolute_url):
                pdf_links.add(absolute_url)

        return sorted(pdf_links)

    finally:
        driver.quit()


def download_all_pdfs(
    webpage_url: str,
    output_folder: str = "downloaded_pdfs",
    wait_seconds: float = 3.0,
) -> None:
    os.makedirs(output_folder, exist_ok=True)

    pdf_links = find_pdf_links_with_selenium(webpage_url, wait_seconds)

    if not pdf_links:
        print("No PDF links found on the rendered page.")
        return

    print(f"Found {len(pdf_links)} PDF link(s).")

    with requests.Session() as session:
        session.headers.update(HEADERS)

        for pdf_url in pdf_links:
            download_file(pdf_url, output_folder, session)


def main() -> None:
    parser = argparse.ArgumentParser(
        description="Render a JavaScript-heavy page with Selenium and download direct PDF links."
    )
    parser.add_argument("url", help="Webpage URL")
    parser.add_argument("output_folder", nargs="?", default="downloaded_pdfs")
    parser.add_argument("--wait", type=float, default=3.0, help="Seconds to wait after page load")
    args = parser.parse_args()

    download_all_pdfs(args.url, args.output_folder, args.wait)


if __name__ == "__main__":
    main()

python -m pip install requests beautifulsoup4 selenium
python download_pdfs_selenium.py "https://example.com/javascript-page"

This uses headless Chrome to render the page before looking for PDF links. Chrome needs to be installed; recent Selenium versions can normally sort out the matching driver themselves.

For slower pages:

python download_pdfs_selenium.py "https://example.com/javascript-page" "my_pdfs" --wait 8

A couple of limits

Direct links, ordinary access.

All three scripts look for direct PDF links. They are not designed to bypass logins, paywalls, access controls or other technical restrictions, and the crawler stays on the domain where it started.

Only download material when you have permission, an appropriate licence, or another lawful basis to make the copy. A file being publicly reachable does not by itself settle copyright, privacy or a website’s terms of use.

Do not use these scripts to collect personal or sensitive information. Respect the site’s published automation guidance and keep request rates modest; stop if the site objects, blocks the requests or asks you to stop.

These scripts are provided as general-purpose examples, without warranty. You are responsible for deciding whether a particular download is lawful and appropriate. This is not legal advice.