cdpfleet / Web scraping
For scrapers
Scrape like a real visitor.
Pick the browser a site should see — real Chrome, Edge, a Camoufox device persona or a stealth-patched Chromium — send it through your proxy or a residential exit by country, and check what the site reads against measurements, not folklore.
- 12real desktop browsers
- $0.50/GBcheapest residential tier
- 0second IPs leaked over WebRTC
Which browser for which protection
| The site checks… | Use | Why — measured |
|---|---|---|
| That it's really Chrome (client hints, TLS) | Google Chrome or Microsoft Edge | Each sends its own real JA4 and HTTP/2 fingerprint, nothing patched (case 01); Chrome's brand is in the client hints, Playwright's Chromium lacks it (Chrome vs Chromium). |
| The device: screen, fonts, WebGL, OS | Camoufox with os and geoip | Spoofs a whole device at the engine level and matches locale and timezone to the exit (Camoufox vs Patchright, fonts in case 52). Based on Firefox source; not affiliated with Mozilla. Needs a proxy. |
| Automation leaks: CDP, the console | Patchright (or Rebrowser on Playwright 1.52) | Chromium with the automation leaks patched; it delivers no console events, so logging costs it less than Playwright's Chromium (case 43, Patchright vs Rebrowser). |
Headless: HeadlessChrome, plugins, permissions | Any Chromium build, headful | Headful removes the obvious tells with the same build, for one extra thread; the software GPU and the server's cores stay visible either way (case 16, case 05). |
A Camoufox Windows persona on a sticky US residential exit
// npm install [email protected]
// Browsers run on cdpfleet, so no `npx playwright install` is needed.
import { firefox } from 'playwright';
const KEY = process.env.CDPFLEET_API_KEY;
// 1. Launch the browser
const res = await fetch('https://starter.cdpfleet.com/camoufox/session', {
method: 'POST',
headers: { 'x-api-key': KEY, 'content-type': 'application/json' },
body: JSON.stringify({
proxy: 'cdpfleet-resi-country-us-session-shop1-lifetime-30',
os: 'windows',
geoip: true,
headless: true,
}),
});
if (!res.ok) throw new Error(`launch failed: ${res.status} ${await res.text()}`);
const { wsUrl } = await res.json();
// 2. Connect and drive it
const browser = await firefox.connect(wsUrl, { headers: { 'x-api-key': KEY } });
const page = await browser.newPage();
await page.goto('https://example.com');
console.log(await page.title());
await browser.close(); // ends the session and stops billing
# pip install playwright==1.60.0 requests
# Browsers run on cdpfleet, so no `playwright install` is needed.
import os
import requests
from playwright.sync_api import sync_playwright
KEY = os.environ["CDPFLEET_API_KEY"]
# 1. Launch the browser
res = requests.post(
"https://starter.cdpfleet.com/camoufox/session",
headers={"x-api-key": KEY},
json={
"proxy": "cdpfleet-resi-country-us-session-shop1-lifetime-30",
"os": "windows",
"geoip": True,
"headless": True,
},
timeout=60,
)
res.raise_for_status()
ws_url = res.json()["wsUrl"]
# 2. Connect and drive it
with sync_playwright() as p:
browser = p.firefox.connect(ws_url, headers={"x-api-key": KEY})
page = browser.new_page()
page.goto("https://example.com")
print(page.title())
browser.close() # ends the session and stops billing
// Maven: com.microsoft.playwright:playwright:1.60.0 and com.google.code.gson:gson:2.11.0
// Run with PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1: browsers run on cdpfleet.
import com.google.gson.JsonParser;
import com.microsoft.playwright.*;
import java.net.URI;
import java.net.http.*;
import java.util.Map;
public class Example {
public static void main(String[] args) throws Exception {
String key = System.getenv("CDPFLEET_API_KEY");
// 1. Launch the browser
String body = """
{
"proxy": "cdpfleet-resi-country-us-session-shop1-lifetime-30",
"os": "windows",
"geoip": true,
"headless": true
}
""";
HttpResponse<String> res = HttpClient.newHttpClient().send(
HttpRequest.newBuilder(URI.create("https://starter.cdpfleet.com/camoufox/session"))
.header("x-api-key", key)
.header("content-type", "application/json")
.POST(HttpRequest.BodyPublishers.ofString(body))
.build(),
HttpResponse.BodyHandlers.ofString());
if (res.statusCode() != 200) {
throw new RuntimeException("launch failed: " + res.statusCode() + " " + res.body());
}
String wsUrl = JsonParser.parseString(res.body()).getAsJsonObject().get("wsUrl").getAsString();
// 2. Connect and drive it
try (Playwright playwright = Playwright.create()) {
Browser browser = playwright.firefox().connect(wsUrl,
new BrowserType.ConnectOptions().setHeaders(Map.of("x-api-key", key)));
Page page = browser.newPage();
page.navigate("https://example.com");
System.out.println(page.title());
browser.close(); // ends the session and stops billing
}
}
}
// dotnet add package Microsoft.Playwright --version 1.60.0
// Browsers run on cdpfleet, so no `playwright install` is needed.
using System.Net.Http.Json;
using System.Text;
using System.Text.Json;
using Microsoft.Playwright;
var key = Environment.GetEnvironmentVariable("CDPFLEET_API_KEY")!;
// 1. Launch the browser
using var http = new HttpClient();
http.DefaultRequestHeaders.Add("x-api-key", key);
var body = """
{
"proxy": "cdpfleet-resi-country-us-session-shop1-lifetime-30",
"os": "windows",
"geoip": true,
"headless": true
}
""";
var res = await http.PostAsync("https://starter.cdpfleet.com/camoufox/session",
new StringContent(body, Encoding.UTF8, "application/json"));
if (!res.IsSuccessStatusCode)
throw new Exception($"launch failed: {(int)res.StatusCode} {await res.Content.ReadAsStringAsync()}");
var wsUrl = (await res.Content.ReadFromJsonAsync<JsonElement>()).GetProperty("wsUrl").GetString()!;
// 2. Connect and drive it
using var playwright = await Playwright.CreateAsync();
var browser = await playwright.Firefox.ConnectAsync(wsUrl, new BrowserTypeConnectOptions
{
Headers = new Dictionary<string, string> { ["x-api-key"] = key },
});
var page = await browser.NewPageAsync();
await page.GotoAsync("https://example.com");
Console.WriteLine(await page.TitleAsync());
await browser.CloseAsync(); // ends the session and stops billing
// go get github.com/playwright-community/[email protected]
//
// One-time driver setup (needs Node.js 18+): playwright-go can no longer download the
// 1.60.0 driver itself, so build it from npm and point PLAYWRIGHT_DRIVER_PATH at it:
// mkdir -p ~/.cache/pw-driver-1.60.0 && cd ~/.cache/pw-driver-1.60.0 &&
// curl -sL https://registry.npmjs.org/playwright-core/-/playwright-core-1.60.0.tgz | tar xz &&
// ln -sf "$(command -v node)" node
// export PLAYWRIGHT_DRIVER_PATH=~/.cache/pw-driver-1.60.0
// Browsers run on cdpfleet, so no browsers are downloaded (SkipInstallBrowsers).
package main
import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
"strings"
"github.com/playwright-community/playwright-go"
)
func main() {
key := os.Getenv("CDPFLEET_API_KEY")
// 1. Launch the browser
body := `{
"proxy": "cdpfleet-resi-country-us-session-shop1-lifetime-30",
"os": "windows",
"geoip": true,
"headless": true
}`
req, _ := http.NewRequest("POST", "https://starter.cdpfleet.com/camoufox/session", strings.NewReader(body))
req.Header.Set("x-api-key", key)
req.Header.Set("content-type", "application/json")
res, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal(err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
msg, _ := io.ReadAll(res.Body)
log.Fatalf("launch failed: %s %s", res.Status, msg)
}
var session struct {
WsURL string `json:"wsUrl"`
}
if err := json.NewDecoder(res.Body).Decode(&session); err != nil {
log.Fatal(err)
}
// 2. Connect and drive it (Install fetches only the Playwright driver, once)
opts := &playwright.RunOptions{SkipInstallBrowsers: true}
if err := playwright.Install(opts); err != nil {
log.Fatal(err)
}
pw, err := playwright.Run(opts)
if err != nil {
log.Fatal(err)
}
defer pw.Stop()
browser, err := pw.Firefox.Connect(session.WsURL, playwright.BrowserTypeConnectOptions{
Headers: map[string]string{"x-api-key": key},
})
if err != nil {
log.Fatal(err)
}
page, err := browser.NewPage()
if err != nil {
log.Fatal(err)
}
if _, err := page.Goto("https://example.com"); err != nil {
log.Fatal(err)
}
title, _ := page.Title()
fmt.Println(title)
browser.Close() // ends the session and stops billing
}
What a site reads before your page even loads
- TLS and HTTP/2 come from the engine, not your code: Chrome and Edge differ by one TLS extension and share their HTTP/2 fingerprint; Camoufox's TLS is Linux Firefox's whatever OS it claims (as a real Firefox's is on every OS); WebKit speaks HTTP/1.1 through a proxy, so use it for rendering tests, not to impersonate (case 01).
- Header order is a fingerprint too: setting
localeorextraHTTPHeadersmovesaccept-languageup, androute.continue({ headers })reorders everything — an order real Chrome doesn't send (case 09). navigator.webdriverisfalseon every Chromium build on the fleet, headless included (case 16).- No WebRTC leak: a session that uses a proxy can't send UDP past it, so no setup exposed a second public IP; Chromium, Chrome and Firefox show only anonymous mDNS names, and Camoufox reports the exit (case 54).
- Treat stealth as a floor, not a guarantee: these are first-line checks; serious vendors add canvas, audio, timing and behaviour (case 05).
Pay for less
- Fetch first: a plain GET through your proxy tells you in a second whether the data is already in the HTML — one of three pages in the test needed no browser at all (case 19).
- Residential by tier: the cdpfleet residential proxy costs $5.00 / $2.50 / $1.00 / $0.50 per GB for Tiers 1–4, from one prepaid balance. Keep the page on Tier 1 and send image hosts through Tier 4 with
proxy_rules: on a gallery, images were about two thirds of the bytes and a load cost well under half (case 51). Or keep your own proxy and send only the one strict host through residential (recipe). - Don't load what you don't need: block images and fonts to save proxy bytes (case 08), and block analytics and beacons so the page doesn't report your visit — the Guardian sent 13–43 telemetry requests per load, some after the scraper had left (case 59).
- Read the API the page reads: log in once, borrow the token the app sends to its own API, and fetch every page of it from inside the browser — all pages in about a second instead of clicking through (case 61).
Pages on Tier 1, image hosts on Tier 4, in one session
// npm install [email protected]
// Browsers run on cdpfleet, so no `npx playwright install` is needed.
import { chromium } from 'playwright';
const KEY = process.env.CDPFLEET_API_KEY;
// 1. Launch the browser
const res = await fetch('https://starter.cdpfleet.com/chromium/session', {
method: 'POST',
headers: { 'x-api-key': KEY, 'content-type': 'application/json' },
body: JSON.stringify({
proxy: 'cdpfleet-resi-country-us',
headless: 'new',
proxy_rules: [
{
name: 'images',
hosts: [
'upload.wikimedia.org',
'*.cloudfront.net',
],
proxy: 'cdpfleet-resi-tier-4-country-us',
},
],
}),
});
if (!res.ok) throw new Error(`launch failed: ${res.status} ${await res.text()}`);
const { wsUrl } = await res.json();
// 2. Connect and drive it
const browser = await chromium.connect(wsUrl, { headers: { 'x-api-key': KEY } });
const page = await browser.newPage();
await page.goto('https://example.com');
console.log(await page.title());
await browser.close(); // ends the session and stops billing
# pip install playwright==1.60.0 requests
# Browsers run on cdpfleet, so no `playwright install` is needed.
import os
import requests
from playwright.sync_api import sync_playwright
KEY = os.environ["CDPFLEET_API_KEY"]
# 1. Launch the browser
res = requests.post(
"https://starter.cdpfleet.com/chromium/session",
headers={"x-api-key": KEY},
json={
"proxy": "cdpfleet-resi-country-us",
"headless": "new",
"proxy_rules": [
{
"name": "images",
"hosts": [
"upload.wikimedia.org",
"*.cloudfront.net",
],
"proxy": "cdpfleet-resi-tier-4-country-us",
},
],
},
timeout=60,
)
res.raise_for_status()
ws_url = res.json()["wsUrl"]
# 2. Connect and drive it
with sync_playwright() as p:
browser = p.chromium.connect(ws_url, headers={"x-api-key": KEY})
page = browser.new_page()
page.goto("https://example.com")
print(page.title())
browser.close() # ends the session and stops billing
// Maven: com.microsoft.playwright:playwright:1.60.0 and com.google.code.gson:gson:2.11.0
// Run with PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1: browsers run on cdpfleet.
import com.google.gson.JsonParser;
import com.microsoft.playwright.*;
import java.net.URI;
import java.net.http.*;
import java.util.Map;
public class Example {
public static void main(String[] args) throws Exception {
String key = System.getenv("CDPFLEET_API_KEY");
// 1. Launch the browser
String body = """
{
"proxy": "cdpfleet-resi-country-us",
"headless": "new",
"proxy_rules": [
{
"name": "images",
"hosts": [
"upload.wikimedia.org",
"*.cloudfront.net"
],
"proxy": "cdpfleet-resi-tier-4-country-us"
}
]
}
""";
HttpResponse<String> res = HttpClient.newHttpClient().send(
HttpRequest.newBuilder(URI.create("https://starter.cdpfleet.com/chromium/session"))
.header("x-api-key", key)
.header("content-type", "application/json")
.POST(HttpRequest.BodyPublishers.ofString(body))
.build(),
HttpResponse.BodyHandlers.ofString());
if (res.statusCode() != 200) {
throw new RuntimeException("launch failed: " + res.statusCode() + " " + res.body());
}
String wsUrl = JsonParser.parseString(res.body()).getAsJsonObject().get("wsUrl").getAsString();
// 2. Connect and drive it
try (Playwright playwright = Playwright.create()) {
Browser browser = playwright.chromium().connect(wsUrl,
new BrowserType.ConnectOptions().setHeaders(Map.of("x-api-key", key)));
Page page = browser.newPage();
page.navigate("https://example.com");
System.out.println(page.title());
browser.close(); // ends the session and stops billing
}
}
}
// dotnet add package Microsoft.Playwright --version 1.60.0
// Browsers run on cdpfleet, so no `playwright install` is needed.
using System.Net.Http.Json;
using System.Text;
using System.Text.Json;
using Microsoft.Playwright;
var key = Environment.GetEnvironmentVariable("CDPFLEET_API_KEY")!;
// 1. Launch the browser
using var http = new HttpClient();
http.DefaultRequestHeaders.Add("x-api-key", key);
var body = """
{
"proxy": "cdpfleet-resi-country-us",
"headless": "new",
"proxy_rules": [
{
"name": "images",
"hosts": [
"upload.wikimedia.org",
"*.cloudfront.net"
],
"proxy": "cdpfleet-resi-tier-4-country-us"
}
]
}
""";
var res = await http.PostAsync("https://starter.cdpfleet.com/chromium/session",
new StringContent(body, Encoding.UTF8, "application/json"));
if (!res.IsSuccessStatusCode)
throw new Exception($"launch failed: {(int)res.StatusCode} {await res.Content.ReadAsStringAsync()}");
var wsUrl = (await res.Content.ReadFromJsonAsync<JsonElement>()).GetProperty("wsUrl").GetString()!;
// 2. Connect and drive it
using var playwright = await Playwright.CreateAsync();
var browser = await playwright.Chromium.ConnectAsync(wsUrl, new BrowserTypeConnectOptions
{
Headers = new Dictionary<string, string> { ["x-api-key"] = key },
});
var page = await browser.NewPageAsync();
await page.GotoAsync("https://example.com");
Console.WriteLine(await page.TitleAsync());
await browser.CloseAsync(); // ends the session and stops billing
// go get github.com/playwright-community/[email protected]
//
// One-time driver setup (needs Node.js 18+): playwright-go can no longer download the
// 1.60.0 driver itself, so build it from npm and point PLAYWRIGHT_DRIVER_PATH at it:
// mkdir -p ~/.cache/pw-driver-1.60.0 && cd ~/.cache/pw-driver-1.60.0 &&
// curl -sL https://registry.npmjs.org/playwright-core/-/playwright-core-1.60.0.tgz | tar xz &&
// ln -sf "$(command -v node)" node
// export PLAYWRIGHT_DRIVER_PATH=~/.cache/pw-driver-1.60.0
// Browsers run on cdpfleet, so no browsers are downloaded (SkipInstallBrowsers).
package main
import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
"strings"
"github.com/playwright-community/playwright-go"
)
func main() {
key := os.Getenv("CDPFLEET_API_KEY")
// 1. Launch the browser
body := `{
"proxy": "cdpfleet-resi-country-us",
"headless": "new",
"proxy_rules": [
{
"name": "images",
"hosts": [
"upload.wikimedia.org",
"*.cloudfront.net"
],
"proxy": "cdpfleet-resi-tier-4-country-us"
}
]
}`
req, _ := http.NewRequest("POST", "https://starter.cdpfleet.com/chromium/session", strings.NewReader(body))
req.Header.Set("x-api-key", key)
req.Header.Set("content-type", "application/json")
res, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal(err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
msg, _ := io.ReadAll(res.Body)
log.Fatalf("launch failed: %s %s", res.Status, msg)
}
var session struct {
WsURL string `json:"wsUrl"`
}
if err := json.NewDecoder(res.Body).Decode(&session); err != nil {
log.Fatal(err)
}
// 2. Connect and drive it (Install fetches only the Playwright driver, once)
opts := &playwright.RunOptions{SkipInstallBrowsers: true}
if err := playwright.Install(opts); err != nil {
log.Fatal(err)
}
pw, err := playwright.Run(opts)
if err != nil {
log.Fatal(err)
}
defer pw.Stop()
browser, err := pw.Chromium.Connect(session.WsURL, playwright.BrowserTypeConnectOptions{
Headers: map[string]string{"x-api-key": key},
})
if err != nil {
log.Fatal(err)
}
page, err := browser.NewPage()
if err != nil {
log.Fatal(err)
}
if _, err := page.Goto("https://example.com"); err != nil {
log.Fatal(err)
}
title, _ := page.Title()
fmt.Println(title)
browser.Close() // ends the session and stops billing
}
When the site pushes back
- Honour
Retry-After, then move: a crawler that retried a 429 at once was challenged and got 3–4 of 8 pages; one that waited as told and swapped the session to another proxy when its exit was burned got 8 of 8, with its login intact (case 53, live swap in case 06). - See the block yourself: open the same browser and exit in live view, click through the challenge or the consent wall by hand, then let your code continue in that page (Launch interactive, case 58).
- Retry the fleet, not the site: a
503from the launch means the fleet is busy and429 threads_exceededmeans no free thread — back off and retry (worker pool).
When not to use a browser
- The data is in the HTML: a plain HTTP client through the same proxy is cheaper and faster (fetch first).
- You need captchas solved for you: there is no built-in solver. Use any solver through Playwright, or hand the browser to a person in live view.
- You want to look like a phone: desktop browsers emulate phone screens, but a real device is a different story — see mobile.