Scrape

The page in the formats you ask for — HTML, Markdown, text, links, a screenshot, its XHR/fetch calls, cookies, console — after optional clicks and typing, with CSS selectors extracted to JSON. POST https://cdpfleet.com/v1/functions/scrape → application/json.

curl -s https://cdpfleet.com/v1/functions/scrape \
  -H "x-api-key: $CDPFLEET_API_KEY" -H "content-type: application/json" \
  -d '{"url":"https://books.toscrape.com/","formats":["markdown","links"],"extract":{"title":"h1","prices":{"selector":".price_color","all":true}}}' -o page.json -D - | grep -i '^x-call-id'

// Node.js 18+: save as scrape.mjs, run node scrape.mjs
const res = await fetch('https://cdpfleet.com/v1/functions/scrape', {
  method: 'POST',
  headers: { 'x-api-key': process.env.CDPFLEET_API_KEY, 'content-type': 'application/json' },
  body: JSON.stringify({
    url: 'https://books.toscrape.com/',
    formats: ['markdown', 'links'],
    extract: {
      title: 'h1',
      prices: { selector: '.price_color', all: true },
    },
  }),
  signal: AbortSignal.timeout(30000),
});
if (!res.ok) throw new Error(`${res.status} ${await res.text()}`); // 429: wait Retry-After seconds
console.log(await res.text());

# Python 3 with requests (pip install requests): save as scrape.py, run python scrape.py
import os
import requests

r = requests.post(
    "https://cdpfleet.com/v1/functions/scrape",
    headers={"x-api-key": os.environ["CDPFLEET_API_KEY"]},
    json={
        "url": "https://books.toscrape.com/",
        "formats": ["markdown", "links"],
        "extract": {
            "title": "h1",
            "prices": {"selector": ".price_color", "all": True},
        },
    },
    timeout=30,
)
if not r.ok:
    raise SystemExit(f"{r.status_code} {r.text}")  # 429: wait r.headers["Retry-After"] seconds
print(r.text)

// Java 17+: save as Main.java, run java Main.java
import java.net.URI;
import java.net.http.*;
import java.time.Duration;

public class Main {
  public static void main(String[] args) throws Exception {
    String body = """
        {
          "url": "https://books.toscrape.com/",
          "formats": ["markdown", "links"],
          "extract": {
            "title": "h1",
            "prices": {"selector": ".price_color", "all": true}
          }
        }
        """;
    HttpResponse<byte[]> res = HttpClient.newHttpClient().send(
        HttpRequest.newBuilder(URI.create("https://cdpfleet.com/v1/functions/scrape"))
            .header("x-api-key", System.getenv("CDPFLEET_API_KEY"))
            .header("content-type", "application/json")
            .timeout(Duration.ofSeconds(30))
            .POST(HttpRequest.BodyPublishers.ofString(body))
            .build(),
        HttpResponse.BodyHandlers.ofByteArray());
    if (res.statusCode() != 200) { // 429: wait Retry-After seconds
      throw new RuntimeException(res.statusCode() + " " + new String(res.body()));
    }
    System.out.writeBytes(res.body()); // UTF-8 as it came
    System.out.println();
  }
}

// .NET 8+: dotnet new console, replace Program.cs with this, run dotnet run
using System.Text;

using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(30) };
http.DefaultRequestHeaders.Add("x-api-key", Environment.GetEnvironmentVariable("CDPFLEET_API_KEY"));
var body = """
    {
      "url": "https://books.toscrape.com/",
      "formats": ["markdown", "links"],
      "extract": {
        "title": "h1",
        "prices": {"selector": ".price_color", "all": true}
      }
    }
    """;
var res = await http.PostAsync("https://cdpfleet.com/v1/functions/scrape",
    new StringContent(body, Encoding.UTF8, "application/json"));
if (!res.IsSuccessStatusCode) // 429: wait Retry-After seconds
    throw new Exception($"{(int)res.StatusCode} {await res.Content.ReadAsStringAsync()}");
Console.WriteLine(await res.Content.ReadAsStringAsync());

// Go 1.21+: save as main.go, run go run main.go
package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

func main() {
	body := `{
	  "url": "https://books.toscrape.com/",
	  "formats": ["markdown", "links"],
	  "extract": {
	    "title": "h1",
	    "prices": {"selector": ".price_color", "all": true}
	  }
	}`
	req, _ := http.NewRequest("POST", "https://cdpfleet.com/v1/functions/scrape", strings.NewReader(body))
	req.Header.Set("x-api-key", os.Getenv("CDPFLEET_API_KEY"))
	req.Header.Set("content-type", "application/json")
	res, err := (&http.Client{Timeout: 30 * time.Second}).Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	data, err := io.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	if res.StatusCode != http.StatusOK { // 429: wait Retry-After seconds
		log.Fatalf("%s %s", res.Status, data)
	}
	fmt.Println(string(data))
}

Fields

Field	Type	Default	What it does
`url`	string	required	The page to load. A public `http` or `https` address: private, loopback, link-local, CGNAT and cloud-metadata addresses, `localhost`, `.local`/`.internal` names and names that resolve to them are refused (`400 invalid_url`).
`country`	string	`"random"`	Two-letter country for the exit IP and for the page's locale, timezone, geolocation and `Accept-Language` (see countries); `"random"` picks one. Unknown codes: `400 invalid_country`.
`max_wait`	number	`20`	Seconds to wait for the page to load, 1–60 (`400 invalid_option` otherwise). The whole call returns within about `max_wait` + 5 s, so a client timeout of `max_wait` + 10 s is safe. A page that is still loading then comes back partial (below); one that never started: `408 page_timeout`.
`proxy`	string or string[]	the Community proxy	Omit it to use the free Community proxy in `country`. Or: your own proxy URL (`http://`, `https://`, `socks5://`), a list (one is picked per call), a `cdpfleet-plan…` token, or a residential `cdpfleet-resi…` token (tier, country, device OS, sticky session) billed per GB to your residential balance (proxies). `proxy_rules` isn't supported yet (`400 invalid_option`).
`formats`	string[]	`["html"]`	Any of `"html"`, `"markdown"`, `"text"`, `"links"`, `"screenshot"` (a base64 PNG of the viewport), `"requests"`, `"cookies"`, `"console"`.
`wait_for`	string or number	none	After the page loads, wait for a CSS selector (`"#results"`) or a number of milliseconds, within `max_wait`.
`actions`	object[]	`[]`	Up to 20 steps before the capture, one key each: `{"click": sel}`, `{"type": [sel, text]}`, `{"press": "Enter"}`, `{"scroll": "bottom"}` (or `"top"`, or pixels), `{"wait": ms or sel}`, `{"hover": sel}`, `{"select": [sel, value]}`. A step that fails: `422 action_failed` with `step` and `message`.
`extract`	object	none	Named CSS selectors returned as JSON: `{"title": "h1"}` gives the text of the first match; `{"links": {"selector": "a", "attr": "href", "all": true}}` gives every match's attribute.

Good to know

Response: url (final), status (the page's HTTP status; null in the rare case a proxy's authentication challenge hid it and the browser couldn't report it either), redirects[] {url, status}, partial: true when the page hadn't finished loading within max_wait, one key per format, and extract. requests[] {url, method, resource_type: "xhr"|"fetch", status, response_headers, body} cuts each body at 64 KB; cookies[] and console[] are as the browser reports them.
Only XHR and fetch calls are listed, not images, scripts or styles. Redirects are the server-side ones; a JavaScript redirect shows as the final url.
rawpage is the same function under its first name: its include is formats.

Billing, errors and countries: Functions.