[Functions](https://cdpfleet.com/docs/functions)

# Scrape

The page in the formats you ask for — HTML, Markdown, text, links, a screenshot, its XHR/fetch calls, cookies, console — after optional clicks and typing, with CSS selectors extracted to JSON. `POST https://cdpfleet.com/v1/functions/scrape` → `application/json`.

 

```bash
curl -s https://cdpfleet.com/v1/functions/scrape \
  -H "x-api-key: $CDPFLEET_API_KEY" -H "content-type: application/json" \
  -d '{"url":"https://books.toscrape.com/","formats":["markdown","links"],"extract":{"title":"h1","prices":{"selector":".price_color","all":true}}}' -o page.json -D - | grep -i '^x-call-id'
```

```js
// Node.js 18+: save as scrape.mjs, run node scrape.mjs
const res = await fetch('https://cdpfleet.com/v1/functions/scrape', {
  method: 'POST',
  headers: { 'x-api-key': process.env.CDPFLEET_API_KEY, 'content-type': 'application/json' },
  body: JSON.stringify({
    url: 'https://books.toscrape.com/',
    formats: ['markdown', 'links'],
    extract: {
      title: 'h1',
      prices: { selector: '.price_color', all: true },
    },
  }),
  signal: AbortSignal.timeout(30000),
});
if (!res.ok) throw new Error(`${res.status} ${await res.text()}`); // 429: wait Retry-After seconds
console.log(await res.text());
```

```python
# Python 3 with requests (pip install requests): save as scrape.py, run python scrape.py
import os
import requests

r = requests.post(
    "https://cdpfleet.com/v1/functions/scrape",
    headers={"x-api-key": os.environ["CDPFLEET_API_KEY"]},
    json={
        "url": "https://books.toscrape.com/",
        "formats": ["markdown", "links"],
        "extract": {
            "title": "h1",
            "prices": {"selector": ".price_color", "all": True},
        },
    },
    timeout=30,
)
if not r.ok:
    raise SystemExit(f"{r.status_code} {r.text}")  # 429: wait r.headers["Retry-After"] seconds
print(r.text)
```

```java
// Java 17+: save as Main.java, run java Main.java
import java.net.URI;
import java.net.http.*;
import java.time.Duration;

public class Main {
  public static void main(String[] args) throws Exception {
    String body = """
        {
          "url": "https://books.toscrape.com/",
          "formats": ["markdown", "links"],
          "extract": {
            "title": "h1",
            "prices": {"selector": ".price_color", "all": true}
          }
        }
        """;
    HttpResponse<byte[]> res = HttpClient.newHttpClient().send(
        HttpRequest.newBuilder(URI.create("https://cdpfleet.com/v1/functions/scrape"))
            .header("x-api-key", System.getenv("CDPFLEET_API_KEY"))
            .header("content-type", "application/json")
            .timeout(Duration.ofSeconds(30))
            .POST(HttpRequest.BodyPublishers.ofString(body))
            .build(),
        HttpResponse.BodyHandlers.ofByteArray());
    if (res.statusCode() != 200) { // 429: wait Retry-After seconds
      throw new RuntimeException(res.statusCode() + " " + new String(res.body()));
    }
    System.out.writeBytes(res.body()); // UTF-8 as it came
    System.out.println();
  }
}
```

```csharp
// .NET 8+: dotnet new console, replace Program.cs with this, run dotnet run
using System.Text;

using var http = new HttpClient { Timeout = TimeSpan.FromSeconds(30) };
http.DefaultRequestHeaders.Add("x-api-key", Environment.GetEnvironmentVariable("CDPFLEET_API_KEY"));
var body = """
    {
      "url": "https://books.toscrape.com/",
      "formats": ["markdown", "links"],
      "extract": {
        "title": "h1",
        "prices": {"selector": ".price_color", "all": true}
      }
    }
    """;
var res = await http.PostAsync("https://cdpfleet.com/v1/functions/scrape",
    new StringContent(body, Encoding.UTF8, "application/json"));
if (!res.IsSuccessStatusCode) // 429: wait Retry-After seconds
    throw new Exception($"{(int)res.StatusCode} {await res.Content.ReadAsStringAsync()}");
Console.WriteLine(await res.Content.ReadAsStringAsync());
```

```go
// Go 1.21+: save as main.go, run go run main.go
package main

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

func main() {
	body := `{
	  "url": "https://books.toscrape.com/",
	  "formats": ["markdown", "links"],
	  "extract": {
	    "title": "h1",
	    "prices": {"selector": ".price_color", "all": true}
	  }
	}`
	req, _ := http.NewRequest("POST", "https://cdpfleet.com/v1/functions/scrape", strings.NewReader(body))
	req.Header.Set("x-api-key", os.Getenv("CDPFLEET_API_KEY"))
	req.Header.Set("content-type", "application/json")
	res, err := (&http.Client{Timeout: 30 * time.Second}).Do(req)
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	data, err := io.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	if res.StatusCode != http.StatusOK { // 429: wait Retry-After seconds
		log.Fatalf("%s %s", res.Status, data)
	}
	fmt.Println(string(data))
}
```

## Fields

| Field | Type | Default | What it does |
| --- | --- | --- | --- |
| `url` | string | required | The page to load. A public `http` or `https` address: private, loopback, link-local, CGNAT and cloud-metadata addresses, `localhost`, `.local`/`.internal` names and names that resolve to them are refused (`400 invalid_url`). |
| `country` | string | `"random"` | Two-letter country for the exit IP and for the page's locale, timezone, geolocation and `Accept-Language` (see [countries](#countries)); `"random"` picks one. Unknown codes: `400 invalid_country`. |
| `max_wait` | number | `20` | Seconds to wait for the page to load, 1–60 (`400 invalid_option` otherwise). The whole call returns within about `max_wait` + 5 s, so a client timeout of `max_wait` + 10 s is safe. A page that is still loading then comes back partial (below); one that never started: `408 page_timeout`. |
| `proxy` | string or string\[\] | the Community proxy | Omit it to use the free Community proxy in `country`. Or: your own proxy URL (`http://`, `https://`, `socks5://`), a list (one is picked per call), a `cdpfleet-plan…` token, or a residential `cdpfleet-resi…` token (tier, country, device OS, sticky session) billed per GB to your residential balance ([proxies](https://cdpfleet.com/docs/proxies)). `proxy_rules` isn't supported yet (`400 invalid_option`). |
| `formats` | string\[\] | `["html"]` | Any of `"html"`, `"markdown"`, `"text"`, `"links"`, `"screenshot"` (a base64 PNG of the viewport), `"requests"`, `"cookies"`, `"console"`. |
| `wait_for` | string or number | none | After the page loads, wait for a CSS selector (`"#results"`) or a number of milliseconds, within `max_wait`. |
| `actions` | object\[\] | `[]` | Up to 20 steps before the capture, one key each: `{"click": sel}`, `{"type": [sel, text]}`, `{"press": "Enter"}`, `{"scroll": "bottom"}` (or `"top"`, or pixels), `{"wait": ms or sel}`, `{"hover": sel}`, `{"select": [sel, value]}`. A step that fails: `422 action_failed` with `step` and `message`. |
| `extract` | object | none | Named CSS selectors returned as JSON: `{"title": "h1"}` gives the text of the first match; `{"links": {"selector": "a", "attr": "href", "all": true}}` gives every match's attribute. |

## Good to know

-   Response: `url` (final), `status` (the page's HTTP status; `null` in the rare case a proxy's authentication challenge hid it and the browser couldn't report it either), `redirects[] {url, status}`, `partial: true` when the page hadn't finished loading within `max_wait`, one key per format, and `extract`. `requests[] {url, method, resource_type: "xhr"|"fetch", status, response_headers, body}` cuts each body at 64 KB; `cookies[]` and `console[]` are as the browser reports them.
-   Only XHR and fetch calls are listed, not images, scripts or styles. Redirects are the server-side ones; a JavaScript redirect shows as the final `url`.
-   `rawpage` is the same function under its first name: its `include` is `formats`.

Billing, errors and countries: [Functions](https://cdpfleet.com/docs/functions).

---
Source: https://cdpfleet.com/docs/functions/scrape · Markdown: https://cdpfleet.com/docs/functions/scrape.md · All docs for AI tools: https://cdpfleet.com/llms.txt