Cases / #33 · 2026-10-10 · Easy

When is a page ready? commit, domcontentloaded, load, networkidle or a selector

Five ways to wait for a page, timed on a static catalogue, a big article and a page that renders its data late — and how many data items were actually there when each wait returned.

Chromium

Run on production on 2026-10-10: ✓ Node.js ✓ Python ✓ Java ✓ C# ✓ Go

The problem

page.goto() waits for load by default, scripts often add networkidle "to be safe", and both feel like "the page is ready". Through a proxy every extra wait costs seconds — and on pages that fetch or render data after load, none of them means the data is there. Which wait gets you the data soonest, and which ones lie?

What we used, and why

What	Why
`waitUntil: "commit"`	Returns as soon as the response starts — nothing rendered yet.
`waitUntil: "domcontentloaded"`	The HTML is parsed; images, scripts started later and data loads may still be running.
`waitUntil: "load"`	Every subresource has loaded (the default). One slow image or ad sets the pace.
`waitUntil: "networkidle"`	No requests for 500 ms. Discouraged by Playwright itself — and not a data signal.
`locator(data).first().waitFor()` after `commit`	Wait for the thing you came for.
A fresh context per measurement	No cache between runs, so every strategy waits for the same work.

How it works

Launch one Chromium session.
For each page and each strategy: open a new context, navigate with that wait, time it, and count the data items present right after.
Print every timing with its item count.

The code

The same program in five languages (also on GitHub, with the raw output). Set these environment variables first:

CDPFLEET_API_KEY — your API key (dashboard → API keys)
PROXY_URL — your proxy, e.g. http://user:[email protected]:8000

// npm install [email protected]
// env: CDPFLEET_API_KEY, PROXY_URL
import { chromium } from 'playwright';

const KEY = process.env.CDPFLEET_API_KEY;
// A static catalogue, a big article and a page that renders its data with a delayed script.
const PAGES = [
  { url: 'https://books.toscrape.com/', data: 'article.product_pod' },
  { url: 'https://en.wikipedia.org/wiki/Web_scraping', data: '#mw-content-text p' },
  { url: 'https://quotes.toscrape.com/js-delayed/', data: '.quote' },
];
const STRATEGIES = ['commit', 'domcontentloaded', 'load', 'networkidle', 'selector'];

const res = await fetch('https://starter.cdpfleet.com/chromium/session', {
  method: 'POST',
  headers: { 'x-api-key': KEY, 'content-type': 'application/json' },
  body: JSON.stringify({ proxy: process.env.PROXY_URL, headless: 'new' }),
});
if (!res.ok) throw new Error(`launch ${res.status} ${await res.text()}`);
const { wsUrl } = await res.json();
const browser = await chromium.connect(wsUrl, { headers: { 'x-api-key': KEY } });

try {
  const rows = [];
  for (const { url, data } of PAGES) {
    for (const strategy of STRATEGIES) {
      // A fresh context each time: no cache, so every strategy waits for the same work.
      const context = await browser.newContext();
      const page = await context.newPage();
      const t = Date.now();
      if (strategy === 'selector') {
        await page.goto(url, { waitUntil: 'commit', timeout: 60000 });
        await page.locator(data).first().waitFor({ timeout: 60000 });
      } else {
        await page.goto(url, { waitUntil: strategy, timeout: 60000 });
      }
      const ms = Date.now() - t;
      rows.push({ url, strategy, ms, items_ready: await page.locator(data).count() });
      await context.close();
    }
  }
  console.log(JSON.stringify({ rows }, null, 2));
} finally {
  await browser.close();
}

# pip install playwright==1.60.0 requests
# env: CDPFLEET_API_KEY, PROXY_URL
import json
import os
import time

import requests
from playwright.sync_api import sync_playwright

KEY = os.environ["CDPFLEET_API_KEY"]
# A static catalogue, a big article and a page that renders its data with a delayed script.
PAGES = [
    {"url": "https://books.toscrape.com/", "data": "article.product_pod"},
    {"url": "https://en.wikipedia.org/wiki/Web_scraping", "data": "#mw-content-text p"},
    {"url": "https://quotes.toscrape.com/js-delayed/", "data": ".quote"},
]
STRATEGIES = ["commit", "domcontentloaded", "load", "networkidle", "selector"]

res = requests.post("https://starter.cdpfleet.com/chromium/session",
    headers={"x-api-key": KEY},
    json={"proxy": os.environ["PROXY_URL"], "headless": "new"}, timeout=60)
if not res.ok:
    raise SystemExit(f"launch {res.status_code} {res.text}")

with sync_playwright() as p:
    browser = p.chromium.connect(res.json()["wsUrl"], headers={"x-api-key": KEY})
    try:
        rows = []
        for pg in PAGES:
            url, data = pg["url"], pg["data"]
            for strategy in STRATEGIES:
                # A fresh context each time: no cache, so every strategy waits for the same work.
                context = browser.new_context()
                page = context.new_page()
                t = time.time()
                if strategy == "selector":
                    page.goto(url, wait_until="commit", timeout=60000)
                    page.locator(data).first.wait_for(timeout=60000)
                else:
                    page.goto(url, wait_until=strategy, timeout=60000)
                ms = round((time.time() - t) * 1000)
                rows.append({"url": url, "strategy": strategy, "ms": ms, "items_ready": page.locator(data).count()})
                context.close()

        print(json.dumps({"rows": rows}, indent=2, ensure_ascii=False))
    finally:
        browser.close()

// Maven: com.microsoft.playwright:playwright:1.60.0, com.google.code.gson:gson:2.11.0
// Run with PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1. env: CDPFLEET_API_KEY, PROXY_URL
import com.google.gson.*;
import com.microsoft.playwright.*;
import com.microsoft.playwright.options.WaitUntilState;
import java.net.URI;
import java.net.http.*;
import java.util.*;

public class Main {
  static final String KEY = System.getenv("CDPFLEET_API_KEY");
  static final Gson GSON = new GsonBuilder().serializeNulls().create();
  // A static catalogue, a big article and a page that renders its data with a delayed script.
  static final String[][] PAGES = {
      {"https://books.toscrape.com/", "article.product_pod"},
      {"https://en.wikipedia.org/wiki/Web_scraping", "#mw-content-text p"},
      {"https://quotes.toscrape.com/js-delayed/", ".quote"},
  };
  static final String[] STRATEGIES = {"commit", "domcontentloaded", "load", "networkidle", "selector"};
  static final Map<String, WaitUntilState> WAIT_UNTIL = Map.of(
      "commit", WaitUntilState.COMMIT,
      "domcontentloaded", WaitUntilState.DOMCONTENTLOADED,
      "load", WaitUntilState.LOAD,
      "networkidle", WaitUntilState.NETWORKIDLE);

  public static void main(String[] args) throws Exception {
    String body = "{\"proxy\": " + GSON.toJson(System.getenv("PROXY_URL")) + ", \"headless\": \"new\"}";
    HttpResponse<String> res = HttpClient.newHttpClient().send(HttpRequest.newBuilder(URI.create("https://starter.cdpfleet.com/chromium/session"))
        .header("x-api-key", KEY).header("content-type", "application/json")
        .POST(HttpRequest.BodyPublishers.ofString(body)).build(), HttpResponse.BodyHandlers.ofString());
    if (res.statusCode() != 200) throw new RuntimeException("launch: " + res.statusCode() + " " + res.body());
    String wsUrl = JsonParser.parseString(res.body()).getAsJsonObject().get("wsUrl").getAsString();

    try (Playwright playwright = Playwright.create()) {
      Browser browser = playwright.chromium().connect(wsUrl, new BrowserType.ConnectOptions().setHeaders(Map.of("x-api-key", KEY)));
      try {
        JsonArray rows = new JsonArray();
        for (String[] p : PAGES) {
          String url = p[0], data = p[1];
          for (String strategy : STRATEGIES) {
            // A fresh context each time: no cache, so every strategy waits for the same work.
            BrowserContext context = browser.newContext();
            Page page = context.newPage();
            long t = System.currentTimeMillis();
            if (strategy.equals("selector")) {
              page.navigate(url, new Page.NavigateOptions().setWaitUntil(WaitUntilState.COMMIT).setTimeout(60000));
              page.locator(data).first().waitFor(new Locator.WaitForOptions().setTimeout(60000));
            } else {
              page.navigate(url, new Page.NavigateOptions().setWaitUntil(WAIT_UNTIL.get(strategy)).setTimeout(60000));
            }
            long ms = System.currentTimeMillis() - t;
            JsonObject row = new JsonObject();
            row.addProperty("url", url);
            row.addProperty("strategy", strategy);
            row.addProperty("ms", ms);
            row.addProperty("items_ready", page.locator(data).count());
            rows.add(row);
            context.close();
          }
        }

        JsonObject out = new JsonObject();
        out.add("rows", rows);
        System.out.println(new GsonBuilder().serializeNulls().setPrettyPrinting().disableHtmlEscaping().create().toJson(out));
      } finally {
        browser.close();
      }
    }
  }
}

// dotnet add package Microsoft.Playwright --version 1.60.0
// env: CDPFLEET_API_KEY, PROXY_URL
using System.Net.Http.Json;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Nodes;
using Microsoft.Playwright;

// A static catalogue, a big article and a page that renders its data with a delayed script.
var pages = new (string Url, string Data)[]
{
    ("https://books.toscrape.com/", "article.product_pod"),
    ("https://en.wikipedia.org/wiki/Web_scraping", "#mw-content-text p"),
    ("https://quotes.toscrape.com/js-delayed/", ".quote"),
};
var strategies = new[] { "commit", "domcontentloaded", "load", "networkidle", "selector" };
var waitUntil = new Dictionary<string, WaitUntilState>
{
    ["commit"] = WaitUntilState.Commit,
    ["domcontentloaded"] = WaitUntilState.DOMContentLoaded,
    ["load"] = WaitUntilState.Load,
    ["networkidle"] = WaitUntilState.NetworkIdle,
};

var key = Environment.GetEnvironmentVariable("CDPFLEET_API_KEY")!;
using var http = new HttpClient();
http.DefaultRequestHeaders.Add("x-api-key", key);
var res = await http.PostAsJsonAsync("https://starter.cdpfleet.com/chromium/session",
    new { proxy = Environment.GetEnvironmentVariable("PROXY_URL"), headless = "new" });
if (!res.IsSuccessStatusCode) throw new Exception($"launch: {(int)res.StatusCode} {await res.Content.ReadAsStringAsync()}");
var wsUrl = (await res.Content.ReadFromJsonAsync<JsonElement>()).GetProperty("wsUrl").GetString()!;

using var playwright = await Playwright.CreateAsync();
var browser = await playwright.Chromium.ConnectAsync(wsUrl, new() { Headers = new Dictionary<string, string> { ["x-api-key"] = key } });
try
{
    var rows = new JsonArray();
    foreach (var (url, data) in pages)
    {
        foreach (var strategy in strategies)
        {
            // A fresh context each time: no cache, so every strategy waits for the same work.
            var context = await browser.NewContextAsync();
            var page = await context.NewPageAsync();
            var sw = System.Diagnostics.Stopwatch.StartNew();
            if (strategy == "selector")
            {
                await page.GotoAsync(url, new() { WaitUntil = WaitUntilState.Commit, Timeout = 60000 });
                await page.Locator(data).First.WaitForAsync(new() { Timeout = 60000 });
            }
            else
            {
                await page.GotoAsync(url, new() { WaitUntil = waitUntil[strategy], Timeout = 60000 });
            }
            var ms = sw.ElapsedMilliseconds;
            rows.Add(new JsonObject
            {
                ["url"] = url,
                ["strategy"] = strategy,
                ["ms"] = ms,
                ["items_ready"] = await page.Locator(data).CountAsync(),
            });
            await context.CloseAsync();
        }
    }

    var result = new JsonObject { ["rows"] = rows };
    Console.WriteLine(result.ToJsonString(new JsonSerializerOptions { WriteIndented = true, Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping }));
}
finally
{
    await browser.CloseAsync();
}

// go get github.com/playwright-community/[email protected]
// Driver: build playwright-core 1.60.0 from npm and set PLAYWRIGHT_DRIVER_PATH (see /docs/quickstart).
// env: CDPFLEET_API_KEY, PROXY_URL
package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/playwright-community/playwright-go"
)

var key = os.Getenv("CDPFLEET_API_KEY")

// A static catalogue, a big article and a page that renders its data with a delayed script.
var pages = []struct{ url, data string }{
	{"https://books.toscrape.com/", "article.product_pod"},
	{"https://en.wikipedia.org/wiki/Web_scraping", "#mw-content-text p"},
	{"https://quotes.toscrape.com/js-delayed/", ".quote"},
}
var strategies = []string{"commit", "domcontentloaded", "load", "networkidle", "selector"}
var waitUntil = map[string]*playwright.WaitUntilState{
	"commit":           playwright.WaitUntilStateCommit,
	"domcontentloaded": playwright.WaitUntilStateDomcontentloaded,
	"load":             playwright.WaitUntilStateLoad,
	"networkidle":      playwright.WaitUntilStateNetworkidle,
}

func launch(name string, options map[string]any) (map[string]any, error) {
	body, _ := json.Marshal(options)
	req, _ := http.NewRequest("POST", "https://starter.cdpfleet.com/"+name+"/session", bytes.NewReader(body))
	req.Header.Set("x-api-key", key)
	req.Header.Set("content-type", "application/json")
	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if res.StatusCode != http.StatusOK {
		msg, _ := io.ReadAll(res.Body)
		return nil, fmt.Errorf("launch %s: %s %s", name, res.Status, msg)
	}
	var session map[string]any
	return session, json.NewDecoder(res.Body).Decode(&session)
}

type row struct {
	URL        string `json:"url"`
	Strategy   string `json:"strategy"`
	Ms         int64  `json:"ms"`
	ItemsReady int    `json:"items_ready"`
}

func measure(browser playwright.Browser, url, data, strategy string) (row, error) {
	// A fresh context each time: no cache, so every strategy waits for the same work.
	context, err := browser.NewContext()
	if err != nil {
		return row{}, err
	}
	defer context.Close()
	page, err := context.NewPage()
	if err != nil {
		return row{}, err
	}
	t := time.Now()
	if strategy == "selector" {
		if _, err := page.Goto(url, playwright.PageGotoOptions{WaitUntil: playwright.WaitUntilStateCommit, Timeout: playwright.Float(60000)}); err != nil {
			return row{}, err
		}
		if err := page.Locator(data).First().WaitFor(playwright.LocatorWaitForOptions{Timeout: playwright.Float(60000)}); err != nil {
			return row{}, err
		}
	} else if _, err := page.Goto(url, playwright.PageGotoOptions{WaitUntil: waitUntil[strategy], Timeout: playwright.Float(60000)}); err != nil {
		return row{}, err
	}
	ms := time.Since(t).Milliseconds()
	count, err := page.Locator(data).Count()
	if err != nil {
		return row{}, err
	}
	return row{URL: url, Strategy: strategy, Ms: ms, ItemsReady: count}, nil
}

func main() {
	session, err := launch("chromium", map[string]any{"proxy": os.Getenv("PROXY_URL"), "headless": "new"})
	if err != nil {
		log.Fatal(err)
	}
	pw, err := playwright.Run(&playwright.RunOptions{SkipInstallBrowsers: true})
	if err != nil {
		log.Fatal(err)
	}
	defer pw.Stop()
	browser, err := pw.Chromium.Connect(session["wsUrl"].(string), playwright.BrowserTypeConnectOptions{Headers: map[string]string{"x-api-key": key}})
	if err != nil {
		log.Fatal(err)
	}
	defer browser.Close()

	rows := []row{}
	for _, p := range pages {
		for _, strategy := range strategies {
			r, err := measure(browser, p.url, p.data, strategy)
			if err != nil {
				browser.Close()
				log.Fatal(err)
			}
			rows = append(rows, r)
		}
	}

	var buf bytes.Buffer
	enc := json.NewEncoder(&buf)
	enc.SetEscapeHTML(false)
	enc.SetIndent("", "  ")
	enc.Encode(map[string]any{"rows": rows})
	fmt.Print(buf.String())
}

What we got

Page	Wait	ms	Data items present
https://books.toscrape.com/	commit	6475	4
https://books.toscrape.com/	domcontentloaded	3491	20
https://books.toscrape.com/	load	3513	20
https://books.toscrape.com/	networkidle	2563	20
https://books.toscrape.com/	selector	3145	17
https://en.wikipedia.org/wiki/Web_scraping	commit	2634	35
https://en.wikipedia.org/wiki/Web_scraping	domcontentloaded	8730	35
https://en.wikipedia.org/wiki/Web_scraping	load	6203	35
https://en.wikipedia.org/wiki/Web_scraping	networkidle	5573	35
https://en.wikipedia.org/wiki/Web_scraping	selector	5110	35
https://quotes.toscrape.com/js-delayed/	commit	1163	0
https://quotes.toscrape.com/js-delayed/	domcontentloaded	2691	0
https://quotes.toscrape.com/js-delayed/	load	6802	0
https://quotes.toscrape.com/js-delayed/	networkidle	11819	0
https://quotes.toscrape.com/js-delayed/	selector	13731	10

From the Node.js run on 2026-10-10. IP addresses are replaced with placeholders (203.0.113.x); equal addresses stay equal. The other languages produced the same findings.

Raw output (Node.js)

{
  "rows": [
    {
      "url": "https://books.toscrape.com/",
      "strategy": "commit",
      "ms": 6475,
      "items_ready": 4
    },
    {
      "url": "https://books.toscrape.com/",
      "strategy": "domcontentloaded",
      "ms": 3491,
      "items_ready": 20
    },
    {
      "url": "https://books.toscrape.com/",
      "strategy": "load",
      "ms": 3513,
      "items_ready": 20
    },
    {
      "url": "https://books.toscrape.com/",
      "strategy": "networkidle",
      "ms": 2563,
      "items_ready": 20
    },
    {
      "url": "https://books.toscrape.com/",
      "strategy": "selector",
      "ms": 3145,
      "items_ready": 17
    },
    {
      "url": "https://en.wikipedia.org/wiki/Web_scraping",
      "strategy": "commit",
      "ms": 2634,
      "items_ready": 35
    },
    {
      "url": "https://en.wikipedia.org/wiki/Web_scraping",
      "strategy": "domcontentloaded",
      "ms": 8730,
      "items_ready": 35
    },
    {
      "url": "https://en.wikipedia.org/wiki/Web_scraping",
      "strategy": "load",
      "ms": 6203,
      "items_ready": 35
    },
    {
      "url": "https://en.wikipedia.org/wiki/Web_scraping",
      "strategy": "networkidle",
      "ms": 5573,
      "items_ready": 35
    },
    {
      "url": "https://en.wikipedia.org/wiki/Web_scraping",
      "strategy": "selector",
      "ms": 5110,
      "items_ready": 35
    },
    {
      "url": "https://quotes.toscrape.com/js-delayed/",
      "strategy": "commit",
      "ms": 1163,
      "items_ready": 0
    },
    {
      "url": "https://quotes.toscrape.com/js-delayed/",
      "strategy": "domcontentloaded",
      "ms": 2691,
      "items_ready": 0
    },
    {
      "url": "https://quotes.toscrape.com/js-delayed/",
      "strategy": "load",
      "ms": 6802,
      "items_ready": 0
    },
    {
      "url": "https://quotes.toscrape.com/js-delayed/",
      "strategy": "networkidle",
      "ms": 11819,
      "items_ready": 0
    },
    {
      "url": "https://quotes.toscrape.com/js-delayed/",
      "strategy": "selector",
      "ms": 13731,
      "items_ready": 10
    }
  ]
}

Takeaways

Only a wait for the data returned with the data: on the page that renders its quotes with a delayed script, commit, domcontentloaded, load and networkidle all returned with zero quotes in all five languages; the selector wait got all 10.
networkidle is not "data ready" — it returned after 4–7 s on that page with nothing on it. It also inherits every slow request: 3.5–49 s on the Wikipedia article across runs.
.first().waitFor() means "at least one": on the catalogue it sometimes returned with 4 or 12 of the 20 books. When you need them all, wait for the count (expect(locator).toHaveCount(20)) or a "done" marker.
Through a proxy, timings swing by 10× between runs (load took 1.9–33 s on the catalogue) — waits tied to all network activity are the least predictable. commit is too early (Wikipedia's paragraphs were missing in 4 of 5 runs); domcontentloaded plus a selector wait is the reliable pair.