How to scrape BBC News

# Scrape BBC News bbc.com news and articles: search and listing pages with the Ujeebu Scrape API.
# Settings and extract rules validated on real pages on 2026-10-28 (Ujeebu site guide).
# Uses the official SDK ujeebu-python 0.1.5: pip install ujeebu-python==0.1.5
# Run: UJEEBU_API_KEY=your_key python3 bbc_listing_sdk.py

import json
import os
import sys
import time

import requests
from ujeebu_python import UjeebuClient

API_KEY = os.environ.get("UJEEBU_API_KEY")
URLS = [
    "https://www.bbc.com/news/technology",
    "https://www.bbc.com/business",
]
SETTINGS = {
    "js": True,
}
EXTRACT_RULES = {
    "articles": {
        "selector": "div[data-testid=\"alaska\"] > div",
        "type": "obj",
        "multiple": True,
        "children": {
            "url": {
                "selector": "a[data-testid=\"internal-link\"]",
                "type": "link",
            },
            "title": {
                "selector": "h2",
                "type": "text",
            },
            "image": {
                "selector": "img.Image-styles__ImageStyled-sc-8c99a12b-0:not(.hide-when-no-script)",
                "type": "image",
            },
            "summary": {
                "selector": "p[data-testid='card-description']",
                "type": "text",
            },
            "published": {
                "selector": "span[data-testid='card-metadata-lastupdated']",
                "type": "text",
            },
        },
    },
}


def scrape(client, url):
    # with extract_rules, client.scrape sends a JSON POST and the API answers {"result": {...}}
    try:
        response = client.scrape(url, params={**SETTINGS, "extract_rules": EXTRACT_RULES})
    except requests.RequestException as error:
        sys.exit(f"Request failed for {url}: {error}")
    if response.status_code != 200:
        try:
            message = response.json().get("message", response.text)
        except ValueError:
            message = response.text
        sys.exit(f"HTTP {response.status_code} for {url}: {str(message)[:300]}")
    print(f"Ujb-Credits: {response.headers.get('Ujb-Credits')}", file=sys.stderr)
    return response.json()["result"]


if not API_KEY:
    sys.exit("Set the UJEEBU_API_KEY environment variable.")

client = UjeebuClient(api_key=API_KEY)

for i, url in enumerate(URLS):
    if i:
        time.sleep(2)  # be polite: one page at a time
    print(json.dumps(scrape(client, url), indent=2, ensure_ascii=False))

// Scrape BBC News bbc.com news and articles: search and listing pages with the Ujeebu Scrape API.
// Settings and extract rules validated on real pages on 2026-10-28 (Ujeebu site guide).
// Uses the official SDK @ujeebu-org/ujeebu-sdk 0.1.2: npm install @ujeebu-org/ujeebu-sdk@0.1.2
// Run: UJEEBU_API_KEY=your_key node bbc_listing_sdk.mjs

import { UjeebuClient } from '@ujeebu-org/ujeebu-sdk';

const API_KEY = process.env.UJEEBU_API_KEY;
const URLS = [
  "https://www.bbc.com/news/technology",
  "https://www.bbc.com/business",
];
const SETTINGS = {
  js: true,
};
const EXTRACT_RULES = {
  articles: {
    selector: "div[data-testid=\"alaska\"] > div",
    type: "obj",
    multiple: true,
    children: {
      url: {
        selector: "a[data-testid=\"internal-link\"]",
        type: "link",
      },
      title: {
        selector: "h2",
        type: "text",
      },
      image: {
        selector: "img.Image-styles__ImageStyled-sc-8c99a12b-0:not(.hide-when-no-script)",
        type: "image",
      },
      summary: {
        selector: "p[data-testid='card-description']",
        type: "text",
      },
      published: {
        selector: "span[data-testid='card-metadata-lastupdated']",
        type: "text",
      },
    },
  },
};

async function scrape(client, url) {
  let res;
  try {
    // client.scrape POSTs JSON when extract_rules is set (scrapeWithRules would send it as a GET query, which the API ignores)
    res = await client.scrape(url, { ...SETTINGS, extract_rules: EXTRACT_RULES });
  } catch (err) {
    const status = err.response?.status ?? 'error';
    const message = err.response?.data?.message ?? err.message;
    console.error(`HTTP ${status} for ${url}: ${String(message).slice(0, 300)}`);
    process.exit(1);
  }
  console.error(`Ujb-Credits: ${res.headers['ujb-credits']}`);
  return res.data.result;
}

if (!API_KEY) {
  console.error('Set the UJEEBU_API_KEY environment variable.');
  process.exit(1);
}

const client = new UjeebuClient(API_KEY);
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

for (const [i, url] of URLS.entries()) {
  if (i) await pause(2000); // be polite: one page at a time
  console.log(JSON.stringify(await scrape(client, url), null, 2));
}

# Scrape BBC News bbc.com news and articles: search and listing pages with the Ujeebu Scrape API.
# Settings and extract rules validated on real pages on 2026-10-28 (Ujeebu site guide).
# Run: UJEEBU_API_KEY=your_key python3 bbc_listing.py   (pip install requests)

import json
import os
import sys
import time

import requests

API_URL = "https://api.ujeebu.com/scrape"
API_KEY = os.environ.get("UJEEBU_API_KEY")
URLS = [
    "https://www.bbc.com/news/technology",
    "https://www.bbc.com/business",
]
SETTINGS = {
    "js": True,
}
EXTRACT_RULES = {
    "articles": {
        "selector": "div[data-testid=\"alaska\"] > div",
        "type": "obj",
        "multiple": True,
        "children": {
            "url": {
                "selector": "a[data-testid=\"internal-link\"]",
                "type": "link",
            },
            "title": {
                "selector": "h2",
                "type": "text",
            },
            "image": {
                "selector": "img.Image-styles__ImageStyled-sc-8c99a12b-0:not(.hide-when-no-script)",
                "type": "image",
            },
            "summary": {
                "selector": "p[data-testid='card-description']",
                "type": "text",
            },
            "published": {
                "selector": "span[data-testid='card-metadata-lastupdated']",
                "type": "text",
            },
        },
    },
}


def scrape(url):
    response = requests.post(
        API_URL,
        headers={"ApiKey": API_KEY},
        json={"url": url, **SETTINGS, "extract_rules": EXTRACT_RULES},
        timeout=180,
    )
    if response.status_code != 200:
        try:
            message = response.json().get("message", response.text)
        except ValueError:
            message = response.text
        sys.exit(f"HTTP {response.status_code} for {url}: {str(message)[:300]}")
    print(f"Ujb-Credits: {response.headers.get('Ujb-Credits')}", file=sys.stderr)
    return response.json()["result"]


if not API_KEY:
    sys.exit("Set the UJEEBU_API_KEY environment variable.")

for i, url in enumerate(URLS):
    if i:
        time.sleep(2)  # be polite: one page at a time
    print(json.dumps(scrape(url), indent=2, ensure_ascii=False))

// Scrape BBC News bbc.com news and articles: search and listing pages with the Ujeebu Scrape API.
// Settings and extract rules validated on real pages on 2026-10-28 (Ujeebu site guide).
// Run: UJEEBU_API_KEY=your_key node bbc_listing.mjs   (Node.js 18 or later)

const API_URL = 'https://api.ujeebu.com/scrape';
const API_KEY = process.env.UJEEBU_API_KEY;
const URLS = [
  "https://www.bbc.com/news/technology",
  "https://www.bbc.com/business",
];
const SETTINGS = {
  js: true,
};
const EXTRACT_RULES = {
  articles: {
    selector: "div[data-testid=\"alaska\"] > div",
    type: "obj",
    multiple: true,
    children: {
      url: {
        selector: "a[data-testid=\"internal-link\"]",
        type: "link",
      },
      title: {
        selector: "h2",
        type: "text",
      },
      image: {
        selector: "img.Image-styles__ImageStyled-sc-8c99a12b-0:not(.hide-when-no-script)",
        type: "image",
      },
      summary: {
        selector: "p[data-testid='card-description']",
        type: "text",
      },
      published: {
        selector: "span[data-testid='card-metadata-lastupdated']",
        type: "text",
      },
    },
  },
};

async function scrape(url) {
  const res = await fetch(API_URL, {
    method: 'POST',
    headers: { ApiKey: API_KEY, 'Content-Type': 'application/json' },
    body: JSON.stringify({ url, ...SETTINGS, extract_rules: EXTRACT_RULES }),
    signal: AbortSignal.timeout(180000),
  });
  const text = await res.text();
  if (res.status !== 200) {
    let message = text;
    try {
      message = JSON.parse(text).message ?? text;
    } catch {}
    console.error(`HTTP ${res.status} for ${url}: ${String(message).slice(0, 300)}`);
    process.exit(1);
  }
  console.error(`Ujb-Credits: ${res.headers.get('ujb-credits')}`);
  return JSON.parse(text).result;
}

if (!API_KEY) {
  console.error('Set the UJEEBU_API_KEY environment variable.');
  process.exit(1);
}

const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

for (const [i, url] of URLS.entries()) {
  if (i) await pause(2000); // be polite: one page at a time
  console.log(JSON.stringify(await scrape(url), null, 2));
}

#!/usr/bin/env bash
# Scrape BBC News bbc.com news and articles: search and listing pages with the Ujeebu Scrape API.
# Settings and extract rules validated on real pages on 2026-10-28 (Ujeebu site guide).
# Run: UJEEBU_API_KEY=your_key bash bbc_listing.sh   (curl 7.76 or later)
set -euo pipefail

curl -sS --fail-with-body --max-time 180 -D headers.txt \
  -X POST 'https://api.ujeebu.com/scrape' \
  -H "ApiKey: ${UJEEBU_API_KEY:?Set the UJEEBU_API_KEY environment variable.}" \
  -H 'Content-Type: application/json' \
  --data-binary @- <<'JSON'
{
  "url": "https://www.bbc.com/news/technology",
  "js": true,
  "extract_rules": {
    "articles": {
      "selector": "div[data-testid=\"alaska\"] > div",
      "type": "obj",
      "multiple": true,
      "children": {
        "url": {
          "selector": "a[data-testid=\"internal-link\"]",
          "type": "link"
        },
        "title": {
          "selector": "h2",
          "type": "text"
        },
        "image": {
          "selector": "img.Image-styles__ImageStyled-sc-8c99a12b-0:not(.hide-when-no-script)",
          "type": "image"
        },
        "summary": {
          "selector": "p[data-testid='card-description']",
          "type": "text"
        },
        "published": {
          "selector": "span[data-testid='card-metadata-lastupdated']",
          "type": "text"
        }
      }
    }
  }
}
JSON
echo
grep -i '^ujb-credits' headers.txt >&2

<?php
// Scrape BBC News bbc.com news and articles: search and listing pages with the Ujeebu Scrape API.
// Settings and extract rules validated on real pages on 2026-10-28 (Ujeebu site guide).
// Run: UJEEBU_API_KEY=your_key php bbc_listing.php   (PHP 7.4 or later with curl)

$apiKey = getenv('UJEEBU_API_KEY') ?: exit("Set the UJEEBU_API_KEY environment variable.\n");
$urls = [
    'https://www.bbc.com/news/technology',
    'https://www.bbc.com/business',
];
$settings = [
    'js' => true,
];
$extractRules = [
    'articles' => [
        'selector' => 'div[data-testid="alaska"] > div',
        'type' => 'obj',
        'multiple' => true,
        'children' => [
            'url' => [
                'selector' => 'a[data-testid="internal-link"]',
                'type' => 'link',
            ],
            'title' => [
                'selector' => 'h2',
                'type' => 'text',
            ],
            'image' => [
                'selector' => 'img.Image-styles__ImageStyled-sc-8c99a12b-0:not(.hide-when-no-script)',
                'type' => 'image',
            ],
            'summary' => [
                'selector' => 'p[data-testid=\'card-description\']',
                'type' => 'text',
            ],
            'published' => [
                'selector' => 'span[data-testid=\'card-metadata-lastupdated\']',
                'type' => 'text',
            ],
        ],
    ],
];

function scrape(string $url, string $apiKey, array $settings, array $extractRules)
{
    $ch = curl_init('https://api.ujeebu.com/scrape');
    $headers = [];
    curl_setopt_array($ch, [
        CURLOPT_POST => true,
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_TIMEOUT => 180,
        CURLOPT_HTTPHEADER => ['ApiKey: ' . $apiKey, 'Content-Type: application/json'],
        CURLOPT_POSTFIELDS => json_encode(['url' => $url] + $settings + ['extract_rules' => $extractRules]),
        CURLOPT_HEADERFUNCTION => function ($ch, $line) use (&$headers) {
            if (strpos($line, ':') !== false) {
                [$k, $v] = explode(':', $line, 2);
                $headers[strtolower(trim($k))] = trim($v);
            }
            return strlen($line);
        },
    ]);
    $body = curl_exec($ch);
    if ($body === false) {
        fwrite(STDERR, 'Request failed: ' . curl_error($ch) . "\n");
        exit(1);
    }
    $status = curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
    if ($status !== 200) {
        $message = json_decode($body, true)['message'] ?? $body;
        fwrite(STDERR, "HTTP $status for $url: " . substr((string) $message, 0, 300) . "\n");
        exit(1);
    }
    fwrite(STDERR, 'Ujb-Credits: ' . ($headers['ujb-credits'] ?? '?') . "\n");
    return json_decode($body, true)['result'];
}

foreach ($urls as $i => $url) {
    if ($i > 0) {
        sleep(2); // be polite: one page at a time
    }
    echo json_encode(scrape($url, $apiKey, $settings, $extractRules), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), "\n";
}

// Scrape BBC News bbc.com news and articles: search and listing pages with the Ujeebu Scrape API.
// Settings and extract rules validated on real pages on 2026-10-28 (Ujeebu site guide).
// Uses the official SDK github.com/ujeebu/ujeebu-go v0.1.8: go mod init scraper && go get github.com/ujeebu/ujeebu-go@v0.1.8
// Run: UJEEBU_API_KEY=your_key go run bbc_listing.go

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"time"

	ujeebu "github.com/ujeebu/ujeebu-go"
)

var urls = []string{
	"https://www.bbc.com/news/technology",
	"https://www.bbc.com/business",
}

// The extract rules of the recipe, as the JSON the API takes.
const extractRulesJSON = `{
    "articles": {
        "selector": "div[data-testid=\"alaska\"] > div",
        "type": "obj",
        "multiple": true,
        "children": {
            "url": {
                "selector": "a[data-testid=\"internal-link\"]",
                "type": "link"
            },
            "title": {
                "selector": "h2",
                "type": "text"
            },
            "image": {
                "selector": "img.Image-styles__ImageStyled-sc-8c99a12b-0:not(.hide-when-no-script)",
                "type": "image"
            },
            "summary": {
                "selector": "p[data-testid='card-description']",
                "type": "text"
            },
            "published": {
                "selector": "span[data-testid='card-metadata-lastupdated']",
                "type": "text"
            }
        }
    }
}`

var extractRules map[string]any

func fail(format string, args ...any) {
	fmt.Fprintf(os.Stderr, format+"\n", args...)
	os.Exit(1)
}

func short(s string) string {
	if len(s) > 300 {
		return s[:300]
	}
	return s
}

func params(target string) ujeebu.ScrapeParams {
	p := ujeebu.ScrapeParams{URL: target}
	p.JS = true
	p.ExtractRules = extractRules
	return p
}

func scrape(client *ujeebu.Client, target string) map[string]any {
	// the int is the credits the request cost (the Ujb-Credits header), not an HTTP status
	resp, credits, err := client.Scrape(params(target))
	if err != nil {
		var apiErr *ujeebu.APIError
		if errors.As(err, &apiErr) {
			fail("HTTP %d for %s: %s", apiErr.StatusCode, target, short(apiErr.Message))
		}
		fail("Request failed for %s: %v", target, err)
	}
	fmt.Fprintf(os.Stderr, "Ujb-Credits: %d\n", credits)
	result, _ := resp.Result.(map[string]any)
	return result
}

func printJSON(v any) {
	enc := json.NewEncoder(os.Stdout)
	enc.SetEscapeHTML(false)
	enc.SetIndent("", "  ")
	if err := enc.Encode(v); err != nil {
		fail("Cannot print the result: %v", err)
	}
}

func main() {
	apiKey := os.Getenv("UJEEBU_API_KEY")
	if apiKey == "" {
		fail("Set the UJEEBU_API_KEY environment variable.")
	}
	if err := json.Unmarshal([]byte(extractRulesJSON), &extractRules); err != nil {
		fail("Bad extract rules: %v", err)
	}
	client, err := ujeebu.NewClient(apiKey, ujeebu.WithTimeout(180*time.Second))
	if err != nil {
		fail("%v", err)
	}

	for i, target := range urls {
		if i > 0 {
			time.Sleep(2 * time.Second) // be polite: one page at a time
		}
		printJSON(scrape(client, target))
	}
}

# Scrape BBC News bbc.com news and articles: articles with the Ujeebu Scrape API.
# Settings and extract rules validated on real pages on 2026-10-28 (Ujeebu site guide).
# Uses the official SDK ujeebu-python 0.1.5: pip install ujeebu-python==0.1.5
# Run: UJEEBU_API_KEY=your_key python3 bbc_detail_sdk.py

import json
import os
import sys
import time

import requests
from ujeebu_python import UjeebuClient

API_KEY = os.environ.get("UJEEBU_API_KEY")
URLS = [
    "https://www.bbc.com/news/articles/cmr4y2yl17x7o",
    "https://www.bbc.com/news/articles/cq74pkelv9rxo",
]
SETTINGS = {
    "js": True,
}
EXTRACT_RULES = {
    "title": {
        "selector": "h1",
        "type": "text",
    },
    "description": {
        "selector": "meta[property=\"og:description\"]",
        "type": "attr",
        "attribute": "content",
    },
    "image": {
        "selector": "meta[property=\"og:image\"]",
        "type": "attr",
        "attribute": "content",
    },
    "url": {
        "selector": "link[rel=\"canonical\"]",
        "type": "attr",
        "attribute": "href",
    },
    "published": {
        "selector": "time[datetime]",
        "type": "attr",
        "attribute": "datetime",
    },
    "summary": {
        "selector": "article p:first-of-type",
        "type": "text",
    },
    "section": {
        "selector": "a[data-testid='internal-link'][href*='/news/topics/']",
        "type": "text",
    },
}


def scrape(client, url):
    # with extract_rules, client.scrape sends a JSON POST and the API answers {"result": {...}}
    try:
        response = client.scrape(url, params={**SETTINGS, "extract_rules": EXTRACT_RULES})
    except requests.RequestException as error:
        sys.exit(f"Request failed for {url}: {error}")
    if response.status_code != 200:
        try:
            message = response.json().get("message", response.text)
        except ValueError:
            message = response.text
        sys.exit(f"HTTP {response.status_code} for {url}: {str(message)[:300]}")
    print(f"Ujb-Credits: {response.headers.get('Ujb-Credits')}", file=sys.stderr)
    return response.json()["result"]


if not API_KEY:
    sys.exit("Set the UJEEBU_API_KEY environment variable.")

client = UjeebuClient(api_key=API_KEY)

for i, url in enumerate(URLS):
    if i:
        time.sleep(2)  # be polite: one page at a time
    print(json.dumps(scrape(client, url), indent=2, ensure_ascii=False))

// Scrape BBC News bbc.com news and articles: articles with the Ujeebu Scrape API.
// Settings and extract rules validated on real pages on 2026-10-28 (Ujeebu site guide).
// Uses the official SDK @ujeebu-org/ujeebu-sdk 0.1.2: npm install @ujeebu-org/ujeebu-sdk@0.1.2
// Run: UJEEBU_API_KEY=your_key node bbc_detail_sdk.mjs

import { UjeebuClient } from '@ujeebu-org/ujeebu-sdk';

const API_KEY = process.env.UJEEBU_API_KEY;
const URLS = [
  "https://www.bbc.com/news/articles/cmr4y2yl17x7o",
  "https://www.bbc.com/news/articles/cq74pkelv9rxo",
];
const SETTINGS = {
  js: true,
};
const EXTRACT_RULES = {
  title: {
    selector: "h1",
    type: "text",
  },
  description: {
    selector: "meta[property=\"og:description\"]",
    type: "attr",
    attribute: "content",
  },
  image: {
    selector: "meta[property=\"og:image\"]",
    type: "attr",
    attribute: "content",
  },
  url: {
    selector: "link[rel=\"canonical\"]",
    type: "attr",
    attribute: "href",
  },
  published: {
    selector: "time[datetime]",
    type: "attr",
    attribute: "datetime",
  },
  summary: {
    selector: "article p:first-of-type",
    type: "text",
  },
  section: {
    selector: "a[data-testid='internal-link'][href*='/news/topics/']",
    type: "text",
  },
};

async function scrape(client, url) {
  let res;
  try {
    // client.scrape POSTs JSON when extract_rules is set (scrapeWithRules would send it as a GET query, which the API ignores)
    res = await client.scrape(url, { ...SETTINGS, extract_rules: EXTRACT_RULES });
  } catch (err) {
    const status = err.response?.status ?? 'error';
    const message = err.response?.data?.message ?? err.message;
    console.error(`HTTP ${status} for ${url}: ${String(message).slice(0, 300)}`);
    process.exit(1);
  }
  console.error(`Ujb-Credits: ${res.headers['ujb-credits']}`);
  return res.data.result;
}

if (!API_KEY) {
  console.error('Set the UJEEBU_API_KEY environment variable.');
  process.exit(1);
}

const client = new UjeebuClient(API_KEY);
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

for (const [i, url] of URLS.entries()) {
  if (i) await pause(2000); // be polite: one page at a time
  console.log(JSON.stringify(await scrape(client, url), null, 2));
}

# Scrape BBC News bbc.com news and articles: articles with the Ujeebu Scrape API.
# Settings and extract rules validated on real pages on 2026-10-28 (Ujeebu site guide).
# Run: UJEEBU_API_KEY=your_key python3 bbc_detail.py   (pip install requests)

import json
import os
import sys
import time

import requests

API_URL = "https://api.ujeebu.com/scrape"
API_KEY = os.environ.get("UJEEBU_API_KEY")
URLS = [
    "https://www.bbc.com/news/articles/cmr4y2yl17x7o",
    "https://www.bbc.com/news/articles/cq74pkelv9rxo",
]
SETTINGS = {
    "js": True,
}
EXTRACT_RULES = {
    "title": {
        "selector": "h1",
        "type": "text",
    },
    "description": {
        "selector": "meta[property=\"og:description\"]",
        "type": "attr",
        "attribute": "content",
    },
    "image": {
        "selector": "meta[property=\"og:image\"]",
        "type": "attr",
        "attribute": "content",
    },
    "url": {
        "selector": "link[rel=\"canonical\"]",
        "type": "attr",
        "attribute": "href",
    },
    "published": {
        "selector": "time[datetime]",
        "type": "attr",
        "attribute": "datetime",
    },
    "summary": {
        "selector": "article p:first-of-type",
        "type": "text",
    },
    "section": {
        "selector": "a[data-testid='internal-link'][href*='/news/topics/']",
        "type": "text",
    },
}


def scrape(url):
    response = requests.post(
        API_URL,
        headers={"ApiKey": API_KEY},
        json={"url": url, **SETTINGS, "extract_rules": EXTRACT_RULES},
        timeout=180,
    )
    if response.status_code != 200:
        try:
            message = response.json().get("message", response.text)
        except ValueError:
            message = response.text
        sys.exit(f"HTTP {response.status_code} for {url}: {str(message)[:300]}")
    print(f"Ujb-Credits: {response.headers.get('Ujb-Credits')}", file=sys.stderr)
    return response.json()["result"]


if not API_KEY:
    sys.exit("Set the UJEEBU_API_KEY environment variable.")

for i, url in enumerate(URLS):
    if i:
        time.sleep(2)  # be polite: one page at a time
    print(json.dumps(scrape(url), indent=2, ensure_ascii=False))

// Scrape BBC News bbc.com news and articles: articles with the Ujeebu Scrape API.
// Settings and extract rules validated on real pages on 2026-10-28 (Ujeebu site guide).
// Run: UJEEBU_API_KEY=your_key node bbc_detail.mjs   (Node.js 18 or later)

const API_URL = 'https://api.ujeebu.com/scrape';
const API_KEY = process.env.UJEEBU_API_KEY;
const URLS = [
  "https://www.bbc.com/news/articles/cmr4y2yl17x7o",
  "https://www.bbc.com/news/articles/cq74pkelv9rxo",
];
const SETTINGS = {
  js: true,
};
const EXTRACT_RULES = {
  title: {
    selector: "h1",
    type: "text",
  },
  description: {
    selector: "meta[property=\"og:description\"]",
    type: "attr",
    attribute: "content",
  },
  image: {
    selector: "meta[property=\"og:image\"]",
    type: "attr",
    attribute: "content",
  },
  url: {
    selector: "link[rel=\"canonical\"]",
    type: "attr",
    attribute: "href",
  },
  published: {
    selector: "time[datetime]",
    type: "attr",
    attribute: "datetime",
  },
  summary: {
    selector: "article p:first-of-type",
    type: "text",
  },
  section: {
    selector: "a[data-testid='internal-link'][href*='/news/topics/']",
    type: "text",
  },
};

async function scrape(url) {
  const res = await fetch(API_URL, {
    method: 'POST',
    headers: { ApiKey: API_KEY, 'Content-Type': 'application/json' },
    body: JSON.stringify({ url, ...SETTINGS, extract_rules: EXTRACT_RULES }),
    signal: AbortSignal.timeout(180000),
  });
  const text = await res.text();
  if (res.status !== 200) {
    let message = text;
    try {
      message = JSON.parse(text).message ?? text;
    } catch {}
    console.error(`HTTP ${res.status} for ${url}: ${String(message).slice(0, 300)}`);
    process.exit(1);
  }
  console.error(`Ujb-Credits: ${res.headers.get('ujb-credits')}`);
  return JSON.parse(text).result;
}

if (!API_KEY) {
  console.error('Set the UJEEBU_API_KEY environment variable.');
  process.exit(1);
}

const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

for (const [i, url] of URLS.entries()) {
  if (i) await pause(2000); // be polite: one page at a time
  console.log(JSON.stringify(await scrape(url), null, 2));
}

#!/usr/bin/env bash
# Scrape BBC News bbc.com news and articles: articles with the Ujeebu Scrape API.
# Settings and extract rules validated on real pages on 2026-10-28 (Ujeebu site guide).
# Run: UJEEBU_API_KEY=your_key bash bbc_detail.sh   (curl 7.76 or later)
set -euo pipefail

curl -sS --fail-with-body --max-time 180 -D headers.txt \
  -X POST 'https://api.ujeebu.com/scrape' \
  -H "ApiKey: ${UJEEBU_API_KEY:?Set the UJEEBU_API_KEY environment variable.}" \
  -H 'Content-Type: application/json' \
  --data-binary @- <<'JSON'
{
  "url": "https://www.bbc.com/news/articles/cmr4y2yl17x7o",
  "js": true,
  "extract_rules": {
    "title": {
      "selector": "h1",
      "type": "text"
    },
    "description": {
      "selector": "meta[property=\"og:description\"]",
      "type": "attr",
      "attribute": "content"
    },
    "image": {
      "selector": "meta[property=\"og:image\"]",
      "type": "attr",
      "attribute": "content"
    },
    "url": {
      "selector": "link[rel=\"canonical\"]",
      "type": "attr",
      "attribute": "href"
    },
    "published": {
      "selector": "time[datetime]",
      "type": "attr",
      "attribute": "datetime"
    },
    "summary": {
      "selector": "article p:first-of-type",
      "type": "text"
    },
    "section": {
      "selector": "a[data-testid='internal-link'][href*='/news/topics/']",
      "type": "text"
    }
  }
}
JSON
echo
grep -i '^ujb-credits' headers.txt >&2

<?php
// Scrape BBC News bbc.com news and articles: articles with the Ujeebu Scrape API.
// Settings and extract rules validated on real pages on 2026-10-28 (Ujeebu site guide).
// Run: UJEEBU_API_KEY=your_key php bbc_detail.php   (PHP 7.4 or later with curl)

$apiKey = getenv('UJEEBU_API_KEY') ?: exit("Set the UJEEBU_API_KEY environment variable.\n");
$urls = [
    'https://www.bbc.com/news/articles/cmr4y2yl17x7o',
    'https://www.bbc.com/news/articles/cq74pkelv9rxo',
];
$settings = [
    'js' => true,
];
$extractRules = [
    'title' => [
        'selector' => 'h1',
        'type' => 'text',
    ],
    'description' => [
        'selector' => 'meta[property="og:description"]',
        'type' => 'attr',
        'attribute' => 'content',
    ],
    'image' => [
        'selector' => 'meta[property="og:image"]',
        'type' => 'attr',
        'attribute' => 'content',
    ],
    'url' => [
        'selector' => 'link[rel="canonical"]',
        'type' => 'attr',
        'attribute' => 'href',
    ],
    'published' => [
        'selector' => 'time[datetime]',
        'type' => 'attr',
        'attribute' => 'datetime',
    ],
    'summary' => [
        'selector' => 'article p:first-of-type',
        'type' => 'text',
    ],
    'section' => [
        'selector' => 'a[data-testid=\'internal-link\'][href*=\'/news/topics/\']',
        'type' => 'text',
    ],
];

function scrape(string $url, string $apiKey, array $settings, array $extractRules)
{
    $ch = curl_init('https://api.ujeebu.com/scrape');
    $headers = [];
    curl_setopt_array($ch, [
        CURLOPT_POST => true,
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_TIMEOUT => 180,
        CURLOPT_HTTPHEADER => ['ApiKey: ' . $apiKey, 'Content-Type: application/json'],
        CURLOPT_POSTFIELDS => json_encode(['url' => $url] + $settings + ['extract_rules' => $extractRules]),
        CURLOPT_HEADERFUNCTION => function ($ch, $line) use (&$headers) {
            if (strpos($line, ':') !== false) {
                [$k, $v] = explode(':', $line, 2);
                $headers[strtolower(trim($k))] = trim($v);
            }
            return strlen($line);
        },
    ]);
    $body = curl_exec($ch);
    if ($body === false) {
        fwrite(STDERR, 'Request failed: ' . curl_error($ch) . "\n");
        exit(1);
    }
    $status = curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
    if ($status !== 200) {
        $message = json_decode($body, true)['message'] ?? $body;
        fwrite(STDERR, "HTTP $status for $url: " . substr((string) $message, 0, 300) . "\n");
        exit(1);
    }
    fwrite(STDERR, 'Ujb-Credits: ' . ($headers['ujb-credits'] ?? '?') . "\n");
    return json_decode($body, true)['result'];
}

foreach ($urls as $i => $url) {
    if ($i > 0) {
        sleep(2); // be polite: one page at a time
    }
    echo json_encode(scrape($url, $apiKey, $settings, $extractRules), JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE), "\n";
}

// Scrape BBC News bbc.com news and articles: articles with the Ujeebu Scrape API.
// Settings and extract rules validated on real pages on 2026-10-28 (Ujeebu site guide).
// Uses the official SDK github.com/ujeebu/ujeebu-go v0.1.8: go mod init scraper && go get github.com/ujeebu/ujeebu-go@v0.1.8
// Run: UJEEBU_API_KEY=your_key go run bbc_detail.go

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"time"

	ujeebu "github.com/ujeebu/ujeebu-go"
)

var urls = []string{
	"https://www.bbc.com/news/articles/cmr4y2yl17x7o",
	"https://www.bbc.com/news/articles/cq74pkelv9rxo",
}

// The extract rules of the recipe, as the JSON the API takes.
const extractRulesJSON = `{
    "title": {
        "selector": "h1",
        "type": "text"
    },
    "description": {
        "selector": "meta[property=\"og:description\"]",
        "type": "attr",
        "attribute": "content"
    },
    "image": {
        "selector": "meta[property=\"og:image\"]",
        "type": "attr",
        "attribute": "content"
    },
    "url": {
        "selector": "link[rel=\"canonical\"]",
        "type": "attr",
        "attribute": "href"
    },
    "published": {
        "selector": "time[datetime]",
        "type": "attr",
        "attribute": "datetime"
    },
    "summary": {
        "selector": "article p:first-of-type",
        "type": "text"
    },
    "section": {
        "selector": "a[data-testid='internal-link'][href*='/news/topics/']",
        "type": "text"
    }
}`

var extractRules map[string]any

func fail(format string, args ...any) {
	fmt.Fprintf(os.Stderr, format+"\n", args...)
	os.Exit(1)
}

func short(s string) string {
	if len(s) > 300 {
		return s[:300]
	}
	return s
}

func params(target string) ujeebu.ScrapeParams {
	p := ujeebu.ScrapeParams{URL: target}
	p.JS = true
	p.ExtractRules = extractRules
	return p
}

func scrape(client *ujeebu.Client, target string) map[string]any {
	// the int is the credits the request cost (the Ujb-Credits header), not an HTTP status
	resp, credits, err := client.Scrape(params(target))
	if err != nil {
		var apiErr *ujeebu.APIError
		if errors.As(err, &apiErr) {
			fail("HTTP %d for %s: %s", apiErr.StatusCode, target, short(apiErr.Message))
		}
		fail("Request failed for %s: %v", target, err)
	}
	fmt.Fprintf(os.Stderr, "Ujb-Credits: %d\n", credits)
	result, _ := resp.Result.(map[string]any)
	return result
}

func printJSON(v any) {
	enc := json.NewEncoder(os.Stdout)
	enc.SetEscapeHTML(false)
	enc.SetIndent("", "  ")
	if err := enc.Encode(v); err != nil {
		fail("Cannot print the result: %v", err)
	}
}

func main() {
	apiKey := os.Getenv("UJEEBU_API_KEY")
	if apiKey == "" {
		fail("Set the UJEEBU_API_KEY environment variable.")
	}
	if err := json.Unmarshal([]byte(extractRulesJSON), &extractRules); err != nil {
		fail("Bad extract rules: %v", err)
	}
	client, err := ujeebu.NewClient(apiKey, ujeebu.WithTimeout(180*time.Second))
	if err != nil {
		fail("%v", err)
	}

	for i, target := range urls {
		if i > 0 {
			time.Sleep(2 * time.Second) // be polite: one page at a time
		}
		printJSON(scrape(client, target))
	}
}

# Fetch a BBC News page with the Ujeebu Scrape API.
# Settings from our health checks (Ujeebu site guide). Prints the start of the page HTML.
# Uses the official SDK ujeebu-python 0.1.5: pip install ujeebu-python==0.1.5
# Run: UJEEBU_API_KEY=your_key python3 bbc_page_sdk.py

import json
import os
import sys
import time

import requests
from ujeebu_python import UjeebuClient

API_KEY = os.environ.get("UJEEBU_API_KEY")
URLS = [
    "https://www.bbc.co.uk/news",
]
SETTINGS = {
    "js": True,
}


def scrape(client, url):
    try:
        response = client.scrape(url, params=SETTINGS)
    except requests.RequestException as error:
        sys.exit(f"Request failed for {url}: {error}")
    if response.status_code != 200:
        try:
            message = response.json().get("message", response.text)
        except ValueError:
            message = response.text
        sys.exit(f"HTTP {response.status_code} for {url}: {str(message)[:300]}")
    print(f"Ujb-Credits: {response.headers.get('Ujb-Credits')}", file=sys.stderr)
    return response.text


if not API_KEY:
    sys.exit("Set the UJEEBU_API_KEY environment variable.")

client = UjeebuClient(api_key=API_KEY)

for i, url in enumerate(URLS):
    if i:
        time.sleep(2)  # be polite: one page at a time
    html = scrape(client, url)
    print(len(html), "bytes")
    print(html[:500])

// Fetch a BBC News page with the Ujeebu Scrape API.
// Settings from our health checks (Ujeebu site guide). Prints the start of the page HTML.
// Uses the official SDK @ujeebu-org/ujeebu-sdk 0.1.2: npm install @ujeebu-org/ujeebu-sdk@0.1.2
// Run: UJEEBU_API_KEY=your_key node bbc_page_sdk.mjs

import { UjeebuClient } from '@ujeebu-org/ujeebu-sdk';

const API_KEY = process.env.UJEEBU_API_KEY;
const URLS = [
  "https://www.bbc.co.uk/news",
];
const SETTINGS = {
  js: true,
};

async function scrape(client, url) {
  let res;
  try {
    res = await client.scrape(url, SETTINGS);
  } catch (err) {
    const status = err.response?.status ?? 'error';
    const message = err.response?.data?.message ?? err.message;
    console.error(`HTTP ${status} for ${url}: ${String(message).slice(0, 300)}`);
    process.exit(1);
  }
  console.error(`Ujb-Credits: ${res.headers['ujb-credits']}`);
  return String(res.data);
}

if (!API_KEY) {
  console.error('Set the UJEEBU_API_KEY environment variable.');
  process.exit(1);
}

const client = new UjeebuClient(API_KEY);
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

for (const [i, url] of URLS.entries()) {
  if (i) await pause(2000); // be polite: one page at a time
  const html = await scrape(client, url);
  console.log(html.length, 'bytes');
  console.log(html.slice(0, 500));
}

# Fetch a BBC News page with the Ujeebu Scrape API.
# Settings from our health checks (Ujeebu site guide). Prints the start of the page HTML.
# Run: UJEEBU_API_KEY=your_key python3 bbc_page.py   (pip install requests)

import json
import os
import sys
import time

import requests

API_URL = "https://api.ujeebu.com/scrape"
API_KEY = os.environ.get("UJEEBU_API_KEY")
URLS = [
    "https://www.bbc.co.uk/news",
]
SETTINGS = {
    "js": True,
}


def scrape(url):
    response = requests.post(
        API_URL,
        headers={"ApiKey": API_KEY},
        json={"url": url, **SETTINGS},
        timeout=180,
    )
    if response.status_code != 200:
        try:
            message = response.json().get("message", response.text)
        except ValueError:
            message = response.text
        sys.exit(f"HTTP {response.status_code} for {url}: {str(message)[:300]}")
    print(f"Ujb-Credits: {response.headers.get('Ujb-Credits')}", file=sys.stderr)
    return response.text


if not API_KEY:
    sys.exit("Set the UJEEBU_API_KEY environment variable.")

for i, url in enumerate(URLS):
    if i:
        time.sleep(2)  # be polite: one page at a time
    html = scrape(url)
    print(len(html), "bytes")
    print(html[:500])

// Fetch a BBC News page with the Ujeebu Scrape API.
// Settings from our health checks (Ujeebu site guide). Prints the start of the page HTML.
// Run: UJEEBU_API_KEY=your_key node bbc_page.mjs   (Node.js 18 or later)

const API_URL = 'https://api.ujeebu.com/scrape';
const API_KEY = process.env.UJEEBU_API_KEY;
const URLS = [
  "https://www.bbc.co.uk/news",
];
const SETTINGS = {
  js: true,
};

async function scrape(url) {
  const res = await fetch(API_URL, {
    method: 'POST',
    headers: { ApiKey: API_KEY, 'Content-Type': 'application/json' },
    body: JSON.stringify({ url, ...SETTINGS }),
    signal: AbortSignal.timeout(180000),
  });
  const text = await res.text();
  if (res.status !== 200) {
    let message = text;
    try {
      message = JSON.parse(text).message ?? text;
    } catch {}
    console.error(`HTTP ${res.status} for ${url}: ${String(message).slice(0, 300)}`);
    process.exit(1);
  }
  console.error(`Ujb-Credits: ${res.headers.get('ujb-credits')}`);
  return text;
}

if (!API_KEY) {
  console.error('Set the UJEEBU_API_KEY environment variable.');
  process.exit(1);
}

const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

for (const [i, url] of URLS.entries()) {
  if (i) await pause(2000); // be polite: one page at a time
  const html = await scrape(url);
  console.log(html.length, 'bytes');
  console.log(html.slice(0, 500));
}

#!/usr/bin/env bash
# Fetch a BBC News page with the Ujeebu Scrape API.
# Settings from our health checks (Ujeebu site guide). Prints the start of the page HTML.
# Run: UJEEBU_API_KEY=your_key bash bbc_page.sh   (curl 7.76 or later)
set -euo pipefail

curl -sS --fail-with-body --max-time 180 -D headers.txt \
  -X POST 'https://api.ujeebu.com/scrape' \
  -H "ApiKey: ${UJEEBU_API_KEY:?Set the UJEEBU_API_KEY environment variable.}" \
  -H 'Content-Type: application/json' \
  -o page.html --data-binary @- <<'JSON'
{
  "url": "https://www.bbc.co.uk/news",
  "js": true
}
JSON
head -c 500 page.html
echo
grep -i '^ujb-credits' headers.txt >&2

<?php
// Fetch a BBC News page with the Ujeebu Scrape API.
// Settings from our health checks (Ujeebu site guide). Prints the start of the page HTML.
// Run: UJEEBU_API_KEY=your_key php bbc_page.php   (PHP 7.4 or later with curl)

$apiKey = getenv('UJEEBU_API_KEY') ?: exit("Set the UJEEBU_API_KEY environment variable.\n");
$urls = [
    'https://www.bbc.co.uk/news',
];
$settings = [
    'js' => true,
];

function scrape(string $url, string $apiKey, array $settings)
{
    $ch = curl_init('https://api.ujeebu.com/scrape');
    $headers = [];
    curl_setopt_array($ch, [
        CURLOPT_POST => true,
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_TIMEOUT => 180,
        CURLOPT_HTTPHEADER => ['ApiKey: ' . $apiKey, 'Content-Type: application/json'],
        CURLOPT_POSTFIELDS => json_encode(['url' => $url] + $settings),
        CURLOPT_HEADERFUNCTION => function ($ch, $line) use (&$headers) {
            if (strpos($line, ':') !== false) {
                [$k, $v] = explode(':', $line, 2);
                $headers[strtolower(trim($k))] = trim($v);
            }
            return strlen($line);
        },
    ]);
    $body = curl_exec($ch);
    if ($body === false) {
        fwrite(STDERR, 'Request failed: ' . curl_error($ch) . "\n");
        exit(1);
    }
    $status = curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
    if ($status !== 200) {
        $message = json_decode($body, true)['message'] ?? $body;
        fwrite(STDERR, "HTTP $status for $url: " . substr((string) $message, 0, 300) . "\n");
        exit(1);
    }
    fwrite(STDERR, 'Ujb-Credits: ' . ($headers['ujb-credits'] ?? '?') . "\n");
    return $body;
}

foreach ($urls as $i => $url) {
    if ($i > 0) {
        sleep(2); // be polite: one page at a time
    }
    $html = scrape($url, $apiKey, $settings);
    echo strlen($html), " bytes\n", substr($html, 0, 500), "\n";
}

// Fetch a BBC News page with the Ujeebu Scrape API.
// Settings from our health checks (Ujeebu site guide). Prints the start of the page HTML.
// Uses the official SDK github.com/ujeebu/ujeebu-go v0.1.8: go mod init scraper && go get github.com/ujeebu/ujeebu-go@v0.1.8
// Run: UJEEBU_API_KEY=your_key go run bbc_page.go

package main

import (
	"errors"
	"fmt"
	"os"
	"time"

	ujeebu "github.com/ujeebu/ujeebu-go"
)

var urls = []string{
	"https://www.bbc.co.uk/news",
}

func fail(format string, args ...any) {
	fmt.Fprintf(os.Stderr, format+"\n", args...)
	os.Exit(1)
}

func short(s string) string {
	if len(s) > 300 {
		return s[:300]
	}
	return s
}

func params(target string) ujeebu.ScrapeParams {
	p := ujeebu.ScrapeParams{URL: target}
	p.JS = true
	return p
}

func scrape(client *ujeebu.Client, target string) string {
	// the int is the credits the request cost (the Ujb-Credits header), not an HTTP status
	resp, credits, err := client.Scrape(params(target))
	if err != nil {
		var apiErr *ujeebu.APIError
		if errors.As(err, &apiErr) {
			fail("HTTP %d for %s: %s", apiErr.StatusCode, target, short(apiErr.Message))
		}
		fail("Request failed for %s: %v", target, err)
	}
	fmt.Fprintf(os.Stderr, "Ujb-Credits: %d\n", credits)
	return resp.HTML
}

func main() {
	apiKey := os.Getenv("UJEEBU_API_KEY")
	if apiKey == "" {
		fail("Set the UJEEBU_API_KEY environment variable.")
	}
	client, err := ujeebu.NewClient(apiKey, ujeebu.WithTimeout(180*time.Second))
	if err != nil {
		fail("%v", err)
	}

	for i, target := range urls {
		if i > 0 {
			time.Sleep(2 * time.Second) // be polite: one page at a time
		}
		html := scrape(client, target)
		fmt.Println(len(html), "bytes")
		fmt.Println(html[:min(len(html), 500)])
	}
}