Proxies for e-commerce data collection

import { ProxyAgent, fetch } from "undici";

const USER = "fp_8s2k4d19";
const PASSWORD = "Xk7mQ2pTz9vRn4Ls";

function proxyUrl({ pool, country }) {
  // "pool-dc" selects rotating datacenter; omitting it uses residential.
  const segments = [USER, "country", country];
  if (pool === "datacenter") segments.push("pool", "dc");
  return `http://${segments.join("-")}:${PASSWORD}@gate.fleetproxy.com:8080`;
}

// Cheapest pool that reliably serves each endpoint class. Re-measure quarterly:
// defences move in both directions and this table decays.
const ROUTES = [
  { match: /\/sitemap.*\.xml$/, pool: "datacenter" },
  { match: /\/api\/catalog\//, pool: "datacenter" },
  { match: /\/s\?k=/, pool: "residential" },
  { match: /\/dp\//, pool: "residential" },
];

const agents = new Map();

function agentFor(url, country) {
  const pool = ROUTES.find((route) => route.match.test(url))?.pool ?? "residential";
  const key = `${pool}:${country}`;
  if (!agents.has(key)) {
    agents.set(key, new ProxyAgent(proxyUrl({ pool, country })));
  }
  return { pool, agent: agents.get(key) };
}

export async function collect(url, country = "us") {
  const { pool, agent } = agentFor(url, country);
  const response = await fetch(url, {
    dispatcher: agent,
    headers: { "accept-language": "en-US,en;q=0.9" },
  });

  // Record which pool served the request. Without it you cannot tell whether a
  // rising block rate is the target changing or the routing table decaying.
  return { status: response.status, pool, body: await response.text() };
}