Scheduled sync

import csv
import os
import time

import requests

API = "https://api.getprice.ai/v1"
# Retry these. Never retry 500: it won't fix itself.
RETRYABLE = {429, 502, 503, 504}
KEY = os.environ["GETPRICE_API_KEY"]
# Your SKUs, straight from your ERP.
SKUS = ["TV-55UHD", "KTL-17-SS", "TEA-RB80-C12"]

session = requests.Session()
session.headers["Authorization"] = f"Bearer {KEY}"


def get_with_retry(url, params, attempts=6):
    """GET, waiting out 429/503 and network errors."""
    for attempt in range(attempts):
        backoff = min(2**attempt, 60)
        try:
            resp = session.get(
                url, params=params, timeout=60
            )
        except requests.RequestException:
            time.sleep(backoff)
            continue
        if resp.status_code in RETRYABLE:
            wait = resp.headers.get("Retry-After")
            time.sleep(int(wait or backoff))
            continue
        if resp.status_code >= 400:
            p = resp.json()
            raise RuntimeError(
                f"{p['code']}: {p['detail']}"
                f" ({p['request_id']})"
            )
        return resp.json()
    raise RuntimeError(f"Gave up on {url}")


def sets_for(skus):
    """Sets for these SKUs, and each SKU's status."""
    url = f"{API}/comparison-sets"
    params = [("sku", s) for s in skus]
    params.append(("limit", "500"))
    page = get_with_retry(url, params)
    statuses = page["skus"]
    sets = page["data"]
    while page["next_cursor"]:
        cursor = [("cursor", page["next_cursor"])]
        page = get_with_retry(url, params + cursor)
        sets += page["data"]
    return sets, statuses


COLUMNS = [
    "sku", "set_id", "competitor", "amount",
    "currency", "last_collected_at", "stale", "review",
]
sets = rows = missing = 0
with open("prices.csv", "w", newline="") as f:
    writer = csv.writer(f)
    writer.writerow(COLUMNS)
    for start in range(0, len(SKUS), 100):
        batch = SKUS[start : start + 100]
        found, statuses = sets_for(batch)
        missing += sum(
            s["status"] != "found" for s in statuses
        )
        for cset in found:
            sets += 1
            for e in cset["competitors"]:
                price = e["price"] or {}
                writer.writerow([
                    cset["sku"],
                    cset["id"],
                    e["competitor"]["name"],
                    price.get("amount"),
                    price.get("currency"),
                    e["last_collected_at"],
                    e["stale"],
                    e["match"]["review"],
                ])
                rows += 1
summary = f"Sets: {sets}. Rows: {rows}. "
print(summary + f"Not found: {missing}.")

import { writeFileSync } from "node:fs";

interface Entry {
  competitor: { name: string };
  price: { amount: string; currency: string } | null;
  last_collected_at: string | null;
  stale: boolean;
  match: { review: string };
}
interface CompSet {
  id: string;
  sku: string | null;
  competitors: Entry[];
}
interface Page {
  data: CompSet[];
  next_cursor: string | null;
  skus?: { status: string }[];
}

const BASE = "https://api.getprice.ai/v1";
const KEY = process.env.GETPRICE_API_KEY;
// Retry these. Never retry 500: it won't fix itself.
const RETRYABLE = new Set([429, 502, 503, 504]);
// Your SKUs, straight from your ERP.
const SKUS = ["TV-55UHD", "KTL-17-SS", "TEA-RB80-C12"];
const sleep = (s: number) =>
  new Promise((r) => setTimeout(r, s * 1000));

async function getWithRetry(
  url: string,
): Promise<Page> {
  for (let attempt = 0; attempt < 6; attempt++) {
    const backoff = Math.min(2 ** attempt, 60);
    const res = await fetch(url, {
      headers: { Authorization: `Bearer ${KEY}` },
    }).catch(() => null);
    if (!res) {
      await sleep(backoff);
      continue;
    }
    if (RETRYABLE.has(res.status)) {
      const wait = res.headers.get("Retry-After");
      await sleep(Number(wait ?? backoff));
      continue;
    }
    if (!res.ok) {
      const p = await res.json();
      throw new Error(`${p.code}: ${p.detail}`);
    }
    return (await res.json()) as Page;
  }
  throw new Error(`Gave up on ${url}`);
}

async function setsFor(skus: string[]) {
  const params = new URLSearchParams();
  for (const sku of skus) params.append("sku", sku);
  params.set("limit", "500");
  const url = `${BASE}/comparison-sets?${params}`;
  let page = await getWithRetry(url);
  const statuses = page.skus ?? [];
  const sets = page.data;
  while (page.next_cursor) {
    params.set("cursor", page.next_cursor);
    page = await getWithRetry(
      `${BASE}/comparison-sets?${params}`,
    );
    sets.push(...page.data);
  }
  return { sets, statuses };
}

const lines = [
  "sku,set_id,competitor,amount,currency," +
    "last_collected_at,stale,review",
];
let sets = 0;
let missing = 0;
for (let i = 0; i < SKUS.length; i += 100) {
  const result = await setsFor(SKUS.slice(i, i + 100));
  missing += result.statuses.filter(
    (s) => s.status !== "found",
  ).length;
  for (const set of result.sets) {
    sets += 1;
    for (const e of set.competitors) {
      lines.push(
        [
          set.sku ?? "",
          set.id,
          e.competitor.name,
          e.price?.amount ?? "",
          e.price?.currency ?? "",
          e.last_collected_at ?? "",
          e.stale,
          e.match.review,
        ].join(","),
      );
    }
  }
}
writeFileSync("prices.csv", lines.join("\n") + "\n");
const rows = lines.length - 1;
console.log(
  `Sets: ${sets}. Rows: ${rows}. ` +
    `Not found: ${missing}.`,
);

<?php
$base = "https://api.getprice.ai/v1";
$key = getenv("GETPRICE_API_KEY");
// Retry these. Never retry 500: it won't fix itself.
$retryable = [429, 502, 503, 504];
// Your SKUs, straight from your ERP.
$skus = ["TV-55UHD", "KTL-17-SS", "TEA-RB80-C12"];

function getWithRetry(string $url): array
{
    global $key, $retryable;
    for ($attempt = 0; $attempt < 6; $attempt++) {
        $backoff = min(2 ** $attempt, 60);
        $ch = curl_init($url);
        curl_setopt_array($ch, [
            CURLOPT_HTTPHEADER => [
                "Authorization: Bearer $key",
            ],
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_HEADER => true,
            CURLOPT_TIMEOUT => 60,
        ]);
        $raw = curl_exec($ch);
        if ($raw === false) {
            sleep($backoff);
            continue;
        }
        $status = curl_getinfo(
            $ch, CURLINFO_RESPONSE_CODE
        );
        $size = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
        $headers = substr($raw, 0, $size);
        $body = substr($raw, $size);
        if (in_array($status, $retryable, true)) {
            $wait = $backoff;
            $re = "/^Retry-After: (\d+)/mi";
            if (preg_match($re, $headers, $m)) {
                $wait = (int) $m[1];
            }
            sleep($wait);
            continue;
        }
        if ($status >= 400) {
            throw new RuntimeException(
                "GetPrice returned $status: $body"
            );
        }
        return json_decode(
            $body, true, flags: JSON_THROW_ON_ERROR
        );
    }
    throw new RuntimeException("Gave up on $url");
}

function setsFor(array $skus): array
{
    global $base;
    $query = "limit=500";
    foreach ($skus as $sku) {
        $query .= "&sku=" . rawurlencode($sku);
    }
    $url = "$base/comparison-sets?$query";
    $page = getWithRetry($url);
    $statuses = $page["skus"] ?? [];
    $sets = $page["data"];
    while ($page["next_cursor"] !== null) {
        $cursor = rawurlencode($page["next_cursor"]);
        $page = getWithRetry("$url&cursor=$cursor");
        $sets = array_merge($sets, $page["data"]);
    }
    return [$sets, $statuses];
}

$out = fopen("prices.csv", "w");
fputcsv($out, [
    "sku", "set_id", "competitor", "amount",
    "currency", "last_collected_at", "stale", "review",
]);
$sets = $rows = $missing = 0;
foreach (array_chunk($skus, 100) as $batch) {
    [$found, $statuses] = setsFor($batch);
    foreach ($statuses as $s) {
        if ($s["status"] !== "found") {
            $missing++;
        }
    }
    foreach ($found as $set) {
        $sets++;
        foreach ($set["competitors"] as $e) {
            fputcsv($out, [
                $set["sku"],
                $set["id"],
                $e["competitor"]["name"],
                $e["price"]["amount"] ?? null,
                $e["price"]["currency"] ?? null,
                $e["last_collected_at"],
                $e["stale"] ? "true" : "false",
                $e["match"]["review"],
            ]);
            $rows++;
        }
    }
}
fclose($out);
echo "Sets: $sets. Rows: $rows. Not found: $missing.\n";

using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Text.Json;
using Naming = System.Text.Json.JsonNamingPolicy;

var key = Environment.GetEnvironmentVariable(
    "GETPRICE_API_KEY");
var api = new Uri("https://api.getprice.ai/v1/");
var http = new HttpClient { BaseAddress = api };
http.Timeout = TimeSpan.FromSeconds(60);
http.DefaultRequestHeaders.Authorization =
    new AuthenticationHeaderValue("Bearer", key);
var json = new JsonSerializerOptions
{
    PropertyNamingPolicy = Naming.SnakeCaseLower,
};
// Retry these. Never retry 500: it won't fix itself.
int[] retryable = { 429, 502, 503, 504 };
// Your SKUs, straight from your ERP.
string[] skus =
{
    "TV-55UHD", "KTL-17-SS", "TEA-RB80-C12",
};

async Task<Page> GetWithRetry(string url)
{
    for (var attempt = 0; attempt < 6; attempt++)
    {
        var backoff = Math.Min(1 << attempt, 60);
        HttpResponseMessage res;
        try
        {
            res = await http.GetAsync(url);
        }
        catch (HttpRequestException)
        {
            await Task.Delay(backoff * 1000);
            continue;
        }
        if (retryable.Contains((int)res.StatusCode))
        {
            var wait = res.Headers.RetryAfter?.Delta
                ?? TimeSpan.FromSeconds(backoff);
            await Task.Delay(wait);
            continue;
        }
        res.EnsureSuccessStatusCode();
        var page = await res.Content
            .ReadFromJsonAsync<Page>(json);
        return page!;
    }
    throw new Exception($"Gave up on {url}");
}

async Task<(List<CompSet>, List<SkuStatus>)> SetsFor(
    IEnumerable<string> batch)
{
    var query = string.Join("&", batch.Select(
        s => "sku=" + Uri.EscapeDataString(s)));
    var url = $"comparison-sets?{query}&limit=500";
    var page = await GetWithRetry(url);
    var statuses = page.Skus ?? new List<SkuStatus>();
    var sets = page.Data;
    while (page.NextCursor is { } cursor)
    {
        var next = Uri.EscapeDataString(cursor);
        var more = $"{url}&cursor={next}";
        page = await GetWithRetry(more);
        sets.AddRange(page.Data);
    }
    return (sets, statuses);
}

var lines = new List<string>
{
    "sku,set_id,competitor,amount,currency,"
    + "last_collected_at,stale,review",
};
var sets = 0;
var missing = 0;
foreach (var batch in skus.Chunk(100))
{
    var (found, statuses) = await SetsFor(batch);
    missing += statuses
        .Count(s => s.Status != "found");
    foreach (var set in found)
    {
        sets++;
        foreach (var e in set.Competitors)
        {
            lines.Add(string.Join(",",
                set.Sku ?? "",
                set.Id,
                e.Competitor.Name,
                e.Price?.Amount ?? "",
                e.Price?.Currency ?? "",
                e.LastCollectedAt ?? "",
                e.Stale ? "true" : "false",
                e.Match.Review));
        }
    }
}
File.WriteAllLines("prices.csv", lines);
var rows = lines.Count - 1;
Console.WriteLine(
    $"Sets: {sets}. Rows: {rows}. "
    + $"Not found: {missing}.");

record Page(
    List<CompSet> Data,
    string? NextCursor,
    List<SkuStatus>? Skus);
record SkuStatus(string Status);
record CompSet(
    string Id,
    string? Sku,
    List<Entry> Competitors);
record Entry(
    Competitor Competitor,
    Price? Price,
    string? LastCollectedAt,
    bool Stale,
    Match Match);
record Competitor(string Name);
record Price(string Amount, string Currency);
record Match(string Review);