Paging through history

import os
from datetime import datetime, timedelta

import requests

BASE_URL = "https://api.sandmark.com/volatility"
session = requests.Session()
session.headers["X-API-Key"] = os.environ["SANDMARK_API_KEY"]


def fetch_all(path: str, **params) -> list[dict]:
    """Every row in the history window, newest first."""
    rows, end = [], None
    while True:
        page_params = {**params, "limit": 5000}
        if end:
            page_params["end"] = end
        response = session.get(f"{BASE_URL}{path}", params=page_params, timeout=60)
        response.raise_for_status()
        page = response.json()["data"]
        rows.extend(page)
        if len(page) < 5000:
            return rows
        oldest = datetime.fromisoformat(page[-1]["DATETIME"].replace("Z", "+00:00"))
        end = (oldest - timedelta(seconds=1)).isoformat()


rows = fetch_all(
    "/v1/timeseries/rv", asset="DER.BTC_USD", frequency="1h", lookback_days=30
)
print(len(rows), "rows from", rows[-1]["DATETIME"], "to", rows[0]["DATETIME"])

const BASE_URL = "https://api.sandmark.com/volatility";

async function fetchAll(path, params) {
  const rows = [];
  let end;
  for (;;) {
    const query = new URLSearchParams({ ...params, limit: "5000", ...(end && { end }) });
    const response = await fetch(`${BASE_URL}${path}?${query}`, {
      headers: { "X-API-Key": process.env.SANDMARK_API_KEY },
    });
    if (!response.ok) throw new Error(`${response.status} ${await response.text()}`);
    const { data } = await response.json();
    rows.push(...data);
    if (data.length < 5000) return rows;
    end = new Date(Date.parse(data.at(-1).DATETIME) - 1000).toISOString();
  }
}

const rows = await fetchAll("/v1/timeseries/rv", {
  asset: "DER.BTC_USD",
  frequency: "1h",
  lookback_days: "30",
});
console.log(rows.length, "rows");