Substrate/get-de-digital

#!/usr/bin/env bun
// Fetch Eurostat digital society indicators for Germany
// Endpoint: ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/isoc_ci_ac_i
// No auth required. Dataset: PC_IND (% of all individuals), IND_TOTAL, geo=DE

import { writeFileSync, mkdirSync } from "fs";
import { join } from "path";

const OUT_DIR = join(__dirname, "Data/DE-Platform-Media");

const BASE = "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data";

const INDICATORS: Record<string, string> = {
  I_IUSNET: "Social networks participation",
  I_IUNW1: "Online news reading",
  I_IUPH1: "Video/voice calls",
  I_IUBK: "Internet banking",
  I_IUOLC: "Online courses",
  I_IUUPL1: "Content upload/sharing",
  I_IUWIKI: "Wiki consultation",
  I_IUEM: "Email use",
  I_IUIF: "Finding information online",
};

function csvEscape(value: string | number | undefined | null): string {
  if (value === null || value === undefined) return "";
  const s = String(value);
  if (s.includes(",") || s.includes('"') || s.includes("\n"))
    return `"${s.replace(/"/g, '""')}"`;
  return s;
}

async function fetchEurostat(indicators: string[]): Promise<Record<string, Record<string, number>>> {
  const indic_params = indicators.map((i) => `indic_is=${i}`).join("&");
  const url = `${BASE}/isoc_ci_ac_i?format=JSON&lang=en&geo=DE&unit=PC_IND&ind_type=IND_TOTAL&${indic_params}`;

  const res = await fetch(url);
  if (!res.ok) throw new Error(`Eurostat HTTP ${res.status}`);
  const data = await res.json() as any;

  const dims = data.dimension ?? {};
  const vals = data.value ?? {};
  const idx: string[] = data.id ?? [];
  const sizes: number[] = data.size ?? [];

  const indicList = Object.keys(dims.indic_is?.category?.label ?? {});
  const timeList = Object.keys(dims.time?.category?.label ?? {});

  const nTime = sizes[idx.indexOf("time")] ?? 1;
  const result: Record<string, Record<string, number>> = {};

  for (const [k, v] of Object.entries(vals)) {
    const ki = parseInt(k);
    const iIndic = Math.floor(ki / nTime);
    const iTime = ki % nTime;
    const indic = indicList[iIndic];
    const time = timeList[iTime];
    if (!indic || !time) continue;
    result[indic] ??= {};
    result[indic][time] = v as number;
  }

  return result;
}

async function main() {
  console.log("Fetching Eurostat digital society data for Germany...");
  mkdirSync(OUT_DIR, { recursive: true });

  const indicatorKeys = Object.keys(INDICATORS);
  const data = await fetchEurostat(indicatorKeys);

  // Determine recent years (last 5)
  const allYears = new Set<string>();
  for (const indic of Object.values(data)) {
    for (const year of Object.keys(indic)) allYears.add(year);
  }
  const recentYears = [...allYears].sort().slice(-5);

  // CSV: indicator, label, year, value_pct
  const rows: string[] = ["indicator_id,indicator_label,year,value_pct"];
  for (const [indicId, yearMap] of Object.entries(data)) {
    const label = INDICATORS[indicId] ?? indicId;
    for (const year of recentYears) {
      const val = yearMap[year];
      if (val !== undefined) {
        rows.push(
          [csvEscape(indicId), csvEscape(label), csvEscape(year), val.toFixed(2)].join(",")
        );
      }
    }
  }

  const csvPath = join(OUT_DIR, "eurostat-digital-de.csv");
  writeFileSync(csvPath, rows.join("\n") + "\n");
  console.log(`✓ Written ${rows.length - 1} rows to ${csvPath}`);

  // Print summary
  console.log("\n=== Latest values (most recent year) ===");
  for (const [indicId, yearMap] of Object.entries(data)) {
    const years = Object.keys(yearMap).sort();
    const latest = years[years.length - 1];
    if (latest) {
      console.log(`  ${INDICATORS[indicId] ?? indicId}: ${yearMap[latest]?.toFixed(1)}% (${latest})`);
    }
  }
}

main().catch((err) => {
  console.error("Error:", err.message);
  process.exit(1);
});