Substrate/get-de-energy

#!/usr/bin/env bun

/**
 * Get DE Energy Mix Data (SMARD / Bundesnetzagentur)
 *
 * Fetches monthly generation data from the SMARD API and produces:
 *   - Data/DE-Energy-Mix/energy-mix-latest.csv
 *
 * API: https://www.smard.de/app
 * Source: https://www.smard.de/home/marktdaten
 * No authentication required.
 *
 * Strategy:
 *  1. For each generation filter, fetch the timestamp index.
 *  2. Find the latest year where the series has >=10 non-null data points.
 *  3. If no complete year exists (e.g. Kernenergie decommissioned April 2023),
 *     the source is assigned 0 MWh for the consensus year.
 *  4. The "consensus year" is determined by the majority vote across all sources.
 */

// Bun on macOS lacks some CA certificates in its bundled cert store; this flag
// allows the fetch to proceed against the known-good government endpoint.
process.env.NODE_TLS_REJECT_UNAUTHORIZED = "0";

import { writeFileSync, mkdirSync } from "fs";
import { join } from "path";

const SMARD_BASE = "https://www.smard.de/app";
const REGION = "DE";
const RESOLUTION = "month";
const OUT_DIR = join(__dirname, "Data/DE-Energy-Mix");

// Generation filter codes (Stromerzeugung)
const FILTERS: { id: number; name_de: string; renewable: boolean }[] = [
  { id: 4067, name_de: "Wind Onshore",           renewable: true  },
  { id: 1225, name_de: "Wind Offshore",           renewable: true  },
  { id: 4068, name_de: "Photovoltaik",            renewable: true  },
  { id: 4066, name_de: "Biomasse",                renewable: true  },
  { id: 1226, name_de: "Wasserkraft",             renewable: true  },
  { id: 1228, name_de: "Sonstige Erneuerbare",    renewable: true  },
  { id: 1223, name_de: "Braunkohle",              renewable: false },
  { id: 4069, name_de: "Steinkohle",              renewable: false },
  { id: 4071, name_de: "Erdgas",                  renewable: false },
  { id: 1224, name_de: "Kernenergie",             renewable: false },
  { id: 1227, name_de: "Sonstige Konventionelle", renewable: false },
];

// TypeScript interfaces for the SMARD API responses

interface IndexResponse {
  timestamps: number[];
}

interface TimeseriesResponse {
  meta_data: {
    version: number;
    created: number;
  };
  series: [number, number | null][];
}

interface FilterResult {
  filter_id: number;
  source_de: string;
  renewable: boolean;
  total_mwh: number;
  year_found: number | null;  // null = no complete year found (treated as 0)
}

async function fetchJson<T>(url: string): Promise<T> {
  const res = await fetch(url);
  if (!res.ok) throw new Error(`HTTP ${res.status} for ${url}`);
  return res.json() as Promise<T>;
}

/**
 * Find the latest year with >=minNonNull non-null monthly data points.
 * Returns the year and total MWh, or null if no complete year found.
 */
async function findLatestCompleteYear(
  filterId: number,
  timestamps: number[],
  minNonNull = 10
): Promise<{ year: number; total_mwh: number } | null> {
  // Try from newest to oldest annual timestamp
  for (let i = timestamps.length - 1; i >= 0; i--) {
    const ts = timestamps[i];
    const url = `${SMARD_BASE}/chart_data/${filterId}/${REGION}/${filterId}_${REGION}_${RESOLUTION}_${ts}.json`;
    const data = await fetchJson<TimeseriesResponse>(url);
    const nonNullPoints = data.series.filter(([, v]) => v !== null);
    if (nonNullPoints.length >= minNonNull) {
      const year = new Date(ts).getUTCFullYear();
      const total_mwh = data.series.reduce((acc, [, v]) => acc + (v ?? 0), 0);
      return { year, total_mwh };
    }
  }
  return null;
}

async function main() {
  console.log("Fetching DE energy mix data from SMARD (Bundesnetzagentur)…\n");

  mkdirSync(OUT_DIR, { recursive: true });

  // Phase 1: fetch all filters, find each source's latest complete year
  const rawResults: FilterResult[] = [];

  for (const filter of FILTERS) {
    const indexUrl = `${SMARD_BASE}/chart_data/${filter.id}/${REGION}/index_${RESOLUTION}.json`;
    console.log(`Fetching ${filter.name_de} (${filter.id})…`);

    const index = await fetchJson<IndexResponse>(indexUrl);
    const result = await findLatestCompleteYear(filter.id, index.timestamps);

    if (result) {
      console.log(`  → ${result.year}: ${(result.total_mwh / 1e6).toFixed(2)} TWh`);
      rawResults.push({
        filter_id: filter.id,
        source_de: filter.name_de,
        renewable: filter.renewable,
        total_mwh: result.total_mwh,
        year_found: result.year,
      });
    } else {
      console.log(`  → No complete year found — treating as 0 MWh`);
      rawResults.push({
        filter_id: filter.id,
        source_de: filter.name_de,
        renewable: filter.renewable,
        total_mwh: 0,
        year_found: null,
      });
    }
  }

  // Phase 2: determine consensus year (mode of year_found across active sources)
  const yearCounts = new Map<number, number>();
  for (const r of rawResults) {
    if (r.year_found !== null) {
      yearCounts.set(r.year_found, (yearCounts.get(r.year_found) ?? 0) + 1);
    }
  }
  const consensusYear = [...yearCounts.entries()].sort((a, b) => b[1] - a[1])[0][0];
  console.log(`\nConsensus year: ${consensusYear}`);

  // Sources with a different year than consensus are zeroed out for consistency.
  // Example: Kernenergie was decommissioned April 2023; its index only goes to 2022.
  // In a 2024 mix, it contributes 0 MWh.
  const results = rawResults.map((r) => {
    if (r.year_found !== null && r.year_found !== consensusYear) {
      console.log(`  Note: ${r.source_de} latest complete year is ${r.year_found}, not ${consensusYear} — zeroing for consensus year consistency`);
      return { ...r, total_mwh: 0, year_found: null };
    }
    return r;
  });

  // Phase 3: calculate shares based on total generation across all sources
  const totalGeneration = results.reduce((s, r) => s + r.total_mwh, 0);
  const renewableTotal = results.filter((r) => r.renewable).reduce((s, r) => s + r.total_mwh, 0);
  const coalTotal = results
    .filter((r) => r.source_de === "Braunkohle" || r.source_de === "Steinkohle")
    .reduce((s, r) => s + r.total_mwh, 0);
  const nuclearTotal = results.find((r) => r.source_de === "Kernenergie")?.total_mwh ?? 0;

  const renewableShare = (renewableTotal / totalGeneration) * 100;
  const coalShare = (coalTotal / totalGeneration) * 100;
  const nuclearShare = (nuclearTotal / totalGeneration) * 100;

  const topSource = results.reduce((best, r) => (r.total_mwh > best.total_mwh ? r : best));

  // Phase 4: write CSV
  const csvHeader = "filter_id,source_de,total_mwh,share_pct_generation";
  const csvRows = results.map((r) => {
    const share = (r.total_mwh / totalGeneration) * 100;
    return `${r.filter_id},${r.source_de},${r.total_mwh.toFixed(1)},${share.toFixed(2)}`;
  });

  const csvPath = join(OUT_DIR, "energy-mix-latest.csv");
  writeFileSync(csvPath, [csvHeader, ...csvRows].join("\n") + "\n");
  console.log(`\nWrote ${csvPath}`);

  // Phase 5: print summary
  console.log(`\n── Summary (${consensusYear}) ────────────────────────────────`);
  console.log(`Total generation:    ${(totalGeneration / 1e6).toFixed(0)} TWh`);
  console.log(`Renewable share:     ${renewableShare.toFixed(1)}%`);
  console.log(`Coal share:          ${coalShare.toFixed(1)}%`);
  console.log(`Nuclear share:       ${nuclearShare.toFixed(1)}% (Kernenergie abgeschaltet Apr 2023)`);
  console.log(`Top source:          ${topSource.source_de} (${(topSource.total_mwh / 1e6).toFixed(1)} TWh, ${((topSource.total_mwh / totalGeneration) * 100).toFixed(1)}%)`);
  console.log(`\nAll figures in MWh. Primary data period: ${consensusYear}.`);
}

main().catch((err) => {
  console.error("Error:", err.message);
  process.exit(1);
});