From 44ed020a133747b1d9ea4ad6fd97c664dab7891f Mon Sep 17 00:00:00 2001 From: Daniel Miessler Date: Wed, 10 Dec 2025 14:27:51 -0800 Subject: [PATCH] refactor: Simplify to US and Global figures without professional/broad split --- CLAUDE.md | 109 ++++++ .../SUMMARY.md | 19 +- .../knowledge-worker-compensation-data.md | 5 +- Data/README.md | 70 +++- Data/US-Common-Metrics/README.md | 101 ++++++ Data/US-Common-Metrics/US-Common-Metrics.md | 315 ++++++++++++++++ Data/US-Common-Metrics/source.md | 263 ++++++++++++++ Data/US-Common-Metrics/us-metrics-current.csv | 67 ++++ .../us-metrics-historical.csv | 329 +++++++++++++++++ .../source.md | 0 .../update.ts | 0 .../DS-00002—UN_SDG_Indicators/source.md | 0 .../DS-00002—UN_SDG_Indicators/update.ts | 0 .../DS-00003—World_Bank_Open_Data/source.md | 0 .../DS-00003—World_Bank_Open_Data/update.ts | 0 .../VALIDATION.md | 0 .../data/README.md | 0 .../source.md | 0 .../update.log | 0 .../update.ts | 0 .../data/README.md | 0 .../DS-00005—CDC_WONDER_Mortality/source.md | 0 .../DS-00005—CDC_WONDER_Mortality/update.ts | 0 .../data/README.md | 0 .../source.md | 0 .../update.ts | 0 .../SETUP_NOTES.md | 0 .../data/README.md | 0 .../data/latest.json | 0 .../data/latest.txt | 0 .../data/permission-to-quit-index.txt | 0 .../DS-00007—BLS_JOLTS_Labor_Market/source.md | 0 .../update.log | 0 .../DS-00007—BLS_JOLTS_Labor_Market/update.ts | 0 .../.env.example | 0 .../.gitignore | 0 .../DS-00008—EPA_Air_Quality_System/README.md | 0 .../data/README.md | 0 .../DS-00008—EPA_Air_Quality_System/source.md | 0 .../DS-00008—EPA_Air_Quality_System/update.ts | 0 .../DS-00009—EIA_Energy_Data/source.md | 336 ++++++++++++++++++ .../DS-00010—Treasury_FiscalData/source.md | 331 +++++++++++++++++ .../sources}/WELLBEING_DATA_SOURCES.md | 0 Questions/Personal/Challenge/README.md | 19 + Questions/README.md | 8 + research/README.md | 168 +++++++++ 46 files changed, 2124 insertions(+), 16 deletions(-) create mode 100644 CLAUDE.md create mode 100644 Data/US-Common-Metrics/README.md create mode 100644 Data/US-Common-Metrics/US-Common-Metrics.md create mode 100644 Data/US-Common-Metrics/source.md create mode 100644 Data/US-Common-Metrics/us-metrics-current.csv create mode 100644 Data/US-Common-Metrics/us-metrics-historical.csv rename {Data-Sources => Data/sources}/DS-00001—WHO_Global_Health_Observatory/source.md (100%) rename {Data-Sources => Data/sources}/DS-00001—WHO_Global_Health_Observatory/update.ts (100%) rename {Data-Sources => Data/sources}/DS-00002—UN_SDG_Indicators/source.md (100%) rename {Data-Sources => Data/sources}/DS-00002—UN_SDG_Indicators/update.ts (100%) rename {Data-Sources => Data/sources}/DS-00003—World_Bank_Open_Data/source.md (100%) rename {Data-Sources => Data/sources}/DS-00003—World_Bank_Open_Data/update.ts (100%) rename {Data-Sources => Data/sources}/DS-00004—FRED_Economic_Wellbeing/VALIDATION.md (100%) rename {Data-Sources => Data/sources}/DS-00004—FRED_Economic_Wellbeing/data/README.md (100%) rename {Data-Sources => Data/sources}/DS-00004—FRED_Economic_Wellbeing/source.md (100%) rename {Data-Sources => Data/sources}/DS-00004—FRED_Economic_Wellbeing/update.log (100%) rename {Data-Sources => Data/sources}/DS-00004—FRED_Economic_Wellbeing/update.ts (100%) rename {Data-Sources => Data/sources}/DS-00005—CDC_WONDER_Mortality/data/README.md (100%) rename {Data-Sources => Data/sources}/DS-00005—CDC_WONDER_Mortality/source.md (100%) rename {Data-Sources => Data/sources}/DS-00005—CDC_WONDER_Mortality/update.ts (100%) rename {Data-Sources => Data/sources}/DS-00006—Census_ACS_Social_Wellbeing/data/README.md (100%) rename {Data-Sources => Data/sources}/DS-00006—Census_ACS_Social_Wellbeing/source.md (100%) rename {Data-Sources => Data/sources}/DS-00006—Census_ACS_Social_Wellbeing/update.ts (100%) rename {Data-Sources => Data/sources}/DS-00007—BLS_JOLTS_Labor_Market/SETUP_NOTES.md (100%) rename {Data-Sources => Data/sources}/DS-00007—BLS_JOLTS_Labor_Market/data/README.md (100%) rename {Data-Sources => Data/sources}/DS-00007—BLS_JOLTS_Labor_Market/data/latest.json (100%) rename {Data-Sources => Data/sources}/DS-00007—BLS_JOLTS_Labor_Market/data/latest.txt (100%) rename {Data-Sources => Data/sources}/DS-00007—BLS_JOLTS_Labor_Market/data/permission-to-quit-index.txt (100%) rename {Data-Sources => Data/sources}/DS-00007—BLS_JOLTS_Labor_Market/source.md (100%) rename {Data-Sources => Data/sources}/DS-00007—BLS_JOLTS_Labor_Market/update.log (100%) rename {Data-Sources => Data/sources}/DS-00007—BLS_JOLTS_Labor_Market/update.ts (100%) rename {Data-Sources => Data/sources}/DS-00008—EPA_Air_Quality_System/.env.example (100%) rename {Data-Sources => Data/sources}/DS-00008—EPA_Air_Quality_System/.gitignore (100%) rename {Data-Sources => Data/sources}/DS-00008—EPA_Air_Quality_System/README.md (100%) rename {Data-Sources => Data/sources}/DS-00008—EPA_Air_Quality_System/data/README.md (100%) rename {Data-Sources => Data/sources}/DS-00008—EPA_Air_Quality_System/source.md (100%) rename {Data-Sources => Data/sources}/DS-00008—EPA_Air_Quality_System/update.ts (100%) create mode 100644 Data/sources/DS-00009—EIA_Energy_Data/source.md create mode 100644 Data/sources/DS-00010—Treasury_FiscalData/source.md rename {Data-Sources => Data/sources}/WELLBEING_DATA_SOURCES.md (100%) create mode 100644 Questions/Personal/Challenge/README.md create mode 100644 Questions/README.md create mode 100644 research/README.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..7e665da --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,109 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +**Substrate** is an open-source framework for human understanding, meaning, and progress. It's a structured knowledge system for capturing, organizing, and analyzing different aspects of civilization—from problems and solutions to values, plans, and experiments. + +The project uses a hierarchical directory structure where each category contains structured markdown files that can reference each other. The primary interface for working with Substrate is AI-powered analysis, which can hold multiple components in context simultaneously and analyze relationships between them. + +## Repository Structure + +The repository is organized into distinct categories, each containing individual entries as markdown files or directories: + +- **Arguments/** - Structured logical arguments (deductive, inductive, etc.) about claims +- **Claims/** - Assertions that can be supported by arguments +- **Data-Sources/** - References to data that supports or refutes claims +- **Experiments/** - Scientific or social experiments related to hypotheses +- **Frames/** - Mental models or worldviews that shape how plans are created +- **Funding-Sources/** - Potential or actual funding for projects +- **Ideas/** - Concepts that could become solutions or projects +- **Models/** - Ways of understanding the world (beliefs, frameworks) +- **Organizations/** - Groups working on related problems/solutions +- **Outcomes/** - Results from projects or experiments +- **People/** - Individuals relevant to the project +- **Plans/** - Comprehensive governance plans (like `us-plan1-miessler.md`) +- **Problems/** - Issues facing humanity at various scales +- **Projects/** - Active initiatives working toward solutions +- **Results/** - Specific outputs from experiments or projects +- **Risks/** - Potential negative consequences of plans or projects +- **Solutions/** - Proposed approaches to problems +- **Threats/** - External challenges to plans or values +- **Values/** - Core principles that guide decision-making + +## File Naming Convention + +Files use a structured naming format: +- **Format**: `[CATEGORY-ID]—[Descriptive_Name]` +- **Example**: `AR-12934—AIs_Are_Capable_of_Understanding` +- The ID helps with cross-referencing between components +- Underscores separate words in the descriptive name + +## Working with Substrate + +### Key Principles + +1. **Interconnected System**: Components reference each other (Problems link to Solutions, Plans reference Values, etc.) +2. **Transparency**: Plans and arguments should be explicit about their reasoning and assumptions +3. **AI-First Interface**: The system is designed to be navigated and analyzed by AI, which can hold multiple components in context +4. **Structured Thinking**: Each category has a specific purpose and format + +### Plans Structure + +Plans (like `us-plan1-miessler.md`) follow the Telos framework and include: +- **SCOPE**: What entity the plan covers +- **CHALLENGES**: Key problems facing the entity +- **MISSION**: Primary objectives +- **STRATEGIES**: High-level approaches +- **IDEAL WORLD**: Vision of desired future state + +Plans can also reference: +- Problems, Solutions, Values, Models, Projects, Budgets, Metrics + +### Arguments Structure + +Arguments follow formal logical structures: +- **Claim**: The assertion being argued +- **Argument Style**: Type (deductive, inductive, abductive, etc.) +- **Argument**: Step-by-step logical progression +- Each step builds on previous ones to reach a conclusion + +## Common Development Tasks + +### Adding New Entries + +When adding new Problems, Solutions, Ideas, etc.: +1. Create file with proper naming convention: `[CATEGORY-ID]—[Name]` +2. Follow the structure of existing entries in that category +3. Include cross-references to related components where appropriate +4. Consider which Plans, Values, or Models the entry relates to + +### Analyzing Relationships + +When analyzing Substrate content: +1. Read multiple related files to understand connections +2. Look for explicit and implicit references between components +3. Consider how Models/Values shape Plans and Solutions +4. Trace Problems → Solutions → Projects → Outcomes chains + +### Contributing + +The project welcomes contributions via pull requests to modify Substrate object files. A web-based interface is planned to make contributions easier for non-coders. + +## Important Context + +- **Single-repo structure**: As of July 27, 2024, all components live in this monorepo +- **Primary creator**: Daniel Miessler +- **Video introduction**: https://www.youtube.com/watch?v=ky7ejowc_qY +- **Blog post**: https://danielmiessler.com/p/introducing-substrate +- **Mission**: Visualize human progress through structured knowledge + +## When Working with This Repo + +- Respect the structured format of each category +- Maintain the naming conventions +- Consider cross-references and relationships between components +- Plans are meant to be comprehensive and transparent +- The system is designed for AI analysis—keep content machine-readable +- Most README.md files in subdirectories are minimal placeholders diff --git a/Data/Knowledge-Worker-Global-Salaries/SUMMARY.md b/Data/Knowledge-Worker-Global-Salaries/SUMMARY.md index 8f89004..31641a5 100644 --- a/Data/Knowledge-Worker-Global-Salaries/SUMMARY.md +++ b/Data/Knowledge-Worker-Global-Salaries/SUMMARY.md @@ -7,10 +7,9 @@ | Metric | Value | Confidence | Last Updated | |--------|-------|------------|--------------| | **Global Knowledge Worker Compensation** | **$35-50 trillion/year** | 65% | December 2025 | -| **U.S. Knowledge Worker Compensation (Professional)** | **$6-8 trillion/year** | 95% | December 2025 | -| **U.S. Knowledge Worker Compensation (Broad)** | **$10-12 trillion/year** | 85% | December 2025 | +| **U.S. Knowledge Worker Compensation** | **$6-12 trillion/year** | 85% | December 2025 | -**One-liner:** Global knowledge workers earn $35-50T annually; U.S. accounts for ~$6-12T depending on definition. +**One-liner:** Global knowledge workers earn $35-50T annually; the U.S. accounts for $6-12T. **Caveat:** "Knowledge worker" has no standard definition—ranges reflect definitional uncertainty more than data uncertainty. @@ -50,12 +49,11 @@ The biggest source of variance isn't data quality—it's definitional ambiguity. | Definition | Workforce Size | Compensation Estimate | |------------|----------------|----------------------| -| **Narrow** | ~230 million | Lower bound | -| **Professional (BLS SOC 11-29)** | ~42 million (US) | $6-8T (US only) | -| **Broad (all white-collar)** | ~100 million (US) | $10-12T (US only) | -| **Expansive** | ~1 billion+ | Upper bound | +| **Narrow** | ~230 million globally | Lower bound | +| **Core** | ~500 million globally | Mid-range | +| **Expansive** | ~1 billion+ globally | Upper bound | -Our $35-50T range spans from conservative professional definitions to broader white-collar inclusion. The U.S. figures ($6-8T professional, $10-12T broad) carry higher confidence because BLS occupational data is excellent. +Our $35-50T range reflects this definitional spectrum. The U.S. figure ($6-12T) carries higher confidence because BLS occupational data is excellent; the range reflects where you draw the line on which roles count. --- @@ -78,8 +76,7 @@ U.S. knowledge workers earn 2-4x their global counterparts on average, explainin | Component | Confidence | Explanation | |-----------|------------|-------------| -| **U.S. Professional ($6-8T)** | 95% (Very High) | BLS OEWS data is authoritative; well-defined occupational codes | -| **U.S. Broad ($10-12T)** | 85% (High) | Good data, some definitional boundary questions | +| **U.S. ($6-12T)** | 85% (High) | BLS OEWS data is authoritative; range reflects definitional choices | | **Global ($35-50T)** | 65% (Medium) | Extrapolation from U.S. weighted by regional wages; limited international occupational data | The wide global range reflects genuine uncertainty in international data, not hedging. We know U.S. numbers well; global figures require more inference. @@ -161,7 +158,7 @@ This estimate synthesizes 40+ parallel research queries across multiple AI resea ## Bottom Line -Global knowledge workers earn **$35-50 trillion annually**. The U.S. accounts for roughly $6-12 trillion of that, depending on how narrowly you define "knowledge worker." These figures represent actual compensation paid—wages, benefits, and equity—just what we pay people to think for a living. +Global knowledge workers earn **$35-50 trillion annually**. The U.S. accounts for $6-12 trillion of that. These figures represent actual compensation paid—wages, benefits, and equity—what we pay people to think for a living. --- diff --git a/Data/Knowledge-Worker-Global-Salaries/knowledge-worker-compensation-data.md b/Data/Knowledge-Worker-Global-Salaries/knowledge-worker-compensation-data.md index 608d706..81a3d7b 100644 --- a/Data/Knowledge-Worker-Global-Salaries/knowledge-worker-compensation-data.md +++ b/Data/Knowledge-Worker-Global-Salaries/knowledge-worker-compensation-data.md @@ -7,10 +7,9 @@ | Metric | Value | Confidence | Last Updated | |--------|-------|------------|--------------| | **Global Knowledge Worker Compensation** | **$35-50 trillion/year** | 65% | December 2025 | -| **U.S. Knowledge Worker Compensation (Professional)** | **$6-8 trillion/year** | 95% | December 2025 | -| **U.S. Knowledge Worker Compensation (Broad)** | **$10-12 trillion/year** | 85% | December 2025 | +| **U.S. Knowledge Worker Compensation** | **$6-12 trillion/year** | 85% | December 2025 | -**One-liner:** Global knowledge workers earn $35-50T annually; U.S. accounts for ~$6-12T depending on definition. +**One-liner:** Global knowledge workers earn $35-50T annually; the U.S. accounts for $6-12T. **Caveat:** "Knowledge worker" has no standard definition - ranges reflect definitional uncertainty more than data uncertainty. diff --git a/Data/README.md b/Data/README.md index 506235d..6e85ee0 100644 --- a/Data/README.md +++ b/Data/README.md @@ -1,8 +1,35 @@ -# Data-Sources +# Data ## Purpose -The Data-Sources directory contains curated, ground-truth datasets about important aspects of human life, society, and progress. This is a collection of reliable, parseable data that can be used for analysis, research, and informed decision-making. +The Data directory contains curated, ground-truth datasets about important aspects of human life, society, and progress, along with documentation for external data sources. This is a collection of reliable, parseable data that can be used for analysis, research, and informed decision-making. + +## Directory Structure + +``` +Data/ +├── sources/ # External data source catalog (APIs, endpoints, metadata) +│ ├── DS-00001—WHO_Global_Health_Observatory/ +│ ├── DS-00002—UN_SDG_Indicators/ +│ ├── DS-00003—World_Bank_Open_Data/ +│ ├── DS-00004—FRED_Economic_Wellbeing/ +│ ├── DS-00005—CDC_WONDER_Mortality/ +│ ├── DS-00006—Census_ACS_Social_Wellbeing/ +│ ├── DS-00007—BLS_JOLTS_Labor_Market/ +│ ├── DS-00008—EPA_Air_Quality_System/ +│ └── WELLBEING_DATA_SOURCES.md +├── Bay-Area-COVID-Wastewater/ # Curated datasets +├── Knowledge-Worker-Global-Salaries/ +├── Pulitzer-Prize-Winners/ +├── US-GDP/ +├── US-Inflation/ +├── README.md +└── UPDATES.md +``` + +**sources/** - Contains documentation and metadata for external data sources (APIs, endpoints, update frequencies, setup instructions). See `sources/WELLBEING_DATA_SOURCES.md` for details. + +**Dataset directories** - Contain curated, processed data collections ready for analysis. ## Philosophy @@ -127,3 +154,42 @@ Data sources support other Substrate components: --- **Mission**: Build a trusted foundation of ground-truth data to support human understanding and progress. + +## Relationship with Research Projects + +The Data directory works in conjunction with `research/` directory to maintain clear traceability between research and resulting datasets. + +**Research → Data Workflow:** + +1. **Input**: Research projects use `Data/sources/` to access external data APIs and endpoints +2. **Analysis**: Research projects perform analysis, synthesis, and investigation +3. **Output**: Research projects produce curated datasets stored in `Data/` top-level +4. **Documentation**: Research projects document their methodology, sources used, and resulting datasets + +**Example Structure:** + +``` +research/knowledge-worker-compensation-study/ +├── README.md # Research overview and methodology +├── SOURCES.md # Links to Data/sources/ used as inputs +├── findings/ # Analysis and insights +└── [references Data/Knowledge-Worker-Global-Salaries/] + +Data/Knowledge-Worker-Global-Salaries/ +├── knowledge-worker-compensation-data.md # Curated dataset (output) +└── source.md # Metadata linking back to research project +``` + +**Key Principles:** + +- Each dataset in `Data/` should include `source.md` documenting origin (research project or direct source) +- Research projects should document which `Data/sources/` they used as inputs in their SOURCES.md +- Research findings and methodology live in `research/`, curated datasets live in `Data/` +- Bidirectional links maintain complete traceability from source → research → dataset + +**Benefits:** + +- Clear provenance: Always know where data came from and how it was produced +- Reproducibility: Research methodology is documented and linked to outputs +- Reusability: Other research can reference existing datasets and their origins +- Quality: Traceability enables verification and validation of data quality diff --git a/Data/US-Common-Metrics/README.md b/Data/US-Common-Metrics/README.md new file mode 100644 index 0000000..9182a8e --- /dev/null +++ b/Data/US-Common-Metrics/README.md @@ -0,0 +1,101 @@ +# US Common Metrics + +**A comprehensive catalog of the most important U.S. economic and social indicators.** + +## Purpose + +This dataset provides a single source of truth for understanding the current state of the U.S. economy and society. Instead of navigating dozens of government websites with different update schedules and formats, you get: + +- **One master document** with 60+ metrics and their current values +- **Full provenance** - every number traced back to its authoritative source +- **Update automation** - TypeScript scripts keep metrics current +- **Transparency** - complete methodology documentation per Substrate philosophy + +## Quick Start + +```bash +# View current metrics +cat US-Common-Metrics.md + +# Get parseable data +cat us-metrics-current.csv + +# Update all metrics (requires API keys) +bun run update.ts +``` + +## Contents + +| File | Description | +|------|-------------| +| `US-Common-Metrics.md` | Master document with all metrics, values, sources | +| `us-metrics-current.csv` | Current values in parseable format | +| `us-metrics-historical.csv` | Historical time series | +| `source.md` | Methodology, data lineage, source links | +| `update.ts` | Automation script to refresh all metrics | + +## Metric Categories + +1. **Economic Output & Growth** - GDP, industrial production, retail sales +2. **Inflation & Prices** - CPI, PCE, gas prices, oil prices +3. **Employment & Labor** - Unemployment, payrolls, jobless claims, quit rate +4. **Housing** - Home prices, mortgage rates, housing starts +5. **Consumer & Personal Finance** - Sentiment, saving rate, credit +6. **Financial Markets** - Interest rates, stock indices, volatility +7. **Trade & International** - Trade balance, USD index +8. **Government & Fiscal** - Federal debt, budget, spending +9. **Demographics & Social** - Population, inequality, poverty +10. **Health & Crisis** - Deaths of despair, air quality, life expectancy + +## Data Sources + +All data comes from authoritative government and institutional sources: + +| Source | Metrics | API | +|--------|---------|-----| +| FRED (Federal Reserve) | Most economic indicators | Free API | +| EIA (Energy Information Admin) | Gas prices, oil, energy | Free API | +| Treasury FiscalData | Federal debt, budget, spending | Free API | +| BLS (Bureau of Labor Statistics) | Employment, wages | Free API | +| Census Bureau | Demographics, housing starts | Free API | +| CDC WONDER | Mortality data | Free access | +| EPA AQS | Air quality | Free API | + +See `source.md` for full documentation of each data source. + +## Update Schedule + +| Frequency | Metrics | +|-----------|---------| +| Daily | Treasury yields, oil prices, stock indices | +| Weekly | Jobless claims, gas prices, mortgage rates | +| Monthly | CPI, employment, retail sales, housing data | +| Quarterly | GDP, home prices, debt service ratio | +| Annual | Population, GINI, poverty rate, mortality | + +## Requirements + +- [Bun](https://bun.sh) runtime for automation scripts +- API keys (all free): + - `FRED_API_KEY` - https://fred.stlouisfed.org/docs/api/api_key.html + - `EIA_API_KEY` - https://www.eia.gov/opendata/register.php + - `CENSUS_API_KEY` - https://api.census.gov/data/key_signup.html + +## Integration with Substrate + +This dataset supports other Substrate components: + +- **Claims** - Back assertions with authoritative data +- **Arguments** - Reference specific metrics and trends +- **Problems** - Quantify the scale of issues (e.g., debt, inequality) +- **Solutions** - Track progress using ground-truth indicators + +## Credits + +**Created:** 2025-12-01 +**Maintainer:** Daniel Miessler +**Sources:** Federal Reserve, BLS, Census Bureau, EIA, Treasury, CDC, EPA + +--- + +**Mission**: Build a trusted foundation of ground-truth data to support human understanding and progress. diff --git a/Data/US-Common-Metrics/US-Common-Metrics.md b/Data/US-Common-Metrics/US-Common-Metrics.md new file mode 100644 index 0000000..dc448d2 --- /dev/null +++ b/Data/US-Common-Metrics/US-Common-Metrics.md @@ -0,0 +1,315 @@ +# US Common Metrics + +**Last Updated:** 2025-12-02 +**Data Sources:** 8+ authoritative government and institutional sources +**Coverage:** 60+ key U.S. economic and social indicators + +--- + +## Quick Reference Dashboard + +| Category | Key Metric | Value | Updated | Trend | +|----------|------------|-------|---------|-------| +| Economy | GDP Growth Rate (QoQ) | 3.8% | 2025-12-02 | → | +| Inflation | CPI-U All Items | 324.368 | 2025-12-02 | → | +| Employment | Unemployment Rate (U-3) | 4.4% | 2025-12-02 | → | +| Housing | 30-Year Mortgage Rate | 6.23% | 2025-12-02 | → | +| Markets | Fed Funds Rate | 3.88% | 2025-12-02 | → | +| Consumer | Consumer Sentiment | 53.6 | 2025-12-02 | → | +| Fiscal | Total Public Debt | +*Values updated8.396T | 2025-12-02 | → | +| Energy | Gas Price (Regular) | +*Values updated.061 | 2025-12-02 | → | + +*Values updated8.396T | 2025-12-02 | → | +| Energy | Gas Price (Regular) | +*Values updated.061 | 2025-12-02 | → | + +*Values updated8.396T | 2025-12-02 | → | +| Energy | Gas Price (Regular) | +*Values updated.061 | 2025-12-02 | → | + +*Values updated8.396T | 2025-12-02 | → | +| Energy | Gas Price (Regular) | -- | -- | -- | + +*Values updated8.396T | 2025-12-02 | → | +| Energy | Gas Price (Regular) | -- | -- | -- | + +*Values updated automatically via `update.ts`* + +--- + +## 1. Economic Output & Growth + +| Metric | Value | Period | Updated | Source | FRED ID | +|--------|-------|--------|---------|--------|---------| +| Real GDP (Billions) | | Q3 2024 | 3770.98B | Q3 2024 | 2025-12-02 | BEA/FRED | GDPC1 | +| Nominal GDP (Billions) | | BEA/FRED | 0485.73B | Q3 2024 | 2025-12-02 | BEA/FRED | GDP | +| GDP Growth Rate (QoQ, SAAR) | 3.8% | Q3 2024 | 2025-12-02 | BEA/FRED | A191RL1Q225SBEA | +| GDP Growth Rate (YoY) | 2.1% | Q3 2024 | 2025-12-02 | BEA/FRED | A191RO1Q156NBEA | +| Industrial Production Index | 101.27 | Monthly | 2025-12-02 | Fed/FRED | INDPRO | +| Capacity Utilization | 75.8% | Monthly | 2025-12-02 | Fed/FRED | TCU | +| Durable Goods Orders | 313653M | Monthly | 2025-12-02 | Census/FRED | DGORDER | +| Retail Sales (Billions) | $733258.0B | Monthly | 2025-12-02 | Census/FRED | RSAFS | +| Retail Sales YoY Change | $733258.0B | Monthly | 2025-12-02 | Census/FRED | RSAFSNA | + +### Source Documentation +- **Primary Source:** Bureau of Economic Analysis (BEA), Federal Reserve +- **Update Frequency:** Quarterly (GDP), Monthly (production, sales) +- **Methodology:** [Data/sources/DS-00004—FRED_Economic_Wellbeing/source.md](../sources/DS-00004—FRED_Economic_Wellbeing/source.md) + +--- + +## 2. Inflation & Prices + +| Metric | Value | Period | Updated | Source | FRED ID | +|--------|-------|--------|---------|--------|---------| +| CPI-U (All Urban Consumers) | 324.368 | Monthly | 2025-12-02 | BLS/FRED | CPIAUCSL | +| CPI YoY Change | -- | Monthly | -- | BLS/FRED | CPIAUCSL (calc) | +| Core CPI (Ex Food/Energy) | 330.542 | Monthly | 2025-12-02 | BLS/FRED | CPILFESL | +| Core CPI YoY Change | 330.542 | Monthly | 2025-12-02 | BLS/FRED | CPILFESL (calc) | +| PCE Price Index | 127.285 | Monthly | 2025-12-02 | BEA/FRED | PCEPI | +| Core PCE (Ex Food/Energy) | 126.705 | Monthly | 2025-12-02 | BEA/FRED | PCEPILFE | +| Producer Price Index (PPI) | 262.34 | Monthly | 2025-12-02 | BLS/FRED | PPIACO | +| Gas Price (Regular, US Avg) | -- | Weekly | -- | EIA | PET.EMM_EPMR_PTE_NUS_DPG.W | +| Oil Price (WTI Crude, $/bbl) | $59.11 | Daily | 2025-12-02 | EIA/FRED | DCOILWTICO | +| Gold Price ($/oz) | -- | Daily | -- | FRED | GOLDAMGBD228NLBM | + +### Source Documentation +- **Primary Sources:** Bureau of Labor Statistics (BLS), Bureau of Economic Analysis (BEA), Energy Information Administration (EIA) +- **Update Frequency:** Monthly (CPI, PCE, PPI), Weekly (gas), Daily (oil, gold) +- **Methodology:** [Data/sources/DS-00009—EIA_Energy_Data/source.md](../sources/DS-00009—EIA_Energy_Data/source.md) + +--- + +## 3. Employment & Labor + +| Metric | Value | Period | Updated | Source | FRED ID | +|--------|-------|--------|---------|--------|---------| +| Unemployment Rate (U-3) | 4.4% | Monthly | 2025-12-02 | BLS/FRED | UNRATE | +| Underemployment Rate (U-6) | 8.0% | Monthly | 2025-12-02 | BLS/FRED | U6RATE | +| Nonfarm Payrolls (Thousands) | 159,626K | Monthly | 2025-12-02 | BLS/FRED | PAYEMS | +| Payroll Change (MoM) | -- | Monthly | -- | BLS/FRED | PAYEMS (calc) | +| Initial Jobless Claims | 216,000K | Weekly | 2025-12-02 | DOL/FRED | ICSA | +| Continuing Claims | 1,960,000K | Weekly | 2025-12-02 | DOL/FRED | CCSA | +| Job Openings (JOLTS, Thousands) | 7,227K | Monthly | 2025-12-02 | BLS/FRED | JTSJOL | +| Quit Rate (JOLTS) | 1.9% | Monthly | 2025-12-02 | BLS/FRED | JTSQUR | +| Hire Rate (JOLTS) | 3.2% | Monthly | 2025-12-02 | BLS/FRED | JTSHIR | +| Labor Force Participation Rate | 62.4% | Monthly | 2025-12-02 | BLS/FRED | CIVPART | +| Employment-Population Ratio | 59.7% | Monthly | 2025-12-02 | BLS/FRED | EMRATIO | +| Average Hourly Earnings | | BLS/FRED | 6.67 | Monthly | 2025-12-02 | BLS/FRED | CES0500000003 | +| Average Weekly Hours | 34.2 | Monthly | 2025-12-02 | BLS/FRED | AWHAETP | +| Long-Term Unemployed (27+ wks) | -- | Monthly | -- | BLS/FRED | UEMP27OV | + +### Source Documentation +- **Primary Sources:** Bureau of Labor Statistics (BLS), Department of Labor (DOL) +- **Update Frequency:** Monthly (employment), Weekly (claims) +- **Methodology:** [Data/sources/DS-00007—BLS_JOLTS_Labor_Market/source.md](../sources/DS-00007—BLS_JOLTS_Labor_Market/source.md) + +--- + +## 4. Housing + +| Metric | Value | Period | Updated | Source | FRED ID | +|--------|-------|--------|---------|--------|---------| +| Median Home Sales Price | MSPUS |10,800 | Quarterly | 2025-12-02 | Census/FRED | MSPUS | +| Case-Shiller Home Price Index | 328.94 | Monthly | 2025-12-02 | S&P/FRED | CSUSHPINSA | +| Existing Home Sales (SAAR) | 4,100,000K | Monthly | 2025-12-02 | NAR/FRED | EXHOSLUSM495S | +| New Home Sales (SAAR) | 800K | Monthly | 2025-12-02 | Census/FRED | HSN1F | +| Housing Starts (SAAR) | 1,307K | Monthly | 2025-12-02 | Census/FRED | HOUST | +| Building Permits (SAAR) | 1,330K | Monthly | 2025-12-02 | Census/FRED | PERMIT | +| Homeownership Rate | -- | Quarterly | -- | Census/FRED | RSAHORUSQ156S | +| 30-Year Mortgage Rate | 6.23% | Weekly | 2025-12-02 | Freddie Mac/FRED | MORTGAGE30US | +| 15-Year Mortgage Rate | 5.51% | Weekly | 2025-12-02 | Freddie Mac/FRED | MORTGAGE15US | +| Median Rent Index | -- | Monthly | -- | BLS/FRED | CUSR0000SEHA | +| Housing Affordability Index | -- | Monthly | -- | NAR/FRED | FIXHAI | + +### Source Documentation +- **Primary Sources:** Census Bureau, National Association of Realtors (NAR), Freddie Mac +- **Update Frequency:** Quarterly (prices), Monthly (sales, starts), Weekly (rates) +- **Methodology:** [Data/sources/DS-00004—FRED_Economic_Wellbeing/source.md](../sources/DS-00004—FRED_Economic_Wellbeing/source.md) + +--- + +## 5. Consumer & Personal Finance + +| Metric | Value | Period | Updated | Source | FRED ID | +|--------|-------|--------|---------|--------|---------| +| Consumer Sentiment (UMich) | 53.6 | Monthly | 2025-12-02 | UMich/FRED | UMCSENT | +| Consumer Confidence (Conf Board) | -- | Monthly | -- | Conf Board | -- | +| Personal Income (Billions) | | Monthly | 6279.9B | Monthly | 2025-12-02 | BEA/FRED | PI | +| Disposable Personal Income | | Monthly | 3033.5B | Monthly | 2025-12-02 | BEA/FRED | DSPI | +| Personal Saving Rate | 4.6% | Monthly | 2025-12-02 | BEA/FRED | PSAVERT | +| Consumer Credit Outstanding | $5076636.0B | Monthly | 2025-12-02 | Fed/FRED | TOTALSL | +| Credit Card Delinquency Rate | 2.98% | Quarterly | 2025-12-02 | Fed/FRED | DRCCLACBS | +| Debt Service Ratio (DSR) | 11.25% | Quarterly | 2025-12-02 | Fed/FRED | TDSP | +| Auto Sales (SAAR, Millions) | 16.80M | Monthly | 2025-12-02 | BEA/FRED | TOTALSA | +| Real Personal Consumption | -- | Monthly | -- | BEA/FRED | PCEC96 | + +### Source Documentation +- **Primary Sources:** University of Michigan, Federal Reserve, Bureau of Economic Analysis +- **Update Frequency:** Monthly (sentiment, income), Quarterly (delinquency, DSR) +- **Methodology:** [Data/sources/DS-00004—FRED_Economic_Wellbeing/source.md](../sources/DS-00004—FRED_Economic_Wellbeing/source.md) + +--- + +## 6. Financial Markets & Interest Rates + +| Metric | Value | Period | Updated | Source | FRED ID | +|--------|-------|--------|---------|--------|---------| +| Federal Funds Rate (Effective) | 3.88% | Daily | 2025-12-02 | Fed/FRED | FEDFUNDS | +| Fed Funds Target (Upper) | 4.00% | -- | 2025-12-02 | Fed/FRED | DFEDTARU | +| 10-Year Treasury Yield | 4.02% | Daily | 2025-12-02 | Treasury/FRED | DGS10 | +| 2-Year Treasury Yield | 3.47% | Daily | 2025-12-02 | Treasury/FRED | DGS2 | +| 10Y-2Y Spread (Yield Curve) | 0.55% | Daily | 2025-12-02 | FRED | T10Y2Y | +| 30-Year Treasury Yield | 4.67% | Daily | 2025-12-02 | Treasury/FRED | DGS30 | +| 3-Month Treasury Bill | 3.73% | Daily | 2025-12-02 | Treasury/FRED | DTB3 | +| Financial Stress Index (StL Fed) | -0.133 | Weekly | 2025-12-02 | StL Fed/FRED | STLFSI4 | +| S&P 500 Index | 6,812.63 | Daily | 2025-12-02 | S&P/FRED | SP500 | +| Dow Jones Industrial Avg | -- | Daily | -- | DJIA | DJIA | +| NASDAQ Composite | -- | Daily | -- | NASDAQ | NASDAQCOM | +| VIX (Volatility Index) | 16.35 | Daily | 2025-12-02 | CBOE/FRED | VIXCLS | +| Corporate Bond Spread (BAA-AAA) | -- | Monthly | -- | Moody's/FRED | BAAFFM | + +### Source Documentation +- **Primary Sources:** Federal Reserve, U.S. Treasury, S&P, CBOE +- **Update Frequency:** Daily (rates, indices), Weekly (stress index) +- **Methodology:** [Data/sources/DS-00004—FRED_Economic_Wellbeing/source.md](../sources/DS-00004—FRED_Economic_Wellbeing/source.md) + +--- + +## 7. Trade & International + +| Metric | Value | Period | Updated | Source | FRED ID | +|--------|-------|--------|---------|--------|---------| +| Trade Balance (Billions) | $-59550.0B | Monthly | 2025-12-02 | Census/BEA/FRED | BOPGSTB | +| Exports (Billions) | | Exports (Billions) | 79019.0B | Monthly | 2025-12-02 | Census/FRED | BOPGEXP | +| Imports (Billions) | | Monthly | 64627.0B | Monthly | 2025-12-02 | Census/FRED | BOPGIMP | +| Current Account Balance | -- | Quarterly | -- | BEA/FRED | NETFI | +| USD Index (DXY) | 121.43 | Daily | 2025-12-02 | ICE/FRED | DTWEXBGS | +| USD/EUR Exchange Rate | 1.1600 | Daily | 2025-12-02 | Fed/FRED | DEXUSEU | +| USD/CNY Exchange Rate | -- | Daily | -- | Fed/FRED | DEXCHUS | + +### Source Documentation +- **Primary Sources:** Census Bureau, Bureau of Economic Analysis, Federal Reserve +- **Update Frequency:** Monthly (trade), Quarterly (current account), Daily (forex) + +--- + +## 8. Government & Fiscal + +| Metric | Value | Period | Updated | Source | ID | +|--------|-------|--------|---------|--------|---------| +| Federal Debt (Total Public, Trillions) | -- | Daily | -- | Treasury | -- | +| Debt Held by Public (Trillions) | -- | Daily | -- | Treasury | FYGFDPUN | +| Debt-to-GDP Ratio | 118.8% | Quarterly | 2025-12-02 | FRED | GFDEGDQ188S | +| Federal Budget Deficit (Billions) | -- | Monthly | -- | Treasury | -- | +| Federal Receipts (Billions) | $5622.8B | Monthly | 2025-12-02 | Treasury | FGRECPT | +| Federal Expenditures (Billions) | $7496.3B | Monthly | 2025-12-02 | Treasury | FGEXPND | +| Interest on Federal Debt | -- | Monthly | -- | Treasury | A091RC1Q027SBEA | + +### Source Documentation +- **Primary Source:** U.S. Treasury (FiscalData.treasury.gov) +- **Update Frequency:** Daily (debt), Monthly (budget, receipts, spending) +- **Methodology:** [Data/sources/DS-00010—Treasury_FiscalData/source.md](../sources/DS-00010—Treasury_FiscalData/source.md) + +--- + +## 9. Demographics & Social + +| Metric | Value | Period | Updated | Source | ID | +|--------|-------|--------|---------|--------|---------| +| US Population (Millions) | 342555.0M | Annual | 2025-12-02 | Census | POPTHM | +| Population Growth Rate | -- | Annual | -- | Census | -- | +| GINI Index (Income Inequality) | 41.800 | Annual | 2025-12-02 | Census/FRED | SIPOVGINIUSA | +| Poverty Rate | -- | Annual | -- | Census | -- | +| Median Household Income | $83,730 | Annual | 2025-12-02 | Census | MEHOINUSA672N | +| Living Alone Rate | -- | Annual | -- | Census ACS | B11001_008E | +| No Internet Access Rate | -- | Annual | -- | Census ACS | B28002_013E | +| Mean Commute Time (Minutes) | -- | Annual | -- | Census ACS | B08303_001E | + +### Source Documentation +- **Primary Source:** U.S. Census Bureau (American Community Survey) +- **Update Frequency:** Annual +- **Methodology:** [Data/sources/DS-00006—Census_ACS_Social_Wellbeing/source.md](../sources/DS-00006—Census_ACS_Social_Wellbeing/source.md) + +--- + +## 10. Health & Crisis Indicators + +| Metric | Value | Period | Updated | Source | ID | +|--------|-------|--------|---------|--------|---------| +| Drug Overdose Deaths | -- | Annual | -- | CDC WONDER | -- | +| Opioid-Specific Deaths | -- | Annual | -- | CDC WONDER | -- | +| Suicide Rate (per 100k) | -- | Annual | -- | CDC WONDER | -- | +| Life Expectancy at Birth | -- | Annual | -- | CDC | -- | +| All-Cause Mortality Rate | -- | Annual | -- | CDC WONDER | -- | +| PM2.5 (Annual Avg, ug/m3) | -- | Annual | -- | EPA AQS | 88101 | +| Days Exceeding AQI 100 | -- | Annual | -- | EPA AQS | -- | + +### Source Documentation +- **Primary Sources:** CDC WONDER, Environmental Protection Agency (EPA) +- **Update Frequency:** Annual (1-2 year lag for mortality) +- **Methodology:** [Data/sources/DS-00005—CDC_WONDER_Mortality/source.md](../sources/DS-00005—CDC_WONDER_Mortality/source.md) + +--- + +## Update Schedule Summary + +| Frequency | Metrics | +|-----------|---------| +| **Daily** | Fed funds, Treasury yields, oil prices, stock indices, forex, federal debt | +| **Weekly** | Jobless claims, gas prices, mortgage rates, financial stress index | +| **Monthly** | CPI, PCE, employment/payrolls, retail sales, housing starts/sales, trade balance | +| **Quarterly** | GDP, home prices, debt service ratio, homeownership rate, current account | +| **Annual** | Population, GINI, poverty rate, life expectancy, mortality data, ACS demographics | + +--- + +## Data Sources Index + +| ID | Name | Metrics | Documentation | +|----|------|---------|---------------| +| DS-00004 | FRED Economic Wellbeing | Most economic indicators | [source.md](../sources/DS-00004—FRED_Economic_Wellbeing/source.md) | +| DS-00005 | CDC WONDER Mortality | Deaths of despair, mortality | [source.md](../sources/DS-00005—CDC_WONDER_Mortality/source.md) | +| DS-00006 | Census ACS Social | Demographics, isolation | [source.md](../sources/DS-00006—Census_ACS_Social_Wellbeing/source.md) | +| DS-00007 | BLS JOLTS Labor Market | Job openings, quit rate | [source.md](../sources/DS-00007—BLS_JOLTS_Labor_Market/source.md) | +| DS-00008 | EPA Air Quality | PM2.5, ozone, air quality | [source.md](../sources/DS-00008—EPA_Air_Quality_System/source.md) | +| DS-00009 | EIA Energy Data | Gas prices, oil, energy | [source.md](../sources/DS-00009—EIA_Energy_Data/source.md) | +| DS-00010 | Treasury FiscalData | Federal debt, budget | [source.md](../sources/DS-00010—Treasury_FiscalData/source.md) | +| DS-00011 | BLS Employment | Unemployment, payrolls | [source.md](../sources/DS-00011—BLS_Employment/source.md) | + +--- + +## Notes on Data Quality + +1. **Revisions**: Many economic indicators are revised multiple times after initial release. Values shown are the most recent available. + +2. **Seasonal Adjustment**: Most monthly/quarterly metrics are seasonally adjusted (SA or SAAR) unless noted otherwise. + +3. **Lag**: Different metrics have different publication lags: + - Financial market data: Same day + - Weekly indicators: 4-7 days + - Monthly indicators: 2-4 weeks + - Quarterly indicators: 1-3 months + - Annual indicators: 6-18 months + +4. **Sources of Truth**: All data comes from authoritative government agencies. See individual source documentation for methodology details. + +--- + +## How to Use This Document + +**For Research**: Find the metric you need, note the FRED ID or source, and access historical data via the source's API. + +**For Fact-Checking**: Every metric includes its authoritative source. Follow the links to verify or dig deeper. + +**For Arguments**: Reference specific metrics with their values, dates, and sources when building Substrate Arguments. + +**For Automation**: Use `update.ts` to refresh all metrics programmatically. + +--- + +**Last Updated:** 2025-12-01 +**Automation:** `bun run update.ts` +**Maintained by:** Daniel Miessler / Kai diff --git a/Data/US-Common-Metrics/source.md b/Data/US-Common-Metrics/source.md new file mode 100644 index 0000000..b5b93c8 --- /dev/null +++ b/Data/US-Common-Metrics/source.md @@ -0,0 +1,263 @@ +# US Common Metrics - Source Documentation + +**Dataset ID:** US-Common-Metrics +**Record Created:** 2025-12-01 +**Last Updated:** 2025-12-01 +**Maintainer:** Daniel Miessler / Kai + +--- + +## Bibliographic Information + +### Title Statement +- **Main Title:** US Common Metrics +- **Subtitle:** Comprehensive U.S. Economic and Social Indicators Dataset +- **Abbreviated Title:** US-CM + +### Responsibility Statement +- **Aggregator:** Substrate Project (danielmiessler/Substrate) +- **Primary Data Sources:** See Data Provenance section below +- **Automation:** TypeScript/Bun automation scripts + +### Publication Information +- **First Published:** 2025-12-01 +- **Update Frequency:** Varies by metric (daily to annual) +- **Current Status:** Active + +--- + +## Purpose & Scope + +### What This Dataset Provides + +A **single source of truth** for the most important U.S. economic and social metrics, aggregating data from multiple authoritative government sources into one coherent document. + +### Problems Solved + +1. **Fragmentation**: Data scattered across dozens of government websites +2. **Inconsistent Formats**: Different agencies use different formats, update schedules +3. **Discovery Friction**: Finding the right metric requires knowing where to look +4. **Verification Burden**: Fact-checking requires navigating multiple sources + +### Target Use Cases + +- **Research & Analysis**: Quick access to authoritative data +- **Fact-Checking**: Verify claims with source citations +- **Substrate Arguments**: Build evidence-based arguments with proper provenance +- **Economic Monitoring**: Track the state of the U.S. economy + +--- + +## Data Provenance + +### Primary Data Sources + +This dataset aggregates from the following authoritative sources: + +| Source | Agency | Metrics | API | +|--------|--------|---------|-----| +| **FRED** | Federal Reserve Bank of St. Louis | Economic indicators, interest rates, employment | [fred.stlouisfed.org/docs/api](https://fred.stlouisfed.org/docs/api/) | +| **EIA** | Energy Information Administration | Gas prices, oil prices, energy data | [api.eia.gov](https://www.eia.gov/opendata/) | +| **Treasury FiscalData** | U.S. Department of Treasury | Federal debt, budget, spending | [fiscaldata.treasury.gov](https://fiscaldata.treasury.gov/api-documentation/) | +| **BLS** | Bureau of Labor Statistics | Employment, wages, prices | [api.bls.gov](https://www.bls.gov/developers/) | +| **Census Bureau** | U.S. Census Bureau | Demographics, housing, social indicators | [api.census.gov](https://api.census.gov/) | +| **CDC WONDER** | Centers for Disease Control | Mortality data | [wonder.cdc.gov](https://wonder.cdc.gov/) | +| **EPA AQS** | Environmental Protection Agency | Air quality data | [aqs.epa.gov/data/api](https://aqs.epa.gov/data/api/) | + +### Source Documentation + +Each data source has comprehensive documentation in `Data/sources/`: + +| ID | Source | Documentation | +|----|--------|---------------| +| DS-00004 | FRED Economic Wellbeing | [source.md](../sources/DS-00004—FRED_Economic_Wellbeing/source.md) | +| DS-00005 | CDC WONDER Mortality | [source.md](../sources/DS-00005—CDC_WONDER_Mortality/source.md) | +| DS-00006 | Census ACS Social Wellbeing | [source.md](../sources/DS-00006—Census_ACS_Social_Wellbeing/source.md) | +| DS-00007 | BLS JOLTS Labor Market | [source.md](../sources/DS-00007—BLS_JOLTS_Labor_Market/source.md) | +| DS-00008 | EPA Air Quality System | [source.md](../sources/DS-00008—EPA_Air_Quality_System/source.md) | +| DS-00009 | EIA Energy Data | [source.md](../sources/DS-00009—EIA_Energy_Data/source.md) | +| DS-00010 | Treasury FiscalData | [source.md](../sources/DS-00010—Treasury_FiscalData/source.md) | + +### Data Flow + +``` +Source Agencies (BEA, BLS, Treasury, etc.) + ↓ +FRED API (aggregates most series) + ↓ +update.ts automation script + ↓ +US-Common-Metrics.md (human-readable) +us-metrics-current.csv (machine-readable) +``` + +--- + +## Methodology + +### Metric Selection Criteria + +Metrics were selected based on: + +1. **Economic Significance**: Core indicators watched by economists, policymakers, markets +2. **Public Interest**: Commonly cited in news, policy discussions +3. **Authoritative Source**: Must come from official government agencies +4. **Regular Updates**: Sufficient update frequency to track changes +5. **Historical Availability**: Long time series for trend analysis + +### Data Aggregation Approach + +1. **FRED-First**: Most metrics sourced via FRED API, which standardizes access to data from multiple agencies +2. **Direct API**: Some metrics (gas prices, federal debt) fetched directly from source agency APIs +3. **No Transformation**: Values presented as-is from source; no smoothing, adjustment, or calculation beyond what source provides +4. **Full Attribution**: Every value includes source, FRED series ID (if applicable), and timestamp + +### Update Process + +1. `update.ts` fetches latest values from all APIs +2. Values written to `US-Common-Metrics.md` in appropriate cells +3. Current values exported to `us-metrics-current.csv` +4. Historical values appended to `us-metrics-historical.csv` +5. Update log written to `update.log` + +### Handling Revisions + +Many economic indicators are revised after initial release: + +- **Advance/Preliminary/Final**: GDP has three releases +- **Benchmark Revisions**: Employment data revised with annual benchmarks +- **Seasonal Factor Updates**: Seasonal adjustments recalculated annually + +We always show the **most recent available value**, which may differ from initially reported figures. + +--- + +## Access Conditions + +### License +All underlying data is **public domain** (U.S. government work). This aggregation is part of the Substrate project under MIT license. + +### API Keys Required + +All APIs are free but most require registration: + +| Source | Registration | Rate Limit | +|--------|--------------|------------| +| FRED | [Get key](https://fred.stlouisfed.org/docs/api/api_key.html) | 120 req/min | +| EIA | [Get key](https://www.eia.gov/opendata/register.php) | 1000 req/hour | +| Census | [Get key](https://api.census.gov/data/key_signup.html) | 500 req/day | +| BLS | Optional (higher limits) | 500 req/day (no key) | +| Treasury | None required | Reasonable use | +| CDC WONDER | None required | Fair use | +| EPA | Email request | 10 req/min | + +### Environment Variables + +```bash +export FRED_API_KEY="your_key" +export EIA_API_KEY="your_key" +export CENSUS_API_KEY="your_key" +export BLS_API_KEY="your_key" # Optional +``` + +--- + +## Known Limitations + +### Coverage Gaps + +1. **Stock Indices**: S&P 500, DJIA not always available via free APIs (licensing restrictions) +2. **Real-Time Data**: Most metrics have publication lag (see individual sources) +3. **Regional Data**: This dataset focuses on national metrics; state/local data available from sources directly + +### Methodological Limitations + +1. **Aggregation Layer**: We don't collect primary data; we aggregate from official sources +2. **Revision Timing**: We fetch current values; historical revisions may not be captured +3. **Definition Changes**: Source agencies sometimes change definitions (noted in source docs) + +### What This Dataset Is NOT + +- **Primary data collection** - we aggregate from authoritative sources +- **Forecasts or projections** - historical and current data only +- **Microdata** - aggregated indicators only; no individual records +- **Real-time trading data** - publication lags vary from hours to months + +--- + +## Integration with Substrate + +### Linked Components + +- **Problems**: Metrics quantify problems (e.g., PR-XXXXX - Economic Inequality → GINI Index) +- **Solutions**: Metrics track outcomes (e.g., SL-XXXXX - Employment Programs → Unemployment Rate) +- **Arguments**: Metrics provide evidence for claims +- **Plans**: Metrics define KPIs for success measurement + +### Usage in Arguments + +When citing metrics in Substrate Arguments: + +```markdown +**Claim**: Unemployment has decreased significantly since 2020. + +**Evidence**: +- Unemployment Rate (U-3): 3.7% (November 2024) +- Source: BLS/FRED (UNRATE) +- Comparison: 14.7% (April 2020, pandemic peak) +- Data: [US-Common-Metrics](../Data/US-Common-Metrics/US-Common-Metrics.md) +``` + +--- + +## Maintenance + +### Update Frequency + +```bash +# Manual update +cd Data/US-Common-Metrics +bun run update.ts + +# Scheduled update (via cron or GitHub Actions) +0 6 * * * cd /path/to/Substrate && bun run Data/US-Common-Metrics/update.ts +``` + +### Quality Checks + +1. **Completeness**: All metrics should have values (missing = API issue) +2. **Freshness**: Values should be within expected update window +3. **Consistency**: Values should be plausible (outlier detection) +4. **Sources**: All values should have source attribution + +### Contributing + +To add new metrics: + +1. Identify authoritative source +2. Add source documentation to `Data/sources/DS-00XXX—Name/source.md` +3. Add metric rows to `US-Common-Metrics.md` +4. Update `update.ts` to fetch new metrics +5. Submit PR with justification for addition + +--- + +## Credits + +**Created:** 2025-12-01 +**Author:** Daniel Miessler / Kai +**Framework:** Substrate (https://github.com/danielmiessler/Substrate) + +**Data Sources:** +- Federal Reserve Bank of St. Louis (FRED) +- Bureau of Labor Statistics (BLS) +- Bureau of Economic Analysis (BEA) +- U.S. Census Bureau +- Energy Information Administration (EIA) +- U.S. Department of Treasury +- Centers for Disease Control (CDC) +- Environmental Protection Agency (EPA) + +--- + +**END OF SOURCE DOCUMENTATION** diff --git a/Data/US-Common-Metrics/us-metrics-current.csv b/Data/US-Common-Metrics/us-metrics-current.csv new file mode 100644 index 0000000..96fda68 --- /dev/null +++ b/Data/US-Common-Metrics/us-metrics-current.csv @@ -0,0 +1,67 @@ +metric_id,metric_name,value,formatted_value,period,updated,source +GDPC1,"Real GDP",23770.976,"$23770.98B",2025-04-01,2025-12-02,"BEA/FRED" +GDP,"Nominal GDP",30485.729,"$30485.73B",2025-04-01,2025-12-02,"BEA/FRED" +A191RL1Q225SBEA,"GDP Growth Rate (QoQ)",3.8,"3.8%",2025-04-01,2025-12-02,"BEA/FRED" +A191RO1Q156NBEA,"GDP Growth Rate (YoY)",2.1,"2.1%",2025-04-01,2025-12-02,"BEA/FRED" +INDPRO,"Industrial Production",101.2704,"101.27",2025-08-01,2025-12-02,"Fed/FRED" +TCU,"Capacity Utilization",75.8401,"75.8%",2025-08-01,2025-12-02,"Fed/FRED" +DGORDER,"Durable Goods Orders",313653,"313653M",2025-09-01,2025-12-02,"Census/FRED" +RSAFS,"Retail Sales",733258,"$733258.0B",2025-09-01,2025-12-02,"Census/FRED" +CPIAUCSL,"CPI-U All Items",324.368,"324.368",2025-09-01,2025-12-02,"BLS/FRED" +CPILFESL,"Core CPI",330.542,"330.542",2025-09-01,2025-12-02,"BLS/FRED" +PCEPI,"PCE Price Index",127.285,"127.285",2025-08-01,2025-12-02,"BEA/FRED" +PCEPILFE,"Core PCE",126.705,"126.705",2025-08-01,2025-12-02,"BEA/FRED" +PPIACO,"Producer Price Index",262.344,"262.34",2025-09-01,2025-12-02,"BLS/FRED" +DCOILWTICO,"WTI Crude Oil",59.11,"$59.11",2025-11-24,2025-12-02,"EIA/FRED" +GAS_PRICE,"Gas Price (Regular)",3.061,"$3.061",2025-11-24,2025-12-02,"EIA" +UNRATE,"Unemployment Rate (U-3)",4.4,"4.4%",2025-09-01,2025-12-02,"BLS/FRED" +U6RATE,"Underemployment Rate (U-6)",8,"8.0%",2025-09-01,2025-12-02,"BLS/FRED" +PAYEMS,"Nonfarm Payrolls",159626,"159,626K",2025-09-01,2025-12-02,"BLS/FRED" +ICSA,"Initial Jobless Claims",216000,"216,000K",2025-11-22,2025-12-02,"DOL/FRED" +CCSA,"Continuing Claims",1960000,"1,960,000K",2025-11-15,2025-12-02,"DOL/FRED" +JTSJOL,"Job Openings (JOLTS)",7227,"7,227K",2025-08-01,2025-12-02,"BLS/FRED" +JTSQUR,"Quit Rate",1.9,"1.9%",2025-08-01,2025-12-02,"BLS/FRED" +JTSHIR,"Hire Rate",3.2,"3.2%",2025-08-01,2025-12-02,"BLS/FRED" +CIVPART,"Labor Force Participation",62.4,"62.4%",2025-09-01,2025-12-02,"BLS/FRED" +EMRATIO,"Employment-Population Ratio",59.7,"59.7%",2025-09-01,2025-12-02,"BLS/FRED" +CES0500000003,"Average Hourly Earnings",36.67,"$36.67",2025-09-01,2025-12-02,"BLS/FRED" +AWHAETP,"Average Weekly Hours",34.2,"34.2",2025-09-01,2025-12-02,"BLS/FRED" +MSPUS,"Median Home Sales Price",410800,"$410,800",2025-04-01,2025-12-02,"Census/FRED" +CSUSHPINSA,"Case-Shiller Index",328.938,"328.94",2025-09-01,2025-12-02,"S&P/FRED" +EXHOSLUSM495S,"Existing Home Sales",4100000,"4,100,000K",2025-10-01,2025-12-02,"NAR/FRED" +HSN1F,"New Home Sales",800,"800K",2025-08-01,2025-12-02,"Census/FRED" +HOUST,"Housing Starts",1307,"1,307K",2025-08-01,2025-12-02,"Census/FRED" +PERMIT,"Building Permits",1330,"1,330K",2025-08-01,2025-12-02,"Census/FRED" +MORTGAGE30US,"30-Year Mortgage Rate",6.23,"6.23%",2025-11-26,2025-12-02,"Freddie Mac/FRED" +MORTGAGE15US,"15-Year Mortgage Rate",5.51,"5.51%",2025-11-26,2025-12-02,"Freddie Mac/FRED" +UMCSENT,"Consumer Sentiment",53.6,"53.6",2025-10-01,2025-12-02,"UMich/FRED" +PI,"Personal Income",26279.9,"$26279.9B",2025-08-01,2025-12-02,"BEA/FRED" +DSPI,"Disposable Personal Income",23033.5,"$23033.5B",2025-08-01,2025-12-02,"BEA/FRED" +PSAVERT,"Personal Saving Rate",4.6,"4.6%",2025-08-01,2025-12-02,"BEA/FRED" +TOTALSL,"Consumer Credit Outstanding",5076636.01,"$5076636.0B",2025-09-01,2025-12-02,"Fed/FRED" +DRCCLACBS,"Credit Card Delinquency",2.98,"2.98%",2025-07-01,2025-12-02,"Fed/FRED" +TDSP,"Debt Service Ratio",11.249194,"11.25%",2025-04-01,2025-12-02,"Fed/FRED" +TOTALSA,"Auto Sales",16.802,"16.80M",2025-08-01,2025-12-02,"BEA/FRED" +FEDFUNDS,"Fed Funds Rate",3.88,"3.88%",2025-11-01,2025-12-02,"Fed/FRED" +DFEDTARU,"Fed Funds Target (Upper)",4,"4.00%",2025-12-01,2025-12-02,"Fed/FRED" +DGS10,"10-Year Treasury",4.02,"4.02%",2025-11-28,2025-12-02,"Treasury/FRED" +DGS2,"2-Year Treasury",3.47,"3.47%",2025-11-28,2025-12-02,"Treasury/FRED" +T10Y2Y,"10Y-2Y Spread",0.55,"0.55%",2025-11-28,2025-12-02,"FRED" +DGS30,"30-Year Treasury",4.67,"4.67%",2025-11-28,2025-12-02,"Treasury/FRED" +DTB3,"3-Month T-Bill",3.73,"3.73%",2025-11-28,2025-12-02,"Treasury/FRED" +STLFSI4,"Financial Stress Index",-0.1331,"-0.133",2025-11-21,2025-12-02,"StL Fed/FRED" +SP500,"S&P 500",6812.63,"6,812.63",2025-12-01,2025-12-02,"S&P/FRED" +VIXCLS,"VIX",16.35,"16.35",2025-11-28,2025-12-02,"CBOE/FRED" +BOPGSTB,"Trade Balance",-59550,"$-59550.0B",2025-08-01,2025-12-02,"Census/BEA/FRED" +BOPGEXP,"Exports",179019,"$179019.0B",2025-08-01,2025-12-02,"Census/FRED" +BOPGIMP,"Imports",264627,"$264627.0B",2025-08-01,2025-12-02,"Census/FRED" +DTWEXBGS,"USD Index",121.4288,"121.43",2025-11-28,2025-12-02,"Fed/FRED" +DEXUSEU,"USD/EUR",1.16,"1.1600",2025-11-28,2025-12-02,"Fed/FRED" +GFDEBTN,"Federal Debt Total",36211469,"$36211469.000T",2025-04-01,2025-12-02,"Treasury/FRED" +GFDEGDQ188S,"Debt-to-GDP Ratio",118.78171,"118.8%",2025-04-01,2025-12-02,"FRED" +FGRECPT,"Federal Receipts",5622.803,"$5622.8B",2025-04-01,2025-12-02,"Treasury/FRED" +FGEXPND,"Federal Expenditures",7496.339,"$7496.3B",2025-04-01,2025-12-02,"Treasury/FRED" +TREASURY_DEBT,"Total Public Debt",38.39621149761365,"$38.396T",2025-11-28,2025-12-02,"Treasury" +POPTHM,"US Population",342555,"342555.0M",2025-08-01,2025-12-02,"Census/FRED" +SIPOVGINIUSA,"GINI Index",41.8,"41.800",2023-01-01,2025-12-02,"Census/FRED" +MEHOINUSA672N,"Median Household Income",83730,"$83,730",2024-01-01,2025-12-02,"Census/FRED" diff --git a/Data/US-Common-Metrics/us-metrics-historical.csv b/Data/US-Common-Metrics/us-metrics-historical.csv new file mode 100644 index 0000000..0e3a7c4 --- /dev/null +++ b/Data/US-Common-Metrics/us-metrics-historical.csv @@ -0,0 +1,329 @@ +fetch_timestamp,metric_id,value,period +2025-12-02T03:11:11.408Z,GDPC1,23770.976,2025-04-01 +2025-12-02T03:11:11.408Z,GDP,30485.729,2025-04-01 +2025-12-02T03:11:11.408Z,A191RL1Q225SBEA,3.8,2025-04-01 +2025-12-02T03:11:11.408Z,A191RO1Q156NBEA,2.1,2025-04-01 +2025-12-02T03:11:11.408Z,INDPRO,101.2704,2025-08-01 +2025-12-02T03:11:11.408Z,TCU,75.8401,2025-08-01 +2025-12-02T03:11:11.408Z,DGORDER,313653,2025-09-01 +2025-12-02T03:11:11.408Z,RSAFS,733258,2025-09-01 +2025-12-02T03:11:11.408Z,CPIAUCSL,324.368,2025-09-01 +2025-12-02T03:11:11.408Z,CPILFESL,330.542,2025-09-01 +2025-12-02T03:11:11.408Z,PCEPI,127.285,2025-08-01 +2025-12-02T03:11:11.408Z,PCEPILFE,126.705,2025-08-01 +2025-12-02T03:11:11.408Z,PPIACO,262.344,2025-09-01 +2025-12-02T03:11:11.408Z,DCOILWTICO,59.11,2025-11-24 +2025-12-02T03:11:11.408Z,UNRATE,4.4,2025-09-01 +2025-12-02T03:11:11.408Z,U6RATE,8,2025-09-01 +2025-12-02T03:11:11.408Z,PAYEMS,159626,2025-09-01 +2025-12-02T03:11:11.408Z,ICSA,216000,2025-11-22 +2025-12-02T03:11:11.408Z,CCSA,1960000,2025-11-15 +2025-12-02T03:11:11.408Z,JTSJOL,7227,2025-08-01 +2025-12-02T03:11:11.408Z,JTSQUR,1.9,2025-08-01 +2025-12-02T03:11:11.408Z,JTSHIR,3.2,2025-08-01 +2025-12-02T03:11:11.408Z,CIVPART,62.4,2025-09-01 +2025-12-02T03:11:11.408Z,EMRATIO,59.7,2025-09-01 +2025-12-02T03:11:11.408Z,CES0500000003,36.67,2025-09-01 +2025-12-02T03:11:11.408Z,AWHAETP,34.2,2025-09-01 +2025-12-02T03:11:11.408Z,MSPUS,410800,2025-04-01 +2025-12-02T03:11:11.408Z,CSUSHPINSA,328.938,2025-09-01 +2025-12-02T03:11:11.408Z,EXHOSLUSM495S,4100000,2025-10-01 +2025-12-02T03:11:11.408Z,HSN1F,800,2025-08-01 +2025-12-02T03:11:11.408Z,HOUST,1307,2025-08-01 +2025-12-02T03:11:11.408Z,PERMIT,1330,2025-08-01 +2025-12-02T03:11:11.408Z,MORTGAGE30US,6.23,2025-11-26 +2025-12-02T03:11:11.408Z,MORTGAGE15US,5.51,2025-11-26 +2025-12-02T03:11:11.408Z,UMCSENT,53.6,2025-10-01 +2025-12-02T03:11:11.408Z,PI,26279.9,2025-08-01 +2025-12-02T03:11:11.408Z,DSPI,23033.5,2025-08-01 +2025-12-02T03:11:11.408Z,PSAVERT,4.6,2025-08-01 +2025-12-02T03:11:11.408Z,TOTALSL,5076636.01,2025-09-01 +2025-12-02T03:11:11.408Z,DRCCLACBS,2.98,2025-07-01 +2025-12-02T03:11:11.408Z,TDSP,11.249194,2025-04-01 +2025-12-02T03:11:11.408Z,TOTALSA,16.802,2025-08-01 +2025-12-02T03:11:11.408Z,FEDFUNDS,3.88,2025-11-01 +2025-12-02T03:11:11.408Z,DFEDTARU,4,2025-12-01 +2025-12-02T03:11:11.408Z,DGS10,4.02,2025-11-28 +2025-12-02T03:11:11.408Z,DGS2,3.47,2025-11-28 +2025-12-02T03:11:11.408Z,T10Y2Y,0.55,2025-11-28 +2025-12-02T03:11:11.408Z,DGS30,4.67,2025-11-28 +2025-12-02T03:11:11.408Z,DTB3,3.73,2025-11-28 +2025-12-02T03:11:11.408Z,STLFSI4,-0.1331,2025-11-21 +2025-12-02T03:11:11.408Z,SP500,6812.63,2025-12-01 +2025-12-02T03:11:11.408Z,VIXCLS,16.35,2025-11-28 +2025-12-02T03:11:11.408Z,BOPGSTB,-59550,2025-08-01 +2025-12-02T03:11:11.408Z,BOPGEXP,179019,2025-08-01 +2025-12-02T03:11:11.408Z,BOPGIMP,264627,2025-08-01 +2025-12-02T03:11:11.408Z,DTWEXBGS,121.4288,2025-11-28 +2025-12-02T03:11:11.408Z,DEXUSEU,1.16,2025-11-28 +2025-12-02T03:11:11.408Z,GFDEBTN,36211469,2025-04-01 +2025-12-02T03:11:11.408Z,GFDEGDQ188S,118.78171,2025-04-01 +2025-12-02T03:11:11.408Z,FGRECPT,5622.803,2025-04-01 +2025-12-02T03:11:11.408Z,FGEXPND,7496.339,2025-04-01 +2025-12-02T03:11:11.408Z,TREASURY_DEBT,38.39621149761365,2025-11-28 +2025-12-02T03:11:11.408Z,POPTHM,342555,2025-08-01 +2025-12-02T03:11:11.408Z,SIPOVGINIUSA,41.8,2023-01-01 +2025-12-02T03:11:11.408Z,MEHOINUSA672N,83730,2024-01-01 +2025-12-02T03:13:11.786Z,GDPC1,23770.976,2025-04-01 +2025-12-02T03:13:11.786Z,GDP,30485.729,2025-04-01 +2025-12-02T03:13:11.786Z,A191RL1Q225SBEA,3.8,2025-04-01 +2025-12-02T03:13:11.786Z,A191RO1Q156NBEA,2.1,2025-04-01 +2025-12-02T03:13:11.786Z,INDPRO,101.2704,2025-08-01 +2025-12-02T03:13:11.786Z,TCU,75.8401,2025-08-01 +2025-12-02T03:13:11.786Z,DGORDER,313653,2025-09-01 +2025-12-02T03:13:11.786Z,RSAFS,733258,2025-09-01 +2025-12-02T03:13:11.786Z,CPIAUCSL,324.368,2025-09-01 +2025-12-02T03:13:11.786Z,CPILFESL,330.542,2025-09-01 +2025-12-02T03:13:11.786Z,PCEPI,127.285,2025-08-01 +2025-12-02T03:13:11.786Z,PCEPILFE,126.705,2025-08-01 +2025-12-02T03:13:11.786Z,PPIACO,262.344,2025-09-01 +2025-12-02T03:13:11.786Z,DCOILWTICO,59.11,2025-11-24 +2025-12-02T03:13:11.786Z,UNRATE,4.4,2025-09-01 +2025-12-02T03:13:11.786Z,U6RATE,8,2025-09-01 +2025-12-02T03:13:11.786Z,PAYEMS,159626,2025-09-01 +2025-12-02T03:13:11.786Z,ICSA,216000,2025-11-22 +2025-12-02T03:13:11.786Z,CCSA,1960000,2025-11-15 +2025-12-02T03:13:11.786Z,JTSJOL,7227,2025-08-01 +2025-12-02T03:13:11.786Z,JTSQUR,1.9,2025-08-01 +2025-12-02T03:13:11.786Z,JTSHIR,3.2,2025-08-01 +2025-12-02T03:13:11.786Z,CIVPART,62.4,2025-09-01 +2025-12-02T03:13:11.786Z,EMRATIO,59.7,2025-09-01 +2025-12-02T03:13:11.786Z,CES0500000003,36.67,2025-09-01 +2025-12-02T03:13:11.786Z,AWHAETP,34.2,2025-09-01 +2025-12-02T03:13:11.786Z,MSPUS,410800,2025-04-01 +2025-12-02T03:13:11.786Z,CSUSHPINSA,328.938,2025-09-01 +2025-12-02T03:13:11.786Z,EXHOSLUSM495S,4100000,2025-10-01 +2025-12-02T03:13:11.786Z,HSN1F,800,2025-08-01 +2025-12-02T03:13:11.786Z,HOUST,1307,2025-08-01 +2025-12-02T03:13:11.786Z,PERMIT,1330,2025-08-01 +2025-12-02T03:13:11.786Z,MORTGAGE30US,6.23,2025-11-26 +2025-12-02T03:13:11.786Z,MORTGAGE15US,5.51,2025-11-26 +2025-12-02T03:13:11.786Z,UMCSENT,53.6,2025-10-01 +2025-12-02T03:13:11.786Z,PI,26279.9,2025-08-01 +2025-12-02T03:13:11.786Z,DSPI,23033.5,2025-08-01 +2025-12-02T03:13:11.786Z,PSAVERT,4.6,2025-08-01 +2025-12-02T03:13:11.786Z,TOTALSL,5076636.01,2025-09-01 +2025-12-02T03:13:11.786Z,DRCCLACBS,2.98,2025-07-01 +2025-12-02T03:13:11.786Z,TDSP,11.249194,2025-04-01 +2025-12-02T03:13:11.786Z,TOTALSA,16.802,2025-08-01 +2025-12-02T03:13:11.786Z,FEDFUNDS,3.88,2025-11-01 +2025-12-02T03:13:11.786Z,DFEDTARU,4,2025-12-01 +2025-12-02T03:13:11.786Z,DGS10,4.02,2025-11-28 +2025-12-02T03:13:11.786Z,DGS2,3.47,2025-11-28 +2025-12-02T03:13:11.786Z,T10Y2Y,0.55,2025-11-28 +2025-12-02T03:13:11.786Z,DGS30,4.67,2025-11-28 +2025-12-02T03:13:11.786Z,DTB3,3.73,2025-11-28 +2025-12-02T03:13:11.786Z,STLFSI4,-0.1331,2025-11-21 +2025-12-02T03:13:11.786Z,SP500,6812.63,2025-12-01 +2025-12-02T03:13:11.786Z,VIXCLS,16.35,2025-11-28 +2025-12-02T03:13:11.786Z,BOPGSTB,-59550,2025-08-01 +2025-12-02T03:13:11.786Z,BOPGEXP,179019,2025-08-01 +2025-12-02T03:13:11.786Z,BOPGIMP,264627,2025-08-01 +2025-12-02T03:13:11.786Z,DTWEXBGS,121.4288,2025-11-28 +2025-12-02T03:13:11.786Z,DEXUSEU,1.16,2025-11-28 +2025-12-02T03:13:11.786Z,GFDEBTN,36211469,2025-04-01 +2025-12-02T03:13:11.786Z,GFDEGDQ188S,118.78171,2025-04-01 +2025-12-02T03:13:11.786Z,FGRECPT,5622.803,2025-04-01 +2025-12-02T03:13:11.786Z,FGEXPND,7496.339,2025-04-01 +2025-12-02T03:13:11.786Z,TREASURY_DEBT,38.39621149761365,2025-11-28 +2025-12-02T03:13:11.786Z,POPTHM,342555,2025-08-01 +2025-12-02T03:13:11.786Z,SIPOVGINIUSA,41.8,2023-01-01 +2025-12-02T03:13:11.786Z,MEHOINUSA672N,83730,2024-01-01 +2025-12-02T03:15:27.132Z,GDPC1,23770.976,2025-04-01 +2025-12-02T03:15:27.132Z,GDP,30485.729,2025-04-01 +2025-12-02T03:15:27.132Z,A191RL1Q225SBEA,3.8,2025-04-01 +2025-12-02T03:15:27.132Z,A191RO1Q156NBEA,2.1,2025-04-01 +2025-12-02T03:15:27.132Z,INDPRO,101.2704,2025-08-01 +2025-12-02T03:15:27.132Z,TCU,75.8401,2025-08-01 +2025-12-02T03:15:27.132Z,DGORDER,313653,2025-09-01 +2025-12-02T03:15:27.132Z,RSAFS,733258,2025-09-01 +2025-12-02T03:15:27.132Z,CPIAUCSL,324.368,2025-09-01 +2025-12-02T03:15:27.132Z,CPILFESL,330.542,2025-09-01 +2025-12-02T03:15:27.132Z,PCEPI,127.285,2025-08-01 +2025-12-02T03:15:27.132Z,PCEPILFE,126.705,2025-08-01 +2025-12-02T03:15:27.132Z,PPIACO,262.344,2025-09-01 +2025-12-02T03:15:27.132Z,DCOILWTICO,59.11,2025-11-24 +2025-12-02T03:15:27.132Z,GAS_PRICE,3.061,2025-11-24 +2025-12-02T03:15:27.132Z,UNRATE,4.4,2025-09-01 +2025-12-02T03:15:27.132Z,U6RATE,8,2025-09-01 +2025-12-02T03:15:27.132Z,PAYEMS,159626,2025-09-01 +2025-12-02T03:15:27.132Z,ICSA,216000,2025-11-22 +2025-12-02T03:15:27.132Z,CCSA,1960000,2025-11-15 +2025-12-02T03:15:27.132Z,JTSJOL,7227,2025-08-01 +2025-12-02T03:15:27.132Z,JTSQUR,1.9,2025-08-01 +2025-12-02T03:15:27.132Z,JTSHIR,3.2,2025-08-01 +2025-12-02T03:15:27.132Z,CIVPART,62.4,2025-09-01 +2025-12-02T03:15:27.132Z,EMRATIO,59.7,2025-09-01 +2025-12-02T03:15:27.132Z,CES0500000003,36.67,2025-09-01 +2025-12-02T03:15:27.132Z,AWHAETP,34.2,2025-09-01 +2025-12-02T03:15:27.132Z,MSPUS,410800,2025-04-01 +2025-12-02T03:15:27.132Z,CSUSHPINSA,328.938,2025-09-01 +2025-12-02T03:15:27.132Z,EXHOSLUSM495S,4100000,2025-10-01 +2025-12-02T03:15:27.132Z,HSN1F,800,2025-08-01 +2025-12-02T03:15:27.132Z,HOUST,1307,2025-08-01 +2025-12-02T03:15:27.132Z,PERMIT,1330,2025-08-01 +2025-12-02T03:15:27.132Z,MORTGAGE30US,6.23,2025-11-26 +2025-12-02T03:15:27.132Z,MORTGAGE15US,5.51,2025-11-26 +2025-12-02T03:15:27.132Z,UMCSENT,53.6,2025-10-01 +2025-12-02T03:15:27.132Z,PI,26279.9,2025-08-01 +2025-12-02T03:15:27.132Z,DSPI,23033.5,2025-08-01 +2025-12-02T03:15:27.132Z,PSAVERT,4.6,2025-08-01 +2025-12-02T03:15:27.132Z,TOTALSL,5076636.01,2025-09-01 +2025-12-02T03:15:27.132Z,DRCCLACBS,2.98,2025-07-01 +2025-12-02T03:15:27.132Z,TDSP,11.249194,2025-04-01 +2025-12-02T03:15:27.132Z,TOTALSA,16.802,2025-08-01 +2025-12-02T03:15:27.132Z,FEDFUNDS,3.88,2025-11-01 +2025-12-02T03:15:27.132Z,DFEDTARU,4,2025-12-01 +2025-12-02T03:15:27.132Z,DGS10,4.02,2025-11-28 +2025-12-02T03:15:27.132Z,DGS2,3.47,2025-11-28 +2025-12-02T03:15:27.132Z,T10Y2Y,0.55,2025-11-28 +2025-12-02T03:15:27.132Z,DGS30,4.67,2025-11-28 +2025-12-02T03:15:27.132Z,DTB3,3.73,2025-11-28 +2025-12-02T03:15:27.132Z,STLFSI4,-0.1331,2025-11-21 +2025-12-02T03:15:27.132Z,SP500,6812.63,2025-12-01 +2025-12-02T03:15:27.132Z,VIXCLS,16.35,2025-11-28 +2025-12-02T03:15:27.132Z,BOPGSTB,-59550,2025-08-01 +2025-12-02T03:15:27.132Z,BOPGEXP,179019,2025-08-01 +2025-12-02T03:15:27.132Z,BOPGIMP,264627,2025-08-01 +2025-12-02T03:15:27.132Z,DTWEXBGS,121.4288,2025-11-28 +2025-12-02T03:15:27.132Z,DEXUSEU,1.16,2025-11-28 +2025-12-02T03:15:27.132Z,GFDEBTN,36211469,2025-04-01 +2025-12-02T03:15:27.132Z,GFDEGDQ188S,118.78171,2025-04-01 +2025-12-02T03:15:27.132Z,FGRECPT,5622.803,2025-04-01 +2025-12-02T03:15:27.132Z,FGEXPND,7496.339,2025-04-01 +2025-12-02T03:15:27.132Z,TREASURY_DEBT,38.39621149761365,2025-11-28 +2025-12-02T03:15:27.132Z,POPTHM,342555,2025-08-01 +2025-12-02T03:15:27.132Z,SIPOVGINIUSA,41.8,2023-01-01 +2025-12-02T03:15:27.132Z,MEHOINUSA672N,83730,2024-01-01 +2025-12-02T03:17:03.588Z,GDPC1,23770.976,2025-04-01 +2025-12-02T03:17:03.588Z,GDP,30485.729,2025-04-01 +2025-12-02T03:17:03.588Z,A191RL1Q225SBEA,3.8,2025-04-01 +2025-12-02T03:17:03.588Z,A191RO1Q156NBEA,2.1,2025-04-01 +2025-12-02T03:17:03.588Z,INDPRO,101.2704,2025-08-01 +2025-12-02T03:17:03.588Z,TCU,75.8401,2025-08-01 +2025-12-02T03:17:03.588Z,DGORDER,313653,2025-09-01 +2025-12-02T03:17:03.588Z,RSAFS,733258,2025-09-01 +2025-12-02T03:17:03.588Z,CPIAUCSL,324.368,2025-09-01 +2025-12-02T03:17:03.588Z,CPILFESL,330.542,2025-09-01 +2025-12-02T03:17:03.588Z,PCEPI,127.285,2025-08-01 +2025-12-02T03:17:03.588Z,PCEPILFE,126.705,2025-08-01 +2025-12-02T03:17:03.588Z,PPIACO,262.344,2025-09-01 +2025-12-02T03:17:03.588Z,DCOILWTICO,59.11,2025-11-24 +2025-12-02T03:17:03.588Z,GAS_PRICE,3.061,2025-11-24 +2025-12-02T03:17:03.588Z,UNRATE,4.4,2025-09-01 +2025-12-02T03:17:03.588Z,U6RATE,8,2025-09-01 +2025-12-02T03:17:03.588Z,PAYEMS,159626,2025-09-01 +2025-12-02T03:17:03.588Z,ICSA,216000,2025-11-22 +2025-12-02T03:17:03.588Z,CCSA,1960000,2025-11-15 +2025-12-02T03:17:03.588Z,JTSJOL,7227,2025-08-01 +2025-12-02T03:17:03.588Z,JTSQUR,1.9,2025-08-01 +2025-12-02T03:17:03.588Z,JTSHIR,3.2,2025-08-01 +2025-12-02T03:17:03.588Z,CIVPART,62.4,2025-09-01 +2025-12-02T03:17:03.588Z,EMRATIO,59.7,2025-09-01 +2025-12-02T03:17:03.588Z,CES0500000003,36.67,2025-09-01 +2025-12-02T03:17:03.588Z,AWHAETP,34.2,2025-09-01 +2025-12-02T03:17:03.588Z,MSPUS,410800,2025-04-01 +2025-12-02T03:17:03.588Z,CSUSHPINSA,328.938,2025-09-01 +2025-12-02T03:17:03.588Z,EXHOSLUSM495S,4100000,2025-10-01 +2025-12-02T03:17:03.588Z,HSN1F,800,2025-08-01 +2025-12-02T03:17:03.588Z,HOUST,1307,2025-08-01 +2025-12-02T03:17:03.588Z,PERMIT,1330,2025-08-01 +2025-12-02T03:17:03.588Z,MORTGAGE30US,6.23,2025-11-26 +2025-12-02T03:17:03.588Z,MORTGAGE15US,5.51,2025-11-26 +2025-12-02T03:17:03.588Z,UMCSENT,53.6,2025-10-01 +2025-12-02T03:17:03.588Z,PI,26279.9,2025-08-01 +2025-12-02T03:17:03.588Z,DSPI,23033.5,2025-08-01 +2025-12-02T03:17:03.588Z,PSAVERT,4.6,2025-08-01 +2025-12-02T03:17:03.588Z,TOTALSL,5076636.01,2025-09-01 +2025-12-02T03:17:03.588Z,DRCCLACBS,2.98,2025-07-01 +2025-12-02T03:17:03.588Z,TDSP,11.249194,2025-04-01 +2025-12-02T03:17:03.588Z,TOTALSA,16.802,2025-08-01 +2025-12-02T03:17:03.588Z,FEDFUNDS,3.88,2025-11-01 +2025-12-02T03:17:03.588Z,DFEDTARU,4,2025-12-01 +2025-12-02T03:17:03.588Z,DGS10,4.02,2025-11-28 +2025-12-02T03:17:03.588Z,DGS2,3.47,2025-11-28 +2025-12-02T03:17:03.588Z,T10Y2Y,0.55,2025-11-28 +2025-12-02T03:17:03.588Z,DGS30,4.67,2025-11-28 +2025-12-02T03:17:03.588Z,DTB3,3.73,2025-11-28 +2025-12-02T03:17:03.588Z,STLFSI4,-0.1331,2025-11-21 +2025-12-02T03:17:03.588Z,SP500,6812.63,2025-12-01 +2025-12-02T03:17:03.588Z,VIXCLS,16.35,2025-11-28 +2025-12-02T03:17:03.588Z,BOPGSTB,-59550,2025-08-01 +2025-12-02T03:17:03.588Z,BOPGEXP,179019,2025-08-01 +2025-12-02T03:17:03.588Z,BOPGIMP,264627,2025-08-01 +2025-12-02T03:17:03.588Z,DTWEXBGS,121.4288,2025-11-28 +2025-12-02T03:17:03.588Z,DEXUSEU,1.16,2025-11-28 +2025-12-02T03:17:03.588Z,GFDEBTN,36211469,2025-04-01 +2025-12-02T03:17:03.588Z,GFDEGDQ188S,118.78171,2025-04-01 +2025-12-02T03:17:03.588Z,FGRECPT,5622.803,2025-04-01 +2025-12-02T03:17:03.588Z,FGEXPND,7496.339,2025-04-01 +2025-12-02T03:17:03.588Z,TREASURY_DEBT,38.39621149761365,2025-11-28 +2025-12-02T03:17:03.588Z,POPTHM,342555,2025-08-01 +2025-12-02T03:17:03.588Z,SIPOVGINIUSA,41.8,2023-01-01 +2025-12-02T03:17:03.588Z,MEHOINUSA672N,83730,2024-01-01 +2025-12-02T03:17:58.606Z,GDPC1,23770.976,2025-04-01 +2025-12-02T03:17:58.606Z,GDP,30485.729,2025-04-01 +2025-12-02T03:17:58.606Z,A191RL1Q225SBEA,3.8,2025-04-01 +2025-12-02T03:17:58.606Z,A191RO1Q156NBEA,2.1,2025-04-01 +2025-12-02T03:17:58.606Z,INDPRO,101.2704,2025-08-01 +2025-12-02T03:17:58.606Z,TCU,75.8401,2025-08-01 +2025-12-02T03:17:58.606Z,DGORDER,313653,2025-09-01 +2025-12-02T03:17:58.606Z,RSAFS,733258,2025-09-01 +2025-12-02T03:17:58.606Z,CPIAUCSL,324.368,2025-09-01 +2025-12-02T03:17:58.606Z,CPILFESL,330.542,2025-09-01 +2025-12-02T03:17:58.606Z,PCEPI,127.285,2025-08-01 +2025-12-02T03:17:58.606Z,PCEPILFE,126.705,2025-08-01 +2025-12-02T03:17:58.606Z,PPIACO,262.344,2025-09-01 +2025-12-02T03:17:58.606Z,DCOILWTICO,59.11,2025-11-24 +2025-12-02T03:17:58.606Z,GAS_PRICE,3.061,2025-11-24 +2025-12-02T03:17:58.606Z,UNRATE,4.4,2025-09-01 +2025-12-02T03:17:58.606Z,U6RATE,8,2025-09-01 +2025-12-02T03:17:58.606Z,PAYEMS,159626,2025-09-01 +2025-12-02T03:17:58.606Z,ICSA,216000,2025-11-22 +2025-12-02T03:17:58.606Z,CCSA,1960000,2025-11-15 +2025-12-02T03:17:58.606Z,JTSJOL,7227,2025-08-01 +2025-12-02T03:17:58.606Z,JTSQUR,1.9,2025-08-01 +2025-12-02T03:17:58.606Z,JTSHIR,3.2,2025-08-01 +2025-12-02T03:17:58.606Z,CIVPART,62.4,2025-09-01 +2025-12-02T03:17:58.606Z,EMRATIO,59.7,2025-09-01 +2025-12-02T03:17:58.606Z,CES0500000003,36.67,2025-09-01 +2025-12-02T03:17:58.606Z,AWHAETP,34.2,2025-09-01 +2025-12-02T03:17:58.606Z,MSPUS,410800,2025-04-01 +2025-12-02T03:17:58.606Z,CSUSHPINSA,328.938,2025-09-01 +2025-12-02T03:17:58.606Z,EXHOSLUSM495S,4100000,2025-10-01 +2025-12-02T03:17:58.606Z,HSN1F,800,2025-08-01 +2025-12-02T03:17:58.606Z,HOUST,1307,2025-08-01 +2025-12-02T03:17:58.606Z,PERMIT,1330,2025-08-01 +2025-12-02T03:17:58.606Z,MORTGAGE30US,6.23,2025-11-26 +2025-12-02T03:17:58.606Z,MORTGAGE15US,5.51,2025-11-26 +2025-12-02T03:17:58.606Z,UMCSENT,53.6,2025-10-01 +2025-12-02T03:17:58.606Z,PI,26279.9,2025-08-01 +2025-12-02T03:17:58.606Z,DSPI,23033.5,2025-08-01 +2025-12-02T03:17:58.606Z,PSAVERT,4.6,2025-08-01 +2025-12-02T03:17:58.606Z,TOTALSL,5076636.01,2025-09-01 +2025-12-02T03:17:58.606Z,DRCCLACBS,2.98,2025-07-01 +2025-12-02T03:17:58.606Z,TDSP,11.249194,2025-04-01 +2025-12-02T03:17:58.606Z,TOTALSA,16.802,2025-08-01 +2025-12-02T03:17:58.606Z,FEDFUNDS,3.88,2025-11-01 +2025-12-02T03:17:58.606Z,DFEDTARU,4,2025-12-01 +2025-12-02T03:17:58.606Z,DGS10,4.02,2025-11-28 +2025-12-02T03:17:58.606Z,DGS2,3.47,2025-11-28 +2025-12-02T03:17:58.606Z,T10Y2Y,0.55,2025-11-28 +2025-12-02T03:17:58.606Z,DGS30,4.67,2025-11-28 +2025-12-02T03:17:58.606Z,DTB3,3.73,2025-11-28 +2025-12-02T03:17:58.606Z,STLFSI4,-0.1331,2025-11-21 +2025-12-02T03:17:58.606Z,SP500,6812.63,2025-12-01 +2025-12-02T03:17:58.606Z,VIXCLS,16.35,2025-11-28 +2025-12-02T03:17:58.606Z,BOPGSTB,-59550,2025-08-01 +2025-12-02T03:17:58.606Z,BOPGEXP,179019,2025-08-01 +2025-12-02T03:17:58.606Z,BOPGIMP,264627,2025-08-01 +2025-12-02T03:17:58.606Z,DTWEXBGS,121.4288,2025-11-28 +2025-12-02T03:17:58.606Z,DEXUSEU,1.16,2025-11-28 +2025-12-02T03:17:58.606Z,GFDEBTN,36211469,2025-04-01 +2025-12-02T03:17:58.606Z,GFDEGDQ188S,118.78171,2025-04-01 +2025-12-02T03:17:58.606Z,FGRECPT,5622.803,2025-04-01 +2025-12-02T03:17:58.606Z,FGEXPND,7496.339,2025-04-01 +2025-12-02T03:17:58.606Z,TREASURY_DEBT,38.39621149761365,2025-11-28 +2025-12-02T03:17:58.606Z,POPTHM,342555,2025-08-01 +2025-12-02T03:17:58.606Z,SIPOVGINIUSA,41.8,2023-01-01 +2025-12-02T03:17:58.606Z,MEHOINUSA672N,83730,2024-01-01 diff --git a/Data-Sources/DS-00001—WHO_Global_Health_Observatory/source.md b/Data/sources/DS-00001—WHO_Global_Health_Observatory/source.md similarity index 100% rename from Data-Sources/DS-00001—WHO_Global_Health_Observatory/source.md rename to Data/sources/DS-00001—WHO_Global_Health_Observatory/source.md diff --git a/Data-Sources/DS-00001—WHO_Global_Health_Observatory/update.ts b/Data/sources/DS-00001—WHO_Global_Health_Observatory/update.ts similarity index 100% rename from Data-Sources/DS-00001—WHO_Global_Health_Observatory/update.ts rename to Data/sources/DS-00001—WHO_Global_Health_Observatory/update.ts diff --git a/Data-Sources/DS-00002—UN_SDG_Indicators/source.md b/Data/sources/DS-00002—UN_SDG_Indicators/source.md similarity index 100% rename from Data-Sources/DS-00002—UN_SDG_Indicators/source.md rename to Data/sources/DS-00002—UN_SDG_Indicators/source.md diff --git a/Data-Sources/DS-00002—UN_SDG_Indicators/update.ts b/Data/sources/DS-00002—UN_SDG_Indicators/update.ts similarity index 100% rename from Data-Sources/DS-00002—UN_SDG_Indicators/update.ts rename to Data/sources/DS-00002—UN_SDG_Indicators/update.ts diff --git a/Data-Sources/DS-00003—World_Bank_Open_Data/source.md b/Data/sources/DS-00003—World_Bank_Open_Data/source.md similarity index 100% rename from Data-Sources/DS-00003—World_Bank_Open_Data/source.md rename to Data/sources/DS-00003—World_Bank_Open_Data/source.md diff --git a/Data-Sources/DS-00003—World_Bank_Open_Data/update.ts b/Data/sources/DS-00003—World_Bank_Open_Data/update.ts similarity index 100% rename from Data-Sources/DS-00003—World_Bank_Open_Data/update.ts rename to Data/sources/DS-00003—World_Bank_Open_Data/update.ts diff --git a/Data-Sources/DS-00004—FRED_Economic_Wellbeing/VALIDATION.md b/Data/sources/DS-00004—FRED_Economic_Wellbeing/VALIDATION.md similarity index 100% rename from Data-Sources/DS-00004—FRED_Economic_Wellbeing/VALIDATION.md rename to Data/sources/DS-00004—FRED_Economic_Wellbeing/VALIDATION.md diff --git a/Data-Sources/DS-00004—FRED_Economic_Wellbeing/data/README.md b/Data/sources/DS-00004—FRED_Economic_Wellbeing/data/README.md similarity index 100% rename from Data-Sources/DS-00004—FRED_Economic_Wellbeing/data/README.md rename to Data/sources/DS-00004—FRED_Economic_Wellbeing/data/README.md diff --git a/Data-Sources/DS-00004—FRED_Economic_Wellbeing/source.md b/Data/sources/DS-00004—FRED_Economic_Wellbeing/source.md similarity index 100% rename from Data-Sources/DS-00004—FRED_Economic_Wellbeing/source.md rename to Data/sources/DS-00004—FRED_Economic_Wellbeing/source.md diff --git a/Data-Sources/DS-00004—FRED_Economic_Wellbeing/update.log b/Data/sources/DS-00004—FRED_Economic_Wellbeing/update.log similarity index 100% rename from Data-Sources/DS-00004—FRED_Economic_Wellbeing/update.log rename to Data/sources/DS-00004—FRED_Economic_Wellbeing/update.log diff --git a/Data-Sources/DS-00004—FRED_Economic_Wellbeing/update.ts b/Data/sources/DS-00004—FRED_Economic_Wellbeing/update.ts similarity index 100% rename from Data-Sources/DS-00004—FRED_Economic_Wellbeing/update.ts rename to Data/sources/DS-00004—FRED_Economic_Wellbeing/update.ts diff --git a/Data-Sources/DS-00005—CDC_WONDER_Mortality/data/README.md b/Data/sources/DS-00005—CDC_WONDER_Mortality/data/README.md similarity index 100% rename from Data-Sources/DS-00005—CDC_WONDER_Mortality/data/README.md rename to Data/sources/DS-00005—CDC_WONDER_Mortality/data/README.md diff --git a/Data-Sources/DS-00005—CDC_WONDER_Mortality/source.md b/Data/sources/DS-00005—CDC_WONDER_Mortality/source.md similarity index 100% rename from Data-Sources/DS-00005—CDC_WONDER_Mortality/source.md rename to Data/sources/DS-00005—CDC_WONDER_Mortality/source.md diff --git a/Data-Sources/DS-00005—CDC_WONDER_Mortality/update.ts b/Data/sources/DS-00005—CDC_WONDER_Mortality/update.ts similarity index 100% rename from Data-Sources/DS-00005—CDC_WONDER_Mortality/update.ts rename to Data/sources/DS-00005—CDC_WONDER_Mortality/update.ts diff --git a/Data-Sources/DS-00006—Census_ACS_Social_Wellbeing/data/README.md b/Data/sources/DS-00006—Census_ACS_Social_Wellbeing/data/README.md similarity index 100% rename from Data-Sources/DS-00006—Census_ACS_Social_Wellbeing/data/README.md rename to Data/sources/DS-00006—Census_ACS_Social_Wellbeing/data/README.md diff --git a/Data-Sources/DS-00006—Census_ACS_Social_Wellbeing/source.md b/Data/sources/DS-00006—Census_ACS_Social_Wellbeing/source.md similarity index 100% rename from Data-Sources/DS-00006—Census_ACS_Social_Wellbeing/source.md rename to Data/sources/DS-00006—Census_ACS_Social_Wellbeing/source.md diff --git a/Data-Sources/DS-00006—Census_ACS_Social_Wellbeing/update.ts b/Data/sources/DS-00006—Census_ACS_Social_Wellbeing/update.ts similarity index 100% rename from Data-Sources/DS-00006—Census_ACS_Social_Wellbeing/update.ts rename to Data/sources/DS-00006—Census_ACS_Social_Wellbeing/update.ts diff --git a/Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/SETUP_NOTES.md b/Data/sources/DS-00007—BLS_JOLTS_Labor_Market/SETUP_NOTES.md similarity index 100% rename from Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/SETUP_NOTES.md rename to Data/sources/DS-00007—BLS_JOLTS_Labor_Market/SETUP_NOTES.md diff --git a/Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/data/README.md b/Data/sources/DS-00007—BLS_JOLTS_Labor_Market/data/README.md similarity index 100% rename from Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/data/README.md rename to Data/sources/DS-00007—BLS_JOLTS_Labor_Market/data/README.md diff --git a/Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/data/latest.json b/Data/sources/DS-00007—BLS_JOLTS_Labor_Market/data/latest.json similarity index 100% rename from Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/data/latest.json rename to Data/sources/DS-00007—BLS_JOLTS_Labor_Market/data/latest.json diff --git a/Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/data/latest.txt b/Data/sources/DS-00007—BLS_JOLTS_Labor_Market/data/latest.txt similarity index 100% rename from Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/data/latest.txt rename to Data/sources/DS-00007—BLS_JOLTS_Labor_Market/data/latest.txt diff --git a/Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/data/permission-to-quit-index.txt b/Data/sources/DS-00007—BLS_JOLTS_Labor_Market/data/permission-to-quit-index.txt similarity index 100% rename from Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/data/permission-to-quit-index.txt rename to Data/sources/DS-00007—BLS_JOLTS_Labor_Market/data/permission-to-quit-index.txt diff --git a/Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/source.md b/Data/sources/DS-00007—BLS_JOLTS_Labor_Market/source.md similarity index 100% rename from Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/source.md rename to Data/sources/DS-00007—BLS_JOLTS_Labor_Market/source.md diff --git a/Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/update.log b/Data/sources/DS-00007—BLS_JOLTS_Labor_Market/update.log similarity index 100% rename from Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/update.log rename to Data/sources/DS-00007—BLS_JOLTS_Labor_Market/update.log diff --git a/Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/update.ts b/Data/sources/DS-00007—BLS_JOLTS_Labor_Market/update.ts similarity index 100% rename from Data-Sources/DS-00007—BLS_JOLTS_Labor_Market/update.ts rename to Data/sources/DS-00007—BLS_JOLTS_Labor_Market/update.ts diff --git a/Data-Sources/DS-00008—EPA_Air_Quality_System/.env.example b/Data/sources/DS-00008—EPA_Air_Quality_System/.env.example similarity index 100% rename from Data-Sources/DS-00008—EPA_Air_Quality_System/.env.example rename to Data/sources/DS-00008—EPA_Air_Quality_System/.env.example diff --git a/Data-Sources/DS-00008—EPA_Air_Quality_System/.gitignore b/Data/sources/DS-00008—EPA_Air_Quality_System/.gitignore similarity index 100% rename from Data-Sources/DS-00008—EPA_Air_Quality_System/.gitignore rename to Data/sources/DS-00008—EPA_Air_Quality_System/.gitignore diff --git a/Data-Sources/DS-00008—EPA_Air_Quality_System/README.md b/Data/sources/DS-00008—EPA_Air_Quality_System/README.md similarity index 100% rename from Data-Sources/DS-00008—EPA_Air_Quality_System/README.md rename to Data/sources/DS-00008—EPA_Air_Quality_System/README.md diff --git a/Data-Sources/DS-00008—EPA_Air_Quality_System/data/README.md b/Data/sources/DS-00008—EPA_Air_Quality_System/data/README.md similarity index 100% rename from Data-Sources/DS-00008—EPA_Air_Quality_System/data/README.md rename to Data/sources/DS-00008—EPA_Air_Quality_System/data/README.md diff --git a/Data-Sources/DS-00008—EPA_Air_Quality_System/source.md b/Data/sources/DS-00008—EPA_Air_Quality_System/source.md similarity index 100% rename from Data-Sources/DS-00008—EPA_Air_Quality_System/source.md rename to Data/sources/DS-00008—EPA_Air_Quality_System/source.md diff --git a/Data-Sources/DS-00008—EPA_Air_Quality_System/update.ts b/Data/sources/DS-00008—EPA_Air_Quality_System/update.ts similarity index 100% rename from Data-Sources/DS-00008—EPA_Air_Quality_System/update.ts rename to Data/sources/DS-00008—EPA_Air_Quality_System/update.ts diff --git a/Data/sources/DS-00009—EIA_Energy_Data/source.md b/Data/sources/DS-00009—EIA_Energy_Data/source.md new file mode 100644 index 0000000..b870723 --- /dev/null +++ b/Data/sources/DS-00009—EIA_Energy_Data/source.md @@ -0,0 +1,336 @@ +# Energy Information Administration - Energy Data + +**Source ID:** DS-00009 +**Record Created:** 2025-12-01 +**Last Updated:** 2025-12-01 +**Cataloger:** DM-001 +**Review Status:** Initial Entry + +--- + +## Bibliographic Information + +### Title Statement +- **Main Title:** U.S. Energy Information Administration Open Data +- **Subtitle:** Comprehensive Energy Statistics for the United States +- **Abbreviated Title:** EIA +- **Variant Titles:** EIA Open Data, EIA API + +### Responsibility Statement +- **Publisher/Issuing Body:** U.S. Energy Information Administration +- **Department/Division:** Office of Energy Statistics, U.S. Department of Energy +- **Contributors:** State energy agencies, private sector energy companies +- **Contact Information:** https://www.eia.gov/about/contact/ + +### Publication Information +- **Place of Publication:** Washington, D.C., United States +- **Date of First Publication:** 1977 (EIA established); API launched 2012 +- **Publication Frequency:** Continuous (weekly, monthly, annual updates depending on data type) +- **Current Status:** Active + +### Edition/Version Information +- **Current Version:** API v2 +- **Version History:** API v1 (2012-2020), API v2 (2020-present) +- **Versioning Scheme:** Major version changes require migration + +--- + +## Authority Statement + +### Organizational Authority + +**Issuing Organization Analysis:** +- **Official Name:** U.S. Energy Information Administration +- **Type:** Federal statistical agency within U.S. Department of Energy +- **Established:** 1977 (Department of Energy Organization Act) +- **Mandate:** Collect, analyze, and disseminate independent and impartial energy information +- **Parent Organization:** U.S. Department of Energy +- **Governance Structure:** Administrator appointed by President, confirmed by Senate + +**Domain Authority:** +- **Subject Expertise:** 45+ years of comprehensive U.S. and international energy data collection +- **Recognition:** Principal federal agency for energy statistics; cited by government, industry, researchers globally +- **Publication History:** Monthly Energy Review (1974-present), Annual Energy Review, Short-Term Energy Outlook, Annual Energy Outlook +- **Peer Recognition:** Used by OPEC, IEA, World Bank; gold standard for U.S. energy statistics + +**Quality Oversight:** +- **Peer Review:** Internal quality review processes; statistical methods peer-reviewed +- **Editorial Board:** Office of Statistical Methods and Research +- **Scientific Committee:** Energy Statistics Advisory Committee +- **External Audit:** Government Accountability Office (GAO), DOE Inspector General +- **Certification:** Follows federal statistical standards; OMB Statistical Policy Directives + +**Independence Assessment:** +- **Funding Model:** Federal appropriations (U.S. Department of Energy budget) +- **Political Independence:** Statutory independence in data collection and analysis (DOE Organization Act) +- **Commercial Interests:** None; public service mission +- **Transparency:** All methodology documented; data freely available + +### Data Authority + +**Provenance Classification:** +- **Source Type:** Primary (collects directly from industry, utilities) and Secondary (aggregates state data) +- **Data Origin:** Mandatory reporting from energy companies, utilities, refineries, importers/exporters +- **Chain of Custody:** Industry reports → EIA validation → Publication via website/API + +**Primary Source Characteristics:** +- Mandatory reporting requirements (legal obligation for energy industry) +- Direct surveys of refineries, utilities, producers +- Administrative records (import/export documentation) +- Real-time price data from market sources + +--- + +## Scope Note + +### Content Description + +**Subject Coverage:** +- **Primary Subjects:** Energy prices, production, consumption, trade, stocks, forecasts +- **Secondary Subjects:** Electricity generation, renewable energy, petroleum, natural gas, coal +- **Subject Classification:** + - LC: TJ (Energy, Fuel), HD9502 (Energy Industry) + - Dewey: 333.79 (Energy Resources), 338.2 (Energy Industry) +- **Keywords:** Gas prices, oil prices, petroleum, natural gas, electricity, renewable energy, energy production, energy consumption + +**Geographic Coverage:** +- **Spatial Scope:** United States (primary), international (petroleum, natural gas trade) +- **Countries/Regions Included:** All 50 states + D.C., U.S. territories; international energy trade partners +- **Geographic Granularity:** National, regional (PADDs), state, some sub-state (refinery, utility level) +- **Coverage Completeness:** 100% for U.S. national data; comprehensive state-level; variable international +- **Notable Exclusions:** Limited real-time retail pricing in some regions + +**Temporal Coverage:** +- **Start Date:** Varies by series; petroleum data to 1949, electricity to 1973, comprehensive data 1977+ +- **End Date:** Present (ongoing data collection) +- **Historical Depth:** 50-75+ years for core petroleum/energy series +- **Frequency of Observations:** Hourly (electricity grid), daily (prices), weekly (petroleum), monthly (most data), annual (comprehensive) +- **Temporal Granularity:** High-frequency data available for prices and electricity +- **Time Series Continuity:** Excellent; breaks documented for methodology changes + +**Population/Cases Covered:** +- **Target Population:** U.S. energy industry, energy markets, consumers +- **Inclusion Criteria:** All energy production, consumption, trade, pricing in U.S. jurisdiction +- **Exclusion Criteria:** Some proprietary trading data; some small producers below reporting thresholds +- **Coverage Rate:** >95% of U.S. energy sector by volume +- **Sample vs. Census:** Mix of census (large reporters) and sample surveys (smaller entities) + +**Variables/Indicators:** + +**Core Indicators for US-Common-Metrics:** +- **PET.EMM_EPMR_PTE_NUS_DPG.W** - U.S. Regular All Formulations Retail Gasoline Prices (Dollars per Gallon) +- **PET.RWTC.W** - Cushing, OK WTI Spot Price FOB (Dollars per Barrel) +- **NG.RNGWHHD.W** - Henry Hub Natural Gas Spot Price (Dollars per Million Btu) +- **ELEC.PRICE.US-ALL.M** - U.S. Average Retail Price of Electricity (Cents per Kilowatthour) + +--- + +## Access Conditions + +### Technical Access + +**API Information:** +- **Endpoint URL:** https://api.eia.gov/v2/ +- **API Type:** REST +- **API Version:** v2 (current) +- **Documentation:** https://www.eia.gov/opendata/documentation.php + +**Authentication:** +- **Authentication Required:** Yes +- **Authentication Type:** API key +- **Registration Process:** Free at https://www.eia.gov/opendata/register.php +- **Approval Required:** No (instant) +- **Approval Timeframe:** Immediate upon registration + +**Rate Limits:** +- **Requests per Second:** Not specified +- **Requests per Hour:** 1,000 requests/hour (registered users) +- **Requests per Day:** No daily limit for reasonable use +- **Throttling Policy:** 429 error if exceeded; exponential backoff recommended + +**Query Capabilities:** +- **Filtering:** By series ID, date range, frequency, facets (state, product, sector) +- **Sorting:** Chronological +- **Pagination:** Supported (offset, length parameters) +- **Aggregation:** API can aggregate by different time periods + +**Data Formats:** +- **Available Formats:** JSON, XML +- **Format Quality:** Well-structured, validated +- **Encoding:** UTF-8 + +**Download Options:** +- **Bulk Download:** Available via Open Data bulk files +- **API Access:** Preferred method for programmatic access +- **Excel Downloads:** Available for individual datasets + +### Legal/Policy Access + +**License:** +- **License Type:** Public Domain (U.S. Government Work) +- **License URL:** https://www.eia.gov/about/copyrights_reuse.php + +**Usage Rights:** +- **Redistribution Allowed:** Yes +- **Commercial Use Allowed:** Yes +- **Modification Allowed:** Yes +- **Attribution Required:** Recommended ("Source: U.S. Energy Information Administration") +- **Share-Alike Required:** No + +**Cost Structure:** +- **Access Cost:** Free + +**Terms of Service:** +- **TOS URL:** https://www.eia.gov/about/copyrights_reuse.php +- **Key Restrictions:** None for data; trademarks/logos require permission +- **Liability Disclaimers:** Data provided "as is"; not liable for decisions based on data + +--- + +## Collection Development Policy Fit + +### Relevance Assessment + +**Substrate Mission Alignment:** +- **Human Progress Focus:** Energy prices directly impact cost of living, transportation, heating/cooling +- **Problem-Solution Connection:** + - Links to Problems: Energy affordability, fossil fuel dependence, energy security + - Links to Solutions: Renewable energy adoption, efficiency improvements, policy interventions +- **Evidence Quality:** Gold-standard government energy statistics; legally mandated reporting + +**Collection Priorities Match:** +- **Priority Level:** HIGH - essential source for energy/price indicators +- **Uniqueness:** Only authoritative source for comprehensive U.S. energy data +- **Comprehensiveness:** Fills critical gap for gas prices, oil prices in US-Common-Metrics + +--- + +## Source Evaluation Narrative + +### Methodological Assessment + +**Data Collection Methodology:** +- **Survey Design:** Mandatory reporting forms for refineries, utilities, producers, importers +- **Census vs. Sample:** Census for large entities; sample surveys for smaller (weighted to population) +- **Validation:** Cross-validation with administrative records, industry data, state reports +- **Quality Control:** Automated edit checks, analyst review, reconciliation with supply/demand balances + +**Error Characteristics:** +- **Sampling Error:** Low (mandatory reporting covers >95% of market by volume) +- **Non-sampling Error:** Minimal (legal reporting requirements reduce non-response) +- **Known Biases:** Small producers below threshold may be undercounted; spot prices may not reflect all transactions + +**Methodology Documentation:** +- **Transparency Level:** 5/5 (Excellent) - comprehensive methodology documentation +- **Documentation URL:** https://www.eia.gov/survey/ +- **Reproducibility:** High - published methodology allows verification + +### Currency Assessment + +**Update Characteristics:** +- **Update Frequency:** + - Gasoline prices: Weekly (Monday release for prior week) + - Crude oil prices: Daily (with 1-day lag) + - Natural gas prices: Weekly + - Electricity prices: Monthly +- **Update Reliability:** Highly consistent; follows published release schedule +- **Timeliness:** 1-7 days for most price data; monthly data within 60 days + +### Objectivity Assessment + +**Potential Biases:** +- **Political Bias:** Statutory independence protects against political interference +- **Commercial Bias:** None (public agency, no commercial interests) +- **Selection Bias:** Comprehensive coverage minimizes selection effects + +### Reliability Assessment + +**Consistency:** +- **Internal Consistency:** Excellent - data validated against supply/demand balances +- **Temporal Consistency:** Long time series with consistent methodology +- **Cross-source Consistency:** Reconciled with OPEC, IEA international data + +### Accuracy Assessment + +**Validation Evidence:** +- **Benchmark Comparisons:** Validated against industry data, international organizations +- **Error Studies:** Published confidence intervals for survey estimates +- **Accuracy for Different Uses:** Excellent for trend analysis and point estimates + +--- + +## Known Limitations and Caveats + +### Coverage Limitations + +**Geographic Gaps:** +- Real-time retail gas prices vary by station; EIA reports regional averages +- Some remote areas underrepresented in price surveys + +**Temporal Gaps:** +- Daily oil prices available; daily gas prices not (weekly only) +- Some historical data less frequent than current + +**Variable Gaps:** +- Retail gas prices are averages; actual prices vary by station +- Some proprietary trading data not captured + +### Methodological Limitations + +**Sampling Limitations:** +- Gas price surveys sample stations; not census of all stations +- Spot prices may not reflect all market transactions + +**Measurement Limitations:** +- Retail prices are survey-based (may lag actual market) +- Some prices are spot/futures; retail prices differ + +### How to Use Responsibly + +1. **Gas prices**: Use as indicator of trends; actual prices vary by location/station +2. **Oil prices**: WTI is benchmark; actual crude prices vary by grade/location +3. **Time lag**: Account for 1-7 day publication lag in analysis +4. **Attribution**: Always cite EIA as source + +--- + +## API Usage Examples + +### Get Weekly Gas Prices + +```bash +curl "https://api.eia.gov/v2/petroleum/pri/gnd/data/?api_key=${EIA_API_KEY}&frequency=weekly&data[0]=value&facets[product][]=EPMR&facets[duession][]=Y&sort[0][column]=period&sort[0][direction]=desc&length=10" +``` + +### Get WTI Crude Oil Price + +```bash +curl "https://api.eia.gov/v2/petroleum/pri/spt/data/?api_key=${EIA_API_KEY}&frequency=daily&data[0]=value&facets[series][]=RWTC&sort[0][column]=period&sort[0][direction]=desc&length=30" +``` + +### TypeScript Example + +```typescript +const EIA_API_KEY = process.env.EIA_API_KEY; + +async function getGasPrices() { + const response = await fetch( + `https://api.eia.gov/v2/petroleum/pri/gnd/data/?api_key=${EIA_API_KEY}&frequency=weekly&data[0]=value&facets[product][]=EPMR&facets[duession][]=Y&sort[0][column]=period&sort[0][direction]=desc&length=1` + ); + const data = await response.json(); + return data.response.data[0]; +} +``` + +--- + +## Credits + +**Data Source:** U.S. Energy Information Administration +**API Documentation:** https://www.eia.gov/opendata/documentation.php +**Cataloged for Substrate:** 2025-12-01 +**Cataloger:** Kai (DM-001) + +--- + +**END OF SOURCE DOCUMENTATION** diff --git a/Data/sources/DS-00010—Treasury_FiscalData/source.md b/Data/sources/DS-00010—Treasury_FiscalData/source.md new file mode 100644 index 0000000..dbb95aa --- /dev/null +++ b/Data/sources/DS-00010—Treasury_FiscalData/source.md @@ -0,0 +1,331 @@ +# Treasury FiscalData - Federal Fiscal Statistics + +**Source ID:** DS-00010 +**Record Created:** 2025-12-01 +**Last Updated:** 2025-12-01 +**Cataloger:** DM-001 +**Review Status:** Initial Entry + +--- + +## Bibliographic Information + +### Title Statement +- **Main Title:** FiscalData - U.S. Treasury Department Open Data +- **Subtitle:** Federal Government Fiscal Information +- **Abbreviated Title:** FiscalData, Treasury Data +- **Variant Titles:** Treasury Open Data, USAspending fiscal data + +### Responsibility Statement +- **Publisher/Issuing Body:** U.S. Department of the Treasury +- **Department/Division:** Bureau of the Fiscal Service +- **Contributors:** Office of Management and Budget (OMB), Congressional Budget Office (CBO) +- **Contact Information:** https://fiscaldata.treasury.gov/contact-us/ + +### Publication Information +- **Place of Publication:** Washington, D.C., United States +- **Date of First Publication:** 2019 (FiscalData portal); Treasury data collection since 1789 +- **Publication Frequency:** Daily (debt), Monthly (budget), Annual (comprehensive) +- **Current Status:** Active + +### Edition/Version Information +- **Current Version:** FiscalData API v2 +- **Version History:** Launched 2019, major updates 2020, 2023 +- **Versioning Scheme:** RESTful API with versioned endpoints + +--- + +## Authority Statement + +### Organizational Authority + +**Issuing Organization Analysis:** +- **Official Name:** U.S. Department of the Treasury +- **Type:** Cabinet-level federal executive department +- **Established:** 1789 (one of the oldest federal agencies) +- **Mandate:** Manage federal finances, collect revenue, pay bills, manage government debt +- **Parent Organization:** Executive Branch of U.S. Federal Government +- **Governance Structure:** Secretary of the Treasury (Cabinet position) + +**Domain Authority:** +- **Subject Expertise:** 235+ years of federal financial management +- **Recognition:** THE authoritative source for U.S. federal fiscal data; no alternative +- **Publication History:** Daily Treasury Statement (1970s-present), Monthly Treasury Statement, Treasury Bulletins +- **Peer Recognition:** Used by Congressional Budget Office, Federal Reserve, Wall Street, international institutions + +**Quality Oversight:** +- **Peer Review:** Internal audit processes; GAO oversight +- **Editorial Board:** Bureau of the Fiscal Service +- **External Audit:** Government Accountability Office (GAO), Treasury Inspector General +- **Certification:** Federal accounting standards (FASAB); GAAP for government + +**Independence Assessment:** +- **Funding Model:** Federal appropriations +- **Political Independence:** Data collection is administrative (not discretionary); political leadership cannot alter debt figures +- **Commercial Interests:** None; constitutional obligation to report +- **Transparency:** Legal requirement to publish debt, receipts, expenditures + +### Data Authority + +**Provenance Classification:** +- **Source Type:** Primary (Treasury is the source; not aggregation) +- **Data Origin:** Treasury's own financial operations, tax collections, debt issuance +- **Chain of Custody:** Treasury operations → Financial systems → Publication + +**Primary Source Characteristics:** +- Direct administrative data from Treasury operations +- Legal requirement to maintain accurate records +- Audited financial statements +- Real-time debt tracking from Treasury securities operations + +--- + +## Scope Note + +### Content Description + +**Subject Coverage:** +- **Primary Subjects:** Federal debt, federal budget, government receipts, government spending +- **Secondary Subjects:** Treasury securities, interest payments, fiscal policy indicators +- **Subject Classification:** + - LC: HJ (Public Finance), HG (Finance) + - Dewey: 336 (Public Finance), 351.72 (Government Budgets) +- **Keywords:** National debt, federal deficit, government spending, tax revenue, Treasury bonds, fiscal policy + +**Geographic Coverage:** +- **Spatial Scope:** United States federal government (national only) +- **Geographic Granularity:** National (federal government only; state/local separate) +- **Coverage Completeness:** 100% of federal fiscal operations +- **Notable Exclusions:** State/local government debt (separate sources); Federal Reserve balance sheet + +**Temporal Coverage:** +- **Start Date:** Various; debt data to 1790s; modern format from 1970s +- **End Date:** Present (daily updates for debt) +- **Historical Depth:** 230+ years for some series +- **Frequency of Observations:** Daily (debt), Monthly (budget statements), Annual (comprehensive) +- **Time Series Continuity:** Excellent; methodology changes documented + +**Variables/Indicators:** + +**Core Indicators for US-Common-Metrics:** +- **Total Public Debt Outstanding** - Total federal debt (daily) +- **Debt Held by the Public** - Marketable Treasury securities held outside government +- **Intragovernmental Holdings** - Debt held by government accounts (Social Security, etc.) +- **Monthly Treasury Statement Receipts** - Federal tax/revenue collections +- **Monthly Treasury Statement Outlays** - Federal government spending +- **Budget Deficit/Surplus** - Receipts minus outlays + +--- + +## Access Conditions + +### Technical Access + +**API Information:** +- **Endpoint URL:** https://api.fiscaldata.treasury.gov/services/api/fiscal_service/ +- **API Type:** REST +- **API Documentation:** https://fiscaldata.treasury.gov/api-documentation/ + +**Authentication:** +- **Authentication Required:** No (public API) +- **Registration Process:** None required +- **Approval Required:** No + +**Rate Limits:** +- **Rate Limiting:** Reasonable use policy (no strict limits published) +- **Throttling Policy:** May throttle excessive requests +- **Recommendation:** Implement caching; batch requests where possible + +**Query Capabilities:** +- **Filtering:** By date, field values, record type +- **Sorting:** By any field (ascending/descending) +- **Pagination:** Supported (page[size], page[number]) +- **Aggregation:** Limited built-in aggregation + +**Data Formats:** +- **Available Formats:** JSON, CSV, XML +- **Format Quality:** Well-structured, validated +- **Encoding:** UTF-8 + +**Download Options:** +- **Bulk Download:** CSV exports available for full datasets +- **API Access:** Recommended for programmatic access +- **Direct Downloads:** Available via FiscalData portal + +### Legal/Policy Access + +**License:** +- **License Type:** Public Domain (U.S. Government Work) +- **License URL:** https://fiscaldata.treasury.gov/about-us/ + +**Usage Rights:** +- **Redistribution Allowed:** Yes +- **Commercial Use Allowed:** Yes +- **Modification Allowed:** Yes +- **Attribution Required:** Recommended ("Source: U.S. Department of the Treasury") + +**Cost Structure:** +- **Access Cost:** Free + +--- + +## Collection Development Policy Fit + +### Relevance Assessment + +**Substrate Mission Alignment:** +- **Human Progress Focus:** Federal fiscal health affects all Americans; debt levels impact future generations +- **Problem-Solution Connection:** + - Links to Problems: National debt burden, deficit spending, fiscal sustainability + - Links to Solutions: Fiscal policy reforms, spending prioritization, revenue policy +- **Evidence Quality:** Authoritative source; legally mandated reporting; audited + +**Collection Priorities Match:** +- **Priority Level:** HIGH - essential for government/fiscal category +- **Uniqueness:** Only authoritative source for federal debt and budget data +- **Comprehensiveness:** Fills critical gap for fiscal metrics in US-Common-Metrics + +--- + +## Source Evaluation Narrative + +### Methodological Assessment + +**Data Collection Methodology:** +- **Administrative Records:** Direct recording of Treasury operations +- **Completeness:** 100% - all federal fiscal transactions recorded +- **Validation:** Internal controls, GAO audit, reconciliation with Fed records +- **Timeliness:** Daily updates for debt; monthly for budget statements + +**Error Characteristics:** +- **Measurement Error:** Minimal (administrative data, not surveys) +- **Known Issues:** Accounting treatments may change (documented); timing differences between cash and accrual +- **Revisions:** Some figures revised as transactions settle + +### Currency Assessment + +**Update Characteristics:** +- **Debt Data:** Daily (by 6 PM ET) +- **Monthly Treasury Statement:** ~8th business day of following month +- **Deficit/Surplus:** Monthly with annual final +- **Reliability:** Highly consistent; legal obligation to publish + +### Objectivity Assessment + +**Potential Biases:** +- **Political Bias:** Data is administrative (not discretionary); political leadership cannot alter figures +- **Presentation Bias:** Different administrations may emphasize different metrics, but underlying data unchanged +- **Transparency:** All data publicly available; methodology documented + +### Reliability Assessment + +**Consistency:** +- **Internal Consistency:** Reconciled with Federal Reserve, CBO figures +- **Temporal Consistency:** Long time series with consistent definitions +- **Cross-source Consistency:** Matches Fed Z.1 Financial Accounts, CBO estimates + +### Accuracy Assessment + +**Validation Evidence:** +- **GAO Audit:** Annual audit of Treasury financial statements +- **Reconciliation:** Daily reconciliation with Federal Reserve +- **Error Rate:** Minimal for administrative data + +--- + +## Known Limitations and Caveats + +### Coverage Limitations + +**What's NOT Included:** +- State and local government debt (separate sources) +- Federal Reserve balance sheet (separate from Treasury debt) +- Off-budget items (some military, emergency spending may be tracked differently) +- Unfunded liabilities (Social Security, Medicare future obligations not counted as debt) + +**Temporal Gaps:** +- Historical data format varies; modern APIs cover recent decades +- Some older data requires accessing archived reports + +### Methodological Limitations + +**Accounting Treatments:** +- Cash basis vs. accrual basis (MTS is cash; some reports accrual) +- Intragovernmental holdings (Social Security surplus) counted differently +- Debt ceiling periods may cause unusual timing patterns + +**Interpretation Cautions:** +- "Debt ceiling" periods create artificial constraints/timing issues +- Debt-to-GDP ratio requires GDP from separate source (BEA) +- Nominal vs. inflation-adjusted comparisons require CPI data + +### How to Use Responsibly + +1. **Debt figures**: Total Public Debt Outstanding is headline number +2. **Debt Held by Public**: Better measure of government's market borrowing +3. **Deficit**: Use 12-month rolling or fiscal year (Oct-Sep) for trends +4. **Context**: Always provide GDP comparison for meaningful interpretation + +--- + +## API Usage Examples + +### Get Total Public Debt Outstanding + +```bash +curl "https://api.fiscaldata.treasury.gov/services/api/fiscal_service/v2/accounting/od/debt_to_penny?sort=-record_date&page[size]=1" +``` + +### Get Monthly Treasury Statement Summary + +```bash +curl "https://api.fiscaldata.treasury.gov/services/api/fiscal_service/v1/accounting/mts/mts_table_5?sort=-record_date&page[size]=12" +``` + +### TypeScript Example + +```typescript +interface DebtResponse { + data: Array<{ + record_date: string; + tot_pub_debt_out_amt: string; + debt_held_public_amt: string; + intragov_hold_amt: string; + }>; +} + +async function getTotalDebt(): Promise { + const response = await fetch( + "https://api.fiscaldata.treasury.gov/services/api/fiscal_service/v2/accounting/od/debt_to_penny?sort=-record_date&page[size]=1" + ); + return response.json(); +} + +// Returns debt in dollars (e.g., "34123456789012.34") +``` + +--- + +## Key Endpoints for US-Common-Metrics + +| Metric | Endpoint | Update Frequency | +|--------|----------|------------------| +| Total Public Debt | `/v2/accounting/od/debt_to_penny` | Daily | +| Debt Held by Public | `/v2/accounting/od/debt_to_penny` | Daily | +| Monthly Receipts | `/v1/accounting/mts/mts_table_4` | Monthly | +| Monthly Outlays | `/v1/accounting/mts/mts_table_5` | Monthly | +| Deficit/Surplus | `/v1/accounting/mts/mts_table_1` | Monthly | +| Interest on Debt | `/v1/accounting/mts/mts_table_5` | Monthly | + +--- + +## Credits + +**Data Source:** U.S. Department of the Treasury, Bureau of the Fiscal Service +**API Documentation:** https://fiscaldata.treasury.gov/api-documentation/ +**Cataloged for Substrate:** 2025-12-01 +**Cataloger:** Kai (DM-001) + +--- + +**END OF SOURCE DOCUMENTATION** diff --git a/Data-Sources/WELLBEING_DATA_SOURCES.md b/Data/sources/WELLBEING_DATA_SOURCES.md similarity index 100% rename from Data-Sources/WELLBEING_DATA_SOURCES.md rename to Data/sources/WELLBEING_DATA_SOURCES.md diff --git a/Questions/Personal/Challenge/README.md b/Questions/Personal/Challenge/README.md new file mode 100644 index 0000000..7fa377d --- /dev/null +++ b/Questions/Personal/Challenge/README.md @@ -0,0 +1,19 @@ +# Personal Challenge Questions + +These are deep introspective questions designed to challenge assumptions and uncover authentic motivations. + +| ID | Question | +|---|---| +| PCQ-00001 | If money and status were no object, what would I do? | +| PCQ-00002 | If I wasn't afraid of other people's opinions what would I say? | +| PCQ-00003 | What actually makes me happy? | +| PCQ-00004 | Does success make me happy? | +| PCQ-00005 | What is my definition of success? | +| PCQ-00006 | What is my definition of success if money was taken out of the equation? | +| PCQ-00007 | What is my definition of success if money and followers were taken out of the equation? | +| PCQ-00008 | What are the things that I'm hiding from myself? | +| PCQ-00009 | What are the emotions that I'm unprepared to feel? | +| PCQ-00010 | What are the things that I feel the most shame and guilt around? | +| PCQ-00011 | What would I do if I knew I couldn't fail? | +| PCQ-00012 | What conversation am I avoiding? | +| PCQ-00013 | If I could only work on one thing for the next year, what would it be? | diff --git a/Questions/README.md b/Questions/README.md new file mode 100644 index 0000000..f48d751 --- /dev/null +++ b/Questions/README.md @@ -0,0 +1,8 @@ +# Questions + +Questions organized by category and purpose. + +## Structure + +- **Personal/** - Questions for personal introspection and growth + - **Challenge/** - Deep questions that challenge assumptions and uncover authentic motivations diff --git a/research/README.md b/research/README.md new file mode 100644 index 0000000..600328c --- /dev/null +++ b/research/README.md @@ -0,0 +1,168 @@ +# Research Projects + +## Purpose + +The `research/` directory contains completed research projects with their findings, methodology, and analysis. Research projects typically use external data sources from `Data/sources/` and produce curated datasets that are stored in `Data/`. + +## Relationship with Data Directory + +Research projects work in conjunction with the `Data/` directory: + +**Research → Data Workflow:** + +1. **Input**: Use `../Data/sources/` to access external data APIs and endpoints +2. **Analysis**: Perform analysis, synthesis, and investigation +3. **Output**: Produce curated datasets stored in `../Data/` top-level +4. **Documentation**: Document methodology, sources used, and resulting datasets + +## Required Research Project Structure + +Each research project should follow this structure: + +``` +research/[project-name-YYYY-MM]/ +├── README.md # Overview, research question, primary findings +├── SOURCES.md # Links to ../Data/sources/ used as inputs +├── METHODOLOGY.md # Research design and methods +├── findings/ # Analysis and insights +│ ├── SYNTHESIS.md # Cross-analysis synthesis +│ └── [topic-specific].md # Individual findings +└── [links to ../Data/[dataset]/] # Reference to resulting dataset +``` + +## README.md Template + +```markdown +# [Research Project Title] + +**Research Study** +**Date:** YYYY-MM-DD +**Researcher:** [Name] +**Research Design:** [Brief description] + +--- + +## Research Question + +[Primary research question or hypothesis] + +--- + +## Methodology + +[Brief methodology summary - link to METHODOLOGY.md for details] + +--- + +## Primary Finding + +[Key finding or answer to research question] + +--- + +## Data Sources Used + +This research used the following external data sources from `../Data/sources/`: +- [DS-00001—WHO_Global_Health_Observatory](../Data/sources/DS-00001—WHO_Global_Health_Observatory/) +- [DS-00002—UN_SDG_Indicators](../Data/sources/DS-00002—UN_SDG_Indicators/) + +See [SOURCES.md](./SOURCES.md) for complete source documentation. + +--- + +## Resulting Dataset + +This research produced the following curated dataset: +- [Dataset Name](../Data/[Dataset-Name]/) + - `[dataset-file].md` - Primary dataset + - `source.md` - Metadata linking back to this research + +--- + +## Findings + +[Links to findings/ directory] + +--- + +## Integration with Substrate + +[How this research connects to Problems, Solutions, Claims, etc.] +``` + +## SOURCES.md Template + +```markdown +# Data Sources Used + +**Research Project:** [Project Name] +**Date:** YYYY-MM-DD + +--- + +## External Data Sources + +This research used the following data sources from `../Data/sources/`: + +### DS-00001 — WHO Global Health Observatory +- **Path:** `../Data/sources/DS-00001—WHO_Global_Health_Observatory/` +- **What we used:** [Specific indicators or datasets] +- **Why we used it:** [Reason for using this source] +- **Date accessed:** YYYY-MM-DD + +### DS-00002 — UN SDG Indicators +- **Path:** `../Data/sources/DS-00002—UN_SDG_Indicators/` +- **What we used:** [Specific indicators or datasets] +- **Why we used it:** [Reason for using this source] +- **Date accessed:** YYYY-MM-DD + +--- + +## Additional Sources + +[Any sources not in Data/sources/ - these should be added to Data/sources/ for future research] + +--- + +## Data Processing + +[Brief description of how raw data from sources was processed, cleaned, and analyzed] +``` + +## Key Principles + +1. **Traceability**: Always document which `Data/sources/` were used +2. **Reproducibility**: Methodology should enable others to reproduce findings +3. **Dataset Production**: Curated outputs go in `../Data/` with `source.md` linking back +4. **Bidirectional Links**: Research → Data and Data → Research connections maintained +5. **Source Citation**: Credit all external sources properly + +## Benefits of This Structure + +- **Provenance**: Clear lineage from source → research → dataset +- **Reproducibility**: Research can be verified and repeated +- **Reusability**: Future research can build on existing work +- **Quality**: Traceability enables validation +- **Discovery**: Easy to find research that used specific sources + +## Example: Knowledge Worker Compensation Study + +``` +research/knowledge-worker-compensation-2025-10/ +├── README.md # Study overview +├── SOURCES.md # BLS, BEA, Census sources used +├── METHODOLOGY.md # 40-agent parallel research design +├── findings/ +│ ├── SYNTHESIS.md +│ ├── variance-analysis.md +│ └── bayesian-reconciliation.md +└── [links to ../Data/Knowledge-Worker-Global-Salaries/] + +../Data/Knowledge-Worker-Global-Salaries/ +├── knowledge-worker-compensation-data.md # Curated dataset +└── source.md # Links back to research project +``` + +--- + +**Mission**: Conduct rigorous research that produces traceable, reproducible, and reusable datasets to support human understanding and progress.