@

Add star-map Angular app, ETL pipeline, and caveman plugin

Angular 3D star map (galaxy/system/body views, Three.js rendering,
navigation store) plus the NASA ETL tooling that builds the star,
exoplanet and solar-system datasets, Playwright e2e suite, and the
cs:caveman Claude Code plugin (command, agent, skill).

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
@

tools/etl/build.ts

@@ -0,0 +1,98 @@
+import { statSync } from 'node:fs';
+
+import { BodyRecord } from '../../src/app/shared/models/body.model';
+import { ExoplanetRecord } from '../../src/app/shared/models/exoplanet.model';
+import { StarRecord } from '../../src/app/shared/models/star.model';
+import { fetchExoplanets } from './fetchExoplanets';
+import { fetchSolarSystem } from './fetchSolarSystem';
+import { fetchStars } from './fetchStars';
+import { dataPath } from './lib/paths';
+
+class ValidationError extends Error {}
+
+function assertCondition(condition: boolean, message: string): void {
+  if (!condition) {
+    throw new ValidationError(message);
+  }
+}
+
+function validateStars(stars: StarRecord[]): void {
+  assertCondition(stars.length > 0, 'No stars were produced.');
+
+  const ids = new Set<number>();
+  for (const star of stars) {
+    assertCondition(Number.isFinite(star.id), `Star has a non-numeric id: ${JSON.stringify(star)}`);
+    assertCondition(!ids.has(star.id), `Duplicate star id: ${star.id}`);
+    ids.add(star.id);
+    assertCondition(!!star.name, `Star ${star.id} has no name.`);
+    assertCondition([star.x, star.y, star.z].every(Number.isFinite), `Star ${star.id} has a non-finite position.`);
+  }
+
+  const binSize = statSync(dataPath('stars.bin')).size;
+  assertCondition(binSize === stars.length * 3 * 4, `stars.bin size (${binSize}) does not match ${stars.length} stars.`);
+}
+
+function validateBodies(bodies: BodyRecord[]): void {
+  assertCondition(bodies.length > 0, 'No solar-system bodies were produced.');
+
+  const ids = new Set(bodies.map((body) => body.id));
+  assertCondition(ids.size === bodies.length, 'Duplicate body ids were found.');
+
+  for (const body of bodies) {
+    const orbitValues = Object.values(body.orbit);
+    assertCondition(orbitValues.every(Number.isFinite), `Body ${body.id} has non-finite orbital elements.`);
+
+    if (body.kind === 'moon') {
+      assertCondition(!!body.parentBodyId && ids.has(body.parentBodyId), `Moon ${body.id} has no valid parentBodyId.`);
+    }
+  }
+
+  const planetCount = bodies.filter((body) => body.kind === 'planet').length;
+  assertCondition(planetCount === 8, `Expected 8 planets, found ${planetCount}.`);
+}
+
+function validateExoplanets(exoplanets: ExoplanetRecord[], starIds: Set<number>): void {
+  assertCondition(exoplanets.length > 0, 'No exoplanets were produced.');
+
+  let crossReferenced = 0;
+  for (const exoplanet of exoplanets) {
+    assertCondition(!!exoplanet.name, `Exoplanet ${exoplanet.id} has no name.`);
+    if (exoplanet.hostStarId !== null) {
+      assertCondition(starIds.has(exoplanet.hostStarId), `Exoplanet ${exoplanet.id} references unknown star id ${exoplanet.hostStarId}.`);
+      crossReferenced++;
+    }
+  }
+
+  console.log(`  ${crossReferenced}/${exoplanets.length} exoplanets cross-referenced to a HYG host star.`);
+}
+
+/**
+ * Orchestrates the whole ETL pipeline: fetches every source (each caches its own raw
+ * responses under `tools/etl/.cache/`), writes the static assets under `src/assets/data/`,
+ * then validates the combined output for completeness before declaring success.
+ */
+async function build(): Promise<void> {
+  console.log('=== NASA star map ETL ===\n');
+
+  const stars = await fetchStars();
+  console.log();
+  const bodies = await fetchSolarSystem();
+  console.log();
+  const exoplanets = await fetchExoplanets(stars);
+  console.log();
+
+  console.log('Validating output...');
+  validateStars(stars);
+  validateBodies(bodies);
+  validateExoplanets(exoplanets, new Set(stars.map((star) => star.id)));
+
+  console.log('\nETL completed successfully:');
+  console.log(`  stars:      ${stars.length}`);
+  console.log(`  bodies:     ${bodies.length}`);
+  console.log(`  exoplanets: ${exoplanets.length}`);
+}
+
+build().catch((error) => {
+  console.error('\nETL failed:', error instanceof Error ? error.message : error);
+  process.exitCode = 1;
+});

tools/etl/fetchExoplanets.ts

@@ -0,0 +1,98 @@
+import { writeFileSync } from 'node:fs';
+
+import { buildStarNameIndex, resolveHostStarId } from '../../src/app/shared/astro/host-star-matching';
+import { ExoplanetRecord } from '../../src/app/shared/models/exoplanet.model';
+import { StarRecord } from '../../src/app/shared/models/star.model';
+import { fetchStars } from './fetchStars';
+import { parseCsvObjects } from './lib/csv';
+import { fetchTextCached } from './lib/http';
+import { dataPath, ensureDataDir } from './lib/paths';
+
+const TAP_BASE_URL = 'https://exoplanetarchive.ipac.caltech.edu/TAP/sync';
+const TAP_COLUMNS = [
+  'pl_name',
+  'hostname',
+  'ra',
+  'dec',
+  'sy_dist',
+  'pl_orbsmax',
+  'pl_orbeccen',
+  'pl_orbincl',
+  'pl_orblper',
+  'pl_orbper',
+  'pl_rade',
+  'pl_bmasse',
+  'disc_year'
+].join(',');
+const TAP_QUERY = `select+${TAP_COLUMNS}+from+ps+where+default_flag=1&format=csv`;
+const TAP_URL = `${TAP_BASE_URL}?query=${TAP_QUERY}`;
+
+// A host star match must be within this many parsecs of the catalog position to be
+// accepted as a cross-reference (guards against coincidental name/position collisions).
+const MATCH_TOLERANCE_PC = 0.5;
+
+/**
+ * Downloads confirmed exoplanets from the NASA Exoplanet Archive (`Planetary Systems` TAP
+ * table), cross-references each host star to the HYG index, and writes `exoplanets.json`.
+ */
+export async function fetchExoplanets(stars?: StarRecord[]): Promise<ExoplanetRecord[]> {
+  console.log('Fetching confirmed exoplanets from the NASA Exoplanet Archive...');
+  const knownStars = stars ?? (await fetchStars());
+  const nameIndex = buildStarNameIndex(knownStars);
+
+  const csv = await fetchTextCached(TAP_URL, 'exoplanet-archive-ps.csv');
+  const rows = parseCsvObjects(csv);
+
+  let matched = 0;
+  const exoplanets: ExoplanetRecord[] = rows.map((row, index) => {
+    const raDeg = Number(row['ra']);
+    const decDeg = Number(row['dec']);
+    const distancePc = Number(row['sy_dist']);
+
+    const hostStarId = resolveHostStarId(
+      { hostname: row['hostname'], raDeg, decDeg, distancePc },
+      knownStars,
+      MATCH_TOLERANCE_PC,
+      nameIndex
+    );
+    if (hostStarId !== null) {
+      matched++;
+    }
+
+    return {
+      id: row['pl_name'] || `exoplanet-${index}`,
+      hostStarId,
+      hostStarName: row['hostname'],
+      name: row['pl_name'],
+      radiusEarth: parseOptionalNumber(row['pl_rade']),
+      massEarth: parseOptionalNumber(row['pl_bmasse']),
+      discoveryYear: parseOptionalNumber(row['disc_year']),
+      orbit: {
+        semiMajorAxisAu: parseOptionalNumber(row['pl_orbsmax']),
+        eccentricity: parseOptionalNumber(row['pl_orbeccen']),
+        inclinationDeg: parseOptionalNumber(row['pl_orbincl']),
+        argumentOfPeriapsisDeg: parseOptionalNumber(row['pl_orblper'])
+      }
+    };
+  });
+
+  ensureDataDir();
+  writeFileSync(dataPath('exoplanets.json'), JSON.stringify(exoplanets));
+  console.log(`  wrote ${exoplanets.length} exoplanets (${matched} cross-referenced to a HYG host star).`);
+  return exoplanets;
+}
+
+function parseOptionalNumber(value: string | undefined): number | undefined {
+  if (!value) {
+    return undefined;
+  }
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : undefined;
+}
+
+if (require.main === module) {
+  fetchExoplanets().catch((error) => {
+    console.error(error);
+    process.exitCode = 1;
+  });
+}

tools/etl/fetchSolarSystem.ts

@@ -0,0 +1,81 @@
+import { writeFileSync } from 'node:fs';
+
+import { BodyRecord } from '../../src/app/shared/models/body.model';
+import { SUN_STAR_ID } from '../../src/app/shared/models/star.model';
+import { fetchHorizonsBody } from './lib/horizons';
+import { dataPath, ensureDataDir } from './lib/paths';
+
+interface BodySpec {
+  id: string;
+  name: string;
+  kind: BodyRecord['kind'];
+  horizonsCommand: string;
+  center: string;
+  parentBodyId?: string;
+}
+
+// Sun-centered planets/dwarf, then their major moons (planetocentric elements).
+const BODY_SPECS: BodySpec[] = [
+  { id: 'mercury', name: 'Mercury', kind: 'planet', horizonsCommand: '199', center: '500@10' },
+  { id: 'venus', name: 'Venus', kind: 'planet', horizonsCommand: '299', center: '500@10' },
+  { id: 'earth', name: 'Earth', kind: 'planet', horizonsCommand: '399', center: '500@10' },
+  { id: 'mars', name: 'Mars', kind: 'planet', horizonsCommand: '499', center: '500@10' },
+  { id: 'jupiter', name: 'Jupiter', kind: 'planet', horizonsCommand: '599', center: '500@10' },
+  { id: 'saturn', name: 'Saturn', kind: 'planet', horizonsCommand: '699', center: '500@10' },
+  { id: 'uranus', name: 'Uranus', kind: 'planet', horizonsCommand: '799', center: '500@10' },
+  { id: 'neptune', name: 'Neptune', kind: 'planet', horizonsCommand: '899', center: '500@10' },
+  { id: 'pluto', name: 'Pluto', kind: 'dwarf', horizonsCommand: '999', center: '500@10' },
+  { id: 'moon', name: 'Moon', kind: 'moon', horizonsCommand: '301', center: '500@399', parentBodyId: 'earth' },
+  { id: 'phobos', name: 'Phobos', kind: 'moon', horizonsCommand: '401', center: '500@499', parentBodyId: 'mars' },
+  { id: 'deimos', name: 'Deimos', kind: 'moon', horizonsCommand: '402', center: '500@499', parentBodyId: 'mars' },
+  { id: 'io', name: 'Io', kind: 'moon', horizonsCommand: '501', center: '500@599', parentBodyId: 'jupiter' },
+  { id: 'europa', name: 'Europa', kind: 'moon', horizonsCommand: '502', center: '500@599', parentBodyId: 'jupiter' },
+  { id: 'ganymede', name: 'Ganymede', kind: 'moon', horizonsCommand: '503', center: '500@599', parentBodyId: 'jupiter' },
+  { id: 'callisto', name: 'Callisto', kind: 'moon', horizonsCommand: '504', center: '500@599', parentBodyId: 'jupiter' },
+  { id: 'titan', name: 'Titan', kind: 'moon', horizonsCommand: '606', center: '500@699', parentBodyId: 'saturn' },
+  { id: 'triton', name: 'Triton', kind: 'moon', horizonsCommand: '801', center: '500@899', parentBodyId: 'neptune' }
+];
+
+/**
+ * Queries JPL Horizons for the osculating orbital elements (and mean radius, where
+ * reported) of the major planets, Pluto, and a curated set of major moons, and writes
+ * `bodies.json`.
+ */
+export async function fetchSolarSystem(): Promise<BodyRecord[]> {
+  console.log(`Fetching ${BODY_SPECS.length} solar-system bodies from JPL Horizons...`);
+  const bodies: BodyRecord[] = [];
+
+  for (const spec of BODY_SPECS) {
+    const result = await fetchHorizonsBody({
+      command: spec.horizonsCommand,
+      center: spec.center,
+      cacheKey: `horizons-${spec.id}.txt`
+    });
+
+    if (result.radiusKm === undefined) {
+      console.warn(`  no physical radius found for ${spec.name}; defaulting to 0.`);
+    }
+
+    bodies.push({
+      id: spec.id,
+      systemStarId: SUN_STAR_ID,
+      name: spec.name,
+      kind: spec.kind,
+      radiusKm: result.radiusKm ?? 0,
+      orbit: result.orbit,
+      ...(spec.parentBodyId ? { parentBodyId: spec.parentBodyId } : {})
+    });
+  }
+
+  ensureDataDir();
+  writeFileSync(dataPath('bodies.json'), JSON.stringify(bodies, null, 2));
+  console.log(`  wrote ${bodies.length} bodies.`);
+  return bodies;
+}
+
+if (require.main === module) {
+  fetchSolarSystem().catch((error) => {
+    console.error(error);
+    process.exitCode = 1;
+  });
+}

tools/etl/fetchStars.ts

@@ -0,0 +1,108 @@
+import { writeFileSync } from 'node:fs';
+
+import { raDecDistanceToXyz } from '../../src/app/shared/astro/coordinates';
+import { StarRecord, SUN_STAR_ID } from '../../src/app/shared/models/star.model';
+import { parseCsvObjects } from './lib/csv';
+import { fetchTextCached } from './lib/http';
+import { dataPath, ensureDataDir } from './lib/paths';
+
+const HYG_CSV_URL = 'https://raw.githubusercontent.com/astronexus/HYG-Database/main/hyg/CURRENT/hygdata_v41.csv';
+const HYG_UNKNOWN_DISTANCE_PC = 100000; // HYG's placeholder for unmeasured/unreliable parallax
+
+/** Stars within this distance (parsecs) of the Sun are kept for the galaxy view. */
+const DISTANCE_CUTOFF_PC = Number(process.env['ETL_STAR_DISTANCE_PC'] ?? 50);
+
+function resolveName(row: Record<string, string>): string {
+  if (row['proper']) {
+    return row['proper'];
+  }
+  if (row['bayer'] && row['con']) {
+    return `${row['bayer']} ${row['con']}`;
+  }
+  if (row['flam'] && row['con']) {
+    return `${row['flam']} ${row['con']}`;
+  }
+  if (row['hd']) {
+    return `HD ${row['hd']}`;
+  }
+  if (row['gl']) {
+    return `Gl ${row['gl']}`;
+  }
+  if (row['hip']) {
+    return `HIP ${row['hip']}`;
+  }
+  return `HYG ${row['id']}`;
+}
+
+/**
+ * Downloads the HYG (Hipparcos/Yale/Gliese) stellar database, converts each star's
+ * RA/Dec/distance into galaxy-scale Cartesian coordinates (parsecs), filters by distance,
+ * and writes `stars.bin` (packed positions) + `stars-index.json` (everything else).
+ */
+export async function fetchStars(): Promise<StarRecord[]> {
+  console.log(`Fetching HYG star catalog (distance cutoff: ${DISTANCE_CUTOFF_PC} pc)...`);
+  const csv = await fetchTextCached(HYG_CSV_URL, 'hygdata_v41.csv');
+  const rows = parseCsvObjects(csv);
+
+  const stars: StarRecord[] = [];
+
+  for (const row of rows) {
+    const id = Number(row['id']);
+    const distancePc = Number(row['dist']);
+
+    if (id === SUN_STAR_ID) {
+      stars.push({ id, name: 'Sol', x: 0, y: 0, z: 0, magnitude: Number(row['mag']), spectralType: row['spect'] || 'G2V', colorIndex: Number(row['ci']) || 0 });
+      continue;
+    }
+
+    if (!Number.isFinite(distancePc) || distancePc >= HYG_UNKNOWN_DISTANCE_PC || distancePc > DISTANCE_CUTOFF_PC) {
+      continue;
+    }
+
+    const raHours = Number(row['ra']);
+    const decDeg = Number(row['dec']);
+    if (!Number.isFinite(raHours) || !Number.isFinite(decDeg)) {
+      continue;
+    }
+
+    const { x, y, z } = raDecDistanceToXyz(raHours, decDeg, distancePc);
+
+    stars.push({
+      id,
+      name: resolveName(row),
+      x,
+      y,
+      z,
+      magnitude: Number(row['mag']) || 0,
+      spectralType: row['spect'] || 'Unknown',
+      colorIndex: Number(row['ci']) || 0
+    });
+  }
+
+  stars.sort((a, b) => a.id - b.id);
+  writeStarAssets(stars);
+
+  console.log(`  kept ${stars.length} stars (of ${rows.length} in the catalog).`);
+  return stars;
+}
+
+function writeStarAssets(stars: StarRecord[]): void {
+  ensureDataDir();
+
+  const positions = new Float32Array(stars.length * 3);
+  stars.forEach((star, index) => {
+    positions[index * 3] = star.x;
+    positions[index * 3 + 1] = star.y;
+    positions[index * 3 + 2] = star.z;
+  });
+
+  writeFileSync(dataPath('stars.bin'), Buffer.from(positions.buffer));
+  writeFileSync(dataPath('stars-index.json'), JSON.stringify(stars));
+}
+
+if (require.main === module) {
+  fetchStars().catch((error) => {
+    console.error(error);
+    process.exitCode = 1;
+  });
+}

tools/etl/lib/csv.ts

@@ -0,0 +1,69 @@
+/**
+ * Minimal RFC4180-ish CSV parser: supports a custom delimiter, quoted fields (with escaped
+ * `""`), and embedded newlines inside quotes. Good enough for the HYG/Exoplanet Archive
+ * exports used by the ETL, without pulling in an extra dependency.
+ */
+export function parseCsv(text: string, delimiter = ','): string[][] {
+  const rows: string[][] = [];
+  let row: string[] = [];
+  let field = '';
+  let inQuotes = false;
+
+  for (let i = 0; i < text.length; i++) {
+    const char = text[i];
+
+    if (inQuotes) {
+      if (char === '"') {
+        if (text[i + 1] === '"') {
+          field += '"';
+          i++;
+        } else {
+          inQuotes = false;
+        }
+      } else {
+        field += char;
+      }
+      continue;
+    }
+
+    if (char === '"') {
+      inQuotes = true;
+    } else if (char === delimiter) {
+      row.push(field);
+      field = '';
+    } else if (char === '\r') {
+      // skip; \n (handled below) terminates the row
+    } else if (char === '\n') {
+      row.push(field);
+      rows.push(row);
+      row = [];
+      field = '';
+    } else {
+      field += char;
+    }
+  }
+
+  if (field.length > 0 || row.length > 0) {
+    row.push(field);
+    rows.push(row);
+  }
+
+  return rows;
+}
+
+/** Parses `text` as CSV and maps each data row to an object keyed by the header row. */
+export function parseCsvObjects(text: string, delimiter = ','): Array<Record<string, string>> {
+  const rows = parseCsv(text, delimiter).filter((row) => row.some((cell) => cell.length > 0));
+  if (rows.length === 0) {
+    return [];
+  }
+
+  const [header, ...dataRows] = rows;
+  return dataRows.map((row) => {
+    const record: Record<string, string> = {};
+    header.forEach((column, index) => {
+      record[column] = row[index] ?? '';
+    });
+    return record;
+  });
+}

tools/etl/lib/horizons.ts

@@ -0,0 +1,96 @@
+import { OrbitalElements } from '../../../src/app/shared/models/body.model';
+import { fetchTextCached } from './http';
+
+const HORIZONS_URL = 'https://ssd.jpl.nasa.gov/api/horizons.api';
+const KM_PER_AU = 149597870.7;
+
+// Fixed reference epoch: keeps the ETL output (and its cache) stable across reruns,
+// consistent with the "bake once at build time" ETL philosophy.
+const REFERENCE_START = '2025-01-01';
+const REFERENCE_STOP = '2025-01-02';
+
+export interface HorizonsQuery {
+  /** Horizons body id, e.g. `'499'` for Mars. */
+  command: string;
+  /** Horizons coordinate center, e.g. `'500@10'` (Sun) or `'500@399'` (Earth). */
+  center: string;
+  cacheKey: string;
+}
+
+export interface HorizonsResult {
+  radiusKm?: number;
+  orbit: OrbitalElements;
+}
+
+const RADIUS_PATTERNS = [
+  /Vol\.?\s*mean\s*radius[^=]*=\s*([\d.]+)/i,
+  /Mean\s*radius[^=]*=\s*([\d.]+)/i,
+  /Radius\s*\(IAU\)[^=]*=\s*([\d.]+)/i,
+  /Radius,?\s*\(km\)\s*=\s*([\d.]+)/i,
+  /Radius\s*\(gravity\),?\s*km\s*=\s*([\d.]+)/i
+];
+
+/**
+ * Queries JPL Horizons for a body's heliocentric (or planetocentric, for moons) osculating
+ * orbital elements plus, when available, its mean physical radius — both in a single
+ * request (`OBJ_DATA=YES` + `EPHEM_TYPE=ELEMENTS`).
+ */
+export async function fetchHorizonsBody(query: HorizonsQuery): Promise<HorizonsResult> {
+  const url =
+    `${HORIZONS_URL}?format=text&COMMAND='${query.command}'&OBJ_DATA='YES'` +
+    `&MAKE_EPHEM='YES'&EPHEM_TYPE='ELEMENTS'&CENTER='${query.center}'` +
+    `&START_TIME='${REFERENCE_START}'&STOP_TIME='${REFERENCE_STOP}'&STEP_SIZE='1d'`;
+
+  const text = await fetchTextCached(url, query.cacheKey);
+  return {
+    radiusKm: extractRadiusKm(text),
+    orbit: extractOrbitalElements(text)
+  };
+}
+
+function extractRadiusKm(text: string): number | undefined {
+  for (const pattern of RADIUS_PATTERNS) {
+    const match = text.match(pattern);
+    if (match) {
+      return Number(match[1]);
+    }
+  }
+  return undefined;
+}
+
+function extractOrbitalElements(text: string): OrbitalElements {
+  const startIndex = text.indexOf('$$SOE');
+  const endIndex = text.indexOf('$$EOE');
+  if (startIndex === -1 || endIndex === -1) {
+    throw new Error('Horizons response did not contain an elements table ($$SOE/$$EOE).');
+  }
+
+  const block = text.slice(startIndex + '$$SOE'.length, endIndex).trim();
+  const firstRecord = block.split(/\n(?=\d)/)[0];
+
+  const epochJd = extractNumber(firstRecord, /^([\d.]+)\s*=/);
+  const eccentricity = extractNumber(firstRecord, /EC\s*=\s*([-\d.Ee+]+)/);
+  const inclinationDeg = extractNumber(firstRecord, /IN\s*=\s*([-\d.Ee+]+)/);
+  const longitudeOfAscendingNodeDeg = extractNumber(firstRecord, /OM\s*=\s*([-\d.Ee+]+)/);
+  const argumentOfPeriapsisDeg = extractNumber(firstRecord, /(?<!\w)W\s*=\s*([-\d.Ee+]+)/);
+  const meanAnomalyAtEpochDeg = extractNumber(firstRecord, /MA\s*=\s*([-\d.Ee+]+)/);
+  const semiMajorAxisKm = extractNumber(firstRecord, /(?<!\w)A\s*=\s*([-\d.Ee+]+)/);
+
+  return {
+    semiMajorAxisAu: semiMajorAxisKm / KM_PER_AU,
+    eccentricity,
+    inclinationDeg,
+    longitudeOfAscendingNodeDeg,
+    argumentOfPeriapsisDeg,
+    meanAnomalyAtEpochDeg,
+    epochJd
+  };
+}
+
+function extractNumber(text: string, pattern: RegExp): number {
+  const match = text.match(pattern);
+  if (!match) {
+    throw new Error(`Could not find pattern ${pattern} in Horizons elements record.`);
+  }
+  return Number(match[1]);
+}

tools/etl/lib/http.ts

@@ -0,0 +1,34 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+
+const CACHE_DIR = join(process.cwd(), 'tools', 'etl', '.cache');
+const FORCE_REFRESH = process.env['ETL_FORCE_REFRESH'] === '1';
+
+/**
+ * Downloads `url` as text, caching the raw response under `tools/etl/.cache/<cacheKey>` so
+ * re-running the ETL doesn't hit live NASA/astronomy endpoints unless the cache is missing
+ * or `ETL_FORCE_REFRESH=1` is set. Keeps the pipeline idempotent and resilient to rate limits.
+ */
+export async function fetchTextCached(url: string, cacheKey: string): Promise<string> {
+  const cachePath = join(CACHE_DIR, cacheKey);
+
+  if (!FORCE_REFRESH && existsSync(cachePath)) {
+    return readFileSync(cachePath, 'utf-8');
+  }
+
+  console.log(`  fetching ${url}`);
+  const response = await fetch(url);
+  if (!response.ok) {
+    throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
+  }
+  const text = await response.text();
+
+  mkdirSync(dirname(cachePath), { recursive: true });
+  writeFileSync(cachePath, text, 'utf-8');
+  return text;
+}
+
+/** Convenience wrapper around {@link fetchTextCached} that parses the cached response as JSON. */
+export async function fetchJsonCached<T>(url: string, cacheKey: string): Promise<T> {
+  return JSON.parse(await fetchTextCached(url, cacheKey)) as T;
+}

tools/etl/lib/paths.ts

@@ -0,0 +1,12 @@
+import { mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+
+export const DATA_DIR = join(process.cwd(), 'src', 'assets', 'data');
+
+export function ensureDataDir(): void {
+  mkdirSync(DATA_DIR, { recursive: true });
+}
+
+export function dataPath(fileName: string): string {
+  return join(DATA_DIR, fileName);
+}

tools/etl/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../../tsconfig.json",
+  "compilerOptions": {
+    "module": "commonjs",
+    "moduleResolution": "bundler",
+    "rootDir": "../..",
+    "outDir": "../../out-tsc/etl",
+    "types": ["node"]
+  },
+  "include": ["**/*.ts", "../../src/app/shared/**/*.ts"],
+  "exclude": ["**/*.spec.ts", "../../src/app/shared/**/*.spec.ts"]
+}