import { statSync } from 'node:fs'; import { BodyRecord, OrbitalElements } from '../../src/app/shared/models/body.model'; import { eclipticToEquatorial, laplacePlaneToEquatorial } from '../../src/app/shared/astro/coordinates'; import { meanElementsAt, positionAtEpoch } from '../../src/app/shared/astro/kepler'; import { DeepSkyRecord } from '../../src/app/shared/models/deepsky.model'; import { ExoplanetRecord } from '../../src/app/shared/models/exoplanet.model'; import { StarRecord, SUN_STAR_ID } from '../../src/app/shared/models/star.model'; import { fetchDeepSky } from './fetchDeepSky'; import { fetchExoplanets } from './fetchExoplanets'; import { fetchSolarSystem, FREELY_SPINNING_MOONS } from './fetchSolarSystem'; import { BYTES_PER_STAR_META, BYTES_PER_STAR_POSITION, decodeStarCatalog, encodeStarCatalog } from '../../src/app/shared/models/star-catalog'; import { fetchStars } from './fetchStars'; import { describeSources } from './sources/registry'; import { dataPath } from './lib/paths'; class ValidationError extends Error {} function assertCondition(condition: boolean, message: string): void { if (!condition) { throw new ValidationError(message); } } function validateStars(stars: StarRecord[]): void { assertCondition(stars.length > 0, 'No stars were produced.'); const ids = new Set(); for (const star of stars) { assertCondition(Number.isFinite(star.id), `Star has a non-numeric id: ${JSON.stringify(star)}`); assertCondition(!ids.has(star.id), `Duplicate star id: ${star.id}`); ids.add(star.id); assertCondition(!!star.name, `Star ${star.id} has no name.`); assertCondition([star.x, star.y, star.z].every(Number.isFinite), `Star ${star.id} has a non-finite position.`); } const positionBytes = statSync(dataPath('stars.bin')).size; assertCondition(positionBytes === stars.length * BYTES_PER_STAR_POSITION, `stars.bin size (${positionBytes}) does not match ${stars.length} stars.`); const metaBytes = statSync(dataPath('stars-meta.bin')).size; assertCondition(metaBytes === stars.length * BYTES_PER_STAR_META, `stars-meta.bin size (${metaBytes}) does not match ${stars.length} stars.`); // Round-trips the written assets back through the decoder the app uses, so a format change // that only half-lands fails here rather than as a silently wrong star map. const { index, positions, meta } = encodeStarCatalog(stars); const decoded = decodeStarCatalog(index, positions, meta); assertCondition(decoded.length === stars.length, `Star catalogue round-trip lost records: ${decoded.length} of ${stars.length}.`); for (let i = 0; i < stars.length; i++) { assertCondition(decoded[i].id === stars[i].id && decoded[i].name === stars[i].name, `Star catalogue round-trip altered record ${i}.`); assertCondition(decoded[i].spectralType === stars[i].spectralType, `Star catalogue round-trip lost the spectral type of star ${stars[i].id}.`); assertCondition(decoded[i].colorIndex === null === (stars[i].colorIndex === null), `Star catalogue round-trip changed whether star ${stars[i].id} has a colour index.`); } } /** * What a good merge looks like, in two numbers the unit suite cannot see. * * The catalogues are regenerated by a scheduled job that pushes straight to `main` once the unit * tests and a production build pass — and both passed, for weeks, on a catalogue carrying 23 000 * stars twice: the suite tests code against fixtures, and no fixture is 400 000 real stars. The * two ways the merge has actually failed both show up here. * * A star kept twice leaves its two entries near each other on the sky, from *different* sources — * one catalogue does not list a star twice. Under an arcsecond that is never two stars at this * depth, so every such pair is a miss. Nineteen survive today, all of them a second HYG row * wanting a Gaia entry that already absorbed one (Gliese lists some doubles twice); the merge * that trusted a Hipparcos parallax over direction left 1 112. * * The other failure leaves no close pair at all, because proper motion had already carried the * two entries tens of arcseconds apart — the 2026-08-24 refresh, where HYG sat at epoch 2000.0 * and Gaia at J2016.0. What it does leave is HYG rows that found no counterpart: 36 056 of them * against the 10 886 today, and no counterpart was possible for most of those. Two thirds of them, * 6 835, are the stars Gaia measures but the main query never downloads, because Gaia's parallax * puts them past `ETL_GAIA_DISTANCE_PC` while Hipparcos put them inside `ETL_STAR_DISTANCE_PC`; * they are every star in the published catalogue beyond 250 pc. The rest are what Gaia genuinely * lacks: bright stars it saturates on, red dwarfs past its magnitude cut. So the headroom left to * the ceiling tracks the gap between those two cutoffs as much as Gaia's completeness. * * This bounds a merge that went wrong, and — loosely — a Gaia download that came back short: a * truncated answer leaves the HYG rows whose counterpart it dropped without one, so survivors go * *up*, not down. Measured against the published catalogue: 10 886 today, 11 004 at nine tenths of * the rows, 12 711 at half, 16 258 at a third. So this ceiling only catches a truncation past about * two thirds, and `fetchGaiaStars` catches the shallower ones with its own row floor. */ const MAX_UNMERGED_TWINS = 100; const MAX_HYG_SURVIVORS = 15_000; const TWIN_TOLERANCE_RAD = (1 / 3600) * (Math.PI / 180); function validateMerge(stars: StarRecord[]): void { // Checked first and on its own: an unreachable Gaia is skipped rather than thrown, and would // otherwise surface below as "68 000 HYG stars found no counterpart" — true, and no help. assertCondition( stars.some((star) => star.source === 'gaia'), 'Gaia DR3 contributed no stars — the archive was unreachable or returned nothing, and a catalogue without it is not one to publish.' ); const survivors = stars.filter((star) => star.source === 'hyg').length; assertCondition( survivors <= MAX_HYG_SURVIVORS, `${survivors} HYG stars found no Gaia counterpart (at most ${MAX_HYG_SURVIVORS} expected) — the two catalogues are not being matched.` ); // Sorted by declination, so each star is only compared against the handful sharing its // parallel — an arcsecond of declination holds one or two of 400 000 stars. const byDec = stars .map((star) => { const distance = Math.hypot(star.x, star.y, star.z); return { star, distance, dec: distance === 0 ? 0 : Math.asin(Math.max(-1, Math.min(1, star.z / distance))) }; }) .filter((entry) => entry.distance > 0) .sort((a, b) => a.dec - b.dec); const cosTolerance = Math.cos(TWIN_TOLERANCE_RAD); let twins = 0; let example = ''; for (let i = 0; i < byDec.length; i++) { const a = byDec[i]; for (let j = i + 1; j < byDec.length && byDec[j].dec - a.dec <= TWIN_TOLERANCE_RAD; j++) { const b = byDec[j]; if (a.star.source === b.star.source) { continue; } const cosine = (a.star.x * b.star.x + a.star.y * b.star.y + a.star.z * b.star.z) / (a.distance * b.distance); if (cosine >= cosTolerance) { twins++; example ||= `${a.star.name} (${a.star.source}) and ${b.star.name} (${b.star.source})`; } } } assertCondition( twins <= MAX_UNMERGED_TWINS, `${twins} stars from different catalogues sit within an arcsecond of each other (at most ${MAX_UNMERGED_TWINS} expected), starting with ${example} — the merge is keeping the same star twice.` ); console.log(` ${survivors} HYG stars have no Gaia counterpart; ${twins} unmerged cross-catalogue pairs within an arcsecond.`); } /** * How far a body's mean elements may put it from where Horizons has it, on the one date the ETL * asks Horizons about (2025-01-01), seen from the Sun for a planet and from its planet for a moon. * * Measured on this catalogue: the planets at most 0.10 degrees (Uranus; Standish's own stated * error for his fit is 2 000 arcseconds, 0.56 degrees), the moons at most 1.41 (the Moon, whose * evection and variation, 1.27 and 0.66 degrees, no mean ellipse has). What this catches is a * table read wrongly: a moon read against the ecliptic instead of its Laplace plane, a precession * run the wrong way, or a column taken for its neighbour, which put Triton 26 degrees out and Io * 0.9. */ const MAX_PLANET_OFFSET_DEG = 0.25; const MAX_MOON_OFFSET_DEG = 2.5; const KM_PER_AU = 149597870.7; /** * The moons whose table row cannot come within that, each for a reason no mean ellipse carries, * with a ceiling just above its worst offset from Horizons at twelve dates from 1980 to 2100: * * - Mimas, 44.7 degrees: its resonance with Tethys swings its mean longitude 44 degrees either * way over 70.8 years, and the table has no column for it (Tethys, on the other end, swings 2). * - Hyperion, 20.2: held in a 4:3 resonance by Titan; the row's eccentricity, 0.0232, is less than * a quarter of the 0.105 JPL's current table gives. * - Iapetus, 10.1: the row sits 9.4 degrees behind Horizons at its own epoch, 2000 Jan 1.5, and * keeps that offset; its plane agrees with Horizons' to 0.07 degrees and its period to 0.001 per * cent, so the fault is in the row's longitude, which this has no second source to correct. * - Nereid, 2.6: an eccentricity of 0.75, the largest here, which a mean ellipse follows least * well: under 0.9 degrees in every year measured but 2025 and 2030 (2.6 and 2.3) and 2100 (1.7). */ const MOON_OFFSET_CEILINGS_DEG: Record = { mimas: 46, hyperion: 21, iapetus: 11, nereid: 3 }; function angleBetweenDeg(a: { x: number; y: number; z: number }, b: { x: number; y: number; z: number }): number { const cosine = (a.x * b.x + a.y * b.y + a.z * b.z) / (Math.hypot(a.x, a.y, a.z) * Math.hypot(b.x, b.y, b.z)); return (Math.acos(Math.min(1, Math.max(-1, cosine))) * 180) / Math.PI; } function validateBodies(bodies: BodyRecord[], horizonsOrbits: Map): void { assertCondition(bodies.length > 0, 'No solar-system bodies were produced.'); const ids = new Set(bodies.map((body) => body.id)); assertCondition(ids.size === bodies.length, 'Duplicate body ids were found.'); const offsets: string[] = []; for (const body of bodies) { const orbitValues = Object.values(body.orbit); assertCondition(orbitValues.every(Number.isFinite), `Body ${body.id} has non-finite orbital elements.`); assertCondition(body.rates.meanMotionDegPerDay > 0, `Body ${body.id} has no mean motion.`); // Horizons' elements are osculating, exact at their own epoch; both sets are placed there. const horizons = horizonsOrbits.get(body.id); assertCondition(horizons !== undefined, `Body ${body.id} has no Horizons elements to be checked against.`); const truth = eclipticToEquatorial(positionAtEpoch(horizons!)); const mean = positionAtEpoch(meanElementsAt(body.orbit, body.rates, horizons!.epochJd)); const offset = angleBetweenDeg(body.laplacePole ? laplacePlaneToEquatorial(mean, body.laplacePole) : eclipticToEquatorial(mean), truth); const ceiling = body.kind === 'moon' ? (MOON_OFFSET_CEILINGS_DEG[body.id] ?? MAX_MOON_OFFSET_DEG) : MAX_PLANET_OFFSET_DEG; assertCondition( offset <= ceiling, `${body.name}'s mean elements put it ${offset.toFixed(2)} degrees from where Horizons has it (at most ${ceiling} expected) — the elements were read wrongly.` ); offsets.push(`${body.id} ${offset.toFixed(3)}`); // A radius of 0 is what a page whose radius no pattern reads comes out as — Charon's did. assertCondition(body.radiusKm > 0, `Body ${body.id} has no radius; its page states it in a form the ETL does not read.`); if (body.kind === 'moon') { const parent = bodies.find((candidate) => candidate.id === body.parentBodyId); assertCondition(parent !== undefined, `Moon ${body.id} has no valid parentBodyId.`); const orbitHours = (360 / body.rates.meanMotionDegPerDay) * 24; if (FREELY_SPINNING_MOONS.has(body.id)) { // Hyperion tumbles and Nereid's page gives no spin, so they have none; Phoebe turns in // 9.27 hours against a 550-day orbit. A lock here would be the rule below misapplied. assertCondition( body.rotationPeriodHours === undefined || Math.abs(body.rotationPeriodHours - orbitHours) > orbitHours * 0.1, `Moon ${body.id} does not keep one face to its planet, yet turns once in ${body.rotationPeriodHours} hours against an orbit of ${orbitHours}.` ); } else { // Every other moon here is tidally locked: its day is its orbit, from the same mean motion // that carries it round, or its face turns away from its planet: the Kepler period of the // osculating orbit this used to take would turn the Moon's five degrees an orbit. assertCondition( body.rotationPeriodHours !== undefined && Math.abs(body.rotationPeriodHours - orbitHours) <= orbitHours * 1e-9, `Moon ${body.id} turns once in ${body.rotationPeriodHours} hours but goes round in ${orbitHours} — it will not keep one face to its planet.` ); } if (body.massRatio !== undefined) { // The pair's barycentre, which the planet's elements place, must lie outside the planet — // that is why the two are drawn going round it — and nearer the planet than the moon. const offsetKm = (body.orbit.semiMajorAxisAu * KM_PER_AU * body.massRatio) / (1 + body.massRatio); assertCondition( body.massRatio > 0 && body.massRatio < 1 && offsetKm > parent!.radiusKm, `${body.name}'s mass ratio ${body.massRatio} puts its barycentre ${offsetKm.toFixed(0)} km from ${parent!.name}'s centre, which is not between its surface, ${parent!.radiusKm} km out, and the moon.` ); } } } const planetCount = bodies.filter((body) => body.kind === 'planet').length; assertCondition(planetCount === 8, `Expected 8 planets, found ${planetCount}.`); const dwarfCount = bodies.filter((body) => body.kind === 'dwarf').length; assertCondition(dwarfCount === 5, `Expected the IAU's 5 dwarf planets, found ${dwarfCount}.`); console.log(` mean elements against Horizons, degrees: ${offsets.join(', ')}.`); } function validateExoplanets(exoplanets: ExoplanetRecord[], starIds: Set): void { assertCondition(exoplanets.length > 0, 'No exoplanets were produced.'); let crossReferenced = 0; for (const exoplanet of exoplanets) { assertCondition(!!exoplanet.name, `Exoplanet ${exoplanet.id} has no name.`); if (exoplanet.hostStarId !== null) { assertCondition(starIds.has(exoplanet.hostStarId), `Exoplanet ${exoplanet.id} references unknown star id ${exoplanet.hostStarId}.`); // The Sun has no exoplanets, so any match to it is a matching failure — historically a // blank distance column parsing as 0, which puts the host at the origin and matches Sol // exactly. Free, permanent tripwire for that whole class of bug. assertCondition( exoplanet.hostStarId !== SUN_STAR_ID, `Exoplanet ${exoplanet.id} was matched to the Sun, which has no exoplanets — the host-star match is wrong.` ); crossReferenced++; } assertCondition( exoplanet.periodDays === undefined || exoplanet.periodDays > 0, `Exoplanet ${exoplanet.id} has a non-positive orbital period.` ); assertCondition( exoplanet.hostStarMassSolar === undefined || exoplanet.hostStarMassSolar > 0, `Exoplanet ${exoplanet.id} has a non-positive host star mass.` ); } console.log(` ${crossReferenced}/${exoplanets.length} exoplanets cross-referenced to a HYG host star.`); // How many can be propagated at their real rate rather than as if the host were the Sun. const withPeriod = exoplanets.filter((exoplanet) => exoplanet.periodDays !== undefined).length; const withHostMass = exoplanets.filter((exoplanet) => exoplanet.hostStarMassSolar !== undefined).length; console.log(` ${withPeriod}/${exoplanets.length} have a measured period, ${withHostMass} a host star mass.`); } const UNIT_VECTOR_TOLERANCE = 1e-6; function validateDeepSky(objects: DeepSkyRecord[]): void { assertCondition(objects.length > 0, 'No deep-sky objects were produced.'); const ids = new Set(); for (const object of objects) { assertCondition(!!object.id, `Deep-sky object has no id: ${JSON.stringify(object)}`); assertCondition(!ids.has(object.id), `Duplicate deep-sky id: ${object.id}`); ids.add(object.id); assertCondition(!!object.name, `Deep-sky object ${object.id} has no name.`); // Positions are directions, so every one of them must be a unit vector — a zero-length // or mis-scaled entry would silently collapse onto the origin on the backdrop shell. const length = Math.hypot(object.x, object.y, object.z); assertCondition(Math.abs(length - 1) < UNIT_VECTOR_TOLERANCE, `Deep-sky object ${object.id} has a non-unit direction (length ${length}).`); assertCondition(object.angularSizeDeg >= 0, `Deep-sky object ${object.id} has a negative angular size.`); assertCondition(object.distancePc === null || object.distancePc > 0, `Deep-sky object ${object.id} has a non-positive distance.`); // The distance and its provenance have to travel together, or the UI cannot say where a // number came from. assertCondition( (object.distancePc === null) === (object.distanceMethod === null), `Deep-sky object ${object.id} has a distance/method mismatch.` ); } const kinds = new Set(objects.map((object) => object.kind)); for (const kind of ['galaxy', 'nebula', 'cluster'] as const) { assertCondition(kinds.has(kind), `No deep-sky objects of kind "${kind}" were produced.`); } const withDistance = objects.filter((object) => object.distancePc !== null).length; console.log(` ${withDistance}/${objects.length} deep-sky objects have a derived distance.`); } /** * Orchestrates the whole ETL pipeline: fetches every source (each caches its own raw * responses under `tools/etl/.cache/`), writes the static assets under `src/assets/data/`, * then validates the combined output for completeness before declaring success. */ async function build(): Promise { console.log('=== NASA star map ETL ===\n'); console.log('Catalogues:'); console.log(describeSources()); console.log(); const stars = await fetchStars(); console.log(); const { bodies, horizonsOrbits } = await fetchSolarSystem(); console.log(); const exoplanets = await fetchExoplanets(stars); console.log(); const deepSky = await fetchDeepSky(); console.log(); console.log('Validating output...'); validateStars(stars); validateMerge(stars); validateBodies(bodies, horizonsOrbits); validateExoplanets(exoplanets, new Set(stars.map((star) => star.id))); validateDeepSky(deepSky); console.log('\nETL completed successfully:'); console.log(` stars: ${stars.length}`); console.log(` bodies: ${bodies.length}`); console.log(` exoplanets: ${exoplanets.length}`); console.log(` deep sky: ${deepSky.length}`); } build().catch((error) => { console.error('\nETL failed:', error instanceof Error ? error.message : error); process.exitCode = 1; });