diff --git a/tools/etl/build.ts b/tools/etl/build.ts index bc82657..53e49ef 100644 --- a/tools/etl/build.ts +++ b/tools/etl/build.ts @@ -49,6 +49,71 @@ function validateStars(stars: StarRecord[]): void { } } +/** + * What a good merge looks like, in two numbers the unit suite cannot see. + * + * The catalogues are regenerated by a scheduled job that pushes straight to `main` once the unit + * tests and a production build pass — and both passed, for weeks, on a catalogue carrying 23 000 + * stars twice: the suite tests code against fixtures, and no fixture is 400 000 real stars. The + * two ways the merge has actually failed both show up here. + * + * A star kept twice leaves its two entries near each other on the sky, from *different* sources — + * one catalogue does not list a star twice. Under an arcsecond that is never two stars at this + * depth, so every such pair is a miss. Nineteen survive today, all of them a second HYG row + * wanting a Gaia entry that already absorbed one (Gliese lists some doubles twice); the merge + * that trusted a Hipparcos parallax over direction left 1 112. + * + * The other failure leaves no close pair at all, because proper motion had already carried the + * two entries tens of arcseconds apart — the 2026-08-24 refresh, where HYG sat at epoch 2000.0 + * and Gaia at J2016.0. What it does leave is HYG rows that found no counterpart: 36 056 of them + * against the 10 876 Gaia genuinely lacks (bright stars it saturates on, red dwarfs past its + * magnitude cut). + */ +const MAX_UNMERGED_TWINS = 100; +const MAX_HYG_SURVIVORS = 15_000; +const TWIN_TOLERANCE_RAD = (1 / 3600) * (Math.PI / 180); + +function validateMerge(stars: StarRecord[]): void { + const survivors = stars.filter((star) => star.source === 'hyg').length; + assertCondition( + survivors <= MAX_HYG_SURVIVORS, + `${survivors} HYG stars found no Gaia counterpart (at most ${MAX_HYG_SURVIVORS} expected) — the two catalogues are not being matched.` + ); + + // Sorted by declination, so each star is only compared against the handful sharing its + // parallel — an arcsecond of declination holds one or two of 400 000 stars. + const byDec = stars + .map((star) => { + const distance = Math.hypot(star.x, star.y, star.z); + return { star, distance, dec: distance === 0 ? 0 : Math.asin(Math.max(-1, Math.min(1, star.z / distance))) }; + }) + .filter((entry) => entry.distance > 0) + .sort((a, b) => a.dec - b.dec); + + const cosTolerance = Math.cos(TWIN_TOLERANCE_RAD); + let twins = 0; + let example = ''; + for (let i = 0; i < byDec.length; i++) { + const a = byDec[i]; + for (let j = i + 1; j < byDec.length && byDec[j].dec - a.dec <= TWIN_TOLERANCE_RAD; j++) { + const b = byDec[j]; + if (a.star.source === b.star.source) { + continue; + } + const cosine = (a.star.x * b.star.x + a.star.y * b.star.y + a.star.z * b.star.z) / (a.distance * b.distance); + if (cosine >= cosTolerance) { + twins++; + example ||= `${a.star.name} (${a.star.source}) and ${b.star.name} (${b.star.source})`; + } + } + } + assertCondition( + twins <= MAX_UNMERGED_TWINS, + `${twins} stars from different catalogues sit within an arcsecond of each other (at most ${MAX_UNMERGED_TWINS} expected), starting with ${example} — the merge is keeping the same star twice.` + ); + console.log(` ${survivors} HYG stars have no Gaia counterpart; ${twins} unmerged cross-catalogue pairs within an arcsecond.`); +} + function validateBodies(bodies: BodyRecord[]): void { assertCondition(bodies.length > 0, 'No solar-system bodies were produced.'); @@ -162,6 +227,7 @@ async function build(): Promise { console.log('Validating output...'); validateStars(stars); + validateMerge(stars); validateBodies(bodies); validateExoplanets(exoplanets, new Set(stars.map((star) => star.id))); validateDeepSky(deepSky);