diff --git a/.github/workflows/data-refresh.yml b/.github/workflows/data-refresh.yml index b1d3b87..15899f9 100644 --- a/.github/workflows/data-refresh.yml +++ b/.github/workflows/data-refresh.yml @@ -41,7 +41,8 @@ jobs: # The ETL's cache directory is gitignored and this is a fresh runner, so every source is # fetched live (~50-100 MB). A failed fetch fails the run by design — no refresh is - # better than a partial one — except Gaia, which the ETL itself treats as best-effort. + # better than a partial one. That includes Gaia: the ETL skips it when unreachable, and + # the merge gate in build.ts then refuses a catalogue it contributed nothing to. - name: Rebuild the datasets run: npm run etl diff --git a/tools/etl/build.ts b/tools/etl/build.ts index bc82657..b213455 100644 --- a/tools/etl/build.ts +++ b/tools/etl/build.ts @@ -49,6 +49,77 @@ function validateStars(stars: StarRecord[]): void { } } +/** + * What a good merge looks like, in two numbers the unit suite cannot see. + * + * The catalogues are regenerated by a scheduled job that pushes straight to `main` once the unit + * tests and a production build pass — and both passed, for weeks, on a catalogue carrying 23 000 + * stars twice: the suite tests code against fixtures, and no fixture is 400 000 real stars. The + * two ways the merge has actually failed both show up here. + * + * A star kept twice leaves its two entries near each other on the sky, from *different* sources — + * one catalogue does not list a star twice. Under an arcsecond that is never two stars at this + * depth, so every such pair is a miss. Nineteen survive today, all of them a second HYG row + * wanting a Gaia entry that already absorbed one (Gliese lists some doubles twice); the merge + * that trusted a Hipparcos parallax over direction left 1 112. + * + * The other failure leaves no close pair at all, because proper motion had already carried the + * two entries tens of arcseconds apart — the 2026-08-24 refresh, where HYG sat at epoch 2000.0 + * and Gaia at J2016.0. What it does leave is HYG rows that found no counterpart: 36 056 of them + * against the 10 876 Gaia genuinely lacks (bright stars it saturates on, red dwarfs past its + * magnitude cut). + */ +const MAX_UNMERGED_TWINS = 100; +const MAX_HYG_SURVIVORS = 15_000; +const TWIN_TOLERANCE_RAD = (1 / 3600) * (Math.PI / 180); + +function validateMerge(stars: StarRecord[]): void { + // Checked first and on its own: an unreachable Gaia is skipped rather than thrown, and would + // otherwise surface below as "68 000 HYG stars found no counterpart" — true, and no help. + assertCondition( + stars.some((star) => star.source === 'gaia'), + 'Gaia DR3 contributed no stars — the archive was unreachable or returned nothing, and a catalogue without it is not one to publish.' + ); + const survivors = stars.filter((star) => star.source === 'hyg').length; + assertCondition( + survivors <= MAX_HYG_SURVIVORS, + `${survivors} HYG stars found no Gaia counterpart (at most ${MAX_HYG_SURVIVORS} expected) — the two catalogues are not being matched.` + ); + + // Sorted by declination, so each star is only compared against the handful sharing its + // parallel — an arcsecond of declination holds one or two of 400 000 stars. + const byDec = stars + .map((star) => { + const distance = Math.hypot(star.x, star.y, star.z); + return { star, distance, dec: distance === 0 ? 0 : Math.asin(Math.max(-1, Math.min(1, star.z / distance))) }; + }) + .filter((entry) => entry.distance > 0) + .sort((a, b) => a.dec - b.dec); + + const cosTolerance = Math.cos(TWIN_TOLERANCE_RAD); + let twins = 0; + let example = ''; + for (let i = 0; i < byDec.length; i++) { + const a = byDec[i]; + for (let j = i + 1; j < byDec.length && byDec[j].dec - a.dec <= TWIN_TOLERANCE_RAD; j++) { + const b = byDec[j]; + if (a.star.source === b.star.source) { + continue; + } + const cosine = (a.star.x * b.star.x + a.star.y * b.star.y + a.star.z * b.star.z) / (a.distance * b.distance); + if (cosine >= cosTolerance) { + twins++; + example ||= `${a.star.name} (${a.star.source}) and ${b.star.name} (${b.star.source})`; + } + } + } + assertCondition( + twins <= MAX_UNMERGED_TWINS, + `${twins} stars from different catalogues sit within an arcsecond of each other (at most ${MAX_UNMERGED_TWINS} expected), starting with ${example} — the merge is keeping the same star twice.` + ); + console.log(` ${survivors} HYG stars have no Gaia counterpart; ${twins} unmerged cross-catalogue pairs within an arcsecond.`); +} + function validateBodies(bodies: BodyRecord[]): void { assertCondition(bodies.length > 0, 'No solar-system bodies were produced.'); @@ -162,6 +233,7 @@ async function build(): Promise { console.log('Validating output...'); validateStars(stars); + validateMerge(stars); validateBodies(bodies); validateExoplanets(exoplanets, new Set(stars.map((star) => star.id))); validateDeepSky(deepSky); diff --git a/tools/etl/fetchStars.ts b/tools/etl/fetchStars.ts index 22e6113..e4da7eb 100644 --- a/tools/etl/fetchStars.ts +++ b/tools/etl/fetchStars.ts @@ -119,9 +119,9 @@ export async function fetchStars(): Promise { /** * Unions HYG with every other positional source that is wired in and reachable. * - * A source that cannot be reached is reported and skipped rather than failing the run. That is - * not defensive padding: the archives this would draw on are frequently unavailable, and a build - * that produces a smaller catalogue is far better than one that produces none. + * A source that cannot be reached is reported and skipped here rather than thrown, so a run still + * gets as far as validation and says what it has. Whether that may be published is decided + * there: `validateMerge` in build.ts refuses a catalogue Gaia contributed nothing to. */ async function mergeWithOtherSources(hygStars: StarRecord[]): Promise { const others = positionalSources().filter((source) => source.id !== 'hyg');