From f935bae3b314fa46ad159003376dce98f96161f0 Mon Sep 17 00:00:00 2001 From: Senrokai Date: Wed, 9 Sep 2026 21:10:23 +0200 Subject: [PATCH] Fail the ETL on a merge that keeps the same star twice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catalogues are regenerated by a scheduled job that pushes straight to main once the unit suite and a production build pass in the same run. Both passed, every Monday, on a catalogue that carried 23 000 stars twice: the suite tests code against fixtures, and no fixture is 400 000 real stars. Nothing between the ETL and the map ever looked at what came out. Two numbers now have to hold, and each is the signature of a way the merge has actually failed here. Different catalogues placing a star within an arcsecond of each other is never two stars at this depth, and one catalogue does not list a star twice, so every cross-source pair that close is a miss. Nineteen survive today — each a second HYG row wanting a Gaia entry that already absorbed one, which is how Gliese lists some doubles — against 1 112 in the catalogue on main, where a Hipparcos parallax off by half outvoted a direction that agreed to a hundredth of an arcsecond. The ceiling is 100. The epoch failure leaves no close pair at all, because sixteen years of proper motion had already carried the two entries tens of arcseconds apart. What it leaves instead is HYG rows that found no counterpart: 36 056 on main against the 10 876 Gaia genuinely lacks — the stars it saturates on and the red dwarfs past its magnitude cut. The ceiling is 15 000. The pair sweep sorts by declination and walks a one-arcsecond window, so it costs about 300 ms on 423 641 stars — cheap enough to run on every ETL, which is the point: the gate has to sit where the bot already is, before the push, because a GITHUB_TOKEN push fires no CI of its own. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016jxMkwA2rbicdGxHosecYi --- tools/etl/build.ts | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tools/etl/build.ts b/tools/etl/build.ts index bc82657..53e49ef 100644 --- a/tools/etl/build.ts +++ b/tools/etl/build.ts @@ -49,6 +49,71 @@ function validateStars(stars: StarRecord[]): void { } } +/** + * What a good merge looks like, in two numbers the unit suite cannot see. + * + * The catalogues are regenerated by a scheduled job that pushes straight to `main` once the unit + * tests and a production build pass — and both passed, for weeks, on a catalogue carrying 23 000 + * stars twice: the suite tests code against fixtures, and no fixture is 400 000 real stars. The + * two ways the merge has actually failed both show up here. + * + * A star kept twice leaves its two entries near each other on the sky, from *different* sources — + * one catalogue does not list a star twice. Under an arcsecond that is never two stars at this + * depth, so every such pair is a miss. Nineteen survive today, all of them a second HYG row + * wanting a Gaia entry that already absorbed one (Gliese lists some doubles twice); the merge + * that trusted a Hipparcos parallax over direction left 1 112. + * + * The other failure leaves no close pair at all, because proper motion had already carried the + * two entries tens of arcseconds apart — the 2026-08-24 refresh, where HYG sat at epoch 2000.0 + * and Gaia at J2016.0. What it does leave is HYG rows that found no counterpart: 36 056 of them + * against the 10 876 Gaia genuinely lacks (bright stars it saturates on, red dwarfs past its + * magnitude cut). + */ +const MAX_UNMERGED_TWINS = 100; +const MAX_HYG_SURVIVORS = 15_000; +const TWIN_TOLERANCE_RAD = (1 / 3600) * (Math.PI / 180); + +function validateMerge(stars: StarRecord[]): void { + const survivors = stars.filter((star) => star.source === 'hyg').length; + assertCondition( + survivors <= MAX_HYG_SURVIVORS, + `${survivors} HYG stars found no Gaia counterpart (at most ${MAX_HYG_SURVIVORS} expected) — the two catalogues are not being matched.` + ); + + // Sorted by declination, so each star is only compared against the handful sharing its + // parallel — an arcsecond of declination holds one or two of 400 000 stars. + const byDec = stars + .map((star) => { + const distance = Math.hypot(star.x, star.y, star.z); + return { star, distance, dec: distance === 0 ? 0 : Math.asin(Math.max(-1, Math.min(1, star.z / distance))) }; + }) + .filter((entry) => entry.distance > 0) + .sort((a, b) => a.dec - b.dec); + + const cosTolerance = Math.cos(TWIN_TOLERANCE_RAD); + let twins = 0; + let example = ''; + for (let i = 0; i < byDec.length; i++) { + const a = byDec[i]; + for (let j = i + 1; j < byDec.length && byDec[j].dec - a.dec <= TWIN_TOLERANCE_RAD; j++) { + const b = byDec[j]; + if (a.star.source === b.star.source) { + continue; + } + const cosine = (a.star.x * b.star.x + a.star.y * b.star.y + a.star.z * b.star.z) / (a.distance * b.distance); + if (cosine >= cosTolerance) { + twins++; + example ||= `${a.star.name} (${a.star.source}) and ${b.star.name} (${b.star.source})`; + } + } + } + assertCondition( + twins <= MAX_UNMERGED_TWINS, + `${twins} stars from different catalogues sit within an arcsecond of each other (at most ${MAX_UNMERGED_TWINS} expected), starting with ${example} — the merge is keeping the same star twice.` + ); + console.log(` ${survivors} HYG stars have no Gaia counterpart; ${twins} unmerged cross-catalogue pairs within an arcsecond.`); +} + function validateBodies(bodies: BodyRecord[]): void { assertCondition(bodies.length > 0, 'No solar-system bodies were produced.'); @@ -162,6 +227,7 @@ async function build(): Promise { console.log('Validating output...'); validateStars(stars); + validateMerge(stars); validateBodies(bodies); validateExoplanets(exoplanets, new Set(stars.map((star) => star.id))); validateDeepSky(deepSky);