From f935bae3b314fa46ad159003376dce98f96161f0 Mon Sep 17 00:00:00 2001 From: Senrokai Date: Wed, 9 Sep 2026 21:10:23 +0200 Subject: [PATCH 1/2] Fail the ETL on a merge that keeps the same star twice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The catalogues are regenerated by a scheduled job that pushes straight to main once the unit suite and a production build pass in the same run. Both passed, every Monday, on a catalogue that carried 23 000 stars twice: the suite tests code against fixtures, and no fixture is 400 000 real stars. Nothing between the ETL and the map ever looked at what came out. Two numbers now have to hold, and each is the signature of a way the merge has actually failed here. Different catalogues placing a star within an arcsecond of each other is never two stars at this depth, and one catalogue does not list a star twice, so every cross-source pair that close is a miss. Nineteen survive today — each a second HYG row wanting a Gaia entry that already absorbed one, which is how Gliese lists some doubles — against 1 112 in the catalogue on main, where a Hipparcos parallax off by half outvoted a direction that agreed to a hundredth of an arcsecond. The ceiling is 100. The epoch failure leaves no close pair at all, because sixteen years of proper motion had already carried the two entries tens of arcseconds apart. What it leaves instead is HYG rows that found no counterpart: 36 056 on main against the 10 876 Gaia genuinely lacks — the stars it saturates on and the red dwarfs past its magnitude cut. The ceiling is 15 000. The pair sweep sorts by declination and walks a one-arcsecond window, so it costs about 300 ms on 423 641 stars — cheap enough to run on every ETL, which is the point: the gate has to sit where the bot already is, before the push, because a GITHUB_TOKEN push fires no CI of its own. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016jxMkwA2rbicdGxHosecYi --- tools/etl/build.ts | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/tools/etl/build.ts b/tools/etl/build.ts index bc82657..53e49ef 100644 --- a/tools/etl/build.ts +++ b/tools/etl/build.ts @@ -49,6 +49,71 @@ function validateStars(stars: StarRecord[]): void { } } +/** + * What a good merge looks like, in two numbers the unit suite cannot see. + * + * The catalogues are regenerated by a scheduled job that pushes straight to `main` once the unit + * tests and a production build pass — and both passed, for weeks, on a catalogue carrying 23 000 + * stars twice: the suite tests code against fixtures, and no fixture is 400 000 real stars. The + * two ways the merge has actually failed both show up here. + * + * A star kept twice leaves its two entries near each other on the sky, from *different* sources — + * one catalogue does not list a star twice. Under an arcsecond that is never two stars at this + * depth, so every such pair is a miss. Nineteen survive today, all of them a second HYG row + * wanting a Gaia entry that already absorbed one (Gliese lists some doubles twice); the merge + * that trusted a Hipparcos parallax over direction left 1 112. + * + * The other failure leaves no close pair at all, because proper motion had already carried the + * two entries tens of arcseconds apart — the 2026-08-24 refresh, where HYG sat at epoch 2000.0 + * and Gaia at J2016.0. What it does leave is HYG rows that found no counterpart: 36 056 of them + * against the 10 876 Gaia genuinely lacks (bright stars it saturates on, red dwarfs past its + * magnitude cut). + */ +const MAX_UNMERGED_TWINS = 100; +const MAX_HYG_SURVIVORS = 15_000; +const TWIN_TOLERANCE_RAD = (1 / 3600) * (Math.PI / 180); + +function validateMerge(stars: StarRecord[]): void { + const survivors = stars.filter((star) => star.source === 'hyg').length; + assertCondition( + survivors <= MAX_HYG_SURVIVORS, + `${survivors} HYG stars found no Gaia counterpart (at most ${MAX_HYG_SURVIVORS} expected) — the two catalogues are not being matched.` + ); + + // Sorted by declination, so each star is only compared against the handful sharing its + // parallel — an arcsecond of declination holds one or two of 400 000 stars. + const byDec = stars + .map((star) => { + const distance = Math.hypot(star.x, star.y, star.z); + return { star, distance, dec: distance === 0 ? 0 : Math.asin(Math.max(-1, Math.min(1, star.z / distance))) }; + }) + .filter((entry) => entry.distance > 0) + .sort((a, b) => a.dec - b.dec); + + const cosTolerance = Math.cos(TWIN_TOLERANCE_RAD); + let twins = 0; + let example = ''; + for (let i = 0; i < byDec.length; i++) { + const a = byDec[i]; + for (let j = i + 1; j < byDec.length && byDec[j].dec - a.dec <= TWIN_TOLERANCE_RAD; j++) { + const b = byDec[j]; + if (a.star.source === b.star.source) { + continue; + } + const cosine = (a.star.x * b.star.x + a.star.y * b.star.y + a.star.z * b.star.z) / (a.distance * b.distance); + if (cosine >= cosTolerance) { + twins++; + example ||= `${a.star.name} (${a.star.source}) and ${b.star.name} (${b.star.source})`; + } + } + } + assertCondition( + twins <= MAX_UNMERGED_TWINS, + `${twins} stars from different catalogues sit within an arcsecond of each other (at most ${MAX_UNMERGED_TWINS} expected), starting with ${example} — the merge is keeping the same star twice.` + ); + console.log(` ${survivors} HYG stars have no Gaia counterpart; ${twins} unmerged cross-catalogue pairs within an arcsecond.`); +} + function validateBodies(bodies: BodyRecord[]): void { assertCondition(bodies.length > 0, 'No solar-system bodies were produced.'); @@ -162,6 +227,7 @@ async function build(): Promise { console.log('Validating output...'); validateStars(stars); + validateMerge(stars); validateBodies(bodies); validateExoplanets(exoplanets, new Set(stars.map((star) => star.id))); validateDeepSky(deepSky); From 29d3ddb6efb39a69bc3d4a9e7cc65ad8d569307e Mon Sep 17 00:00:00 2001 From: Senrokai Date: Fri, 11 Sep 2026 18:41:09 +0200 Subject: [PATCH 2/2] Say so when Gaia is missing, rather than as 68 000 unmatched stars With the merge gate in place, a Gaia DR3 outage no longer ships a HYG-only catalogue: the ETL skips the unreachable source, and validateMerge then fails on the survivor count. That is the right outcome and the wrong message: "68 000 HYG stars found no Gaia counterpart" sends the reader looking at the merge. Gaia contributing nothing is now checked first, by name. Two comments said Gaia was best-effort, in data-refresh.yml and on the merge in fetchStars. They now say what happens instead. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016jxMkwA2rbicdGxHosecYi --- .github/workflows/data-refresh.yml | 3 ++- tools/etl/build.ts | 6 ++++++ tools/etl/fetchStars.ts | 6 +++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/data-refresh.yml b/.github/workflows/data-refresh.yml index b1d3b87..15899f9 100644 --- a/.github/workflows/data-refresh.yml +++ b/.github/workflows/data-refresh.yml @@ -41,7 +41,8 @@ jobs: # The ETL's cache directory is gitignored and this is a fresh runner, so every source is # fetched live (~50-100 MB). A failed fetch fails the run by design — no refresh is - # better than a partial one — except Gaia, which the ETL itself treats as best-effort. + # better than a partial one. That includes Gaia: the ETL skips it when unreachable, and + # the merge gate in build.ts then refuses a catalogue it contributed nothing to. - name: Rebuild the datasets run: npm run etl diff --git a/tools/etl/build.ts b/tools/etl/build.ts index 53e49ef..b213455 100644 --- a/tools/etl/build.ts +++ b/tools/etl/build.ts @@ -74,6 +74,12 @@ const MAX_HYG_SURVIVORS = 15_000; const TWIN_TOLERANCE_RAD = (1 / 3600) * (Math.PI / 180); function validateMerge(stars: StarRecord[]): void { + // Checked first and on its own: an unreachable Gaia is skipped rather than thrown, and would + // otherwise surface below as "68 000 HYG stars found no counterpart" — true, and no help. + assertCondition( + stars.some((star) => star.source === 'gaia'), + 'Gaia DR3 contributed no stars — the archive was unreachable or returned nothing, and a catalogue without it is not one to publish.' + ); const survivors = stars.filter((star) => star.source === 'hyg').length; assertCondition( survivors <= MAX_HYG_SURVIVORS, diff --git a/tools/etl/fetchStars.ts b/tools/etl/fetchStars.ts index 22e6113..e4da7eb 100644 --- a/tools/etl/fetchStars.ts +++ b/tools/etl/fetchStars.ts @@ -119,9 +119,9 @@ export async function fetchStars(): Promise { /** * Unions HYG with every other positional source that is wired in and reachable. * - * A source that cannot be reached is reported and skipped rather than failing the run. That is - * not defensive padding: the archives this would draw on are frequently unavailable, and a build - * that produces a smaller catalogue is far better than one that produces none. + * A source that cannot be reached is reported and skipped here rather than thrown, so a run still + * gets as far as validation and says what it has. Whether that may be published is decided + * there: `validateMerge` in build.ts refuses a catalogue Gaia contributed nothing to. */ async function mergeWithOtherSources(hygStars: StarRecord[]): Promise { const others = positionalSources().filter((source) => source.id !== 'hyg');