Merge pull request #17 from avalon-vanguard/star-map/guard/merge-quality

Fail the ETL on a merge that keeps the same star twice

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016jxMkwA2rbicdGxHosecYi

.github/workflows/data-refresh.yml

@@ -41,7 +41,8 @@ jobs:
 
       # The ETL's cache directory is gitignored and this is a fresh runner, so every source is
       # fetched live (~50-100 MB). A failed fetch fails the run by design — no refresh is
-      # better than a partial one — except Gaia, which the ETL itself treats as best-effort.
+      # better than a partial one. That includes Gaia: the ETL skips it when unreachable, and
+      # the merge gate in build.ts then refuses a catalogue it contributed nothing to.
       - name: Rebuild the datasets
         run: npm run etl
 

tools/etl/build.ts

@@ -49,6 +49,77 @@ function validateStars(stars: StarRecord[]): void {
   }
 }
 
+/**
+ * What a good merge looks like, in two numbers the unit suite cannot see.
+ *
+ * The catalogues are regenerated by a scheduled job that pushes straight to `main` once the unit
+ * tests and a production build pass — and both passed, for weeks, on a catalogue carrying 23 000
+ * stars twice: the suite tests code against fixtures, and no fixture is 400 000 real stars. The
+ * two ways the merge has actually failed both show up here.
+ *
+ * A star kept twice leaves its two entries near each other on the sky, from *different* sources —
+ * one catalogue does not list a star twice. Under an arcsecond that is never two stars at this
+ * depth, so every such pair is a miss. Nineteen survive today, all of them a second HYG row
+ * wanting a Gaia entry that already absorbed one (Gliese lists some doubles twice); the merge
+ * that trusted a Hipparcos parallax over direction left 1 112.
+ *
+ * The other failure leaves no close pair at all, because proper motion had already carried the
+ * two entries tens of arcseconds apart — the 2026-08-24 refresh, where HYG sat at epoch 2000.0
+ * and Gaia at J2016.0. What it does leave is HYG rows that found no counterpart: 36 056 of them
+ * against the 10 876 Gaia genuinely lacks (bright stars it saturates on, red dwarfs past its
+ * magnitude cut).
+ */
+const MAX_UNMERGED_TWINS = 100;
+const MAX_HYG_SURVIVORS = 15_000;
+const TWIN_TOLERANCE_RAD = (1 / 3600) * (Math.PI / 180);
+
+function validateMerge(stars: StarRecord[]): void {
+  // Checked first and on its own: an unreachable Gaia is skipped rather than thrown, and would
+  // otherwise surface below as "68 000 HYG stars found no counterpart" — true, and no help.
+  assertCondition(
+    stars.some((star) => star.source === 'gaia'),
+    'Gaia DR3 contributed no stars — the archive was unreachable or returned nothing, and a catalogue without it is not one to publish.'
+  );
+  const survivors = stars.filter((star) => star.source === 'hyg').length;
+  assertCondition(
+    survivors <= MAX_HYG_SURVIVORS,
+    `${survivors} HYG stars found no Gaia counterpart (at most ${MAX_HYG_SURVIVORS} expected) — the two catalogues are not being matched.`
+  );
+
+  // Sorted by declination, so each star is only compared against the handful sharing its
+  // parallel — an arcsecond of declination holds one or two of 400 000 stars.
+  const byDec = stars
+    .map((star) => {
+      const distance = Math.hypot(star.x, star.y, star.z);
+      return { star, distance, dec: distance === 0 ? 0 : Math.asin(Math.max(-1, Math.min(1, star.z / distance))) };
+    })
+    .filter((entry) => entry.distance > 0)
+    .sort((a, b) => a.dec - b.dec);
+
+  const cosTolerance = Math.cos(TWIN_TOLERANCE_RAD);
+  let twins = 0;
+  let example = '';
+  for (let i = 0; i < byDec.length; i++) {
+    const a = byDec[i];
+    for (let j = i + 1; j < byDec.length && byDec[j].dec - a.dec <= TWIN_TOLERANCE_RAD; j++) {
+      const b = byDec[j];
+      if (a.star.source === b.star.source) {
+        continue;
+      }
+      const cosine = (a.star.x * b.star.x + a.star.y * b.star.y + a.star.z * b.star.z) / (a.distance * b.distance);
+      if (cosine >= cosTolerance) {
+        twins++;
+        example ||= `${a.star.name} (${a.star.source}) and ${b.star.name} (${b.star.source})`;
+      }
+    }
+  }
+  assertCondition(
+    twins <= MAX_UNMERGED_TWINS,
+    `${twins} stars from different catalogues sit within an arcsecond of each other (at most ${MAX_UNMERGED_TWINS} expected), starting with ${example} — the merge is keeping the same star twice.`
+  );
+  console.log(`  ${survivors} HYG stars have no Gaia counterpart; ${twins} unmerged cross-catalogue pairs within an arcsecond.`);
+}
+
 function validateBodies(bodies: BodyRecord[]): void {
   assertCondition(bodies.length > 0, 'No solar-system bodies were produced.');
 
@@ -162,6 +233,7 @@ async function build(): Promise<void> {
 
   console.log('Validating output...');
   validateStars(stars);
+  validateMerge(stars);
   validateBodies(bodies);
   validateExoplanets(exoplanets, new Set(stars.map((star) => star.id)));
   validateDeepSky(deepSky);

tools/etl/fetchStars.ts

@@ -119,9 +119,9 @@ export async function fetchStars(): Promise<StarRecord[]> {
 /**
  * Unions HYG with every other positional source that is wired in and reachable.
  *
- * A source that cannot be reached is reported and skipped rather than failing the run. That is
- * not defensive padding: the archives this would draw on are frequently unavailable, and a build
- * that produces a smaller catalogue is far better than one that produces none.
+ * A source that cannot be reached is reported and skipped here rather than thrown, so a run still
+ * gets as far as validation and says what it has. Whether that may be published is decided
+ * there: `validateMerge` in build.ts refuses a catalogue Gaia contributed nothing to.
  */
 async function mergeWithOtherSources(hygStars: StarRecord[]): Promise<StarRecord[]> {
   const others = positionalSources().filter((source) => source.id !== 'hyg');