Merge pull request #18 from avalon-vanguard/star-map/fix/hyg-gaia-distances

Draw each HYG star at Gaia's distance, and keep the ones Hipparcos misplaced

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016jxMkwA2rbicdGxHosecYi

.github/workflows/data-refresh.yml

@@ -39,10 +39,19 @@ jobs:
 
       - run: npm ci
 
-      # The ETL's cache directory is gitignored and this is a fresh runner, so every source is
-      # fetched live (~50-100 MB). A failed fetch fails the run by design — no refresh is
-      # better than a partial one. That includes Gaia: the ETL skips it when unreachable, and
-      # the merge gate in build.ts then refuses a catalogue it contributed nothing to.
+      # Gaia DR3 is a frozen release: the same query returns the same bytes (a live re-fetch has
+      # reproduced stars.bin exactly), so its responses are carried from one run to the next
+      # rather than re-downloaded every week from an archive that times out under load. The key
+      # follows gaia.ts, where the queries are written, so a changed query is fetched afresh.
+      - uses: actions/cache@v4
+        with:
+          path: tools/etl/.cache/gaia-dr3-*.csv
+          key: gaia-dr3-${{ hashFiles('tools/etl/sources/gaia.ts') }}
+
+      # Every other source is fetched live on this fresh runner. A failed fetch fails the run by
+      # design — no refresh is better than a partial one. That includes Gaia on a cold cache:
+      # the ETL skips it when unreachable, and the merge gate in build.ts then refuses a
+      # catalogue it contributed nothing to.
       - name: Rebuild the datasets
         run: npm run etl
 

src/app/features/galaxy-system/galaxy-system-scene.component.ts

@@ -1178,7 +1178,7 @@ export class GalaxySystemSceneComponent implements AfterViewInit, OnDestroy {
         { label: 'Catalogued', value: `${this.stars.length} stars` }
       ]);
       // Quotes the catalogue's own reach rather than a figure that has already been raised once.
-      this.hudNote.set(`Galactic structure is an illustrative model built on measured arm geometry — no catalogue holds the Galaxy’s stars. The ${this.stars.length} catalogued stars within ${LOCAL_GRID_RINGS_PC[LOCAL_GRID_RINGS_PC.length - 1]} pc are real.`);
+      this.hudNote.set(`Galactic structure is an illustrative model built on measured arm geometry — no catalogue holds the Galaxy’s stars. The ${this.stars.length} catalogued stars are real.`);
       return;
     }
 
@@ -1189,7 +1189,9 @@ export class GalaxySystemSceneComponent implements AfterViewInit, OnDestroy {
       // Both numbers, because they differ: the catalogue is what the map knows and the first is
       // what it draws. See `STAR_RENDER_BUDGET`.
       { label: 'Stars', value: this.starField && this.starField.drawnCount < this.stars.length ? `${this.starField.drawnCount} / ${this.stars.length}` : `${this.stars.length}` },
-      { label: 'Radius', value: `${LOCAL_GRID_RINGS_PC[LOCAL_GRID_RINGS_PC.length - 1]} pc` },
+      // The radius Gaia is surveyed to, not the edge of the map: the Hipparcos stars Gaia places
+      // further out are drawn where it places them.
+      { label: 'Survey radius', value: `${LOCAL_GRID_RINGS_PC[LOCAL_GRID_RINGS_PC.length - 1]} pc` },
       { label: 'Exoplanets', value: `${this.exoplanets.length}` },
       // The one thing the field itself cannot show: which of those points can be flown into.
       { label: 'Systems', value: `${this.enterableSystems}` }

src/app/shared/astro/star-merge.spec.ts

@@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest';
 
 import { raDegDecDistanceToXyz } from './coordinates';
 import { StarRecord } from '../models/star.model';
-import { directionCosine, isSameStar, MERGE_ANGULAR_TOLERANCE_DEG, mergeStarCatalogues } from './star-merge';
+import { directionCosine, isSameStar, MERGE_ANGULAR_TOLERANCE_DEG, mergeStarCatalogues, placementDistancePc } from './star-merge';
 
 /** A star at a given sky position and distance, which is how catalogues actually report them. */
 function at(id: number, raDeg: number, decDeg: number, distancePc: number, overrides: Partial<StarRecord> = {}): StarRecord {
@@ -237,3 +237,35 @@ describe('mergeStarCatalogues', () => {
     expect(Date.now() - started).toBeLessThan(10000);
   });
 });
+
+describe('placementDistancePc', () => {
+  it("draws a star both surveys measured at Gaia's distance", () => {
+    expect(placementDistancePc(120, 118.4, 250)).toBe(118.4);
+  });
+
+  // The case the old cut got wrong: Hipparcos inside, Gaia outside. Kept, at the distance Gaia
+  // gives, rather than at one a third short or dropped for having been misplaced.
+  it('keeps a star Hipparcos put inside the cutoff, where Gaia puts it, even past the cutoff', () => {
+    expect(placementDistancePc(200, 306, 250)).toBe(306);
+  });
+
+  // The mirror image: Hipparcos outside, Gaia inside. The Gaia download already holds the star,
+  // and keeping the HYG row is what lets the merge give that entry its name.
+  it('keeps a star only Gaia puts inside the cutoff', () => {
+    expect(placementDistancePc(262, 241, 250)).toBe(241);
+  });
+
+  it('keeps a star Gaia measured and Hipparcos gave no distance for', () => {
+    expect(placementDistancePc(undefined, 180, 250)).toBe(180);
+  });
+
+  it('falls back to Hipparcos where Gaia has no usable distance', () => {
+    expect(placementDistancePc(90, undefined, 250)).toBe(90);
+  });
+
+  it('drops a star both surveys put outside, or neither measured', () => {
+    expect(placementDistancePc(300, 410, 250)).toBeNull();
+    expect(placementDistancePc(300, undefined, 250)).toBeNull();
+    expect(placementDistancePc(undefined, undefined, 250)).toBeNull();
+  });
+});

src/app/shared/astro/star-merge.ts

@@ -67,6 +67,28 @@ export const MERGE_BRIGHTER_TOLERANCE = 1;
  */
 export const MERGE_DISTANCE_RATIO_TOLERANCE = 0.5;
 
+/**
+ * Where to draw a star Hipparcos and Gaia both measured, and whether the map keeps it at all.
+ *
+ * Gaia's distance wherever it has a usable one, since its parallaxes are fifty times more
+ * precise; Hipparcos's otherwise. The two catalogues used to be cut at the same radius, each on
+ * its own distance, so a star Hipparcos put at 200 pc and Gaia at 300 was kept by one, never
+ * downloaded from the other, and drawn at 200. That was 83% of the HYG stars left without a
+ * Gaia counterpart, and at the median Hipparcos had them at two-thirds of Gaia's distance.
+ *
+ * Now a star either survey places inside `cutoffPc` is kept, and every kept star sits where the
+ * better measurement puts it, inside the cutoff or not. `null` for a star neither survey places
+ * inside, or that no survey gives a distance for.
+ */
+export function placementDistancePc(hipparcosPc: number | undefined, gaiaPc: number | undefined, cutoffPc: number): number | null {
+  const best = gaiaPc ?? hipparcosPc;
+  if (best === undefined) {
+    return null;
+  }
+  const inside = best <= cutoffPc || (hipparcosPc !== undefined && hipparcosPc <= cutoffPc);
+  return inside ? best : null;
+}
+
 export interface MergeCandidate {
   readonly sourceId: string;
   /** Lower is better — the parallax precision this source measures with, in milliarcseconds. */

tools/etl/fetchStars.ts

@@ -1,8 +1,9 @@
 import { writeFileSync } from 'node:fs';
 
-import { mergeStarCatalogues } from '../../src/app/shared/astro/star-merge';
+import { mergeStarCatalogues, placementDistancePc } from '../../src/app/shared/astro/star-merge';
 import { encodeStarCatalog } from '../../src/app/shared/models/star-catalog';
 import { StarRecord, SUN_STAR_ID } from '../../src/app/shared/models/star.model';
+import { fetchGaiaDistancesByHip } from './sources/gaia';
 import { positionalSources } from './sources/registry';
 import { PARALLAX_PRECISION_MAS } from './sources/star-sources';
 import { parseCsvObjects, parseOptionalNumber } from './lib/csv';
@@ -19,13 +20,14 @@ const HYG_UNKNOWN_DISTANCE_PC = 100000; // HYG's placeholder for unmeasured/unre
 const UNKNOWN_MAGNITUDE = 15;
 
 /**
- * Stars within this distance (parsecs) of the Sun are kept for the galaxy view.
+ * Stars either survey places within this distance (parsecs) of the Sun are kept for the galaxy
+ * view; `placementDistancePc` decides which distance a kept star is drawn at.
  *
- * Set at the range HYG's own measurements reach rather than at a round number. 98.6% of its
- * rows carry a Hipparcos identifier, and Hipparcos parallaxes are good to roughly a
- * milliarcsecond — so at 250 pc (4 mas) a star's distance is uncertain by some tens of per
- * cent, and beyond it the catalogue is plotting noise. Note that only the *radial* placement
- * blurs: a star's direction on the sky stays exact at any distance.
+ * Set at the range Hipparcos's own measurements reach rather than at a round number: its
+ * parallaxes are good to roughly a milliarcsecond, so at 250 pc (4 mas) a distance is uncertain
+ * by some tens of per cent. That is why it is not applied to the Hipparcos distance alone:
+ * Gaia puts 6 833 of the stars Hipparcos places inside it outside, and 3 666 the other way
+ * round. Only the *radial* placement blurs; a star's direction on the sky stays exact.
  *
  * The catalogue is also magnitude-limited, so this is not a volume-complete sample beyond about
  * 50 pc: it thins to the intrinsically bright, which is the same selection the naked eye makes.
@@ -58,27 +60,35 @@ function resolveName(row: Record<string, string>): string {
 }
 
 /**
- * Downloads the HYG (Hipparcos/Yale/Gliese) stellar database, takes each star's equatorial
- * Cartesian position (parsecs, epoch J2000.0), filters by distance, unions the other positional
- * sources, and writes `stars.bin` (packed positions) + `stars-index.json` (everything else).
+ * Downloads the HYG (Hipparcos/Yale/Gliese) stellar database, places each star along its
+ * equatorial direction (epoch J2000.0) at the better of its Hipparcos and Gaia distances, keeps
+ * the ones either survey puts within range, unions the other positional sources, and writes
+ * `stars.bin` (packed positions) + `stars-index.json` (everything else).
  */
 export async function fetchStars(): Promise<StarRecord[]> {
   console.log(`Fetching HYG star catalog (distance cutoff: ${DISTANCE_CUTOFF_PC} pc)...`);
   const csv = await fetchTextCached(HYG_CSV_URL, 'hygdata_v41.csv');
   const rows = parseCsvObjects(csv);
+  // Not skipped when unreachable, unlike the positional sources below; see its own comment.
+  const gaiaPcByHip = await fetchGaiaDistancesByHip();
 
   const stars: StarRecord[] = [];
+  let atGaiaDistance = 0;
+  let pastCutoff = 0;
 
   for (const row of rows) {
     const id = Number(row['id']);
-    const distancePc = Number(row['dist']);
 
     if (id === SUN_STAR_ID) {
       stars.push({ id, name: 'Sol', x: 0, y: 0, z: 0, magnitude: parseOptionalNumber(row['mag']) ?? UNKNOWN_MAGNITUDE, spectralType: row['spect'] || 'G2V', colorIndex: parseOptionalNumber(row['ci']) ?? null });
       continue;
     }
 
-    if (!Number.isFinite(distancePc) || distancePc >= HYG_UNKNOWN_DISTANCE_PC || distancePc > DISTANCE_CUTOFF_PC) {
+    const hygPc = Number(row['dist']);
+    const hipparcosPc = Number.isFinite(hygPc) && hygPc > 0 && hygPc < HYG_UNKNOWN_DISTANCE_PC ? hygPc : undefined;
+    const gaiaPc = row['hip'] ? gaiaPcByHip.get(Number(row['hip'])) : undefined;
+    const distancePc = placementDistancePc(hipparcosPc, gaiaPc, DISTANCE_CUTOFF_PC);
+    if (distancePc === null) {
       continue;
     }
 
@@ -89,26 +99,37 @@ export async function fetchStars(): Promise<StarRecord[]> {
     // once brought to the same epoch — have it. 1813 stars differ by over an arcsecond, and the
     // Cartesian columns are the ones Gaia agrees with for 1155 of them against 156 (one of those,
     // HIP 57146, has x/y/z 161″ from its own ra/dec and stays double).
+    //
+    // Only their direction is used. They sit at HYG's own distance, or at its 100 000 pc
+    // placeholder where it has none, and are carried along that direction to the one chosen above.
     const x = Number(row['x']);
     const y = Number(row['y']);
     const z = Number(row['z']);
-    if (![x, y, z].every(Number.isFinite)) {
+    const length = Math.hypot(x, y, z);
+    if (![x, y, z].every(Number.isFinite) || length === 0) {
       continue;
     }
+    const scale = distancePc / length;
+    if (gaiaPc !== undefined) {
+      atGaiaDistance++;
+    }
+    if (distancePc > DISTANCE_CUTOFF_PC) {
+      pastCutoff++;
+    }
 
     stars.push({
       id,
       name: resolveName(row),
-      x,
-      y,
-      z,
+      x: x * scale,
+      y: y * scale,
+      z: z * scale,
       magnitude: parseOptionalNumber(row['mag']) ?? UNKNOWN_MAGNITUDE,
       spectralType: row['spect'] || 'Unknown',
       colorIndex: parseOptionalNumber(row['ci']) ?? null
     });
   }
 
-  console.log(`  kept ${stars.length} stars (of ${rows.length} in the catalog).`);
+  console.log(`  kept ${stars.length} stars (of ${rows.length} in the catalog): ${atGaiaDistance} at Gaia's distance, ${pastCutoff} of them past ${DISTANCE_CUTOFF_PC} pc.`);
 
   const merged = await mergeWithOtherSources(stars);
   merged.sort((a, b) => a.id - b.id);

tools/etl/lib/http.ts

@@ -17,17 +17,38 @@ export async function fetchTextCached(url: string, cacheKey: string): Promise<st
   }
 
   console.log(`  fetching ${url}`);
-  const response = await fetch(url);
-  if (!response.ok) {
-    throw new Error(`Failed to fetch ${url}: ${response.status} ${response.statusText}`);
-  }
-  const text = await response.text();
+  const text = await fetchText(url);
 
   mkdirSync(dirname(cachePath), { recursive: true });
   writeFileSync(cachePath, text, 'utf-8');
   return text;
 }
 
+/**
+ * How long to wait before each retry of a failed request. The archives this reads are public
+ * services that time out under load — the Gaia TAP has answered a five-row join in two and a
+ * half minutes and a full one with a 500 — and a weekly refresh that gives up on the first of
+ * those publishes nothing that week.
+ */
+const RETRY_DELAYS_MS = [30_000, 120_000];
+
+async function fetchText(url: string): Promise<string> {
+  for (let attempt = 0; ; attempt++) {
+    const response = await fetch(url).catch((error: unknown) => (error instanceof Error ? error : new Error(String(error))));
+    if (!(response instanceof Error) && response.ok) {
+      return response.text();
+    }
+    const reason = response instanceof Error ? response.message : `${response.status} ${response.statusText}`;
+    // A 4xx is the request's own fault, and waiting will not change the answer.
+    const retryable = response instanceof Error || response.status >= 500;
+    if (!retryable || attempt >= RETRY_DELAYS_MS.length) {
+      throw new Error(`Failed to fetch ${url}: ${reason}`);
+    }
+    console.log(`  ${reason}; trying again in ${RETRY_DELAYS_MS[attempt] / 1000} s`);
+    await new Promise((resolve) => setTimeout(resolve, RETRY_DELAYS_MS[attempt]));
+  }
+}
+
 /** Convenience wrapper around {@link fetchTextCached} that parses the cached response as JSON. */
 export async function fetchJsonCached<T>(url: string, cacheKey: string): Promise<T> {
   return JSON.parse(await fetchTextCached(url, cacheKey)) as T;

tools/etl/sources/gaia.ts

@@ -125,3 +125,52 @@ export async function fetchGaiaStars(): Promise<StarRecord[]> {
   console.log(`  kept ${stars.length} Gaia stars (of ${rows.length} rows).`);
   return stars;
 }
+
+/**
+ * DR3's Hipparcos cross-match is a fixed table of 99 525 rows, 97 751 of them with a usable
+ * parallax. Far fewer means the answer was an error page served with a 200, or was cut short,
+ * and either would pass for "Gaia does not know these stars" and put every one of them back at
+ * its Hipparcos distance.
+ */
+const MIN_USABLE_HIP_DISTANCES = 90_000;
+
+/**
+ * Gaia's distance for every Hipparcos star it has a usable parallax for, keyed by HIP number.
+ *
+ * Taken from the archive's own cross-match (`hipparcos2_best_neighbour`) rather than from
+ * matching positions here, since Gaia's team made that identification star by star with the
+ * proper motions and photometry in hand. It is deliberately not bounded by distance: the stars
+ * it exists for are the ones Hipparcos put inside the map and Gaia puts outside, which the main
+ * query above never fetches.
+ *
+ * Required rather than best effort. Without it every HYG star falls back to its Hipparcos
+ * distance, the 6 833 that Gaia puts past 250 pc move back inside, and the published map would
+ * flip between the two with the archive's availability.
+ */
+export async function fetchGaiaDistancesByHip(): Promise<Map<number, number>> {
+  const query = [
+    'select top 200000 b.original_ext_source_id as hip, g.parallax, g.parallax_over_error',
+    'from gaiadr3.hipparcos2_best_neighbour b join gaiadr3.gaia_source g on g.source_id = b.source_id'
+  ].join(' ');
+  const url = `${GAIA_TAP_URL}?REQUEST=doQuery&LANG=ADQL&FORMAT=csv&QUERY=${encodeURIComponent(query)}`;
+  console.log('Fetching Gaia DR3 distances for Hipparcos stars (archive cross-match)...');
+  const rows = parseCsvObjects(await fetchTextCached(url, `gaia-dr3-hip-${createHash('sha1').update(url).digest('hex').slice(0, 8)}.csv`));
+
+  const distances = new Map<number, number>();
+  for (const row of rows) {
+    const hip = parseOptionalNumber(row['hip']);
+    const parallaxMas = parseOptionalNumber(row['parallax']);
+    const overError = parseOptionalNumber(row['parallax_over_error']);
+    if (hip !== undefined && parallaxMas !== undefined && parallaxMas > 0 && overError !== undefined && overError > 1 / MAX_PARALLAX_ERROR_RATIO) {
+      distances.set(hip, 1000 / parallaxMas);
+    }
+  }
+  if (distances.size < MIN_USABLE_HIP_DISTANCES) {
+    throw new Error(
+      `the Hipparcos cross-match gave ${distances.size} usable distances (of ${rows.length} rows), not the ~97 751 it holds; ` +
+        'delete tools/etl/.cache/gaia-dr3-hip-*.csv once the archive answers properly'
+    );
+  }
+  console.log(`  ${distances.size} Hipparcos stars have a Gaia distance (of ${rows.length} cross-matched).`);
+  return distances;
+}

tools/etl/sources/registry.ts

@@ -14,7 +14,8 @@ export const STAR_SOURCES: readonly StarSource[] = [
     name: 'HYG database (Hipparcos, Yale Bright Star, Gliese)',
     role: 'positional',
     endpoint: 'https://raw.githubusercontent.com/astronexus/HYG-Database',
-    contributes: 'A complete, named, spectrally classified bright-star catalogue with parallaxes — 68388 stars within 250 pc.',
+    contributes:
+      'A complete, named, spectrally classified bright-star catalogue. Its Hipparcos parallaxes give way to Gaia’s wherever Gaia has a usable one, so its stars sit where the better measurement puts them.',
     unimplementedBecause: null
   },
   {