Bring Gaia to HYG's epoch before merging, and keep a star's name when it matches

Gaia DR3 gives positions for J2016.0, HYG for 2000.0, and the merge matched
them on the sky to one arcsecond without propagating any proper motion.
Sixteen years of motion is 62" for Proxima and 166" for Barnard's Star, so
every star faster than ~62 mas/yr — most of the nearest ones — was kept twice,
some 23 000 in all. The slow ones were matched, and lost: the merge kept
Gaia's row whole, so 102 proper names, 1 336 Bayer/Flamsteed names and
32 000 spectral types became "Gaia DR3 <id>" and "Unknown", and 92 named
exoplanet hosts handed their planets to their anonymous twin.

Gaia is now asked for its proper motions and carried back to J2000 before it
leaves the fetcher. HYG is placed from its own x/y/z columns, which are right
where its `ra` is not: that column was carried from the Hipparcos epoch
without the cos δ its motion needs, 17.9" off for Proxima. A match combines
the two entries — Gaia's position, HYG's name, type, magnitude, colour and id
— instead of choosing one. The tolerance is 15" with a five-magnitude guard,
both set by measurement: 55 457 pairs sit under 1" once the epochs agree, the
Gliese-only entries up to 12" (Ross 248), shifting every entry a quarter of
a degree finds 16 chance neighbours at 15", and the guard keeps Sirius out
of Sirius B's entry. Entries of one source are never merged with each other:
the 1 411 Gaia doubles resolved under 1" are two stars, not one.

Regenerated: 425 071 stars (was 447 410), 56 082 of them Gaia positions
carrying HYG identities; no HYG id or name lost; the sixteen stars nearest
the Sun carry no survey designation; 196 residual doubles, all components
17" or more from their counterpart. Five planets of four bright giants
(7 CMa, HD 81688, omi UMa, xi Aql) lose their host link: their Gaia distance
sits 0.7–1.1 pc from the archive's Hipparcos-based one, past the 0.5 pc the
host match allows. Matching hosts on the sky rather than in space, as the
merge does, is the follow-up.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01QL6F9Bgfh8SgAiAAcPB9Hw

tools/etl/fetchStars.ts

@@ -1,6 +1,5 @@
 import { writeFileSync } from 'node:fs';
 
-import { raDecDistanceToXyz } from '../../src/app/shared/astro/coordinates';
 import { mergeStarCatalogues } from '../../src/app/shared/astro/star-merge';
 import { encodeStarCatalog } from '../../src/app/shared/models/star-catalog';
 import { StarRecord, SUN_STAR_ID } from '../../src/app/shared/models/star.model';
@@ -59,9 +58,9 @@ function resolveName(row: Record<string, string>): string {
 }
 
 /**
- * Downloads the HYG (Hipparcos/Yale/Gliese) stellar database, converts each star's
- * RA/Dec/distance into galaxy-scale Cartesian coordinates (parsecs), filters by distance,
- * and writes `stars.bin` (packed positions) + `stars-index.json` (everything else).
+ * Downloads the HYG (Hipparcos/Yale/Gliese) stellar database, takes each star's equatorial
+ * Cartesian position (parsecs, epoch J2000.0), filters by distance, unions the other positional
+ * sources, and writes `stars.bin` (packed positions) + `stars-index.json` (everything else).
  */
 export async function fetchStars(): Promise<StarRecord[]> {
   console.log(`Fetching HYG star catalog (distance cutoff: ${DISTANCE_CUTOFF_PC} pc)...`);
@@ -83,14 +82,18 @@ export async function fetchStars(): Promise<StarRecord[]> {
       continue;
     }
 
-    const raHours = Number(row['ra']);
-    const decDeg = Number(row['dec']);
-    if (!Number.isFinite(raHours) || !Number.isFinite(decDeg)) {
+    // HYG's own Cartesian columns rather than its `ra`/`dec`, which are in the same frame as
+    // `raDecDistanceToXyz` and would be redundant if the two agreed. They do not, for the stars
+    // that move: the right ascension was carried from the Hipparcos epoch to 2000.0 without the
+    // cos δ its motion needs, which puts Proxima 17.9″ from where HYG's own x/y/z — and Gaia,
+    // once brought to the same epoch — have it. 1813 stars differ by over an arcsecond.
+    const x = Number(row['x']);
+    const y = Number(row['y']);
+    const z = Number(row['z']);
+    if (![x, y, z].every(Number.isFinite)) {
       continue;
     }
 
-    const { x, y, z } = raDecDistanceToXyz(raHours, decDeg, distancePc);
-
     stars.push({
       id,
       name: resolveName(row),
@@ -143,7 +146,7 @@ async function mergeWithOtherSources(hygStars: StarRecord[]): Promise<StarRecord
   }
 
   const { stars, summary } = mergeStarCatalogues(candidates);
-  console.log(`  merged ${summary.total} stars from ${candidates.length} catalogues (${summary.duplicates} duplicates resolved to the better parallax):`);
+  console.log(`  merged ${summary.total} stars from ${candidates.length} catalogues (${summary.duplicates} entries folded into a better-measured one):`);
   for (const [sourceId, count] of Object.entries(summary.bySource)) {
     console.log(`    ${sourceId}: ${count}`);
   }

tools/etl/sources/gaia.ts

@@ -1,4 +1,4 @@
-import { raDegDecDistanceToXyz } from '../../../src/app/shared/astro/coordinates';
+import { propagateProperMotion, raDegDecDistanceToXyz } from '../../../src/app/shared/astro/coordinates';
 import { StarRecord } from '../../../src/app/shared/models/star.model';
 import { parseCsvObjects, parseOptionalNumber } from '../lib/csv';
 import { fetchTextCached } from '../lib/http';
@@ -11,14 +11,24 @@ import { fetchTextCached } from '../lib/http';
  * Galaxy — no catalogue is close to the rest — but within a few hundred parsecs it is complete
  * in a way Hipparcos never was, and its parallaxes are fifty times more precise.
  *
- * **This has never been run.** Every ESA, NOIRLab, SDSS and Euclid endpoint is unreachable from
- * the environment this was written in, so the query below is written against the published DR3
- * schema and has not been executed against it. Treat the column names as the first thing to
- * check if a real run misbehaves.
+ * Written blind against the published DR3 schema, since no ESA endpoint was reachable from the
+ * environment it was written in; first run for real by the scheduled refresh of 2026-08-24, which
+ * fetched 412 765 rows.
  */
 
 const GAIA_TAP_URL = 'https://gea.esac.esa.int/tap-server/tap/sync';
 
+/**
+ * Gaia DR3 gives positions for J2016.0; HYG for J2000.0, which is the epoch this map keeps.
+ * Sixteen years of proper motion is over an arcsecond for anything faster than ~62 mas/yr —
+ * which is most of the nearest stars: 62″ for Proxima, 166″ for Barnard's — so as published, the
+ * two catalogues never agree on where those stars are, and a merge that matched them on the sky
+ * kept every one of them twice. Each position is therefore carried back to J2000.0 with Gaia's
+ * own proper motion before it leaves here.
+ */
+const GAIA_DR3_EPOCH = 2016.0;
+const CATALOGUE_EPOCH = 2000.0;
+
 /**
  * How far out to take Gaia, in parsecs, and the faintest star to keep.
  *
@@ -44,7 +54,7 @@ function parallaxFloorMas(distancePc: number): number {
 function buildQuery(): string {
   return [
     `select top ${ROW_LIMIT}`,
-    'source_id, ra, dec, parallax, parallax_error, phot_g_mean_mag, bp_rp',
+    'source_id, ra, dec, pmra, pmdec, parallax, parallax_error, phot_g_mean_mag, bp_rp',
     'from gaiadr3.gaia_source',
     `where parallax > ${parallaxFloorMas(DISTANCE_CUTOFF_PC).toFixed(6)}`,
     `and parallax_over_error > ${(1 / MAX_PARALLAX_ERROR_RATIO).toFixed(1)}`,
@@ -72,7 +82,9 @@ export async function fetchGaiaStars(): Promise<StarRecord[]> {
   const url = `${GAIA_TAP_URL}?REQUEST=doQuery&LANG=ADQL&FORMAT=csv&QUERY=${encodeURIComponent(buildQuery())}`;
   console.log(`Fetching Gaia DR3 (within ${DISTANCE_CUTOFF_PC} pc, G < ${MAGNITUDE_LIMIT}, at most ${ROW_LIMIT} rows)...`);
 
-  const csv = await fetchTextCached(url, 'gaia-dr3.csv');
+  // The cache key names the columns, so a response cached before proper motions were asked for
+  // cannot be mistaken for one that has them.
+  const csv = await fetchTextCached(url, 'gaia-dr3-pm.csv');
   const rows = parseCsvObjects(csv);
   const stars: StarRecord[] = [];
 
@@ -89,7 +101,8 @@ export async function fetchGaiaStars(): Promise<StarRecord[]> {
       return;
     }
 
-    const { x, y, z } = raDegDecDistanceToXyz(raDeg, decDeg, distancePc);
+    const j2000 = propagateProperMotion(raDeg, decDeg, parseOptionalNumber(row['pmra']) ?? 0, parseOptionalNumber(row['pmdec']) ?? 0, CATALOGUE_EPOCH - GAIA_DR3_EPOCH);
+    const { x, y, z } = raDegDecDistanceToXyz(j2000.raDeg, j2000.decDeg, distancePc);
     stars.push({
       id: GAIA_ID_BASE + index,
       name: `Gaia DR3 ${row['source_id']}`,