star-map/tools/etl/sources/gaia.ts

import { createHash } from 'node:crypto';

import { propagateProperMotion, raDegDecDistanceToXyz } from '../../../src/app/shared/astro/coordinates';
import { StarRecord } from '../../../src/app/shared/models/star.model';
import { parseCsvObjects, parseOptionalNumber } from '../lib/csv';
import { fetchTextCached } from '../lib/http';

/**
 * Gaia DR3, via the ESA archive's TAP service.
 *
 * The only one of the large modern surveys that can add stars to a *3D* map, because it is the
 * only one that measures parallaxes for them. Its 1.8 billion sources are roughly 1% of the
 * Galaxy — no catalogue is close to the rest — but within a few hundred parsecs it is complete
 * in a way Hipparcos never was, and its parallaxes are fifty times more precise.
 *
 * Written blind against the published DR3 schema, since no ESA endpoint was reachable from the
 * environment it was written in; first run for real by the scheduled refresh of 2026-08-24, which
 * fetched 412 765 rows.
 */

const GAIA_TAP_URL = 'https://gea.esac.esa.int/tap-server/tap/sync';

/**
 * Gaia DR3 gives positions for J2016.0; HYG for J2000.0, which is the epoch this map keeps.
 * Sixteen years of proper motion is over an arcsecond for anything faster than ~62 mas/yr —
 * which is most of the nearest stars: 62″ for Proxima, 166″ for Barnard's — so as published, the
 * two catalogues never agree on where those stars are, and a merge that matched them on the sky
 * kept every one of them twice. Each position is therefore carried back to J2000.0 with Gaia's
 * own proper motion before it leaves here.
 */
const GAIA_DR3_EPOCH = 2016.0;
const CATALOGUE_EPOCH = 2000.0;

/**
 * How far out to take Gaia, in parsecs, and the faintest star to keep.
 *
 * Both exist to bound the download rather than the science. Gaia's parallaxes stay useful far
 * past anything this map draws, so the limit here is a payload decision: the catalogue is baked
 * into a static asset that a browser downloads before the first frame.
 */
const DEFAULT_DISTANCE_CUTOFF_PC = 250;
const DEFAULT_MAGNITUDE_LIMIT = 12;
const DEFAULT_ROW_LIMIT = 500_000;
const DISTANCE_CUTOFF_PC = Number(process.env['ETL_GAIA_DISTANCE_PC'] ?? DEFAULT_DISTANCE_CUTOFF_PC);
const MAGNITUDE_LIMIT = Number(process.env['ETL_GAIA_MAGNITUDE_LIMIT'] ?? DEFAULT_MAGNITUDE_LIMIT);
const ROW_LIMIT = Number(process.env['ETL_GAIA_ROW_LIMIT'] ?? DEFAULT_ROW_LIMIT);

/**
 * Relative parallax error above which a star is dropped: a parallax measured to worse than 20%
 * gives a distance that is not worth plotting, and inverting a noisy parallax biases it badly.
 */
const MAX_PARALLAX_ERROR_RATIO = 0.2;

/** Parallax in milliarcseconds for a given distance — the query's cutoff, expressed as Gaia has it. */
function parallaxFloorMas(distancePc: number): number {
  return 1000 / distancePc;
}

function buildQuery(): string {
  return [
    `select top ${ROW_LIMIT}`,
    'source_id, ra, dec, pmra, pmdec, parallax, parallax_error, phot_g_mean_mag, bp_rp',
    'from gaiadr3.gaia_source',
    `where parallax > ${parallaxFloorMas(DISTANCE_CUTOFF_PC).toFixed(6)}`,
    `and parallax_over_error > ${(1 / MAX_PARALLAX_ERROR_RATIO).toFixed(1)}`,
    `and phot_g_mean_mag < ${MAGNITUDE_LIMIT}`,
    // source_id breaks the ties — 20 064 groups share a G at the published precision — so the
    // row order, and with it the ids assigned below, is a pure function of the archive's content.
    'order by phot_g_mean_mag asc, source_id asc'
  ].join(' ');
}

/**
 * How many rows the query above holds when nothing is overridden: 412 765, and DR3 is a finished
 * data release, so that number only moves when the query does. It lives here, under the query, so
 * that an edit to any of its filters is made with the count it invalidates in view.
 *
 * Checked because a short answer looks exactly like a complete one. The TAP service truncates on
 * its own timeout and still serves a well-formed CSV with a 200, and the rows are ordered by
 * magnitude, so what comes back is the bright half — the half HYG overlaps. The merge gate in
 * `build.ts` would then see Gaia stars present and a survivor count barely moved, and pass a
 * catalogue missing two hundred thousand stars, which the weekly job would publish and the runner
 * would cache for the weeks after it. Same failure, and same guard, as
 * {@link MIN_USABLE_HIP_DISTANCES} below.
 *
 * Only checked when nothing is overridden: the environment overrides exist to fetch a smaller
 * slice on purpose.
 */
const DEFAULT_QUERY_ROWS = 412_765;
const MIN_ROW_SHARE = 0.95;

/**
 * An answer the archive gave that cannot be worked with, as against an archive that gave none.
 *
 * `fetchStars` skips a source it cannot reach and leaves the merge gate to judge the result. That
 * is right for an outage and wrong for a truncated CSV, which would be skipped, cached, and land
 * as "the archive was unreachable" long after the assets had been overwritten — so these throws
 * are marked, and rethrown there.
 */
export class GaiaAnswerError extends Error {}

/**
 * Gaia publishes no spectral classifications, but `bp_rp` is a colour index on the same footing
 * as HYG's `ci` — so the app's existing colour and spectral-class handling works unchanged, and
 * the spectral type is left as unknown rather than invented from the colour.
 */
const UNKNOWN_SPECTRAL_TYPE = 'Unknown';

/**
 * Gaia source ids are 19 digits and there are no proper names, so a star's identity here is its
 * catalogue designation. The app's star ids are 32-bit, which a Gaia source id overflows, so the
 * two are kept apart: `id` is assigned within this run's own range and the designation carries
 * the real identifier in the name.
 */
const GAIA_ID_BASE = 1_000_000_000;

export async function fetchGaiaStars(): Promise<StarRecord[]> {
  const query = buildQuery();
  const url = `${GAIA_TAP_URL}?REQUEST=doQuery&LANG=ADQL&FORMAT=csv&QUERY=${encodeURIComponent(query)}`;
  console.log(`Fetching Gaia DR3 (within ${DISTANCE_CUTOFF_PC} pc, G < ${MAGNITUDE_LIMIT}, at most ${ROW_LIMIT} rows)...`);

  // Keyed by the whole request, so a response cached for other columns, another order, or
  // another endpoint can never be mistaken for this one — the cache records only that some
  // response arrived, not what it answered.
  const cacheKey = `gaia-dr3-${createHash('sha1').update(url).digest('hex').slice(0, 8)}.csv`;
  const csv = await fetchTextCached(url, cacheKey);
  const rows = parseCsvObjects(csv);
  const jobsQuery = DISTANCE_CUTOFF_PC === DEFAULT_DISTANCE_CUTOFF_PC && MAGNITUDE_LIMIT === DEFAULT_MAGNITUDE_LIMIT && ROW_LIMIT === DEFAULT_ROW_LIMIT;
  if (jobsQuery && rows.length < DEFAULT_QUERY_ROWS * MIN_ROW_SHARE) {
    throw new GaiaAnswerError(
      `Gaia returned ${rows.length} rows, not the ~${DEFAULT_QUERY_ROWS} this query holds — the answer was cut short, it was an error page ` +
        `served with a 200, or the query was edited without updating DEFAULT_QUERY_ROWS; delete tools/etl/.cache/${cacheKey} once the archive answers properly`
    );
  }
  // Not gated on `jobsQuery` like the floor above it: the only ways to reach this cap are the
  // overrides that *widen* the query, and they are exactly when it is worth saying. What it must
  // not fire on is a deliberately smaller slice, where filling the limit is the whole point.
  if (ROW_LIMIT >= DEFAULT_ROW_LIMIT && rows.length >= ROW_LIMIT) {
    throw new GaiaAnswerError(`Gaia returned the query's own ${ROW_LIMIT}-row limit, so it is the limit deciding what the map holds; raise ETL_GAIA_ROW_LIMIT.`);
  }
  const stars: StarRecord[] = [];

  rows.forEach((row, index) => {
    const parallaxMas = parseOptionalNumber(row['parallax']);
    const raDeg = parseOptionalNumber(row['ra']);
    const decDeg = parseOptionalNumber(row['dec']);
    if (!parallaxMas || parallaxMas <= 0 || raDeg === undefined || decDeg === undefined) {
      return;
    }

    const distancePc = 1000 / parallaxMas;
    if (distancePc > DISTANCE_CUTOFF_PC) {
      return;
    }

    const j2000 = propagateProperMotion(raDeg, decDeg, parseOptionalNumber(row['pmra']) ?? 0, parseOptionalNumber(row['pmdec']) ?? 0, CATALOGUE_EPOCH - GAIA_DR3_EPOCH);
    const { x, y, z } = raDegDecDistanceToXyz(j2000.raDeg, j2000.decDeg, distancePc);
    stars.push({
      id: GAIA_ID_BASE + index,
      name: `Gaia DR3 ${row['source_id']}`,
      x,
      y,
      z,
      magnitude: parseOptionalNumber(row['phot_g_mean_mag']) ?? MAGNITUDE_LIMIT,
      spectralType: UNKNOWN_SPECTRAL_TYPE,
      colorIndex: parseOptionalNumber(row['bp_rp']) ?? null,
      source: 'gaia'
    });
  });

  console.log(`  kept ${stars.length} Gaia stars (of ${rows.length} rows).`);
  return stars;
}

/**
 * DR3's Hipparcos cross-match is a fixed table of 99 525 rows, 97 751 of them with a usable
 * parallax. Far fewer means the answer was an error page served with a 200, or was cut short,
 * and either would pass for "Gaia does not know these stars" and put every one of them back at
 * its Hipparcos distance.
 */
const MIN_USABLE_HIP_DISTANCES = 90_000;

/**
 * Gaia's distance for every Hipparcos star it has a usable parallax for, keyed by HIP number.
 *
 * Taken from the archive's own cross-match (`hipparcos2_best_neighbour`) rather than from
 * matching positions here, since Gaia's team made that identification star by star with the
 * proper motions and photometry in hand. It is deliberately not bounded by distance: the stars
 * it exists for are the ones Hipparcos put inside the map and Gaia puts outside, which the main
 * query above never fetches.
 *
 * Required rather than best effort. Without it every HYG star falls back to its Hipparcos
 * distance, the 6 833 that Gaia puts past 250 pc move back inside, and the published map would
 * flip between the two with the archive's availability.
 */
export async function fetchGaiaDistancesByHip(): Promise<Map<number, number>> {
  const query = [
    'select top 200000 b.original_ext_source_id as hip, g.parallax, g.parallax_over_error',
    'from gaiadr3.hipparcos2_best_neighbour b join gaiadr3.gaia_source g on g.source_id = b.source_id'
  ].join(' ');
  const url = `${GAIA_TAP_URL}?REQUEST=doQuery&LANG=ADQL&FORMAT=csv&QUERY=${encodeURIComponent(query)}`;
  console.log('Fetching Gaia DR3 distances for Hipparcos stars (archive cross-match)...');
  const rows = parseCsvObjects(await fetchTextCached(url, `gaia-dr3-hip-${createHash('sha1').update(url).digest('hex').slice(0, 8)}.csv`));

  const distances = new Map<number, number>();
  for (const row of rows) {
    const hip = parseOptionalNumber(row['hip']);
    const parallaxMas = parseOptionalNumber(row['parallax']);
    const overError = parseOptionalNumber(row['parallax_over_error']);
    if (hip !== undefined && parallaxMas !== undefined && parallaxMas > 0 && overError !== undefined && overError > 1 / MAX_PARALLAX_ERROR_RATIO) {
      distances.set(hip, 1000 / parallaxMas);
    }
  }
  if (distances.size < MIN_USABLE_HIP_DISTANCES) {
    throw new Error(
      `the Hipparcos cross-match gave ${distances.size} usable distances (of ${rows.length} rows), not the ~97 751 it holds; ` +
        'delete tools/etl/.cache/gaia-dr3-hip-*.csv once the archive answers properly'
    );
  }
  console.log(`  ${distances.size} Hipparcos stars have a Gaia distance (of ${rows.length} cross-matched).`);
  return distances;
}