star-map/tools/etl/build.ts

import { statSync } from 'node:fs';

import { BodyRecord } from '../../src/app/shared/models/body.model';
import { DeepSkyRecord } from '../../src/app/shared/models/deepsky.model';
import { ExoplanetRecord } from '../../src/app/shared/models/exoplanet.model';
import { StarRecord, SUN_STAR_ID } from '../../src/app/shared/models/star.model';
import { fetchDeepSky } from './fetchDeepSky';
import { fetchExoplanets } from './fetchExoplanets';
import { fetchSolarSystem } from './fetchSolarSystem';
import { BYTES_PER_STAR_META, BYTES_PER_STAR_POSITION, decodeStarCatalog, encodeStarCatalog, isDesignation } from '../../src/app/shared/models/star-catalog';
import { fetchStars } from './fetchStars';
import { describeSources } from './sources/registry';
import { dataPath } from './lib/paths';

class ValidationError extends Error {}

function assertCondition(condition: boolean, message: string): void {
  if (!condition) {
    throw new ValidationError(message);
  }
}

/**
 * Stars whose magnitude is a stand-in, and distances published without an error. Measured 309 and
 * 439: the 44 Gaia sources with no G and the 265 archive hosts with neither V nor G; the 357 Gliese
 * distances (no Hipparcos parallax behind them) and 82 archive hosts the archive gives no error
 * for. Losing either field loses it for hundreds of thousands of stars.
 */
const MAX_STARS_WITHOUT_BAND = 1_000;
const MAX_STARS_WITHOUT_DISTANCE_ERROR = 1_000;
/**
 * What the errors themselves come to, which the two counts above cannot see: Gaia's parallax_error
 * stored in milliarcseconds rather than over the parallax still encodes, decodes and passes both.
 * Measured: the median relative error of the stars at Gaia's distance is 0.33 %, and 1 109 parallax
 * distances (0.24 % of the stars) have one of a fifth or more, which the card prints as a range.
 * The mistake above gives 1.92 % and 17 689.
 */
const MAX_MEDIAN_GAIA_DISTANCE_ERROR = 0.01;
const MAX_RANGED_DISTANCE_SHARE = 0.01;
/**
 * HYG stars that keep their own row but sit at Gaia's distance, which fetchStars flags for the card
 * to say "HYG, Gaia DR3 distance". Measured 8 129; the flag dropped leaves none, and nothing else
 * notices — the other 62 002 carry it from their Gaia row.
 */
const MIN_HYG_STARS_AT_GAIA_DISTANCE = 6_000;

function validateStars(stars: StarRecord[]): void {
  assertCondition(stars.length > 0, 'No stars were produced.');

  const ids = new Set<number>();
  for (const star of stars) {
    assertCondition(Number.isFinite(star.id), `Star has a non-numeric id: ${JSON.stringify(star)}`);
    // Under 2^30, which V8 keeps unboxed; past it every id is a heap number, and the app's boot
    // task grew by 230 ms when the nearby Gaia stars were numbered from 2 000 000 000.
    assertCondition(star.id >= 0 && star.id < 2 ** 30, `Star ${star.id} has an id outside 0 to 2^30.`);
    assertCondition(!ids.has(star.id), `Duplicate star id: ${star.id}`);
    ids.add(star.id);
    assertCondition(!!star.name, `Star ${star.id} has no name.`);
    assertCondition([star.x, star.y, star.z].every(Number.isFinite), `Star ${star.id} has a non-finite position.`);
  }

  const positionBytes = statSync(dataPath('stars.bin')).size;
  assertCondition(positionBytes === stars.length * BYTES_PER_STAR_POSITION, `stars.bin size (${positionBytes}) does not match ${stars.length} stars.`);
  const metaBytes = statSync(dataPath('stars-meta.bin')).size;
  assertCondition(metaBytes === stars.length * BYTES_PER_STAR_META, `stars-meta.bin size (${metaBytes}) does not match ${stars.length} stars.`);

  // Round-trips the written assets back through the decoder the app uses, so a format change
  // that only half-lands fails here rather than as a silently wrong star map.
  const { index, positions, meta } = encodeStarCatalog(stars);
  const decoded = decodeStarCatalog(index, positions, meta);
  assertCondition(decoded.length === stars.length, `Star catalogue round-trip lost records: ${decoded.length} of ${stars.length}.`);
  for (let i = 0; i < stars.length; i++) {
    assertCondition(decoded[i].id === stars[i].id && decoded[i].name === stars[i].name, `Star catalogue round-trip altered record ${i}.`);
    assertCondition(decoded[i].spectralType === stars[i].spectralType, `Star catalogue round-trip lost the spectral type of star ${stars[i].id}.`);
    assertCondition(decoded[i].colorIndex === null === (stars[i].colorIndex === null), `Star catalogue round-trip changed whether star ${stars[i].id} has a colour index.`);
    assertCondition(
      decoded[i].magnitudeBand === stars[i].magnitudeBand &&
        decoded[i].colorSystem === stars[i].colorSystem &&
        decoded[i].distanceFromGaia === !!stars[i].distanceFromGaia &&
        decoded[i].colorFromTemperature === !!stars[i].colorFromTemperature,
      `Star catalogue round-trip changed the photometry of star ${stars[i].id}.`
    );
    // Stored as its square root in 255ths, up to 100 %; see `star-catalog.ts`.
    const error = stars[i].distanceError;
    assertCondition(
      error === undefined ? decoded[i].distanceError === undefined : Math.abs(Math.sqrt(decoded[i].distanceError!) - Math.sqrt(Math.min(1, error))) <= 0.5 / 255 + 1e-9,
      `Star catalogue round-trip changed the distance error of star ${stars[i].id}.`
    );
  }

  // What each star says it was measured in. A band or an error dropped on the way still encodes,
  // decodes and draws; it shows only as a card reading "Not measured" for a star that was.
  const withoutBand = stars.filter((star) => star.magnitudeBand === undefined).length;
  assertCondition(withoutBand <= MAX_STARS_WITHOUT_BAND, `${withoutBand} stars have no magnitude band (at most ${MAX_STARS_WITHOUT_BAND} expected) — the band is being lost.`);
  const withoutError = stars.filter((star) => star.id !== SUN_STAR_ID && star.distanceError === undefined).length;
  assertCondition(
    withoutError <= MAX_STARS_WITHOUT_DISTANCE_ERROR,
    `${withoutError} stars have no distance error (at most ${MAX_STARS_WITHOUT_DISTANCE_ERROR} expected) — the parallax errors are being lost.`
  );
  console.log(`  ${withoutBand} stars with a stand-in magnitude; ${withoutError} distances without a published error.`);

  const gaiaErrors = stars
    .filter((star) => star.distanceFromGaia && star.distanceError !== undefined)
    .map((star) => star.distanceError!)
    .sort((a, b) => a - b);
  const medianGaiaError = gaiaErrors[Math.floor(gaiaErrors.length / 2)] ?? 0;
  assertCondition(
    medianGaiaError <= MAX_MEDIAN_GAIA_DISTANCE_ERROR,
    `The median error of Gaia's distances is ${(medianGaiaError * 100).toFixed(2)} % (at most ${MAX_MEDIAN_GAIA_DISTANCE_ERROR * 100} % expected) — the parallax errors are no longer relative.`
  );
  // The archive's errors are on the distance and never printed as a range; see formatDistance.
  const ranged = stars.filter((star) => star.source !== 'exoplanet-archive' && (star.distanceError ?? 0) >= 0.2).length;
  assertCondition(
    ranged <= stars.length * MAX_RANGED_DISTANCE_SHARE,
    `${ranged} parallax distances have an error of a fifth or more (at most ${MAX_RANGED_DISTANCE_SHARE * 100} % of stars expected).`
  );
  const hygAtGaiaDistance = stars.filter((star) => star.source === 'hyg' && star.distanceFromGaia).length;
  assertCondition(
    hygAtGaiaDistance >= MIN_HYG_STARS_AT_GAIA_DISTANCE,
    `Only ${hygAtGaiaDistance} HYG stars are flagged at Gaia's distance (at least ${MIN_HYG_STARS_AT_GAIA_DISTANCE} expected) — fetchStars no longer says whose parallax it placed them by.`
  );
  console.log(
    `  Gaia distances a median ${(medianGaiaError * 100).toFixed(2)} % uncertain; ${ranged} parallax distances ranged; ${hygAtGaiaDistance} HYG stars at Gaia's distance.`
  );

  // Hipparcos's mark on a classification it does not print in full reached the card as "Spectral
  // type A0m...", for Sirius and 2 126 other stars, which reads as text the app cut short.
  const dotted = stars.filter((star) => star.spectralType.endsWith('...')).length;
  assertCondition(dotted === 0, `${dotted} spectral types end in "..." — fetchStars no longer trims Hipparcos's mark from them.`);
}

/**
 * What a good merge looks like, in two numbers the unit suite cannot see.
 *
 * The catalogues are regenerated by a scheduled job that pushes straight to `main` once the unit
 * tests and a production build pass — and both passed, for weeks, on a catalogue carrying 23 000
 * stars twice: the suite tests code against fixtures, and no fixture is 400 000 real stars. The
 * two ways the merge has actually failed both show up here.
 *
 * A star kept twice leaves its two entries near each other on the sky, from *different* sources —
 * one catalogue does not list a star twice. Under an arcsecond that is never two stars at this
 * depth, so every such pair is a miss. Twenty-three survive today; the nineteen first counted were
 * all a second HYG row wanting a Gaia entry that already absorbed one (Gliese lists some doubles
 * twice), and the merge that trusted a Hipparcos parallax over direction left 1 112.
 *
 * The other failure leaves no close pair at all, because proper motion had already carried the
 * two entries tens of arcseconds apart — the 2026-08-24 refresh, where HYG sat at epoch 2000.0
 * and Gaia at J2016.0. What it does leave is HYG rows that found no counterpart: 36 056 of them
 * against the 11 554 today, and no counterpart was possible for most of those. 8 301 of them are
 * every star in the published catalogue beyond 250 pc, which the main query never downloads: 6 835
 * that Gaia's parallax puts past `ETL_GAIA_DISTANCE_PC` while Hipparcos put them inside
 * `ETL_STAR_DISTANCE_PC`, and 1 466 naked-eye stars kept at any distance. The other 3 252 are what
 * Gaia genuinely lacks: 1 194 brighter than V 8, which it saturates on or measures poorly, 1 832
 * between 8 and 12, and 226 fainter, 184 of them Gliese stars within 50 pc that neither of its
 * queries holds. So the headroom left to the ceiling tracks the gap between those two cutoffs as
 * much as Gaia's completeness.
 *
 * This bounds a merge that went wrong, and — loosely — a Gaia download that came back short: a
 * truncated answer leaves the HYG rows whose counterpart it dropped without one, so survivors go
 * *up*, not down. Measured on the main query when it was the only one, with 10 886 survivors
 * against today's 11 554: 11 004 at nine tenths of its rows, 12 711 at half, 16 258 at a third. So
 * this ceiling only catches a deep truncation, and `fetchGaiaStars` catches the shallower ones
 * with a row floor on each query.
 */
const MAX_UNMERGED_TWINS = 100;
const MAX_HYG_SURVIVORS = 15_000;
const TWIN_TOLERANCE_RAD = (1 / 3600) * (Math.PI / 180);

function validateMerge(stars: StarRecord[]): void {
  // Checked first and on its own: an unreachable Gaia is skipped rather than thrown, and would
  // otherwise surface below as "68 000 HYG stars found no counterpart" — true, and no help.
  assertCondition(
    stars.some((star) => star.source === 'gaia'),
    'Gaia DR3 contributed no stars — the archive was unreachable or returned nothing, and a catalogue without it is not one to publish.'
  );
  const survivors = stars.filter((star) => star.source === 'hyg').length;
  assertCondition(
    survivors <= MAX_HYG_SURVIVORS,
    `${survivors} HYG stars found no Gaia counterpart (at most ${MAX_HYG_SURVIVORS} expected) — the two catalogues are not being matched.`
  );

  // Sorted by declination, so each star is only compared against the handful sharing its
  // parallel — an arcsecond of declination holds one or two of 400 000 stars.
  const byDec = stars
    .map((star) => {
      const distance = Math.hypot(star.x, star.y, star.z);
      return { star, distance, dec: distance === 0 ? 0 : Math.asin(Math.max(-1, Math.min(1, star.z / distance))) };
    })
    .filter((entry) => entry.distance > 0)
    .sort((a, b) => a.dec - b.dec);

  const cosTolerance = Math.cos(TWIN_TOLERANCE_RAD);
  let twins = 0;
  let example = '';
  for (let i = 0; i < byDec.length; i++) {
    const a = byDec[i];
    for (let j = i + 1; j < byDec.length && byDec[j].dec - a.dec <= TWIN_TOLERANCE_RAD; j++) {
      const b = byDec[j];
      if (a.star.source === b.star.source) {
        continue;
      }
      const cosine = (a.star.x * b.star.x + a.star.y * b.star.y + a.star.z * b.star.z) / (a.distance * b.distance);
      if (cosine >= cosTolerance) {
        twins++;
        example ||= `${a.star.name} (${a.star.source}) and ${b.star.name} (${b.star.source})`;
      }
    }
  }
  assertCondition(
    twins <= MAX_UNMERGED_TWINS,
    `${twins} stars from different catalogues sit within an arcsecond of each other (at most ${MAX_UNMERGED_TWINS} expected), starting with ${example} — the merge is keeping the same star twice.`
  );
  console.log(`  ${survivors} HYG stars have no Gaia counterpart; ${twins} unmerged cross-catalogue pairs within an arcsecond.`);
}

function validateBodies(bodies: BodyRecord[]): void {
  assertCondition(bodies.length > 0, 'No solar-system bodies were produced.');

  const ids = new Set(bodies.map((body) => body.id));
  assertCondition(ids.size === bodies.length, 'Duplicate body ids were found.');

  for (const body of bodies) {
    const orbitValues = Object.values(body.orbit);
    assertCondition(orbitValues.every(Number.isFinite), `Body ${body.id} has non-finite orbital elements.`);

    if (body.kind === 'moon') {
      assertCondition(!!body.parentBodyId && ids.has(body.parentBodyId), `Moon ${body.id} has no valid parentBodyId.`);
    }
  }

  const planetCount = bodies.filter((body) => body.kind === 'planet').length;
  assertCondition(planetCount === 8, `Expected 8 planets, found ${planetCount}.`);
}

/**
 * The share of planets that must have a star on the map: 6 327 of 6 354 did when the ETL began
 * adding the hosts the catalogue lacks from the archive's own figures, up from 2 071. The other
 * 27 have no distance in either archive table, so nothing can place them; the floor leaves room
 * for a few more of those, not for the matching or the additions to stop working.
 */
const MIN_HOSTED_SHARE = 0.995;

/**
 * The share of planets whose host's radius and temperature the archive gives, which is what
 * starSurfaceOf draws a host with before deriving one. Measured 6 030 and 6 054 of 6 354 (0.95).
 * Both come from the composite table only; a column lost on the way leaves every host derived —
 * Proxima 0.105 R☉ instead of 0.141 — and nothing else fails.
 */
const MIN_HOST_SURFACE_SHARE = 0.9;

function validateExoplanets(exoplanets: ExoplanetRecord[], stars: StarRecord[]): void {
  assertCondition(exoplanets.length > 0, 'No exoplanets were produced.');
  const starsById = new Map(stars.map((star) => [star.id, star]));

  let crossReferenced = 0;
  for (const exoplanet of exoplanets) {
    assertCondition(!!exoplanet.name, `Exoplanet ${exoplanet.id} has no name.`);
    if (exoplanet.hostStarId !== null) {
      const host = starsById.get(exoplanet.hostStarId);
      assertCondition(host !== undefined, `Exoplanet ${exoplanet.id} references unknown star id ${exoplanet.hostStarId}.`);
      // A host known only as "Gaia DR3 2635476908753563008" cannot be found by searching for
      // TRAPPIST-1; fetchExoplanets names it after its host, and 575 were renamed.
      assertCondition(!isDesignation(host!), `Exoplanet ${exoplanet.id}'s host is only a designation, ${host!.name}, not named after ${exoplanet.hostStarName}.`);
      // The Sun has no exoplanets, so any match to it is a matching failure — historically a
      // blank distance column parsing as 0, which puts the host at the origin and matches Sol
      // exactly. Free, permanent tripwire for that whole class of bug.
      assertCondition(
        exoplanet.hostStarId !== SUN_STAR_ID,
        `Exoplanet ${exoplanet.id} was matched to the Sun, which has no exoplanets — the host-star match is wrong.`
      );
      crossReferenced++;
    }

    assertCondition(
      exoplanet.periodDays === undefined || exoplanet.periodDays > 0,
      `Exoplanet ${exoplanet.id} has a non-positive orbital period.`
    );
    assertCondition(
      exoplanet.hostStarMassSolar === undefined || exoplanet.hostStarMassSolar > 0,
      `Exoplanet ${exoplanet.id} has a non-positive host star mass.`
    );
  }

  console.log(`  ${crossReferenced}/${exoplanets.length} exoplanets have a host star on the map.`);
  assertCondition(
    crossReferenced >= exoplanets.length * MIN_HOSTED_SHARE,
    `Only ${crossReferenced} of ${exoplanets.length} exoplanets have a host star (at least ${MIN_HOSTED_SHARE * 100} % expected) — hosts are no longer being matched or added.`
  );

  const withRadius = exoplanets.filter((exoplanet) => exoplanet.hostStarRadiusSolar !== undefined).length;
  const withTemperature = exoplanets.filter((exoplanet) => exoplanet.hostStarTemperatureK !== undefined).length;
  assertCondition(
    Math.min(withRadius, withTemperature) >= exoplanets.length * MIN_HOST_SURFACE_SHARE,
    `Only ${withRadius} of ${exoplanets.length} exoplanets carry their host's radius and ${withTemperature} its temperature (at least ${MIN_HOST_SURFACE_SHARE * 100} % expected) — st_rad or st_teff is being lost.`
  );
  console.log(`  ${withRadius}/${exoplanets.length} carry their host's radius, ${withTemperature} its temperature.`);

  // How many can be propagated at their real rate rather than as if the host were the Sun.
  const withPeriod = exoplanets.filter((exoplanet) => exoplanet.periodDays !== undefined).length;
  const withHostMass = exoplanets.filter((exoplanet) => exoplanet.hostStarMassSolar !== undefined).length;
  console.log(`  ${withPeriod}/${exoplanets.length} have a measured period, ${withHostMass} a host star mass.`);
}

const UNIT_VECTOR_TOLERANCE = 1e-6;

/**
 * Objects the backdrop cannot ship without, two from each of OpenNGC's files: the Andromeda
 * Galaxy and the Small Magellanic Cloud from NGC.csv, the Large Magellanic Cloud and the
 * Pleiades from addendum.csv. The addendum went unread for as long as the ETL has existed,
 * because 463 objects without the brightest deep-sky object in the sky validated cleanly.
 */
const REQUIRED_DEEP_SKY_IDS = ['NGC0224', 'NGC0292', 'ESO056-115', 'Mel022'];
/**
 * 107 of the 110 Messier objects. OpenNGC types the other three as what they are: M40 a double
 * star, M73 an asterism and M102 a duplicate of M101, none of them a deep-sky object to draw.
 */
const MIN_MESSIER_OBJECTS = 107;

function validateDeepSky(objects: DeepSkyRecord[]): void {
  assertCondition(objects.length > 0, 'No deep-sky objects were produced.');

  const ids = new Set<string>();
  for (const object of objects) {
    assertCondition(!!object.id, `Deep-sky object has no id: ${JSON.stringify(object)}`);
    assertCondition(!ids.has(object.id), `Duplicate deep-sky id: ${object.id}`);
    ids.add(object.id);
    assertCondition(!!object.name, `Deep-sky object ${object.id} has no name.`);

    // Positions are directions, so every one of them must be a unit vector — a zero-length
    // or mis-scaled entry would silently collapse onto the origin on the backdrop shell.
    const length = Math.hypot(object.x, object.y, object.z);
    assertCondition(Math.abs(length - 1) < UNIT_VECTOR_TOLERANCE, `Deep-sky object ${object.id} has a non-unit direction (length ${length}).`);

    assertCondition(object.angularSizeDeg >= 0, `Deep-sky object ${object.id} has a negative angular size.`);
    assertCondition(object.distancePc === null || object.distancePc > 0, `Deep-sky object ${object.id} has a non-positive distance.`);
    // The distance and its provenance have to travel together, or the UI cannot say where a
    // number came from.
    assertCondition(
      (object.distancePc === null) === (object.distanceMethod === null),
      `Deep-sky object ${object.id} has a distance/method mismatch.`
    );
  }

  const kinds = new Set(objects.map((object) => object.kind));
  for (const kind of ['galaxy', 'nebula', 'cluster'] as const) {
    assertCondition(kinds.has(kind), `No deep-sky objects of kind "${kind}" were produced.`);
  }

  for (const id of REQUIRED_DEEP_SKY_IDS) {
    assertCondition(ids.has(id), `Deep-sky object ${id} is missing — one of OpenNGC's two files was not read.`);
  }
  const messier = new Set(objects.map((object) => object.messier).filter((designation) => designation !== null)).size;
  assertCondition(messier >= MIN_MESSIER_OBJECTS, `Only ${messier} Messier objects were produced (at least ${MIN_MESSIER_OBJECTS} expected).`);

  const withDistance = objects.filter((object) => object.distancePc !== null).length;
  console.log(`  ${withDistance}/${objects.length} deep-sky objects have a derived distance.`);
}

/**
 * Orchestrates the whole ETL pipeline: fetches every source (each caches its own raw
 * responses under `tools/etl/.cache/`), writes the static assets under `src/assets/data/`,
 * then validates the combined output for completeness before declaring success.
 */
async function build(): Promise<void> {
  console.log('=== NASA star map ETL ===\n');
  console.log('Catalogues:');
  console.log(describeSources());
  console.log();

  const catalogueStars = await fetchStars();
  console.log();
  const bodies = await fetchSolarSystem();
  console.log();
  // Adds the hosts the catalogue lacks, so it is this list, not the one above, that is published.
  const { exoplanets, stars } = await fetchExoplanets(catalogueStars);
  console.log();
  const deepSky = await fetchDeepSky();
  console.log();

  console.log('Validating output...');
  validateStars(stars);
  validateMerge(stars);
  validateBodies(bodies);
  validateExoplanets(exoplanets, stars);
  validateDeepSky(deepSky);

  console.log('\nETL completed successfully:');
  console.log(`  stars:      ${stars.length}`);
  console.log(`  bodies:     ${bodies.length}`);
  console.log(`  exoplanets: ${exoplanets.length}`);
  console.log(`  deep sky:   ${deepSky.length}`);
}

build().catch((error) => {
  console.error('\nETL failed:', error instanceof Error ? error.message : error);
  process.exitCode = 1;
});