star-map/tools/etl/fetchExoplanets.ts

import { createHash } from 'node:crypto';
import { writeFileSync } from 'node:fs';

import { buildStarNameIndex, resolveHostStarId } from '../../src/app/shared/astro/host-star-matching';
import { ExoplanetRecord } from '../../src/app/shared/models/exoplanet.model';
import { StarRecord } from '../../src/app/shared/models/star.model';
import { fetchStars } from './fetchStars';
import { parseCsvObjects, parseOptionalNumber } from './lib/csv';
import { fetchTextCached } from './lib/http';
import { dataPath, ensureDataDir } from './lib/paths';

const TAP_BASE_URL = 'https://exoplanetarchive.ipac.caltech.edu/TAP/sync';
const TAP_COLUMNS = [
  'pl_name',
  'hostname',
  'ra',
  'dec',
  'sy_dist',
  'sy_pmra',
  'sy_pmdec',
  'pl_orbsmax',
  'pl_orbeccen',
  'pl_orbincl',
  'pl_orblper',
  'pl_orbper',
  'pl_rade',
  'pl_bmasse',
  'st_mass',
  'disc_year'
].join(',');
// Ordered explicitly: without it the archive is free to return rows in any order, and a
// scheduled re-run of the ETL would then rewrite exoplanets.json — and commit a diff — when
// nothing was actually published. pl_name is unique among default_flag=1 rows, so the order
// is total and the output is a pure function of the archive's content.
const TAP_QUERY = `select+${TAP_COLUMNS}+from+ps+where+default_flag=1+order+by+pl_name&format=csv`;
const TAP_URL = `${TAP_BASE_URL}?query=${TAP_QUERY}`;

// The cache is keyed by the request it answers — endpoint included, since the cache records
// only that some response arrived: one cached before a column was added would otherwise keep
// serving rows without it, and a missing proper-motion cell reads as "does not move",
// silently wrong rather than visibly broken.
const CACHE_FILE = `exoplanet-archive-ps-${createHash('sha1').update(TAP_URL).digest('hex').slice(0, 8)}.csv`;

/**
 * Downloads confirmed exoplanets from the NASA Exoplanet Archive (`Planetary Systems` TAP
 * table), cross-references each host star to the HYG index, and writes `exoplanets.json`.
 */
export async function fetchExoplanets(stars?: StarRecord[]): Promise<ExoplanetRecord[]> {
  console.log('Fetching confirmed exoplanets from the NASA Exoplanet Archive...');
  const knownStars = stars ?? (await fetchStars());
  const nameIndex = buildStarNameIndex(knownStars);

  const csv = await fetchTextCached(TAP_URL, CACHE_FILE);
  const rows = parseCsvObjects(csv);

  let matched = 0;
  const exoplanets: ExoplanetRecord[] = rows.map((row, index) => {
    // `parseOptionalNumber`, not `Number`: a blank cell would otherwise become 0, which is a
    // finite, plausible-looking coordinate rather than the "not measured" it actually means.
    const raDeg = parseOptionalNumber(row['ra']) ?? Number.NaN;
    const decDeg = parseOptionalNumber(row['dec']) ?? Number.NaN;
    const distancePc = parseOptionalNumber(row['sy_dist']) ?? Number.NaN;
    const pmRaMasPerYear = parseOptionalNumber(row['sy_pmra']);
    const pmDecMasPerYear = parseOptionalNumber(row['sy_pmdec']);

    const hostStarId = resolveHostStarId(
      { hostname: row['hostname'], raDeg, decDeg, distancePc, pmRaMasPerYear, pmDecMasPerYear },
      knownStars,
      nameIndex
    );
    if (hostStarId !== null) {
      matched++;
    }

    return {
      id: row['pl_name'] || `exoplanet-${index}`,
      hostStarId,
      hostStarName: row['hostname'],
      name: row['pl_name'],
      radiusEarth: parseOptionalNumber(row['pl_rade']),
      massEarth: parseOptionalNumber(row['pl_bmasse']),
      discoveryYear: parseOptionalNumber(row['disc_year']),
      // The period was already being downloaded and thrown away. With the semi-major axis it
      // determines the host's gravitational parameter, so keeping it is the difference between
      // propagating a planet at its real rate and pretending every host is the Sun.
      periodDays: parseOptionalNumber(row['pl_orbper']),
      hostStarMassSolar: parseOptionalNumber(row['st_mass']),
      // Kept so the cross-reference can be redone without the archive; see the record's own
      // documentation. Undefined rather than NaN, which JSON cannot represent.
      hostRaDeg: parseOptionalNumber(row['ra']),
      hostDecDeg: parseOptionalNumber(row['dec']),
      hostDistancePc: parseOptionalNumber(row['sy_dist']),
      hostPmRaMasPerYear: pmRaMasPerYear,
      hostPmDecMasPerYear: pmDecMasPerYear,
      orbit: {
        semiMajorAxisAu: parseOptionalNumber(row['pl_orbsmax']),
        eccentricity: parseOptionalNumber(row['pl_orbeccen']),
        inclinationDeg: parseOptionalNumber(row['pl_orbincl']),
        argumentOfPeriapsisDeg: parseOptionalNumber(row['pl_orblper'])
      }
    };
  });

  ensureDataDir();
  writeFileSync(dataPath('exoplanets.json'), JSON.stringify(exoplanets));
  console.log(`  wrote ${exoplanets.length} exoplanets (${matched} cross-referenced to a HYG host star).`);
  return exoplanets;
}

if (require.main === module) {
  fetchExoplanets().catch((error) => {
    console.error(error);
    process.exitCode = 1;
  });
}