Widen the star catalogue, and separate what is drawn from what is known

The map held 8750 stars within 50 pc and rendered 371 systems. Both were
lower than they needed to be, for different reasons.

The star catalogue was capped by its own encoding as much as by the
cutoff: one JSON object per star, eight key names repeated each time, 157
bytes a star. At the range HYG actually reaches that is 17 MB to download
and parse before the first frame. So the numbers move into two binary
column stores — positions in stars.bin, which the GPU is handed verbatim,
and id/magnitude/colour/spectral index in stars-meta.bin — and the JSON
keeps only the strings, with 2600 distinct spectral classifications
collapsed to a dictionary. The layout is defined once, in star-catalog.ts,
and the ETL and the app both use it, so the writer and the reader cannot
drift.

The cutoff then goes to 250 pc: 68388 stars, 7.8x as many for 1.7x the
bytes. That is where HYG's measurements stop rather than a round number —
98.6% of its rows are Hipparcos, whose parallaxes are good to about a
milliarcsecond, so beyond 250 pc it would be plotting noise.

Drawing all of them is a separate question from knowing them, and it is
answered separately. The field draws a budget: every star inside 25 pc,
because the nearest are faint red dwarfs and Proxima Centauri is magnitude
11, then the brightest of everything beyond. Search, navigation and the
planet cross-reference still see the whole catalogue. A real GPU would
draw all 68388 without noticing; the budget is for the machines that would
not, and it is one constant.

Systems were limited by something else entirely. The archive data already
shipped named 4735 host stars and only 388 resolved, because the rest lay
outside a 50 pc catalogue — and the cross-reference kept only its own
result, so redoing it meant re-downloading an archive that is not
reachable from here. Host coordinates are now stored with each planet, and
the match is re-resolved at build time against whatever catalogue the run
produced. Even name matching alone, which needs no coordinates and so
works on the records already shipped, rescues 335 planets across 238
systems: 371 renderable systems become 609.

Two selection rules were tuned for a 50 pc bubble and no longer fit.
Tethers followed the Sun's nearest neighbours, which are a speck at this
range, and now follow the brightest; labels were ranked by proximity,
which named whatever sat nearest the middle of the screen, and are now
ranked by brightness — so the view names Canopus, Achernar and Spica
rather than a clump of catalogue designations.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WaySiNst4HhDXBHnMy8p5G

src/app/shared/astro/host-star-matching.ts

@@ -1,4 +1,5 @@
 import { CartesianCoordinates, distanceBetween, raDegDecDistanceToXyz } from './coordinates';
+import { ExoplanetRecord } from '../models/exoplanet.model';
 import { StarRecord } from '../models/star.model';
 
 /** Normalizes a star name for comparison: lowercase, alphanumeric characters only. */
@@ -65,3 +66,68 @@ function findNearestStarWithin(position: CartesianCoordinates, stars: readonly S
 
   return closest ? closest.id : null;
 }
+
+/**
+ * Re-resolves every exoplanet's host star against a star catalogue.
+ *
+ * The cross-reference is a *derived* fact: it depends as much on which stars were loaded as on
+ * the archive itself. When the catalogue reached 50 pc, 388 of the archive's 4735 named hosts
+ * found a match and the other 4347 were carried and never drawn — not because their planets are
+ * unknown, but because their star was out of range. Widening the catalogue rescues some of them,
+ * and until the host coordinates were stored alongside each planet that meant re-downloading an
+ * archive which is not always reachable.
+ *
+ * Records written before those coordinates were kept can still be matched *by name*, which needs
+ * no coordinates at all — and that alone is worth doing, because a wider catalogue contains more
+ * names. What such a record cannot do is disprove its existing match: a name miss means only
+ * that the name missed, not that the star is absent. So those are upgraded where a match is
+ * found and left alone otherwise, while records that do carry coordinates take the new result
+ * outright, match or no match.
+ */
+
+/** A host must sit within this many parsecs of a catalogue star to count as the same object. */
+export const HOST_MATCH_TOLERANCE_PC = 2;
+
+export interface RematchSummary {
+  total: number;
+  /** Records carrying host coordinates, and therefore eligible to be re-matched in full. */
+  resolvable: number;
+  matched: number;
+  gained: number;
+  lost: number;
+}
+
+export function rematchHostStars(exoplanets: ExoplanetRecord[], stars: readonly StarRecord[]): RematchSummary {
+  const nameIndex = buildStarNameIndex(stars);
+  const summary: RematchSummary = { total: exoplanets.length, resolvable: 0, matched: 0, gained: 0, lost: 0 };
+
+  for (const exoplanet of exoplanets) {
+    const { hostRaDeg, hostDecDeg, hostDistancePc } = exoplanet;
+    const positioned = hostRaDeg !== undefined && hostDecDeg !== undefined && hostDistancePc !== undefined;
+    if (positioned) {
+      summary.resolvable++;
+    }
+
+    const previous = exoplanet.hostStarId;
+    // With no coordinates the query still carries the host's name, and `resolveHostStarId` tries
+    // that first; the positional fallback simply declines to run on non-finite coordinates.
+    const resolved = resolveHostStarId(
+      { hostname: exoplanet.hostStarName, raDeg: hostRaDeg ?? Number.NaN, decDeg: hostDecDeg ?? Number.NaN, distancePc: hostDistancePc ?? Number.NaN },
+      stars,
+      HOST_MATCH_TOLERANCE_PC,
+      nameIndex
+    );
+
+    exoplanet.hostStarId = positioned ? resolved : (resolved ?? previous);
+    if (exoplanet.hostStarId !== null) {
+      summary.matched++;
+    }
+    if (previous === null && exoplanet.hostStarId !== null) {
+      summary.gained++;
+    } else if (previous !== null && exoplanet.hostStarId === null) {
+      summary.lost++;
+    }
+  }
+
+  return summary;
+}

src/app/shared/astro/host-star-rematch.spec.ts

@@ -0,0 +1,82 @@
+import { describe, expect, it } from 'vitest';
+
+import { ExoplanetRecord } from '../models/exoplanet.model';
+import { StarRecord } from '../models/star.model';
+import { rematchHostStars } from './host-star-matching';
+
+/** Two catalogue stars, one of which is only present in the wider of the two catalogues. */
+const NEARBY: StarRecord = { id: 100, name: 'Gl 357', x: 9, y: 0, z: 0, magnitude: 10.9, spectralType: 'K', colorIndex: 1.4 };
+const DISTANT: StarRecord = { id: 200, name: 'HD 33844', x: 0, y: 120, z: 0, magnitude: 7.7, spectralType: 'K0', colorIndex: 1.0 };
+
+const NARROW_CATALOGUE = [NEARBY];
+const WIDE_CATALOGUE = [NEARBY, DISTANT];
+
+function planet(overrides: Partial<ExoplanetRecord> = {}): ExoplanetRecord {
+  return { id: 'p', hostStarId: null, hostStarName: 'HD 33844', name: 'HD 33844 b', orbit: { semiMajorAxisAu: 1 }, ...overrides };
+}
+
+describe('rematchHostStars', () => {
+  it('rescues a host that the wider catalogue now contains, by name alone', () => {
+    // The whole point: the cross-reference is a fact about the catalogue as much as about the
+    // archive, so widening one ought to resolve hosts the other already knew about.
+    const planets = [planet()];
+    const summary = rematchHostStars(planets, WIDE_CATALOGUE);
+
+    expect(planets[0].hostStarId).toBe(DISTANT.id);
+    expect(summary.gained).toBe(1);
+    expect(summary.matched).toBe(1);
+  });
+
+  it('needs no coordinates to do it', () => {
+    // Which matters, because the shipped records were written before coordinates were kept.
+    const planets = [planet()];
+    expect(planets[0].hostRaDeg).toBeUndefined();
+    rematchHostStars(planets, WIDE_CATALOGUE);
+    expect(planets[0].hostStarId).toBe(DISTANT.id);
+  });
+
+  it('will not clear an existing match on a name miss when it has no coordinates', () => {
+    // A name miss says the name missed, not that the star is absent — and the earlier match may
+    // have been positional, from data this record no longer carries.
+    const planets = [planet({ hostStarId: 999, hostStarName: 'Some Survey Designation' })];
+    const summary = rematchHostStars(planets, WIDE_CATALOGUE);
+
+    expect(planets[0].hostStarId).toBe(999);
+    expect(summary.lost).toBe(0);
+    expect(summary.matched).toBe(1);
+  });
+
+  it('takes the new answer outright when the record does carry coordinates', () => {
+    // With coordinates the match can be redone in full, so its result is authoritative — a host
+    // that no longer resolves is cleared rather than left pointing at a star that may be gone.
+    const planets = [planet({ hostStarId: 999, hostStarName: 'Nowhere', hostRaDeg: 10, hostDecDeg: 10, hostDistancePc: 500 })];
+    const summary = rematchHostStars(planets, WIDE_CATALOGUE);
+
+    expect(planets[0].hostStarId).toBeNull();
+    expect(summary.resolvable).toBe(1);
+    expect(summary.lost).toBe(1);
+  });
+
+  it('matches a positioned host to the catalogue star at its coordinates', () => {
+    const planets = [planet({ hostStarName: 'unlisted alias', hostRaDeg: 90, hostDecDeg: 0, hostDistancePc: 120 })];
+    rematchHostStars(planets, WIDE_CATALOGUE);
+    expect(planets[0].hostStarId).toBe(DISTANT.id);
+  });
+
+  it('leaves a host that neither catalogue contains unmatched', () => {
+    const planets = [planet()];
+    const summary = rematchHostStars(planets, NARROW_CATALOGUE);
+
+    expect(planets[0].hostStarId).toBeNull();
+    expect(summary.matched).toBe(0);
+    expect(summary.gained).toBe(0);
+  });
+
+  it('counts every record it was given', () => {
+    const planets = [planet(), planet({ id: 'q', hostStarName: 'Gl 357' }), planet({ id: 'r', hostStarName: 'nobody' })];
+    const summary = rematchHostStars(planets, WIDE_CATALOGUE);
+
+    expect(summary.total).toBe(3);
+    expect(summary.matched).toBe(2);
+  });
+});

src/app/shared/models/exoplanet.model.ts

@@ -20,5 +20,18 @@ export interface ExoplanetRecord {
   periodDays?: number;
   /** Host star mass in solar masses (`st_mass`); the fallback when no period is published. */
   hostStarMassSolar?: number;
+  /**
+   * The host star's own published position (`ra`, `dec`, `sy_dist`) — the coordinates the
+   * cross-reference above is resolved from.
+   *
+   * Kept rather than consumed and discarded. `hostStarId` is the *result* of a match against
+   * whatever star catalogue was loaded at the time, so widening that catalogue ought to rescue
+   * some of the 4347 hosts that currently resolve to nothing — but with only the result stored,
+   * redoing the match meant re-downloading the archive. These three numbers make it a local
+   * operation. See `rematchHostStars`.
+   */
+  hostRaDeg?: number;
+  hostDecDeg?: number;
+  hostDistancePc?: number;
   orbit: Partial<OrbitalElements>;
 }

src/app/shared/models/star-catalog.spec.ts

@@ -0,0 +1,93 @@
+import { describe, expect, it } from 'vitest';
+
+import { BYTES_PER_STAR_META, BYTES_PER_STAR_POSITION, decodeStarCatalog, encodeStarCatalog } from './star-catalog';
+import { StarRecord } from './star.model';
+
+const STARS: StarRecord[] = [
+  { id: 0, name: 'Sol', x: 0, y: 0, z: 0, magnitude: -26.7, spectralType: 'G2V', colorIndex: 0.656 },
+  { id: 71456, name: 'Rigil Kentaurus', x: -1.35, y: -0.04, z: -0.98, magnitude: -0.01, spectralType: 'G2V', colorIndex: 0.71 },
+  { id: 32263, name: 'Sirius', x: -0.49, y: 2.47, z: -0.75, magnitude: -1.44, spectralType: 'A0m...', colorIndex: 0.009 },
+  // The case a plain number cannot carry: about a tenth of the catalogue was never photometered.
+  { id: 118554, name: 'GJ 3512', x: 20.1, y: -3.4, z: 8.8, magnitude: 15, spectralType: 'Unknown', colorIndex: null }
+];
+
+describe('encodeStarCatalog / decodeStarCatalog', () => {
+  const encoded = encodeStarCatalog(STARS);
+  const decoded = decodeStarCatalog(encoded.index, encoded.positions, encoded.meta);
+
+  it('round-trips every field of every star', () => {
+    expect(decoded).toHaveLength(STARS.length);
+    decoded.forEach((star, index) => {
+      const original = STARS[index];
+      expect(star.id).toBe(original.id);
+      expect(star.name).toBe(original.name);
+      expect(star.spectralType).toBe(original.spectralType);
+      expect(star.x).toBeCloseTo(original.x, 4);
+      expect(star.y).toBeCloseTo(original.y, 4);
+      expect(star.z).toBeCloseTo(original.z, 4);
+      expect(star.magnitude).toBeCloseTo(original.magnitude, 4);
+    });
+  });
+
+  it('carries an absent colour index through as null, not as zero', () => {
+    // Zero is a real colour index meaning a hot blue-white A-type star, so it cannot double as
+    // "not measured" — the float column uses NaN, which nothing else can be.
+    expect(decoded[3].colorIndex).toBeNull();
+    expect(decoded[0].colorIndex).toBeCloseTo(0.656, 5);
+    expect(decoded[2].colorIndex).toBeCloseTo(0.009, 5);
+  });
+
+  it('sizes both binaries exactly to the star count', () => {
+    expect(encoded.positions.byteLength).toBe(STARS.length * BYTES_PER_STAR_POSITION);
+    expect(encoded.meta.byteLength).toBe(STARS.length * BYTES_PER_STAR_META);
+  });
+
+  it('hands positions over as a bare xyz buffer, which is what the GPU is given', () => {
+    expect(Array.from(encoded.positions.slice(0, 3))).toEqual([0, 0, 0]);
+    expect(encoded.positions[3]).toBeCloseTo(-1.35, 4);
+  });
+
+  it('stores each distinct spectral type once and refers to it by index', () => {
+    // Two of the four stars are G2V. Across the real catalogue this is 68000 stars sharing
+    // about 2600 strings, which is why the dictionary is worth having.
+    expect(encoded.index.spectralTypes).toEqual(['G2V', 'A0m...', 'Unknown']);
+  });
+
+  it('keeps the index free of anything that is not a string, since the numbers are elsewhere', () => {
+    expect(Object.keys(encoded.index).sort()).toEqual(['count', 'names', 'spectralTypes']);
+    expect(encoded.index.count).toBe(STARS.length);
+    expect(encoded.index.names).toEqual(STARS.map((star) => star.name));
+  });
+
+  it('is smaller than the array of objects it replaced', () => {
+    // The whole reason for the format: the old encoding repeated eight key names per star.
+    const asObjects = JSON.stringify(STARS).length;
+    const asCatalogue = JSON.stringify(encoded.index).length + encoded.positions.byteLength + encoded.meta.byteLength;
+    expect(asCatalogue).toBeLessThan(asObjects);
+  });
+
+  it('handles an empty catalogue without producing a malformed buffer', () => {
+    const empty = encodeStarCatalog([]);
+    expect(empty.positions.byteLength).toBe(0);
+    expect(empty.meta.byteLength).toBe(0);
+    expect(decodeStarCatalog(empty.index, empty.positions, empty.meta)).toEqual([]);
+  });
+
+  it('survives more distinct spectral types than a handful, up to the column width', () => {
+    // The dictionary index is 16-bit, and the real catalogue has about 2600 distinct types.
+    const many: StarRecord[] = Array.from({ length: 5000 }, (_, i) => ({
+      id: i,
+      name: `HYG ${i}`,
+      x: i,
+      y: 0,
+      z: 0,
+      magnitude: 10,
+      spectralType: `S${i}`,
+      colorIndex: null
+    }));
+    const round = decodeStarCatalog(...(({ index, positions, meta }) => [index, positions, meta] as const)(encodeStarCatalog(many)));
+
+    expect(round[4999].spectralType).toBe('S4999');
+    expect(round[4999].id).toBe(4999);
+  });
+});

src/app/shared/models/star-catalog.ts

@@ -0,0 +1,131 @@
+import { StarRecord } from './star.model';
+
+/**
+ * On-disk format for the star catalogue, shared by the ETL that writes it and the app that
+ * reads it so the two cannot drift apart.
+ *
+ * The catalogue outgrew a plain array of JSON objects. At the 50 pc cutoff it held 8750 stars
+ * and cost 157 bytes each — most of that the same eight key names repeated once per star. At
+ * the distance Hipparcos parallaxes actually reach, that same encoding would have been about
+ * 17 MB of JSON to parse before the first frame.
+ *
+ * So the numbers move to a binary column store and the strings stay in JSON, where the two
+ * repetitive ones — spectral types, of which 68000 stars share about 2600 distinct values —
+ * collapse into a dictionary. The result is roughly a quarter of the size for eight times the
+ * stars, and the numeric columns arrive as typed arrays with no parsing at all.
+ */
+
+/**
+ * Positions stay in their own file rather than joining the columns below.
+ *
+ * They are the one column handed to the GPU verbatim: `StarFieldRenderer` binds the buffer
+ * straight from `stars.bin` as an instanced attribute, so keeping it a bare `Float32Array` of
+ * xyz triples means the star field costs one fetch and no repacking.
+ */
+export const STAR_POSITION_COMPONENTS = 3;
+export const BYTES_PER_STAR_POSITION = STAR_POSITION_COMPONENTS * Float32Array.BYTES_PER_ELEMENT;
+
+/**
+ * Columns in `stars-meta.bin`, in order: catalogue id, apparent magnitude, colour index, and an
+ * index into the spectral-type dictionary. Stored column by column rather than record by record
+ * so each one is a single typed-array view over the buffer, with no per-record stride or
+ * alignment padding.
+ */
+export const BYTES_PER_STAR_META =
+  Int32Array.BYTES_PER_ELEMENT + Float32Array.BYTES_PER_ELEMENT + Float32Array.BYTES_PER_ELEMENT + Uint16Array.BYTES_PER_ELEMENT;
+
+/** `stars-index.json`: everything that is a string, plus the count the columns are sized by. */
+export interface StarCatalogIndex {
+  count: number;
+  /** One per star, in catalogue order. */
+  names: string[];
+  /** Distinct spectral classifications; the meta column holds indices into this. */
+  spectralTypes: string[];
+}
+
+interface StarMetaColumns {
+  ids: Int32Array;
+  magnitudes: Float32Array;
+  colorIndices: Float32Array;
+  spectralTypeIndices: Uint16Array;
+}
+
+/** Lays typed-array views over the meta buffer at the offsets the format defines. */
+function metaColumns(buffer: ArrayBuffer, count: number): StarMetaColumns {
+  let offset = 0;
+  const ids = new Int32Array(buffer, offset, count);
+  offset += count * Int32Array.BYTES_PER_ELEMENT;
+  const magnitudes = new Float32Array(buffer, offset, count);
+  offset += count * Float32Array.BYTES_PER_ELEMENT;
+  const colorIndices = new Float32Array(buffer, offset, count);
+  offset += count * Float32Array.BYTES_PER_ELEMENT;
+  const spectralTypeIndices = new Uint16Array(buffer, offset, count);
+
+  return { ids, magnitudes, colorIndices, spectralTypeIndices };
+}
+
+/**
+ * Packs the string and numeric halves of a star list into the two files the app loads.
+ *
+ * `colorIndex` is genuinely nullable — about a tenth of the catalogue was never photometered —
+ * and `NaN` carries that through the float column. It is the one value a float can hold that
+ * means "no measurement" without colliding with a real one, and 0 emphatically does not: it is
+ * a real colour index meaning a hot blue-white A-type star.
+ */
+export function encodeStarCatalog(stars: readonly StarRecord[]): {
+  index: StarCatalogIndex;
+  positions: Float32Array;
+  meta: ArrayBuffer;
+} {
+  const count = stars.length;
+  const positions = new Float32Array(count * STAR_POSITION_COMPONENTS);
+  const meta = new ArrayBuffer(count * BYTES_PER_STAR_META);
+  const columns = metaColumns(meta, count);
+
+  const spectralTypes: string[] = [];
+  const spectralTypeIds = new Map<string, number>();
+  const names: string[] = [];
+
+  stars.forEach((star, index) => {
+    positions[index * 3] = star.x;
+    positions[index * 3 + 1] = star.y;
+    positions[index * 3 + 2] = star.z;
+
+    names.push(star.name);
+
+    let spectralTypeId = spectralTypeIds.get(star.spectralType);
+    if (spectralTypeId === undefined) {
+      spectralTypeId = spectralTypes.push(star.spectralType) - 1;
+      spectralTypeIds.set(star.spectralType, spectralTypeId);
+    }
+
+    columns.ids[index] = star.id;
+    columns.magnitudes[index] = star.magnitude;
+    columns.colorIndices[index] = star.colorIndex ?? Number.NaN;
+    columns.spectralTypeIndices[index] = spectralTypeId;
+  });
+
+  return { index: { count, names, spectralTypes }, positions, meta };
+}
+
+/** Rebuilds the star records the app works with from the three loaded assets. */
+export function decodeStarCatalog(index: StarCatalogIndex, positions: Float32Array, meta: ArrayBuffer): StarRecord[] {
+  const columns = metaColumns(meta, index.count);
+  const stars: StarRecord[] = new Array(index.count);
+
+  for (let i = 0; i < index.count; i++) {
+    const colorIndex = columns.colorIndices[i];
+    stars[i] = {
+      id: columns.ids[i],
+      name: index.names[i],
+      x: positions[i * 3],
+      y: positions[i * 3 + 1],
+      z: positions[i * 3 + 2],
+      magnitude: columns.magnitudes[i],
+      spectralType: index.spectralTypes[columns.spectralTypeIndices[i]],
+      colorIndex: Number.isNaN(colorIndex) ? null : colorIndex
+    };
+  }
+
+  return stars;
+}