Fill in the Sun's faint neighbours from Gaia, and fold the Gliese entries they repeat
Gaia's G < 12 cut took a quarter of what lies within 10 pc: the red, brown and white dwarfs most of the neighbourhood is made of, which the map only had where Gliese happened to list them. Teegarden's Star was missing, and with it its three planets' host. A second query fetches the complement out to 50 pc: G >= 12 or no G, parallax > 20 mas, with pmra/pmdec so the J2016 -> J2000 propagation applies. The quality filter was chosen by counting. Parallax over error > 5 keeps 39 751 of the 39 764 sources that pass the floor, but the Gaia Catalogue of Nearby Stars (GCNS; Gaia Collaboration, Smart et al. 2021) rejects 11 752 of them as spurious: median G 20.2, astrometric excess noise 5.4 mas against 0.15 for the ones it keeps, 6 933 toward the Galactic centre. So the query joins the GCNS main table (EDR3 astrometry and source ids, which DR3 carries unchanged) and keeps 28 012 rows; the error cut stays and costs no GCNS source, brown dwarfs included. The query has its own row-count floor (28 012) and the row-limit cap, and is ordered by (phot_g_mean_mag, source_id); a second ETL run reproduced stars.bin, stars-meta.bin and stars-index.json byte for byte. Its ids start at 1 050 000 000, clear of the main query's and under 2^30, which V8 keeps unboxed: numbered from 2 000 000 000 they made the app's boot task 230 ms longer (medians of five interleaved runs, 1.41 s against 1.18). validateStars now refuses an id outside 0 to 2^30. The Gliese entries these stars duplicate were not folded: HYG carries them with positions off by up to a minute of arc and photometric distances, so they missed the 15" tolerance or failed the distance test. Their proper motions, which Gliese measured well, give them away: isSameStar now takes two entries moving within 20 % of each other as one star up to 60" apart, whatever their distances, brightness still permitting. Of the 602 Gliese-only rows left without a counterpart, 253 have such a Gaia entry; with every entry shifted a quarter degree, none does. fetchStars passes HYG's motions only for rows without Hipparcos astrometry: given them too, 15 Hipparcos stars took a co-moving companion's Gaia entry and the cross-catalogue pairs under an arcsecond went from 23 to 35. Of HYG's 1 200 stars fainter than V 12 within 25 pc, 1 024 now sit on a Gaia position (325 before); of the 176 left alone, 43 still have a Gaia entry 3-60" away (218 without the motion rule), some of them real companions. Measured on the rebuilt catalogue, against the GCNS (sources with parallax > 100, 40 and 20 mas): within 10 pc 336 -> 372 (GCNS 312; the map adds 60 HYG-only stars) within 25 pc 3 652 -> 5 560 (GCNS 5 111) within 50 pc 13 702 -> 40 916 (GCNS 40 231) 452 331 stars (+27 214). Proxima, Barnard's Star, Wolf 359, Rigel, Deneb and Alnilam are all present by name; Teegarden's Star is Gaia DR3 35227046884571776 at 3.83 pc and hosts its three planets. Luhman 16 is not in Gaia DR3 with a parallax (5353626573555863424 has a two-parameter solution) and stays absent. 94 more exoplanets find a host (2 071), none changes host. TRAPPIST-1 is now drawn (Gaia DR3 2635476908753563008, 12.47 pc) but its planets are not yet matched to it. Merge gate, ceilings unchanged: HYG rows without a Gaia counterpart 12 352 -> 11 554 (ceiling 15 000; 10 886 before the naked-eye stars), cross-catalogue pairs under an arcsecond 23 -> 23 (ceiling 100). The gate's comment now accounts for the survivors by magnitude. gzip -9 sizes against the catalogue before both changes: stars.bin 4 711 922 -> 5 030 741 B, stars-meta.bin 2 576 213 -> 2 784 614 B, stars-index.json 3 724 855 -> 4 003 584 B (+806 KB, 7.3 %). Boot on the dev server, five interleaved cold runs: the task that indexes the catalogue after the data lands, median 1 072 -> 1 182 ms; HUD shown, median 2 622 -> 2 716 ms. This machine measured 0.82-1.30 s for the same baseline task today, above audit #25's 627-843 ms. The drawn-star budget is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -79,6 +79,32 @@ describe('isSameStar', () => {
|
||||
expect(isSameStar(companion, at(43109, 131.69 + arcsecOfRa(2.7, 6.42), 6.42, 40, { name: 'Ashlesha', magnitude: 3.38 }))).toBe(false);
|
||||
});
|
||||
|
||||
// GJ 1035 and GJ 3052 as HYG has them from Gliese, against their Gaia entries: 21″ away at
|
||||
// about the same distance, and 6.6″ away at half Gaia's distance. Their motions agree to 2 and 3 %.
|
||||
it('matches a Gliese entry to the Gaia entry moving with it, a minute of arc away or at another distance', () => {
|
||||
const gj1035 = at(1, 19.9245, 84.1612, 14.4, { magnitude: 13.1, source: 'gaia', pmRaMasYr: -981.9, pmDecMasYr: 475.6 });
|
||||
const gliese1035 = at(118058, 19.9245 + arcsecOfRa(21.2, 84.1612), 84.1612, 13.7, { name: 'GJ 1035', magnitude: 14.77, pmRaMasYr: -978.0, pmDecMasYr: 458.1 });
|
||||
expect(isSameStar(gj1035, gliese1035)).toBe(true);
|
||||
expect(isSameStar(gj1035, { ...gliese1035, pmRaMasYr: undefined, pmDecMasYr: undefined })).toBe(false);
|
||||
|
||||
const gj3052 = at(2, 11.0897, 9.1262, 25.0, { magnitude: 12.6, source: 'gaia', pmRaMasYr: 813.1, pmDecMasYr: -2.6 });
|
||||
const gliese3052 = at(118014, 11.0897 + arcsecOfRa(6.6, 9.1262), 9.1262, 12.3, { name: 'GJ 3052', magnitude: 13.8, pmRaMasYr: 799.7, pmDecMasYr: -20.9 });
|
||||
expect(isSameStar(gj3052, gliese3052)).toBe(true);
|
||||
});
|
||||
|
||||
it('does not match two entries moving differently past fifteen arcseconds, nor co-moving ones past a minute', () => {
|
||||
const kept = at(1, 120, 30, 10, { magnitude: 12, source: 'gaia', pmRaMasYr: 1000, pmDecMasYr: 0 });
|
||||
expect(isSameStar(kept, at(2, 120 + arcsecOfRa(20, 30), 30, 10, { magnitude: 13, pmRaMasYr: 750, pmDecMasYr: 0 }))).toBe(false);
|
||||
expect(isSameStar(kept, at(2, 120 + arcsecOfRa(20, 30), 30, 10, { magnitude: 13, pmRaMasYr: 850, pmDecMasYr: 0 }))).toBe(true);
|
||||
expect(isSameStar(kept, at(2, 120 + arcsecOfRa(65, 30), 30, 10, { magnitude: 13, pmRaMasYr: 1000, pmDecMasYr: 0 }))).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps a co-moving primary out of its companion's entry", () => {
|
||||
// A binary shares its motion, so only the brightness tells GJ 9160 from its companion 13.5″ away.
|
||||
const companion = at(1, 69.54, -14.3, 24.4, { magnitude: 15.1, source: 'gaia', pmRaMasYr: -78.5, pmDecMasYr: -150.8 });
|
||||
expect(isSameStar(companion, at(2, 69.54 + arcsecOfRa(13.5, -14.3), -14.3, 24.4, { magnitude: 7.3, pmRaMasYr: -78.5, pmDecMasYr: -150.8 }))).toBe(false);
|
||||
});
|
||||
|
||||
it('matches on direction rather than on 3D proximity', () => {
|
||||
// The distinction the merge rests on. These two are 60 pc apart in space and are the same
|
||||
// star; a 3D-proximity test would have to be so loose it swallowed real neighbours.
|
||||
|
||||
@@ -47,6 +47,22 @@ export const MERGE_ANGULAR_TOLERANCE_DEG = 15 / 3600;
|
||||
*/
|
||||
export const MERGE_CERTAIN_ANGULAR_TOLERANCE_DEG = 3 / 3600;
|
||||
|
||||
/**
|
||||
* Angular separation, in degrees, within which two entries that cross the sky together are one
|
||||
* star, and how closely their proper motions must agree (as a fraction of the kept entry's) to
|
||||
* say so. Past fifteen arcseconds, and past a distance conflict, a Gliese-only entry is still
|
||||
* often the same star: its position is off by up to a minute of arc and its distance is
|
||||
* photometric — GJ 1035 sits 21″ from its Gaia entry, GJ 3052 at half Gaia's distance. What
|
||||
* gives them away is their motion, which Gliese measured well: a few per cent from Gaia's.
|
||||
*
|
||||
* Once the nearby faint Gaia stars joined, 253 of the 602 Gliese-only stars left without a
|
||||
* counterpart had a Gaia entry within a minute of arc moving within a fifth of their own motion;
|
||||
* shifted a quarter of a degree, none did. Brightness still has its say, so a co-moving companion
|
||||
* is not folded into its primary, and `fetchStars` gives HYG's motions only to those rows.
|
||||
*/
|
||||
export const MERGE_COMOVING_ANGULAR_TOLERANCE_DEG = 60 / 3600;
|
||||
export const MERGE_PROPER_MOTION_TOLERANCE = 0.2;
|
||||
|
||||
/**
|
||||
* How much fainter, and how much brighter, an entry may be than the one it is folded into and
|
||||
* still be the same star. Bands differ, and not symmetrically: a red dwarf is three magnitudes
|
||||
@@ -159,10 +175,19 @@ export function directionCosine(a: StarRecord, b: StarRecord): number {
|
||||
return Math.max(-1, Math.min(1, ax * bx + ay * by + az * bz));
|
||||
}
|
||||
|
||||
/** Whether both entries have a proper motion and `entry`'s is within tolerance of `kept`'s. */
|
||||
function movesWith(kept: StarRecord, entry: StarRecord): boolean {
|
||||
if (kept.pmRaMasYr === undefined || kept.pmDecMasYr === undefined || entry.pmRaMasYr === undefined || entry.pmDecMasYr === undefined) {
|
||||
return false;
|
||||
}
|
||||
const difference = Math.hypot(entry.pmRaMasYr - kept.pmRaMasYr, entry.pmDecMasYr - kept.pmDecMasYr);
|
||||
return difference < MERGE_PROPER_MOTION_TOLERANCE * Math.hypot(kept.pmRaMasYr, kept.pmDecMasYr);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether `entry` describes the star already `kept`: the same direction, the brightness not in
|
||||
* conflict and — unless the directions agree closely enough to settle it — the distance not in
|
||||
* conflict either.
|
||||
* conflict and — unless the directions agree closely enough to settle it, or the two move
|
||||
* together — the distance not in conflict either.
|
||||
*/
|
||||
export function isSameStar(kept: StarRecord, entry: StarRecord): boolean {
|
||||
const [near, far] = [distanceOf(kept), distanceOf(entry)].sort((p, q) => p - q);
|
||||
@@ -175,7 +200,8 @@ export function isSameStar(kept: StarRecord, entry: StarRecord): boolean {
|
||||
}
|
||||
|
||||
const separationDeg = Math.acos(directionCosine(kept, entry)) / DEG_TO_RAD;
|
||||
if (separationDeg > MERGE_ANGULAR_TOLERANCE_DEG) {
|
||||
const comoving = movesWith(kept, entry);
|
||||
if (separationDeg > (comoving ? MERGE_COMOVING_ANGULAR_TOLERANCE_DEG : MERGE_ANGULAR_TOLERANCE_DEG)) {
|
||||
return false;
|
||||
}
|
||||
const fainterBy = entry.magnitude - kept.magnitude;
|
||||
@@ -183,7 +209,7 @@ export function isSameStar(kept: StarRecord, entry: StarRecord): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (separationDeg <= MERGE_CERTAIN_ANGULAR_TOLERANCE_DEG) {
|
||||
if (comoving || separationDeg <= MERGE_CERTAIN_ANGULAR_TOLERANCE_DEG) {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@@ -26,6 +26,13 @@ export interface StarRecord {
|
||||
* the better-measured parallax wins.
|
||||
*/
|
||||
source?: string;
|
||||
/**
|
||||
* Proper motion in milliarcseconds a year, in right ascension (times cos δ) and declination,
|
||||
* where the source measured one. Only the ETL sets these, for `star-merge.ts` to recognise two
|
||||
* entries of one star whose positions disagree; the assets do not carry them.
|
||||
*/
|
||||
pmRaMasYr?: number;
|
||||
pmDecMasYr?: number;
|
||||
}
|
||||
|
||||
/** HYG id used for the Sun itself, so solar-system bodies can reference their host star. */
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Binary file not shown.
Binary file not shown.
+16
-11
@@ -26,6 +26,9 @@ function validateStars(stars: StarRecord[]): void {
|
||||
const ids = new Set<number>();
|
||||
for (const star of stars) {
|
||||
assertCondition(Number.isFinite(star.id), `Star has a non-numeric id: ${JSON.stringify(star)}`);
|
||||
// Under 2^30, which V8 keeps unboxed; past it every id is a heap number, and the app's boot
|
||||
// task grew by 230 ms when the nearby Gaia stars were numbered from 2 000 000 000.
|
||||
assertCondition(star.id >= 0 && star.id < 2 ** 30, `Star ${star.id} has an id outside 0 to 2^30.`);
|
||||
assertCondition(!ids.has(star.id), `Duplicate star id: ${star.id}`);
|
||||
ids.add(star.id);
|
||||
assertCondition(!!star.name, `Star ${star.id} has no name.`);
|
||||
@@ -59,26 +62,28 @@ function validateStars(stars: StarRecord[]): void {
|
||||
*
|
||||
* A star kept twice leaves its two entries near each other on the sky, from *different* sources —
|
||||
* one catalogue does not list a star twice. Under an arcsecond that is never two stars at this
|
||||
* depth, so every such pair is a miss. Nineteen survive today, all of them a second HYG row
|
||||
* wanting a Gaia entry that already absorbed one (Gliese lists some doubles twice); the merge
|
||||
* that trusted a Hipparcos parallax over direction left 1 112.
|
||||
* depth, so every such pair is a miss. Twenty-three survive today; the nineteen first counted were
|
||||
* all a second HYG row wanting a Gaia entry that already absorbed one (Gliese lists some doubles
|
||||
* twice), and the merge that trusted a Hipparcos parallax over direction left 1 112.
|
||||
*
|
||||
* The other failure leaves no close pair at all, because proper motion had already carried the
|
||||
* two entries tens of arcseconds apart — the 2026-08-24 refresh, where HYG sat at epoch 2000.0
|
||||
* and Gaia at J2016.0. What it does leave is HYG rows that found no counterpart: 36 056 of them
|
||||
* against the 12 352 today, and no counterpart was possible for most of those. 8 301 of them are
|
||||
* against the 11 554 today, and no counterpart was possible for most of those. 8 301 of them are
|
||||
* every star in the published catalogue beyond 250 pc, which the main query never downloads: 6 835
|
||||
* that Gaia's parallax puts past `ETL_GAIA_DISTANCE_PC` while Hipparcos put them inside
|
||||
* `ETL_STAR_DISTANCE_PC`, and 1 466 naked-eye stars kept at any distance. The rest are what Gaia
|
||||
* genuinely lacks: bright stars it saturates on, red dwarfs past its magnitude cut. So the headroom
|
||||
* left to the ceiling tracks the gap between those two cutoffs as much as Gaia's completeness.
|
||||
* `ETL_STAR_DISTANCE_PC`, and 1 466 naked-eye stars kept at any distance. The other 3 252 are what
|
||||
* Gaia genuinely lacks: 1 194 brighter than V 8, which it saturates on or measures poorly, 1 832
|
||||
* between 8 and 12, and 226 fainter, 184 of them Gliese stars within 50 pc that neither of its
|
||||
* queries holds. So the headroom left to the ceiling tracks the gap between those two cutoffs as
|
||||
* much as Gaia's completeness.
|
||||
*
|
||||
* This bounds a merge that went wrong, and — loosely — a Gaia download that came back short: a
|
||||
* truncated answer leaves the HYG rows whose counterpart it dropped without one, so survivors go
|
||||
* *up*, not down. Measured before the naked-eye stars joined, which add about 1 466 to each
|
||||
* figure: 10 886, 11 004 at nine tenths of the rows, 12 711 at half, 16 258 at a third. So this
|
||||
* ceiling only catches a deep truncation, and `fetchGaiaStars` catches the shallower ones with its
|
||||
* own row floor.
|
||||
* *up*, not down. Measured on the main query when it was the only one, with 10 886 survivors
|
||||
* against today's 11 554: 11 004 at nine tenths of its rows, 12 711 at half, 16 258 at a third. So
|
||||
* this ceiling only catches a deep truncation, and `fetchGaiaStars` catches the shallower ones
|
||||
* with a row floor on each query.
|
||||
*/
|
||||
const MAX_UNMERGED_TWINS = 100;
|
||||
const MAX_HYG_SURVIVORS = 15_000;
|
||||
|
||||
@@ -127,7 +127,11 @@ export async function fetchStars(): Promise<StarRecord[]> {
|
||||
z: z * scale,
|
||||
magnitude,
|
||||
spectralType: row['spect'] || 'Unknown',
|
||||
colorIndex: parseOptionalNumber(row['ci']) ?? null
|
||||
colorIndex: parseOptionalNumber(row['ci']) ?? null,
|
||||
// Only for the Gliese-only rows, whose positions are what the merge needs the motion to see
|
||||
// past. A Hipparcos position is good to under an arcsecond; given its motion too, 15 stars
|
||||
// took their co-moving companion's Gaia entry, and the companion was kept twice.
|
||||
...(row['hip'] ? {} : { pmRaMasYr: parseOptionalNumber(row['pmra']), pmDecMasYr: parseOptionalNumber(row['pmdec']) })
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+77
-15
@@ -70,6 +70,37 @@ function buildQuery(): string {
|
||||
].join(' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* The nearby complement of the query above: every source it leaves out for being fainter than its
|
||||
* magnitude limit, or for having no G at all, out to 50 pc.
|
||||
*
|
||||
* The limit took a quarter of what lies within 10 pc — 84 of the 312 sources the Gaia Catalogue of
|
||||
* Nearby Stars (GCNS; Gaia Collaboration, Smart et al. 2021, A&A 649, A6) has there, Teegarden's
|
||||
* Star among them — and 28 012 within 50 pc: the red, brown and white dwarfs most of the Sun's
|
||||
* neighbourhood is made of, which the map only had where Gliese happened to list them. Further out
|
||||
* the same band is far too many rows to ship (G 12–13 alone is 188 408 within 250 pc).
|
||||
*
|
||||
* Only sources the GCNS keeps. Its classifier rejects 11 752 of the 39 764 that pass this parallax
|
||||
* floor, and the parallax error the main query filters on would keep all but 13 of them: they are
|
||||
* spurious parallaxes — median G 20.2 and 5.4 mas of astrometric excess noise, against 0.15 mas for
|
||||
* the ones it keeps; 6 933 of them toward the crowded Galactic centre. GCNS was built on EDR3,
|
||||
* whose astrometry and source ids DR3 carries unchanged. The error cut is kept for consistency; it
|
||||
* costs no GCNS source here, brown dwarfs included.
|
||||
*/
|
||||
const NEARBY_DISTANCE_PC = 50;
|
||||
|
||||
function buildNearbyQuery(): string {
|
||||
return [
|
||||
`select top ${ROW_LIMIT}`,
|
||||
'g.source_id, g.ra, g.dec, g.pmra, g.pmdec, g.parallax, g.parallax_error, g.phot_g_mean_mag, g.bp_rp',
|
||||
'from gaiadr3.gaia_source g join external.gaiaedr3_gcns_main_1 n on n.source_id = g.source_id',
|
||||
`where g.parallax > ${parallaxFloorMas(NEARBY_DISTANCE_PC).toFixed(6)}`,
|
||||
`and g.parallax_over_error > ${(1 / MAX_PARALLAX_ERROR_RATIO).toFixed(1)}`,
|
||||
`and (g.phot_g_mean_mag >= ${MAGNITUDE_LIMIT} or g.phot_g_mean_mag is null)`,
|
||||
'order by g.phot_g_mean_mag asc, g.source_id asc'
|
||||
].join(' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* How many rows the query above holds when nothing is overridden: 412 765, and DR3 is a finished
|
||||
* data release, so that number only moves when the query does. It lives here, under the query, so
|
||||
@@ -87,6 +118,8 @@ function buildQuery(): string {
|
||||
* slice on purpose.
|
||||
*/
|
||||
const DEFAULT_QUERY_ROWS = 412_765;
|
||||
/** The same count for the nearby query, which the same truncation would cut from its faint end. */
|
||||
const DEFAULT_NEARBY_QUERY_ROWS = 28_012;
|
||||
const MIN_ROW_SHARE = 0.95;
|
||||
|
||||
/**
|
||||
@@ -113,31 +146,54 @@ const UNKNOWN_SPECTRAL_TYPE = 'Unknown';
|
||||
* the real identifier in the name.
|
||||
*/
|
||||
const GAIA_ID_BASE = 1_000_000_000;
|
||||
/**
|
||||
* Where the nearby query's ids start: fifty million past the main query's, and under 2^30, the
|
||||
* largest integer V8 keeps unboxed. The Int32 id column would take up to 2^31, but every id past
|
||||
* 2^30 is a heap number in the app: starting these at 2 000 000 000 made the boot task that
|
||||
* indexes the catalogue 230 ms longer (1.41 s against 1.18, medians of five interleaved runs).
|
||||
*/
|
||||
const NEARBY_ID_BASE = 1_050_000_000;
|
||||
|
||||
export async function fetchGaiaStars(): Promise<StarRecord[]> {
|
||||
const query = buildQuery();
|
||||
const url = `${GAIA_TAP_URL}?REQUEST=doQuery&LANG=ADQL&FORMAT=csv&QUERY=${encodeURIComponent(query)}`;
|
||||
const unchanged = MAGNITUDE_LIMIT === DEFAULT_MAGNITUDE_LIMIT && ROW_LIMIT === DEFAULT_ROW_LIMIT;
|
||||
console.log(`Fetching Gaia DR3 (within ${DISTANCE_CUTOFF_PC} pc, G < ${MAGNITUDE_LIMIT}, at most ${ROW_LIMIT} rows)...`);
|
||||
const rows = await fetchQueryRows(buildQuery(), unchanged && DISTANCE_CUTOFF_PC === DEFAULT_DISTANCE_CUTOFF_PC ? DEFAULT_QUERY_ROWS : undefined);
|
||||
console.log(`Fetching Gaia DR3's nearby complement (within ${NEARBY_DISTANCE_PC} pc, G >= ${MAGNITUDE_LIMIT} or none, kept by the GCNS)...`);
|
||||
const nearbyRows = await fetchQueryRows(buildNearbyQuery(), unchanged ? DEFAULT_NEARBY_QUERY_ROWS : undefined);
|
||||
|
||||
const stars = [...rowsToStars(rows, GAIA_ID_BASE), ...rowsToStars(nearbyRows, NEARBY_ID_BASE)];
|
||||
console.log(` kept ${stars.length} Gaia stars (of ${rows.length} + ${nearbyRows.length} rows).`);
|
||||
return stars;
|
||||
}
|
||||
|
||||
/**
|
||||
* One query's rows, refused when there are fewer than `expectedRows` — see
|
||||
* {@link DEFAULT_QUERY_ROWS} — or as many as the row limit.
|
||||
*/
|
||||
async function fetchQueryRows(query: string, expectedRows: number | undefined): Promise<Record<string, string>[]> {
|
||||
const url = `${GAIA_TAP_URL}?REQUEST=doQuery&LANG=ADQL&FORMAT=csv&QUERY=${encodeURIComponent(query)}`;
|
||||
// Keyed by the whole request, so a response cached for other columns, another order, or
|
||||
// another endpoint can never be mistaken for this one — the cache records only that some
|
||||
// response arrived, not what it answered.
|
||||
const cacheKey = `gaia-dr3-${createHash('sha1').update(url).digest('hex').slice(0, 8)}.csv`;
|
||||
const csv = await fetchTextCached(url, cacheKey);
|
||||
const rows = parseCsvObjects(csv);
|
||||
const jobsQuery = DISTANCE_CUTOFF_PC === DEFAULT_DISTANCE_CUTOFF_PC && MAGNITUDE_LIMIT === DEFAULT_MAGNITUDE_LIMIT && ROW_LIMIT === DEFAULT_ROW_LIMIT;
|
||||
if (jobsQuery && rows.length < DEFAULT_QUERY_ROWS * MIN_ROW_SHARE) {
|
||||
const rows = parseCsvObjects(await fetchTextCached(url, cacheKey));
|
||||
if (expectedRows !== undefined && rows.length < expectedRows * MIN_ROW_SHARE) {
|
||||
throw new GaiaAnswerError(
|
||||
`Gaia returned ${rows.length} rows, not the ~${DEFAULT_QUERY_ROWS} this query holds — the answer was cut short, it was an error page ` +
|
||||
`served with a 200, or the query was edited without updating DEFAULT_QUERY_ROWS; delete tools/etl/.cache/${cacheKey} once the archive answers properly`
|
||||
`Gaia returned ${rows.length} rows, not the ~${expectedRows} this query holds — the answer was cut short, it was an error page ` +
|
||||
`served with a 200, or the query was edited without updating its row count; delete tools/etl/.cache/${cacheKey} once the archive answers properly`
|
||||
);
|
||||
}
|
||||
// Not gated on `jobsQuery` like the floor above it: the only ways to reach this cap are the
|
||||
// overrides that *widen* the query, and they are exactly when it is worth saying. What it must
|
||||
// not fire on is a deliberately smaller slice, where filling the limit is the whole point.
|
||||
// Not gated like the floor above it: the only ways to reach this cap are the overrides that
|
||||
// *widen* the query, and they are exactly when it is worth saying. What it must not fire on is
|
||||
// a deliberately smaller slice, where filling the limit is the whole point.
|
||||
if (ROW_LIMIT >= DEFAULT_ROW_LIMIT && rows.length >= ROW_LIMIT) {
|
||||
throw new GaiaAnswerError(`Gaia returned the query's own ${ROW_LIMIT}-row limit, so it is the limit deciding what the map holds; raise ETL_GAIA_ROW_LIMIT.`);
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
/** Places each row at J2000.0, numbering them from `idBase` in the query's own order. */
|
||||
function rowsToStars(rows: readonly Record<string, string>[], idBase: number): StarRecord[] {
|
||||
const stars: StarRecord[] = [];
|
||||
|
||||
rows.forEach((row, index) => {
|
||||
@@ -153,22 +209,28 @@ export async function fetchGaiaStars(): Promise<StarRecord[]> {
|
||||
return;
|
||||
}
|
||||
|
||||
const j2000 = propagateProperMotion(raDeg, decDeg, parseOptionalNumber(row['pmra']) ?? 0, parseOptionalNumber(row['pmdec']) ?? 0, CATALOGUE_EPOCH - GAIA_DR3_EPOCH);
|
||||
const pmRaMasYr = parseOptionalNumber(row['pmra']);
|
||||
const pmDecMasYr = parseOptionalNumber(row['pmdec']);
|
||||
const j2000 = propagateProperMotion(raDeg, decDeg, pmRaMasYr ?? 0, pmDecMasYr ?? 0, CATALOGUE_EPOCH - GAIA_DR3_EPOCH);
|
||||
const { x, y, z } = raDegDecDistanceToXyz(j2000.raDeg, j2000.decDeg, distancePc);
|
||||
stars.push({
|
||||
id: GAIA_ID_BASE + index,
|
||||
id: idBase + index,
|
||||
name: `Gaia DR3 ${row['source_id']}`,
|
||||
x,
|
||||
y,
|
||||
z,
|
||||
// Only the nearby query has sources without a G, 45 of them, and they are given its limit.
|
||||
// That errs bright: 26 of the 31 that 2MASS measured are at J 12.6–15.7, fainter still in G
|
||||
// for stars this red, and 5 at J 7–8.
|
||||
magnitude: parseOptionalNumber(row['phot_g_mean_mag']) ?? MAGNITUDE_LIMIT,
|
||||
spectralType: UNKNOWN_SPECTRAL_TYPE,
|
||||
colorIndex: parseOptionalNumber(row['bp_rp']) ?? null,
|
||||
source: 'gaia'
|
||||
source: 'gaia',
|
||||
pmRaMasYr,
|
||||
pmDecMasYr
|
||||
});
|
||||
});
|
||||
|
||||
console.log(` kept ${stars.length} Gaia stars (of ${rows.length} rows).`);
|
||||
return stars;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user