diff --git a/.github/workflows/data-refresh.yml b/.github/workflows/data-refresh.yml index fa24a64..bf90aa5 100644 --- a/.github/workflows/data-refresh.yml +++ b/.github/workflows/data-refresh.yml @@ -49,9 +49,11 @@ jobs: key: gaia-dr3-${{ hashFiles('tools/etl/sources/gaia.ts') }} # Every other source is fetched live on this fresh runner. A failed fetch fails the run by - # design — no refresh is better than a partial one. That includes Gaia on a cold cache: - # the ETL skips it when unreachable, and the merge gate in build.ts then refuses a - # catalogue it contributed nothing to. + # design — no refresh is better than a partial one. That includes Gaia on a cold cache, by + # two different paths: its Hipparcos cross-match is required, so an unreachable archive + # fails the run from fetchStars itself, while its main query is skipped when unreachable and + # the merge gate in build.ts then refuses a catalogue it contributed nothing to. An archive + # that answers short rather than not at all is caught in fetchGaiaStars. - name: Rebuild the datasets run: npm run etl diff --git a/tools/etl/build.ts b/tools/etl/build.ts index b213455..6f11c09 100644 --- a/tools/etl/build.ts +++ b/tools/etl/build.ts @@ -66,8 +66,15 @@ function validateStars(stars: StarRecord[]): void { * The other failure leaves no close pair at all, because proper motion had already carried the * two entries tens of arcseconds apart — the 2026-08-24 refresh, where HYG sat at epoch 2000.0 * and Gaia at J2016.0. What it does leave is HYG rows that found no counterpart: 36 056 of them - * against the 10 876 Gaia genuinely lacks (bright stars it saturates on, red dwarfs past its - * magnitude cut). + * against the 10 886 today, and no counterpart was possible for most of those. Two thirds of them, + * 6 835, are the stars Gaia measures but the main query never downloads, because Gaia's parallax + * puts them past `ETL_GAIA_DISTANCE_PC` while Hipparcos put them inside `ETL_STAR_DISTANCE_PC`; + * they are every star in the published catalogue beyond 250 pc. The rest are what Gaia genuinely + * lacks: bright stars it saturates on, red dwarfs past its magnitude cut. So the headroom left to + * the ceiling tracks the gap between those two cutoffs as much as Gaia's completeness. + * + * This bounds a merge that went wrong. It cannot bound a Gaia download that came back short: that + * makes *fewer* survivors, not more, and is guarded where it can be seen, in `fetchGaiaStars`. */ const MAX_UNMERGED_TWINS = 100; const MAX_HYG_SURVIVORS = 15_000; diff --git a/tools/etl/lib/http.ts b/tools/etl/lib/http.ts index 9410ec0..a214332 100644 --- a/tools/etl/lib/http.ts +++ b/tools/etl/lib/http.ts @@ -34,9 +34,15 @@ const RETRY_DELAYS_MS = [30_000, 120_000]; async function fetchText(url: string): Promise { for (let attempt = 0; ; attempt++) { - const response = await fetch(url).catch((error: unknown) => (error instanceof Error ? error : new Error(String(error)))); + let response = await fetch(url).catch((error: unknown) => (error instanceof Error ? error : new Error(String(error)))); if (!(response instanceof Error) && response.ok) { - return response.text(); + // Read inside the loop, because the body is where these downloads fail: the Gaia CSV is + // 57 MB, and a connection reset part-way through rejects here, long after the 200. + const body = await response.text().catch((error: unknown) => (error instanceof Error ? error : new Error(String(error)))); + if (typeof body === 'string') { + return body; + } + response = body; } const reason = response instanceof Error ? response.message : `${response.status} ${response.statusText}`; // A 4xx is the request's own fault, and waiting will not change the answer. diff --git a/tools/etl/sources/gaia.ts b/tools/etl/sources/gaia.ts index c70d6a6..6d02dc9 100644 --- a/tools/etl/sources/gaia.ts +++ b/tools/etl/sources/gaia.ts @@ -38,9 +38,29 @@ const CATALOGUE_EPOCH = 2000.0; * past anything this map draws, so the limit here is a payload decision: the catalogue is baked * into a static asset that a browser downloads before the first frame. */ -const DISTANCE_CUTOFF_PC = Number(process.env['ETL_GAIA_DISTANCE_PC'] ?? 250); -const MAGNITUDE_LIMIT = Number(process.env['ETL_GAIA_MAGNITUDE_LIMIT'] ?? 12); -const ROW_LIMIT = Number(process.env['ETL_GAIA_ROW_LIMIT'] ?? 500000); +const DEFAULT_DISTANCE_CUTOFF_PC = 250; +const DEFAULT_MAGNITUDE_LIMIT = 12; +const DEFAULT_ROW_LIMIT = 500_000; +const DISTANCE_CUTOFF_PC = Number(process.env['ETL_GAIA_DISTANCE_PC'] ?? DEFAULT_DISTANCE_CUTOFF_PC); +const MAGNITUDE_LIMIT = Number(process.env['ETL_GAIA_MAGNITUDE_LIMIT'] ?? DEFAULT_MAGNITUDE_LIMIT); +const ROW_LIMIT = Number(process.env['ETL_GAIA_ROW_LIMIT'] ?? DEFAULT_ROW_LIMIT); + +/** + * How many rows the scheduled job's own query holds: 412 765, and DR3 is a finished data release, + * so that number only moves when the query does. + * + * Checked because a short answer looks exactly like a complete one. The TAP service truncates on + * its own timeout and still serves a well-formed CSV with a 200, and the rows are ordered by + * magnitude, so what comes back is the bright half — the half HYG overlaps. The merge gate in + * `build.ts` would then see Gaia stars present, fewer HYG survivors and fewer unmerged twins, and + * pass a catalogue missing two hundred thousand stars, which the weekly job would publish and the + * runner would cache for the weeks after it. Same failure, and same guard, as + * {@link MIN_USABLE_HIP_DISTANCES} below. + * + * Only checked for that query: the environment overrides exist to fetch a smaller slice on purpose. + */ +const DEFAULT_QUERY_ROWS = 412_765; +const MIN_ROW_SHARE = 0.95; /** * Relative parallax error above which a star is dropped: a parallax measured to worse than 20% @@ -92,6 +112,16 @@ export async function fetchGaiaStars(): Promise { // response arrived, not what it answered. const csv = await fetchTextCached(url, `gaia-dr3-${createHash('sha1').update(url).digest('hex').slice(0, 8)}.csv`); const rows = parseCsvObjects(csv); + const jobsQuery = DISTANCE_CUTOFF_PC === DEFAULT_DISTANCE_CUTOFF_PC && MAGNITUDE_LIMIT === DEFAULT_MAGNITUDE_LIMIT && ROW_LIMIT === DEFAULT_ROW_LIMIT; + if (jobsQuery && rows.length < DEFAULT_QUERY_ROWS * MIN_ROW_SHARE) { + throw new Error( + `Gaia returned ${rows.length} rows, not the ~${DEFAULT_QUERY_ROWS} this query holds — the answer was cut short, ` + + 'or was an error page served with a 200; delete tools/etl/.cache/gaia-dr3-*.csv once the archive answers properly' + ); + } + if (rows.length >= ROW_LIMIT) { + throw new Error(`Gaia returned the query's own ${ROW_LIMIT}-row limit, so it is the limit deciding what the map holds; raise ETL_GAIA_ROW_LIMIT.`); + } const stars: StarRecord[] = []; rows.forEach((row, index) => {