From 44f6a8d0864b826cbe20e293f289150f7e18d5b3 Mon Sep 17 00:00:00 2001 From: Senrokai Date: Fri, 18 Sep 2026 12:12:49 +0200 Subject: [PATCH 1/3] Refuse a Gaia answer that came back short, and read the body inside the retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge gate asks whether Gaia contributed any stars, never how many. The TAP service truncates on its own timeout and still serves a well-formed CSV with a 200, ordered by magnitude — so a half answer is the bright half, which is the half HYG overlaps. Every gate passes: Gaia stars are present, HYG survivors go down rather than up, unmerged twins can only fall. The weekly job would publish a catalogue missing two hundred thousand stars and the runner would cache it for the weeks after. `fetchGaiaStars` now refuses fewer than 95% of the 412 765 rows its query holds, as its sibling query already did, and refuses an answer that fills the row limit. `fetchText` retried the request but not the body: a connection reset part-way through the 57 MB CSV rejected out of the loop, with no wait and no second attempt. The read now happens inside it. Also corrected: the merge gate's account of the HYG survivors (two thirds of them are stars Gaia measures but the main query never downloads, since Gaia puts them past the 250 pc cutoff), and the refresh workflow's comment on what happens when the archive is unreachable. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016jxMkwA2rbicdGxHosecYi --- .github/workflows/data-refresh.yml | 8 ++++--- tools/etl/build.ts | 11 +++++++-- tools/etl/lib/http.ts | 10 +++++++-- tools/etl/sources/gaia.ts | 36 +++++++++++++++++++++++++++--- 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/.github/workflows/data-refresh.yml b/.github/workflows/data-refresh.yml index fa24a64..bf90aa5 100644 --- a/.github/workflows/data-refresh.yml +++ b/.github/workflows/data-refresh.yml @@ -49,9 +49,11 @@ jobs: key: gaia-dr3-${{ hashFiles('tools/etl/sources/gaia.ts') }} # Every other source is fetched live on this fresh runner. A failed fetch fails the run by - # design — no refresh is better than a partial one. That includes Gaia on a cold cache: - # the ETL skips it when unreachable, and the merge gate in build.ts then refuses a - # catalogue it contributed nothing to. + # design — no refresh is better than a partial one. That includes Gaia on a cold cache, by + # two different paths: its Hipparcos cross-match is required, so an unreachable archive + # fails the run from fetchStars itself, while its main query is skipped when unreachable and + # the merge gate in build.ts then refuses a catalogue it contributed nothing to. An archive + # that answers short rather than not at all is caught in fetchGaiaStars. - name: Rebuild the datasets run: npm run etl diff --git a/tools/etl/build.ts b/tools/etl/build.ts index b213455..6f11c09 100644 --- a/tools/etl/build.ts +++ b/tools/etl/build.ts @@ -66,8 +66,15 @@ function validateStars(stars: StarRecord[]): void { * The other failure leaves no close pair at all, because proper motion had already carried the * two entries tens of arcseconds apart — the 2026-08-24 refresh, where HYG sat at epoch 2000.0 * and Gaia at J2016.0. What it does leave is HYG rows that found no counterpart: 36 056 of them - * against the 10 876 Gaia genuinely lacks (bright stars it saturates on, red dwarfs past its - * magnitude cut). + * against the 10 886 today, and no counterpart was possible for most of those. Two thirds of them, + * 6 835, are the stars Gaia measures but the main query never downloads, because Gaia's parallax + * puts them past `ETL_GAIA_DISTANCE_PC` while Hipparcos put them inside `ETL_STAR_DISTANCE_PC`; + * they are every star in the published catalogue beyond 250 pc. The rest are what Gaia genuinely + * lacks: bright stars it saturates on, red dwarfs past its magnitude cut. So the headroom left to + * the ceiling tracks the gap between those two cutoffs as much as Gaia's completeness. + * + * This bounds a merge that went wrong. It cannot bound a Gaia download that came back short: that + * makes *fewer* survivors, not more, and is guarded where it can be seen, in `fetchGaiaStars`. */ const MAX_UNMERGED_TWINS = 100; const MAX_HYG_SURVIVORS = 15_000; diff --git a/tools/etl/lib/http.ts b/tools/etl/lib/http.ts index 9410ec0..a214332 100644 --- a/tools/etl/lib/http.ts +++ b/tools/etl/lib/http.ts @@ -34,9 +34,15 @@ const RETRY_DELAYS_MS = [30_000, 120_000]; async function fetchText(url: string): Promise { for (let attempt = 0; ; attempt++) { - const response = await fetch(url).catch((error: unknown) => (error instanceof Error ? error : new Error(String(error)))); + let response = await fetch(url).catch((error: unknown) => (error instanceof Error ? error : new Error(String(error)))); if (!(response instanceof Error) && response.ok) { - return response.text(); + // Read inside the loop, because the body is where these downloads fail: the Gaia CSV is + // 57 MB, and a connection reset part-way through rejects here, long after the 200. + const body = await response.text().catch((error: unknown) => (error instanceof Error ? error : new Error(String(error)))); + if (typeof body === 'string') { + return body; + } + response = body; } const reason = response instanceof Error ? response.message : `${response.status} ${response.statusText}`; // A 4xx is the request's own fault, and waiting will not change the answer. diff --git a/tools/etl/sources/gaia.ts b/tools/etl/sources/gaia.ts index c70d6a6..6d02dc9 100644 --- a/tools/etl/sources/gaia.ts +++ b/tools/etl/sources/gaia.ts @@ -38,9 +38,29 @@ const CATALOGUE_EPOCH = 2000.0; * past anything this map draws, so the limit here is a payload decision: the catalogue is baked * into a static asset that a browser downloads before the first frame. */ -const DISTANCE_CUTOFF_PC = Number(process.env['ETL_GAIA_DISTANCE_PC'] ?? 250); -const MAGNITUDE_LIMIT = Number(process.env['ETL_GAIA_MAGNITUDE_LIMIT'] ?? 12); -const ROW_LIMIT = Number(process.env['ETL_GAIA_ROW_LIMIT'] ?? 500000); +const DEFAULT_DISTANCE_CUTOFF_PC = 250; +const DEFAULT_MAGNITUDE_LIMIT = 12; +const DEFAULT_ROW_LIMIT = 500_000; +const DISTANCE_CUTOFF_PC = Number(process.env['ETL_GAIA_DISTANCE_PC'] ?? DEFAULT_DISTANCE_CUTOFF_PC); +const MAGNITUDE_LIMIT = Number(process.env['ETL_GAIA_MAGNITUDE_LIMIT'] ?? DEFAULT_MAGNITUDE_LIMIT); +const ROW_LIMIT = Number(process.env['ETL_GAIA_ROW_LIMIT'] ?? DEFAULT_ROW_LIMIT); + +/** + * How many rows the scheduled job's own query holds: 412 765, and DR3 is a finished data release, + * so that number only moves when the query does. + * + * Checked because a short answer looks exactly like a complete one. The TAP service truncates on + * its own timeout and still serves a well-formed CSV with a 200, and the rows are ordered by + * magnitude, so what comes back is the bright half — the half HYG overlaps. The merge gate in + * `build.ts` would then see Gaia stars present, fewer HYG survivors and fewer unmerged twins, and + * pass a catalogue missing two hundred thousand stars, which the weekly job would publish and the + * runner would cache for the weeks after it. Same failure, and same guard, as + * {@link MIN_USABLE_HIP_DISTANCES} below. + * + * Only checked for that query: the environment overrides exist to fetch a smaller slice on purpose. + */ +const DEFAULT_QUERY_ROWS = 412_765; +const MIN_ROW_SHARE = 0.95; /** * Relative parallax error above which a star is dropped: a parallax measured to worse than 20% @@ -92,6 +112,16 @@ export async function fetchGaiaStars(): Promise { // response arrived, not what it answered. const csv = await fetchTextCached(url, `gaia-dr3-${createHash('sha1').update(url).digest('hex').slice(0, 8)}.csv`); const rows = parseCsvObjects(csv); + const jobsQuery = DISTANCE_CUTOFF_PC === DEFAULT_DISTANCE_CUTOFF_PC && MAGNITUDE_LIMIT === DEFAULT_MAGNITUDE_LIMIT && ROW_LIMIT === DEFAULT_ROW_LIMIT; + if (jobsQuery && rows.length < DEFAULT_QUERY_ROWS * MIN_ROW_SHARE) { + throw new Error( + `Gaia returned ${rows.length} rows, not the ~${DEFAULT_QUERY_ROWS} this query holds — the answer was cut short, ` + + 'or was an error page served with a 200; delete tools/etl/.cache/gaia-dr3-*.csv once the archive answers properly' + ); + } + if (rows.length >= ROW_LIMIT) { + throw new Error(`Gaia returned the query's own ${ROW_LIMIT}-row limit, so it is the limit deciding what the map holds; raise ETL_GAIA_ROW_LIMIT.`); + } const stars: StarRecord[] = []; rows.forEach((row, index) => { From b7f277ea04714a46cf6b40ad7a59818b12676cfc Mon Sep 17 00:00:00 2001 From: Senrokai Date: Fri, 18 Sep 2026 14:35:17 +0200 Subject: [PATCH 2/3] Answer the review: a short answer makes more survivors, and must not be skipped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three things this got wrong. The direction: truncating Gaia leaves the HYG rows whose counterpart it dropped without one, so survivors rise — 10 886 today, 12 711 at half the rows, 16 258 at a third — which the comment claimed was the other way, and which decides whether the 15 000 ceiling can be leaned on at all (it catches a truncation past about two thirds, and nothing shallower). The throw: `fetchStars` catches everything a source throws and skips it, so a truncated CSV was reported as "the archive was unreachable" one step after `writeStarAssets` had already overwritten the published catalogue. Marked with `GaiaAnswerError` and rethrown there, so an answer that cannot be worked with fails the run where it happened. Measured end to end in a throwaway working directory, 300 000 rows in the cache: fails, names the cache file to delete, assets untouched. With the rethrow taken back out again: assets written, then "the archive was unreachable". The row limit: `rows.length >= ROW_LIMIT` is true for every reduced ETL_GAIA_ROW_LIMIT, so the tripwire fired on exactly the deliberate slice the override exists for — and told the operator to raise it. Gated on the same flag as its neighbour. `ETL_GAIA_ROW_LIMIT=20000` now runs through; without the gate it dies on the limit it was given. Also: the row floor names the one cache file it is about rather than a glob that takes the Hipparcos cross-match with it, and says an edited query is a third reason it can fire — DEFAULT_QUERY_ROWS now sits under the query it counts. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016jxMkwA2rbicdGxHosecYi --- tools/etl/build.ts | 7 +++-- tools/etl/fetchStars.ts | 11 ++++++-- tools/etl/sources/gaia.ts | 59 ++++++++++++++++++++++++--------------- 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/tools/etl/build.ts b/tools/etl/build.ts index 6f11c09..5ca1709 100644 --- a/tools/etl/build.ts +++ b/tools/etl/build.ts @@ -73,8 +73,11 @@ function validateStars(stars: StarRecord[]): void { * lacks: bright stars it saturates on, red dwarfs past its magnitude cut. So the headroom left to * the ceiling tracks the gap between those two cutoffs as much as Gaia's completeness. * - * This bounds a merge that went wrong. It cannot bound a Gaia download that came back short: that - * makes *fewer* survivors, not more, and is guarded where it can be seen, in `fetchGaiaStars`. + * This bounds a merge that went wrong, and — loosely — a Gaia download that came back short: a + * truncated answer leaves the HYG rows whose counterpart it dropped without one, so survivors go + * *up*, not down. Measured against the published catalogue: 10 886 today, 11 004 at nine tenths of + * the rows, 12 711 at half, 16 258 at a third. So this ceiling only catches a truncation past about + * two thirds, and `fetchGaiaStars` catches the shallower ones with its own row floor. */ const MAX_UNMERGED_TWINS = 100; const MAX_HYG_SURVIVORS = 15_000; diff --git a/tools/etl/fetchStars.ts b/tools/etl/fetchStars.ts index 1f2eb66..0225fba 100644 --- a/tools/etl/fetchStars.ts +++ b/tools/etl/fetchStars.ts @@ -3,7 +3,7 @@ import { writeFileSync } from 'node:fs'; import { mergeStarCatalogues, placementDistancePc } from '../../src/app/shared/astro/star-merge'; import { encodeStarCatalog } from '../../src/app/shared/models/star-catalog'; import { StarRecord, SUN_STAR_ID } from '../../src/app/shared/models/star.model'; -import { fetchGaiaDistancesByHip } from './sources/gaia'; +import { fetchGaiaDistancesByHip, GaiaAnswerError } from './sources/gaia'; import { positionalSources } from './sources/registry'; import { PARALLAX_PRECISION_MAS } from './sources/star-sources'; import { parseCsvObjects, parseOptionalNumber } from './lib/csv'; @@ -142,7 +142,9 @@ export async function fetchStars(): Promise { * * A source that cannot be reached is reported and skipped here rather than thrown, so a run still * gets as far as validation and says what it has. Whether that may be published is decided - * there: `validateMerge` in build.ts refuses a catalogue Gaia contributed nothing to. + * there: `validateMerge` in build.ts refuses a catalogue Gaia contributed nothing to. A source + * that answered with something unusable ({@link GaiaAnswerError}) is a different matter, and stops + * the run where it happened rather than being reported later as an outage. */ async function mergeWithOtherSources(hygStars: StarRecord[]): Promise { const others = positionalSources().filter((source) => source.id !== 'hyg'); @@ -160,6 +162,11 @@ async function mergeWithOtherSources(hygStars: StarRecord[]): Promise { // Keyed by the whole request, so a response cached for other columns, another order, or // another endpoint can never be mistaken for this one — the cache records only that some // response arrived, not what it answered. - const csv = await fetchTextCached(url, `gaia-dr3-${createHash('sha1').update(url).digest('hex').slice(0, 8)}.csv`); + const cacheKey = `gaia-dr3-${createHash('sha1').update(url).digest('hex').slice(0, 8)}.csv`; + const csv = await fetchTextCached(url, cacheKey); const rows = parseCsvObjects(csv); const jobsQuery = DISTANCE_CUTOFF_PC === DEFAULT_DISTANCE_CUTOFF_PC && MAGNITUDE_LIMIT === DEFAULT_MAGNITUDE_LIMIT && ROW_LIMIT === DEFAULT_ROW_LIMIT; if (jobsQuery && rows.length < DEFAULT_QUERY_ROWS * MIN_ROW_SHARE) { - throw new Error( - `Gaia returned ${rows.length} rows, not the ~${DEFAULT_QUERY_ROWS} this query holds — the answer was cut short, ` + - 'or was an error page served with a 200; delete tools/etl/.cache/gaia-dr3-*.csv once the archive answers properly' + throw new GaiaAnswerError( + `Gaia returned ${rows.length} rows, not the ~${DEFAULT_QUERY_ROWS} this query holds — the answer was cut short, it was an error page ` + + `served with a 200, or the query was edited without updating DEFAULT_QUERY_ROWS; delete tools/etl/.cache/${cacheKey} once the archive answers properly` ); } - if (rows.length >= ROW_LIMIT) { - throw new Error(`Gaia returned the query's own ${ROW_LIMIT}-row limit, so it is the limit deciding what the map holds; raise ETL_GAIA_ROW_LIMIT.`); + if (jobsQuery && rows.length >= ROW_LIMIT) { + throw new GaiaAnswerError(`Gaia returned the query's own ${ROW_LIMIT}-row limit, so it is the limit deciding what the map holds; raise ETL_GAIA_ROW_LIMIT.`); } const stars: StarRecord[] = []; From 1a5785fb63a2c7d2dab0e47f6eae9cc1894d7b9a Mon Sep 17 00:00:00 2001 From: Senrokai Date: Fri, 18 Sep 2026 15:46:42 +0200 Subject: [PATCH 3/3] Keep the row cap live for the queries that can reach it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gating it on `jobsQuery` switched it off for every override that *widens* the query — which is the only way to fill `select top N` at all. `ETL_GAIA_MAGNITUDE_LIMIT=14` asks for 500 000 rows, the sky holds more, and the answer is the limit rather than the filters: exactly what the tripwire is for, and it no longer fired. It now reads the row limit itself, so only a deliberately smaller slice is silent. Measured with a synthetic answer of exactly 500 000 rows in the cache, under the key the widened query hashes to: refused. With the `jobsQuery` gate back, the same run keeps 500 000 Gaia stars and goes on to publish them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016jxMkwA2rbicdGxHosecYi --- tools/etl/sources/gaia.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/etl/sources/gaia.ts b/tools/etl/sources/gaia.ts index 1df52bc..bdd6626 100644 --- a/tools/etl/sources/gaia.ts +++ b/tools/etl/sources/gaia.ts @@ -132,7 +132,10 @@ export async function fetchGaiaStars(): Promise { `served with a 200, or the query was edited without updating DEFAULT_QUERY_ROWS; delete tools/etl/.cache/${cacheKey} once the archive answers properly` ); } - if (jobsQuery && rows.length >= ROW_LIMIT) { + // Not gated on `jobsQuery` like the floor above it: the only ways to reach this cap are the + // overrides that *widen* the query, and they are exactly when it is worth saying. What it must + // not fire on is a deliberately smaller slice, where filling the limit is the whole point. + if (ROW_LIMIT >= DEFAULT_ROW_LIMIT && rows.length >= ROW_LIMIT) { throw new GaiaAnswerError(`Gaia returned the query's own ${ROW_LIMIT}-row limit, so it is the limit deciding what the map holds; raise ETL_GAIA_ROW_LIMIT.`); } const stars: StarRecord[] = [];