diff --git a/tools/etl/build.ts b/tools/etl/build.ts index 6f11c09..5ca1709 100644 --- a/tools/etl/build.ts +++ b/tools/etl/build.ts @@ -73,8 +73,11 @@ function validateStars(stars: StarRecord[]): void { * lacks: bright stars it saturates on, red dwarfs past its magnitude cut. So the headroom left to * the ceiling tracks the gap between those two cutoffs as much as Gaia's completeness. * - * This bounds a merge that went wrong. It cannot bound a Gaia download that came back short: that - * makes *fewer* survivors, not more, and is guarded where it can be seen, in `fetchGaiaStars`. + * This bounds a merge that went wrong, and — loosely — a Gaia download that came back short: a + * truncated answer leaves the HYG rows whose counterpart it dropped without one, so survivors go + * *up*, not down. Measured against the published catalogue: 10 886 today, 11 004 at nine tenths of + * the rows, 12 711 at half, 16 258 at a third. So this ceiling only catches a truncation past about + * two thirds, and `fetchGaiaStars` catches the shallower ones with its own row floor. */ const MAX_UNMERGED_TWINS = 100; const MAX_HYG_SURVIVORS = 15_000; diff --git a/tools/etl/fetchStars.ts b/tools/etl/fetchStars.ts index 1f2eb66..0225fba 100644 --- a/tools/etl/fetchStars.ts +++ b/tools/etl/fetchStars.ts @@ -3,7 +3,7 @@ import { writeFileSync } from 'node:fs'; import { mergeStarCatalogues, placementDistancePc } from '../../src/app/shared/astro/star-merge'; import { encodeStarCatalog } from '../../src/app/shared/models/star-catalog'; import { StarRecord, SUN_STAR_ID } from '../../src/app/shared/models/star.model'; -import { fetchGaiaDistancesByHip } from './sources/gaia'; +import { fetchGaiaDistancesByHip, GaiaAnswerError } from './sources/gaia'; import { positionalSources } from './sources/registry'; import { PARALLAX_PRECISION_MAS } from './sources/star-sources'; import { parseCsvObjects, parseOptionalNumber } from './lib/csv'; @@ -142,7 +142,9 @@ export async function fetchStars(): Promise { * * A source that cannot be reached is reported and skipped here rather than thrown, so a run still * gets as far as validation and says what it has. Whether that may be published is decided - * there: `validateMerge` in build.ts refuses a catalogue Gaia contributed nothing to. + * there: `validateMerge` in build.ts refuses a catalogue Gaia contributed nothing to. A source + * that answered with something unusable ({@link GaiaAnswerError}) is a different matter, and stops + * the run where it happened rather than being reported later as an outage. */ async function mergeWithOtherSources(hygStars: StarRecord[]): Promise { const others = positionalSources().filter((source) => source.id !== 'hyg'); @@ -160,6 +162,11 @@ async function mergeWithOtherSources(hygStars: StarRecord[]): Promise { // Keyed by the whole request, so a response cached for other columns, another order, or // another endpoint can never be mistaken for this one — the cache records only that some // response arrived, not what it answered. - const csv = await fetchTextCached(url, `gaia-dr3-${createHash('sha1').update(url).digest('hex').slice(0, 8)}.csv`); + const cacheKey = `gaia-dr3-${createHash('sha1').update(url).digest('hex').slice(0, 8)}.csv`; + const csv = await fetchTextCached(url, cacheKey); const rows = parseCsvObjects(csv); const jobsQuery = DISTANCE_CUTOFF_PC === DEFAULT_DISTANCE_CUTOFF_PC && MAGNITUDE_LIMIT === DEFAULT_MAGNITUDE_LIMIT && ROW_LIMIT === DEFAULT_ROW_LIMIT; if (jobsQuery && rows.length < DEFAULT_QUERY_ROWS * MIN_ROW_SHARE) { - throw new Error( - `Gaia returned ${rows.length} rows, not the ~${DEFAULT_QUERY_ROWS} this query holds — the answer was cut short, ` + - 'or was an error page served with a 200; delete tools/etl/.cache/gaia-dr3-*.csv once the archive answers properly' + throw new GaiaAnswerError( + `Gaia returned ${rows.length} rows, not the ~${DEFAULT_QUERY_ROWS} this query holds — the answer was cut short, it was an error page ` + + `served with a 200, or the query was edited without updating DEFAULT_QUERY_ROWS; delete tools/etl/.cache/${cacheKey} once the archive answers properly` ); } - if (rows.length >= ROW_LIMIT) { - throw new Error(`Gaia returned the query's own ${ROW_LIMIT}-row limit, so it is the limit deciding what the map holds; raise ETL_GAIA_ROW_LIMIT.`); + if (jobsQuery && rows.length >= ROW_LIMIT) { + throw new GaiaAnswerError(`Gaia returned the query's own ${ROW_LIMIT}-row limit, so it is the limit deciding what the map holds; raise ETL_GAIA_ROW_LIMIT.`); } const stars: StarRecord[] = [];