From 1a5785fb63a2c7d2dab0e47f6eae9cc1894d7b9a Mon Sep 17 00:00:00 2001 From: Senrokai Date: Fri, 18 Sep 2026 15:46:42 +0200 Subject: [PATCH] Keep the row cap live for the queries that can reach it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gating it on `jobsQuery` switched it off for every override that *widens* the query — which is the only way to fill `select top N` at all. `ETL_GAIA_MAGNITUDE_LIMIT=14` asks for 500 000 rows, the sky holds more, and the answer is the limit rather than the filters: exactly what the tripwire is for, and it no longer fired. It now reads the row limit itself, so only a deliberately smaller slice is silent. Measured with a synthetic answer of exactly 500 000 rows in the cache, under the key the widened query hashes to: refused. With the `jobsQuery` gate back, the same run keeps 500 000 Gaia stars and goes on to publish them. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016jxMkwA2rbicdGxHosecYi --- tools/etl/sources/gaia.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/etl/sources/gaia.ts b/tools/etl/sources/gaia.ts index 1df52bc..bdd6626 100644 --- a/tools/etl/sources/gaia.ts +++ b/tools/etl/sources/gaia.ts @@ -132,7 +132,10 @@ export async function fetchGaiaStars(): Promise { `served with a 200, or the query was edited without updating DEFAULT_QUERY_ROWS; delete tools/etl/.cache/${cacheKey} once the archive answers properly` ); } - if (jobsQuery && rows.length >= ROW_LIMIT) { + // Not gated on `jobsQuery` like the floor above it: the only ways to reach this cap are the + // overrides that *widen* the query, and they are exactly when it is worth saying. What it must + // not fire on is a deliberately smaller slice, where filling the limit is the whole point. + if (ROW_LIMIT >= DEFAULT_ROW_LIMIT && rows.length >= ROW_LIMIT) { throw new GaiaAnswerError(`Gaia returned the query's own ${ROW_LIMIT}-row limit, so it is the limit deciding what the map holds; raise ETL_GAIA_ROW_LIMIT.`); } const stars: StarRecord[] = [];