From 9a5e41495b350dc0660f4d44fa4fc29c1d44bc83 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 14:05:41 +0000 Subject: [PATCH] Make a cold-cache ETL run a pure function of the archives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two order dependencies surfaced while designing the scheduled refresh, both invisible on a developer machine with a warm cache and both guaranteed to haunt a weekly CI run that starts cold. The Exoplanet Archive query had no ORDER BY, so the archive was free to return rows in any order it liked — and exoplanets.json preserves row order, so a re-run could rewrite the file, and commit a diff, when nothing was actually published. Ordered by pl_name, which is unique among default_flag=1 rows, so the order is total. The deep-sky sort tie-broke with localeCompare, whose collation belongs to the ICU build of whichever Node runs the ETL. Compared code points instead: the file must not reorder because the runner's ICU disagrees with the machine that wrote it last. Both may reorder the committed files once, on the next real refresh. After that, byte-identity between runs means what it should: the archives published nothing. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WaySiNst4HhDXBHnMy8p5G --- tools/etl/fetchDeepSky.ts | 5 ++++- tools/etl/fetchExoplanets.ts | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/etl/fetchDeepSky.ts b/tools/etl/fetchDeepSky.ts index f2d4caa..4ba285b 100644 --- a/tools/etl/fetchDeepSky.ts +++ b/tools/etl/fetchDeepSky.ts @@ -126,7 +126,10 @@ export async function fetchDeepSky(): Promise { // Brightest first, so a consumer taking a prefix gets the most prominent objects. Objects // with no measured magnitude sort last rather than being treated as infinitely bright. - records.sort((a, b) => (a.magnitude ?? Infinity) - (b.magnitude ?? Infinity) || a.id.localeCompare(b.id)); + // The tie-break compares code points rather than localeCompare: locale collation depends on + // the ICU build of whichever Node runs the ETL, and a scheduled re-run must not reorder the + // file just because the runner's ICU disagrees with the machine that wrote it last. + records.sort((a, b) => (a.magnitude ?? Infinity) - (b.magnitude ?? Infinity) || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0)); ensureDataDir(); writeFileSync(dataPath('deepsky.json'), JSON.stringify(records)); diff --git a/tools/etl/fetchExoplanets.ts b/tools/etl/fetchExoplanets.ts index ade5d34..dac2b21 100644 --- a/tools/etl/fetchExoplanets.ts +++ b/tools/etl/fetchExoplanets.ts @@ -25,7 +25,11 @@ const TAP_COLUMNS = [ 'st_mass', 'disc_year' ].join(','); -const TAP_QUERY = `select+${TAP_COLUMNS}+from+ps+where+default_flag=1&format=csv`; +// Ordered explicitly: without it the archive is free to return rows in any order, and a +// scheduled re-run of the ETL would then rewrite exoplanets.json — and commit a diff — when +// nothing was actually published. pl_name is unique among default_flag=1 rows, so the order +// is total and the output is a pure function of the archive's content. +const TAP_QUERY = `select+${TAP_COLUMNS}+from+ps+where+default_flag=1+order+by+pl_name&format=csv`; const TAP_URL = `${TAP_BASE_URL}?query=${TAP_QUERY}`; // A host star match must be within this many parsecs of the catalog position to be