From 8e55964b14aa2e5f1515201fa936908fc9bd6fd7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 17 Aug 2026 14:05:42 +0000 Subject: [PATCH] Refresh the catalogues on a schedule, and republish when they change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mondays 05:23 UTC: re-run the ETL cold against the live archives, and if the output differs by a byte, gate it on the unit suite and a production build, commit it to main, and dispatch the Pages deploy. If nothing changed, say so in the run summary and touch nothing. The gates run inside this workflow because they cannot run after it: a push made with GITHUB_TOKEN fires no push workflows at all — GitHub's recursion guard — so an unguarded push would deploy nothing and be checked by nothing. The same guard is why the deploy and a visible CI record are dispatched explicitly afterwards; dispatch events do go through where push events do not. ci.yml gains a workflow_dispatch trigger for exactly that call. Also corrects pages.yml's claim that configure-pages enables Pages on first run. It cannot: the action's `enablement` input requires an admin-scoped token, which GITHUB_TOKEN is not. If the site has never been enabled, the first deploy fails at that step and the one-time fix is Settings → Pages → Source: GitHub Actions. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WaySiNst4HhDXBHnMy8p5G --- .github/workflows/ci.yml | 3 + .github/workflows/data-refresh.yml | 89 ++++++++++++++++++++++++++++++ .github/workflows/pages.yml | 5 +- 3 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/data-refresh.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7fdc7ad..4932d7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,9 @@ on: push: branches: [main, develop] pull_request: + # For the data-refresh workflow: its bot push to main fires no `push` events (GitHub's + # recursion guard), so it dispatches CI here explicitly to put checks on the new commit. + workflow_dispatch: # A second push to the same branch makes the first run's answer irrelevant. concurrency: diff --git a/.github/workflows/data-refresh.yml b/.github/workflows/data-refresh.yml new file mode 100644 index 0000000..b1d3b87 --- /dev/null +++ b/.github/workflows/data-refresh.yml @@ -0,0 +1,89 @@ +name: Refresh the catalogues + +# Re-runs the ETL against the live archives on a schedule and republishes the site when the data +# actually changed, so the catalogues track NASA without anyone touching the code. The archives +# this reads — HYG, the Exoplanet Archive, JPL Horizons, OpenNGC, Gaia — are all anonymous +# public endpoints; no keys are involved. +on: + schedule: + # Mondays 05:23 UTC. An arbitrary minute rather than :00, which is the busiest minute on + # GitHub's cron fleet and the most likely to be delayed or dropped. + - cron: '23 5 * * 1' + workflow_dispatch: + +# Never two refreshes at once, and never cancel one mid-push. +concurrency: + group: data-refresh + cancel-in-progress: false + +permissions: + contents: write # push the regenerated catalogues to main + actions: write # dispatch the deploy and CI afterwards — see the final step + +jobs: + refresh: + name: Fetch, gate, publish + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v5 + with: + # The data lives on main and the push below goes to main, whichever ref the workflow + # file itself ran from. + ref: main + + - uses: actions/setup-node@v5 + with: + node-version: 22 + cache: npm + + - run: npm ci + + # The ETL's cache directory is gitignored and this is a fresh runner, so every source is + # fetched live (~50-100 MB). A failed fetch fails the run by design — no refresh is + # better than a partial one — except Gaia, which the ETL itself treats as best-effort. + - name: Rebuild the datasets + run: npm run etl + + - name: Detect a real change + id: diff + run: | + if git diff --quiet -- src/assets/data; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "The archives published nothing new — catalogues are byte-identical." >> "$GITHUB_STEP_SUMMARY" + else + echo "changed=true" >> "$GITHUB_OUTPUT" + { echo "Catalogue changes:"; echo '```'; git diff --stat -- src/assets/data; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + fi + + # The same gates CI runs, run here instead: the push below is made with GITHUB_TOKEN, and + # GitHub deliberately fires no workflows for such pushes, so the data must be proven + # before it lands rather than checked after. + - name: Unit tests against the new data + if: steps.diff.outputs.changed == 'true' + run: npm test -- --no-watch + + - name: Production build against the new data + if: steps.diff.outputs.changed == 'true' + run: npm run build + + - name: Commit to main + if: steps.diff.outputs.changed == 'true' + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git add src/assets/data + git commit -m "Refresh the astronomical catalogues" \ + -m "Scheduled re-run of the ETL against the live archives. Gated on the unit suite and a production build in this same run, because a GITHUB_TOKEN push triggers no CI of its own." + git push origin HEAD:main + + # The recursion guard that keeps the bot push from triggering `push` workflows also keeps + # it from deploying, so the deploy — and a visible CI record on the new commit — are + # dispatched explicitly. Dispatch does go through, unlike push events. + - name: Redeploy the site, and put checks on the commit + if: steps.diff.outputs.changed == 'true' + env: + GH_TOKEN: ${{ github.token }} + run: | + gh workflow run pages.yml --ref main + gh workflow run ci.yml --ref main diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index ee46039..d26e0c6 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -34,7 +34,10 @@ jobs: - run: npm ci - # Enables Pages on first run if it is not on yet, and reports the URL the site will live at. + # Verifies Pages is enabled and reports the URL the site will live at. It cannot *enable* + # Pages itself: the action's `enablement` input requires an admin-scoped token, which the + # workflow's GITHUB_TOKEN is not. If this step fails with "Get Pages site failed", the + # one-time fix is Settings → Pages → Source: GitHub Actions, then re-run. - uses: actions/configure-pages@v6 # A project site is served from a subdirectory, so the app cannot assume it sits at the