diff --git a/README.md b/README.md index ad57822..e5302ec 100644 --- a/README.md +++ b/README.md @@ -209,9 +209,9 @@ The star catalogue ships as two binary column stores plus a small JSON file, not objects. At 68 388 stars the old encoding — one JSON object per star, its eight key names repeated each time — would have been about 17 MB to download and parse before the first frame. Splitting it puts the numbers in `stars.bin` (positions, handed to the GPU verbatim) and -`stars-meta.bin` (id, magnitude, colour index, spectral-type index, the band and colour system -those were measured in or whether the colour was read off a temperature, and the distance's -relative error), and leaves `stars-index.json` +`stars-meta.bin` (id, magnitude, colour index, spectral-type index, the distance's relative +error in two bytes, and the band and colour system those were measured in or whether the colour +was read off a temperature), and leaves `stars-index.json` holding only the strings, with the ~2 600 distinct spectral classifications collapsed into a dictionary. The result is 2.6 MB for 7.8× the stars. `star-catalog.ts` defines the layout once and both the ETL and the app use it, so the writer and the reader cannot drift apart. diff --git a/src/app/shared/models/star-catalog.spec.ts b/src/app/shared/models/star-catalog.spec.ts index 012d531..a788dc0 100644 --- a/src/app/shared/models/star-catalog.spec.ts +++ b/src/app/shared/models/star-catalog.spec.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { BYTES_PER_STAR_META, BYTES_PER_STAR_POSITION, decodeStarCatalog, encodeStarCatalog, isDesignation } from './star-catalog'; import { StarRecord } from './star.model'; +import { formatDistance } from '../format/quantity'; const STARS: StarRecord[] = [ { id: 0, name: 'Sol', x: 0, y: 0, z: 0, magnitude: -26.7, spectralType: 'G2V', colorIndex: 0.656 }, @@ -129,6 +130,15 @@ describe('the photometry and distance error columns', () => { // Past the parallax itself there is no upper bound on the distance, which is what 100 % says. expect(decoded[4].distanceError).toBe(1); }); + + it('keeps an error close enough that the card prints the published one', () => { + // Rigel, 264.55 pc at van Leeuwen's 3.78 ± 0.34 mas: 23.8 pc, which one byte stored as 23.5. + const rigel: StarRecord = { id: 7, name: 'Rigel', x: 264.55, y: 0, z: 0, magnitude: 0.18, magnitudeBand: 'V', spectralType: 'B8Ia', colorIndex: -0.03, colorSystem: 'B-V', distanceError: 0.34 / 3.78, source: 'hyg' }; + const packed = encodeStarCatalog([rigel]); + const [kept] = decodeStarCatalog(packed.index, packed.positions, packed.meta); + expect(formatDistance(264.55, kept.distanceError)).toBe('265 ± 24 pc'); + expect(Math.abs(kept.distanceError! / rigel.distanceError! - 1)).toBeLessThan(1e-4); + }); }); describe('star catalogue provenance and derived names', () => { diff --git a/src/app/shared/models/star-catalog.ts b/src/app/shared/models/star-catalog.ts index db850b9..7d23d1a 100644 --- a/src/app/shared/models/star-catalog.ts +++ b/src/app/shared/models/star-catalog.ts @@ -37,7 +37,7 @@ export const BYTES_PER_STAR_META = Float32Array.BYTES_PER_ELEMENT + Float32Array.BYTES_PER_ELEMENT + Uint16Array.BYTES_PER_ELEMENT + - Uint8Array.BYTES_PER_ELEMENT + + Uint16Array.BYTES_PER_ELEMENT + Uint8Array.BYTES_PER_ELEMENT; /** @@ -50,14 +50,19 @@ export const BYTES_PER_STAR_META = const PHOTOMETRY = { bandV: 1, bandG: 2, bandMask: 3, colorBpRp: 4, distanceFromGaia: 8, colorFromTemperature: 16 } as const; /** - * The distance error column holds the square root of the relative error, in 255ths, and 0 where + * The distance error column holds the square root of the relative error, in 65 535ths, and 0 where * none was published. The errors span three orders of magnitude — Gaia's are a median 0.3 % and * at most 20 %, the cut its queries make, while a Hipparcos parallax the map keeps for a bright - * star can be as large as itself — and the square root keeps a step small at each end: 0.08 % of - * distance at 1 %, 0.35 % at 20 %, under 1 % at 100 %. Anything past 100 % is stored as that, - * where it no longer bounds the distance from above. + * star can be as large as itself — and the square root keeps a step small at each end. Anything + * past 100 % is stored as that, where it no longer bounds the distance from above. + * + * In 255ths, one byte, a step was 0.35 % of the distance at a 20 % error, a few per cent of the + * error itself, and the card printed another error than the published one for 2 822 of the 53 209 + * Gaia stars it prints one for; Rigel read ± 23 pc where van Leeuwen's 3.78 ± 0.34 mas gives 24. + * In two bytes, placed before the photometry byte so the column stays aligned for its view, 12 do, + * each on a rounding half. */ -const DISTANCE_ERROR_STEPS = 255; +const DISTANCE_ERROR_STEPS = 65_535; /** `stars-index.json`: everything that is a string, plus the count the columns are sized by. */ export interface StarCatalogIndex { @@ -107,7 +112,7 @@ interface StarMetaColumns { colorIndices: Float32Array; spectralTypeIndices: Uint16Array; photometry: Uint8Array; - distanceErrors: Uint8Array; + distanceErrors: Uint16Array; } /** Lays typed-array views over the meta buffer at the offsets the format defines. */ @@ -121,9 +126,9 @@ function metaColumns(buffer: ArrayBuffer, count: number): StarMetaColumns { offset += count * Float32Array.BYTES_PER_ELEMENT; const spectralTypeIndices = new Uint16Array(buffer, offset, count); offset += count * Uint16Array.BYTES_PER_ELEMENT; + const distanceErrors = new Uint16Array(buffer, offset, count); + offset += count * Uint16Array.BYTES_PER_ELEMENT; const photometry = new Uint8Array(buffer, offset, count); - offset += count * Uint8Array.BYTES_PER_ELEMENT; - const distanceErrors = new Uint8Array(buffer, offset, count); return { ids, magnitudes, colorIndices, spectralTypeIndices, photometry, distanceErrors }; } diff --git a/src/assets/data/stars-meta.bin b/src/assets/data/stars-meta.bin index f2cf1ce..8dc1d73 100644 Binary files a/src/assets/data/stars-meta.bin and b/src/assets/data/stars-meta.bin differ diff --git a/tools/etl/build.ts b/tools/etl/build.ts index e5a7aa9..2dae57d 100644 --- a/tools/etl/build.ts +++ b/tools/etl/build.ts @@ -120,10 +120,10 @@ function validateStars(stars: StarRecord[]): void { decoded[i].colorFromTemperature === !!stars[i].colorFromTemperature, `Star catalogue round-trip changed the photometry of star ${stars[i].id}.` ); - // Stored as its square root in 255ths, up to 100 %; see `star-catalog.ts`. + // Stored as its square root in 65 535ths, up to 100 %; see `star-catalog.ts`. const error = stars[i].distanceError; assertCondition( - error === undefined ? decoded[i].distanceError === undefined : Math.abs(Math.sqrt(decoded[i].distanceError!) - Math.sqrt(Math.min(1, error))) <= 0.5 / 255 + 1e-9, + error === undefined ? decoded[i].distanceError === undefined : Math.abs(Math.sqrt(decoded[i].distanceError!) - Math.sqrt(Math.min(1, error))) <= 0.5 / 65_535 + 1e-9, `Star catalogue round-trip changed the distance error of star ${stars[i].id}.` ); }