From c401957b510bd0f8881790ba814ec85ee46a866b Mon Sep 17 00:00:00 2001 From: Christian Johannesen Date: Thu, 8 Oct 2026 00:08:48 +0200 Subject: [PATCH] Complete Milestone 0 source reconnaissance --- .env.example | 13 + .gitignore | 53 +++ AGENTS.md | 413 ++++++++++++++++++++ README.md | 124 ++++++ START_PROMPT.md | 47 +++ data/interim/.gitkeep | 0 data/processed/.gitkeep | 0 data/raw/README.md | 19 + data/reports/.gitkeep | 0 docs/DATA_MODEL.md | 77 ++++ docs/FEATURE_SPEC_V1.md | 168 ++++++++ docs/IMPLEMENTATION_PLAN.md | 133 +++++++ docs/PIPELINE.md | 92 +++++ docs/SOURCE_DATA.md | 89 +++++ docs/SOURCE_RECONNAISSANCE.md | 125 ++++++ pyproject.toml | 33 ++ src/smk_playground/__init__.py | 0 src/smk_playground/canonical/__init__.py | 0 src/smk_playground/cli.py | 23 ++ src/smk_playground/features/__init__.py | 0 src/smk_playground/ingest/__init__.py | 0 src/smk_playground/normalize/__init__.py | 0 src/smk_playground/quality/__init__.py | 0 src/smk_playground/similarity/__init__.py | 0 src/smk_playground/sonification/__init__.py | 0 tests/test_smoke.py | 12 + tools/source-reconnaissance.mjs | 267 +++++++++++++ 27 files changed, 1688 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 README.md create mode 100644 START_PROMPT.md create mode 100644 data/interim/.gitkeep create mode 100644 data/processed/.gitkeep create mode 100644 data/raw/README.md create mode 100644 data/reports/.gitkeep create mode 100644 docs/DATA_MODEL.md create mode 100644 docs/FEATURE_SPEC_V1.md create mode 100644 docs/IMPLEMENTATION_PLAN.md create mode 100644 docs/PIPELINE.md create mode 100644 docs/SOURCE_DATA.md create mode 100644 docs/SOURCE_RECONNAISSANCE.md create mode 100644 pyproject.toml create mode 100644 src/smk_playground/__init__.py create mode 100644 src/smk_playground/canonical/__init__.py create mode 100644 src/smk_playground/cli.py create mode 100644 src/smk_playground/features/__init__.py create mode 100644 src/smk_playground/ingest/__init__.py create mode 100644 src/smk_playground/normalize/__init__.py create mode 100644 src/smk_playground/quality/__init__.py create mode 100644 src/smk_playground/similarity/__init__.py create mode 100644 src/smk_playground/sonification/__init__.py create mode 100644 tests/test_smoke.py create mode 100644 tools/source-reconnaissance.mjs diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..e545b9f --- /dev/null +++ b/.env.example @@ -0,0 +1,13 @@ +SMK_DATA_DIR=./data +SMK_RAW_DIR=./data/raw +SMK_INTERIM_DIR=./data/interim +SMK_PROCESSED_DIR=./data/processed +SMK_REPORTS_DIR=./data/reports + +SMK_DA_FILE=./data/raw/smk_all_da.json +SMK_EN_FILE=./data/raw/smk_all_en.json + +# Live services are for verification, enrichment spot-checks and future refresh tools. +# They are NOT the primary bulk-ingest path for v1. +SMK_API_BASE=https://api.smk.dk/api/v1 +SMK_ENRICHMENT_BASE=https://enrichment.api.smk.dk/api/enrichment diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0210521 --- /dev/null +++ b/.gitignore @@ -0,0 +1,53 @@ +# Python +__pycache__/ +*.py[cod] +*.pyd +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.coverage +htmlcov/ +.venv/ +venv/ +dist/ +build/ +*.egg-info/ + +# Environment / secrets +.env +.env.* +!.env.example + +# IDE / OS +.vscode/ +.idea/ +.DS_Store +Thumbs.db + +# Large SMK source snapshots: NEVER commit these. +data/raw/* +!data/raw/README.md + +# Generated analytical data: reproducible, local artifacts. +data/interim/* +data/processed/* +data/reports/* +!data/interim/.gitkeep +!data/processed/.gitkeep +!data/reports/.gitkeep + +# Local analytical databases / caches +*.duckdb +*.duckdb.wal +*.db +.cache/ + +# Generated columnar/export formats +*.parquet +*.arrow +*.feather + +# Logs / temporary files +*.log +*.tmp +*.temp diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..6500b9e --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,413 @@ +# AGENTS.md — SMK Playground + +## Mission + +Build a small, reproducible analysis pipeline for visual and colour features in Statens Museum for Kunst (SMK) artwork data. The first deliverable is **not** a web application. It is a trustworthy, reusable analytical dataset that can support later colour search, similarity, clustering, timelines, mosaics, experimental interfaces and data sonification. + +Keep the project easy to understand, quick to run and cheap to develop. Prefer explicit data code over frameworks, services and premature abstraction. + +## Read first + +Before coding, read: + +- `README.md` +- `docs/SOURCE_DATA.md` +- `docs/DATA_MODEL.md` +- `docs/FEATURE_SPEC_V1.md` +- `docs/PIPELINE.md` +- `docs/IMPLEMENTATION_PLAN.md` + +The two real source snapshots already exist locally: + +```text +data/raw/smk_all_da.json +data/raw/smk_all_en.json +``` + +They are large (~900 MB each) and intentionally ignored by Git. + +## Core architecture + +Treat the project as four analytical layers: + +1. `source` — untouched SMK snapshot/API/enrichment input + provenance. +2. `canonical` — cleaned, language-aware values with stable field names. +3. `features` — deterministic per-artwork derived features. These MUST NOT depend on other artworks. +4. `normalized` — corpus-relative values based on a named, versioned normalization profile. + +Never overwrite source data. Never silently invent missing values. + +## Primary bulk source + +v1 bulk ingestion uses the local JSON snapshots, not tens of thousands of API calls. + +Expected paths: + +- Danish: `data/raw/smk_all_da.json` +- English: `data/raw/smk_all_en.json` + +Use the live SMK API/enrichment API for verification, source understanding, spot checks and future refresh utilities. + +SMK documents its API as JSON/REST and states that the downloadable full JSON dataset is updated nightly. Therefore local files are immutable time-specific snapshots and MUST be fingerprinted for reproducibility. + +Relevant public endpoints include: + +- `https://api.smk.dk/api/v1/art?object_number=` +- `https://enrichment.api.smk.dk/api/enrichment/` +- `https://open.smk.dk/artwork/image/` + +## First task: inspect, do not guess + +Before implementing the final parser, inspect the real two files and establish: + +- SHA-256 and byte size, +- top-level JSON shape, +- actual record container/envelope, +- record counts, +- identity field presence, +- duplicate/missing `object_number` cases, +- DA/EN identity-set overlap, +- representative field shapes, +- whether language-independent fields differ between DA/EN. + +Do **not** assume the two files have identical row order. Join by stable identity, preferably `object_number`. + +Do **not** use `json.load()` over a ~900 MB file as the default approach. Prefer bounded-memory streaming (`ijson`) and/or a one-time manifest-bound conversion to Parquet/Arrow. + +## Corpus size + +Do not hardcode `39,508` as a required record count. The intended public/open visual collection may currently be around that size, but the complete dump can contain other records and SMK changes over time. + +Always derive and report actual counts from the source snapshot: + +- raw DA/EN records, +- unique artwork identities, +- public-domain artworks, +- records with images/visual data, +- records eligible for a given downstream profile, +- exclusions/null reasons. + +## Source manifest + +Create `source-manifest.json` before producing authoritative derived output. Include at least filename, language, bytes, SHA-256, observed structure, record count, generation timestamp and parser/tool version. + +All authoritative derived artifacts should be traceable to a source manifest. + +## Technology + +Preferred v1 stack: + +- Python 3.12+ +- `ijson` for bounded-memory JSON streaming +- `polars` for dataframe transforms +- `pyarrow` for Parquet interoperability +- `duckdb` for local analytical querying +- `numpy` for numerical work +- a small, verified colour conversion implementation or lightweight reputable colour library +- `typer` for CLI +- `pytest` for focused tests + +Avoid pandas unless a dependency forces it. Avoid Spark, Airflow, Celery, Kafka, Kubernetes, microservices and other infrastructure that this corpus does not justify. + +## Repository structure + +```text +smk-playground/ +├─ AGENTS.md +├─ README.md +├─ START_PROMPT.md +├─ pyproject.toml +├─ .gitignore +├─ .env.example +├─ docs/ +│ ├─ SOURCE_DATA.md +│ ├─ DATA_MODEL.md +│ ├─ PIPELINE.md +│ ├─ FEATURE_SPEC_V1.md +│ └─ IMPLEMENTATION_PLAN.md +├─ src/smk_playground/ +│ ├─ ingest/ +│ ├─ canonical/ +│ ├─ features/ +│ ├─ normalize/ +│ ├─ quality/ +│ ├─ similarity/ +│ └─ sonification/ +├─ tests/ +│ └─ fixtures/ +└─ data/ + ├─ raw/ + ├─ interim/ + ├─ processed/ + └─ reports/ +``` + +## v1 derived outputs + +Aim to produce: + +- `source-manifest.json` +- `artworks.parquet` +- `palette.parquet` +- `features.parquet` +- `normalization-profile.json` +- `normalized-features.parquet` +- `quality-report.json` + +Parquet is the portable source of truth for derived tabular output. DuckDB may provide local views/tables for exploration. + +## Danish + English merge rules + +- Never join DA/EN by row position. +- Prefer `object_number` as identity. +- Preserve localized textual metadata separately where useful. +- Do not run duplicate visual feature extraction just because both language snapshots contain the same artwork. +- If language-independent values disagree, preserve provenance and report/resolve via an explicit deterministic rule. +- Missing one language must not automatically discard an otherwise valid artwork. + +## Canonical artwork rules + +Use stable internal field names independent of exact SMK JSON layout. + +Retain at minimum when present/useful: + +- `object_number` +- public-domain flag +- localized title/object type/classification fields +- source/provenance/version information +- image width/height in pixels +- physical Netto width/height/depth in centimetres +- palette/source visual fields + +### Dimensions + +- Prefer **Netto** dimensions over Brutto when both exist and the semantic meaning is suitable. +- Convert supported physical units to centimetres. +- Keep unknown/ambiguous dimensions null. +- Never represent missing dimensions as zero. +- `physical_area_cm2` requires meaningful width AND height. +- Digital pixel dimensions are never a substitute for physical dimensions. + +## Palette rules + +Preserve original SMK palette/source fields in source/interim provenance. + +Canonical palette rows should include: + +- `object_number` +- `rank` +- `hex` +- `weight` +- `weight_source` +- `srgb_r`, `srgb_g`, `srgb_b` +- OKLab `l`, `a`, `b` +- OKLCH `l`, `c`, `h_deg` +- palette provenance/source/version + +All explicit canonical weights must be non-negative and normalized so the sum is approximately `1.0`. + +### `colors_palette_weighted_s` + +Treat this defensively. The observed enrichment representation can be a space-separated sequence containing repeated hex colours. Parse valid tokens, count occurrences and convert counts into relative weights. Preserve the original source representation/provenance. + +If absent or malformed, record a quality flag. Never pretend equal/fallback weights are original SMK weights. + +## Colour space + +Use: + +```text +sRGB -> linear RGB -> OKLab -> OKLCH +``` + +Use OKLab as the primary Cartesian perceptual calculation space. + +Hue is circular. Do not average hue degrees arithmetically. Aggregate colour centroids are calculated in OKLab and converted afterward to OKLCH. + +## Feature vocabulary v1 + +### Colour centroid + +- `centroid_l` +- `centroid_a` +- `centroid_b` +- `centroid_chroma` +- `centroid_hue_deg` +- `centroid_hue_x = cos(h)` +- `centroid_hue_y = sin(h)` + +### Palette distribution + +- `dominant_color_weight` +- `palette_entropy_01` +- `effective_palette_size` +- `palette_spread` +- `lightness_stddev` +- `chroma_stddev` +- `hue_concentration_01` + +### SMK visual features + +Preserve, when available and with provenance: + +- `smk_brightness` +- `smk_contrast` +- `smk_saturation` +- `smk_colortemp` +- `smk_entropy` + +Do not confuse SMK image entropy with our palette entropy. + +### Geometry + +- `physical_width_cm` +- `physical_height_cm` +- `physical_depth_cm` +- `physical_area_cm2` +- `physical_aspect_ratio` +- `image_aspect_ratio` +- `log_width_cm` +- `log_height_cm` +- `log_area_cm2` + +Only calculate logarithms for valid positive values. + +## Feature formulas + +### Weighted OKLab centroid + +```text +L = sum(w_i * L_i) +a = sum(w_i * a_i) +b = sum(w_i * b_i) +``` + +### Palette entropy + +```text +H = -sum(p_i * ln(p_i)) +H01 = H / ln(n) +``` + +For `n <= 1`, `palette_entropy_01 = 0.0`. + +### Effective palette size + +```text +N_eff = exp(H) +``` + +### Palette spread + +Weighted mean Euclidean OKLab distance of palette colours from their weighted OKLab centroid. + +### Hue concentration + +Use a chroma-aware circular resultant with contribution weight: + +```text +palette_weight * chroma +``` + +Return null and flag achromatic/undefined hue when total chromatic weight is effectively zero. + +## Normalization v1 + +Normalization is corpus-level and MUST be versioned independently from feature extraction. + +Initial profile: + +```text +smk-public-visual-v1 +``` + +Its exact inclusion predicate and actual membership counts must be written into the normalization profile. Do not define membership by a hardcoded expected count. + +For suitable scalar features calculate: + +1. `robust_01`: P01 -> 0, P99 -> 1, clamp outside. +2. `percentile_01`: empirical percentile rank. +3. `z_robust`: `(x - median) / (1.4826 * MAD)`. + +Do not corpus-normalize hue as an ordinary linear scalar. + +The profile JSON should record per-feature valid/null counts, min/max, median, MAD, P01/P05/P25/P50/P75/P95/P99, profile version, feature schema version, source-manifest reference and generation timestamp. + +## Missing data + +Missing means null. + +Never: + +- replace missing with zero, +- silently impute medians, +- fabricate physical measurements from pixels, +- invent palette weights and present them as SMK data. + +Downstream similarity code may compare only shared available dimensions and renormalize configured weights, but must not mutate stored features. + +## Quality + +At minimum support/report flags such as: + +- `PALETTE_MISSING` +- `PALETTE_WEIGHTS_MISSING` +- `PALETTE_MALFORMED` +- `VERY_LOW_CHROMA` +- `DIMENSIONS_MISSING` +- `PHYSICAL_IMAGE_ASPECT_MISMATCH` +- `NON_2D_OBJECT` +- `SOURCE_LANGUAGE_MISMATCH` + +Quality flags are observations, not automatic deletion instructions. + +## Similarity + +Do not invent one universal similarity score. Later support named profiles such as: + +- `color` +- `visual` +- `composition` + +Use fast feature-vector candidate retrieval, then optionally rerank top candidates with a weighted-palette optimal-transport/Wasserstein-style distance. Do not precompute all pairwise distances. + +## Sonification + +Sonification is a separate mapping layer, never an objective artwork feature. + +Initial profile: `art-tone-v1`. + +Tentative mapping: + +- centroid hue -> chromatic pitch class +- physical area percentile -> register +- chroma -> velocity/harmonic richness +- lightness -> timbral brightness +- palette entropy -> complexity +- contrast -> attack +- hue concentration -> tonal purity + +Mappings must be versioned, deterministic and explainable by recording their exact feature inputs. + +## Testing philosophy + +Do not over-test or repeatedly process the whole corpus while developing small changes. + +Focus tests on: + +- source-shape/streaming parser fixtures, +- DA/EN identity merge behavior, +- colour conversion reference values, +- weighted-palette parsing, +- feature formulas, +- invariants (`weights ~= 1`, normalized values in range), +- deterministic output, +- null/quality behavior. + +Use small fixtures and targeted milestone checks. Full-corpus runs belong at milestone boundaries. + +## Development workflow + +Work milestone by milestone. Make coherent commits rather than a stream of trivial commits. Keep docs aligned with material implementation decisions. + +Do not add a web UI/API/service stack before the analytical Dataset v1 works and has been inspected. diff --git a/README.md b/README.md new file mode 100644 index 0000000..90711f2 --- /dev/null +++ b/README.md @@ -0,0 +1,124 @@ +# SMK Playground + +A small, reproducible analysis pipeline for visual and colour features in artworks from **Statens Museum for Kunst (SMK)**. + +The first goal is deliberately not a web app. The goal is to turn the locally downloaded SMK dataset into a trustworthy, versioned analytical foundation that can be reused for colour search, similarity, clustering, timelines, mosaics, visual experiments and data sonification. + +## Local source files + +The project uses these two local SMK snapshots as the primary bulk source: + +```text +data/raw/smk_all_da.json +data/raw/smk_all_en.json +``` + +They are large (~900 MB each), must stay out of Git, and must be treated as immutable source data. + +The Danish and English files should be inspected before coding assumptions about their exact envelope/record layout. The pipeline joins language variants by stable artwork identity such as `object_number`; it must never rely on row order being identical between the two files. + +The live SMK API remains useful for spot-checking, enrichment lookups and future refresh tooling, but v1 bulk processing starts from the local snapshots. + +## Why two languages? + +Visual features are language-independent, but retaining both snapshots gives us: + +- reproducible source provenance, +- Danish and English text fields for future interfaces/search, +- a way to detect language-specific or structural differences rather than assuming the dumps are identical apart from text. + +For v1, Danish can act as the default canonical metadata source where a single textual value is needed, while English is retained as a parallel localized value. Do not duplicate language-independent feature extraction. + +## Data model + +The pipeline has four analytical layers: + +```text +SMK source snapshots + ↓ +canonical artwork records + ↓ +per-artwork visual features + ↓ +corpus-relative normalization +``` + +### Source +Untouched values plus snapshot provenance (filename, byte size, SHA-256 and observed structure). + +### Canonical +Stable internal fields, normalized units and explicit nulls. Source-language values remain attributable. + +### Features +Deterministic properties calculated from one artwork only: weighted palette, OKLab/OKLCH colour statistics, perceptual centroid, palette entropy/spread, dimensions and SMK-provided visual features. + +### Normalized +Values calculated relative to a named, frozen corpus/profile. Per-artwork features never depend on corpus membership; normalized values explicitly do. + +## Target corpus + +Do **not** hardcode `39,508` as a validation constant. The local SMK dump can contain records outside the project's intended open/visual corpus and SMK's collection changes over time. + +The build must report: + +1. raw record count per source snapshot, +2. unique `object_number` count, +3. public-domain count, +4. records with usable image/visual data, +5. records eligible for each downstream profile, +6. explicit exclusion/null reasons. + +The first normalization profile is named: + +```text +smk-public-visual-v1 +``` + +Its exact membership must be recorded when it is frozen. + +## v1 outputs + +The pipeline should eventually produce: + +```text +source-manifest.json +artworks.parquet +palette.parquet +features.parquet +normalization-profile.json +normalized-features.parquet +quality-report.json +``` + +`source-manifest.json` is important: it identifies exactly which local JSON snapshots produced the outputs. + +## Colour model + +Canonical calculations use: + +```text +sRGB -> linear RGB -> OKLab -> OKLCH +``` + +OKLab is used for Cartesian averaging and distance. OKLCH is retained for human-friendly lightness, chroma and hue. + +SMK weighted-palette data is converted into explicit normalized weights. Hue is circular and is never averaged naïvely as degrees. + +## Development philosophy + +This is a modest analytical dataset, not an enterprise platform. + +Use Python + streaming JSON ingestion + Polars/PyArrow + Parquet + DuckDB. Keep stages independently runnable and inspectable. Do not build a web application, API service, worker queue, PostgreSQL deployment or vector database before Dataset v1 is trustworthy. + +See: + +- [`AGENTS.md`](AGENTS.md) +- [`docs/SOURCE_DATA.md`](docs/SOURCE_DATA.md) +- [`docs/DATA_MODEL.md`](docs/DATA_MODEL.md) +- [`docs/FEATURE_SPEC_V1.md`](docs/FEATURE_SPEC_V1.md) +- [`docs/PIPELINE.md`](docs/PIPELINE.md) +- [`docs/IMPLEMENTATION_PLAN.md`](docs/IMPLEMENTATION_PLAN.md) + +## Current status + +Repository foundation and Dataset v1 specification. The next action is source reconnaissance against the two real local JSON files before implementing the final parser. diff --git a/START_PROMPT.md b/START_PROMPT.md new file mode 100644 index 0000000..78cd383 --- /dev/null +++ b/START_PROMPT.md @@ -0,0 +1,47 @@ +# Codex start prompt + +Read `AGENTS.md`, `README.md` and every file under `docs/` before changing code. + +The two real SMK source snapshots are already present locally: + +```text +data/raw/smk_all_da.json +data/raw/smk_all_en.json +``` + +Start with **Milestone 0 only**. Do not jump directly into a guessed parser or full feature build. + +Your first job is source reconnaissance: + +1. verify both files exist, +2. compute file byte sizes and SHA-256 hashes, +3. inspect their actual top-level JSON/envelope structure using bounded-memory techniques, +4. determine how artwork records are reached, +5. count/summarize records without loading an entire ~900 MB file into one Python object, +6. inspect representative DA/EN records, +7. compare identity sets using `object_number` (including duplicates/missing IDs and DA-only/EN-only cases), +8. identify where the fields relevant to Dataset v1 actually occur, +9. write `source-manifest.json` and a concise source-reconnaissance report. + +Important constraints: + +- Treat `data/raw/` as immutable. Never rewrite the source JSON files. +- Do not use live API calls to rebuild information already present locally. +- Live SMK API/enrichment calls are acceptable for a few explicit verification spot-checks only. +- Do not assume DA and EN files have the same row order or identical non-language fields. +- Do not hardcode `39,508` as an expected count. Report actual counts and later define corpus membership explicitly. +- Prefer streaming (`ijson`) / bounded-memory inspection; avoid `json.load()` on the entire source dump. +- Keep the solution small. No web UI, API server, PostgreSQL, job queue or distributed infrastructure. +- Add only focused tests. Do not repeatedly run expensive full-file/full-corpus operations while making small edits. +- Make a coherent Milestone 0 commit when the source structure is understood and documented. + +STOP after Milestone 0 and report: + +1. exact structure of both files, +2. record/identity counts and DA/EN overlap, +3. important field locations and discrepancies, +4. source-manifest path, +5. any ambiguities that affect Milestones 1–3, +6. the concrete parser/canonicalization approach you recommend next. + +Do not implement Milestones 1–3 until the actual source structure has been established from the files. diff --git a/data/interim/.gitkeep b/data/interim/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/processed/.gitkeep b/data/processed/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/data/raw/README.md b/data/raw/README.md new file mode 100644 index 0000000..24c53cf --- /dev/null +++ b/data/raw/README.md @@ -0,0 +1,19 @@ +# Raw SMK source snapshots + +Place the locally downloaded SMK JSON snapshots here: + +```text +data/raw/smk_all_da.json +data/raw/smk_all_en.json +``` + +These files are deliberately ignored by Git. + +Rules: + +- Treat them as immutable source snapshots. +- Never rewrite, normalize, pretty-print or otherwise modify them in place. +- Record file size and SHA-256 before processing. +- Derived/intermediate data belongs under `data/interim/` or `data/processed/`. +- The pipeline must inspect the real top-level JSON structure before assuming array vs object/envelope shape. +- Do not load a ~900 MB file into Python as one giant object when a streaming parser can be used. diff --git a/data/reports/.gitkeep b/data/reports/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md new file mode 100644 index 0000000..8425dce --- /dev/null +++ b/docs/DATA_MODEL.md @@ -0,0 +1,77 @@ +# Data Model + +## Layers + +### 0 — Source +Untouched Danish/English SMK snapshots plus provenance. No rewriting in place. + +### 1 — Canonical +Stable internal representation with consistent units, explicit nulls and language-aware textual fields. + +### 2 — Features +Deterministic values derived solely from one artwork and its source/enrichment data. + +### 3 — Normalized +Values calculated relative to a named, frozen corpus/profile. + +## Main artifacts + +### `source-manifest.json` +Identifies exact input snapshots by filename, size, SHA-256, observed JSON structure and record counts. + +### `artworks.parquet` +One canonical row per artwork identity. + +Suggested fields include: + +- `object_number` +- `public_domain` +- localized title/object-name fields when useful (`*_da`, `*_en`) +- stable classification/object type fields +- `image_width_px`, `image_height_px`, `image_aspect_ratio` +- `physical_width_cm`, `physical_height_cm`, `physical_depth_cm` +- `physical_area_cm2`, `physical_aspect_ratio` +- source/enrichment provenance +- source-presence flags such as `present_in_da`, `present_in_en` + +### `palette.parquet` +One row per artwork + canonical palette colour. + +Suggested fields: + +- `object_number` +- `rank` +- `hex` +- `weight` +- `weight_source` +- `srgb_r`, `srgb_g`, `srgb_b` +- `oklab_l`, `oklab_a`, `oklab_b` +- `oklch_l`, `oklch_c`, `oklch_h_deg` +- palette provenance/source/version + +### `features.parquet` +One row per artwork with Feature Specification v1 values and quality flags. + +### `normalized-features.parquet` +One row per artwork per normalization profile/version, or a wide equivalent if simpler to query. + +### `quality-report.json` +Corpus-wide counts, missingness, exclusions, warnings and distribution diagnostics. + +## Identity + +Prefer `object_number` as the stable artwork key where present. Detect and report missing/duplicate IDs before silently deduplicating. + +If duplicate IDs exist, inspect them and implement an explicit deterministic policy with provenance; do not keep an arbitrary “first row”. + +## Null semantics + +Null means unavailable/not safely derivable. + +Zero is a real value and must never stand in for missing data. + +## Corpus membership + +Canonical ingestion and feature extraction may include records that are not eligible for `smk-public-visual-v1`. + +Normalization membership is a separate, explicit decision. Store inclusion/exclusion reasons rather than deleting records merely because a downstream profile cannot use them. diff --git a/docs/FEATURE_SPEC_V1.md b/docs/FEATURE_SPEC_V1.md new file mode 100644 index 0000000..6ab670e --- /dev/null +++ b/docs/FEATURE_SPEC_V1.md @@ -0,0 +1,168 @@ +# Feature Specification v1 + +## Scope + +Per-artwork feature extraction is deterministic and corpus-independent. Running the extractor for the same canonical inputs and extractor version must produce the same output regardless of how many other artworks exist. + +## Palette input + +Prefer an explicit SMK weighted palette when valid. Preserve the source representation. + +For `colors_palette_weighted_s`, the observed format may contain repeated space-separated hex tokens. Parse defensively: + +1. tokenize valid colours, +2. count each colour, +3. divide by total valid token count, +4. retain an explicit `weight_source`, +5. quality-flag malformed/partial input. + +If weights are unavailable, do not label equal/fallback weights as original SMK weights. A fallback can be supported if explicitly identified. + +Canonical weights must satisfy approximately: + +```text +weight >= 0 +sum(weight) = 1 +``` + +## Colour conversion + +Use: + +```text +sRGB -> linear RGB -> OKLab -> OKLCH +``` + +Use a single documented conversion implementation throughout v1 and test it against known reference colours. + +## Perceptual colour centroid + +Calculate the weighted mean in OKLab: + +```text +L = Σ(wᵢLᵢ) +a = Σ(wᵢaᵢ) +b = Σ(wᵢbᵢ) +``` + +Store: + +- `centroid_l` +- `centroid_a` +- `centroid_b` +- `centroid_chroma` +- `centroid_hue_deg` +- `centroid_hue_x` +- `centroid_hue_y` + +Convert the centroid to OKLCH for chroma/hue. Hue is undefined for sufficiently achromatic centroids; use null rather than inventing a direction. + +## Palette statistics + +### `dominant_color_weight` +Largest normalized palette weight. + +### `palette_entropy_01` +Normalized Shannon entropy: + +```text +H = -Σ(pᵢ ln pᵢ) +H01 = H / ln(n) +``` + +For `n <= 1`, define `palette_entropy_01 = 0.0`. + +### `effective_palette_size` + +```text +exp(H) +``` + +using unnormalized natural-log entropy. + +### `palette_spread` +Weighted mean Euclidean OKLab distance from palette colours to the weighted centroid. + +### `lightness_stddev` +Weighted standard deviation of OKLab L. + +### `chroma_stddev` +Weighted standard deviation of OKLCH chroma. + +### `hue_concentration_01` +Magnitude of a chroma-aware circular resultant where each hue contribution is weighted by: + +```text +palette_weight * chroma +``` + +Return null when total chromatic mass is effectively zero. + +## SMK-provided visual features + +Preserve independently from our own calculations when present: + +- `smk_brightness` +- `smk_contrast` +- `smk_saturation` +- `smk_colortemp` +- `smk_entropy` + +SMK image entropy and our palette entropy are different concepts. + +If multiple enrichment components/versions provide competing values, retain provenance and use an explicit precedence rule. Never silently mix components. + +## Geometry + +Canonical geometry/features: + +- `physical_width_cm` +- `physical_height_cm` +- `physical_depth_cm` +- `physical_area_cm2` +- `physical_aspect_ratio` +- `image_aspect_ratio` +- `log_width_cm` +- `log_height_cm` +- `log_area_cm2` + +Rules: + +- prefer meaningful **Netto** dimensions over Brutto when both are available, +- normalize supported units to centimetres, +- do not infer physical dimensions from pixel dimensions, +- calculate area/aspect only where width + height are meaningful, +- calculate logarithms only for positive valid values. + +## Aspect sanity check + +For suitable 2D works where both physical and image aspect ratios exist, compare them using a symmetric log-ratio measure, e.g.: + +```text +abs(log(physical_aspect / image_aspect)) +``` + +Flag large mismatches; do not automatically “correct” either source. + +## Quality/support fields + +Include: + +- `feature_schema_version` +- `extractor_version` +- source snapshot references +- enrichment component/version provenance when available +- `quality_flags` + +Initial quality-flag vocabulary may include: + +- `PALETTE_MISSING` +- `PALETTE_WEIGHTS_MISSING` +- `PALETTE_MALFORMED` +- `VERY_LOW_CHROMA` +- `DIMENSIONS_MISSING` +- `PHYSICAL_IMAGE_ASPECT_MISMATCH` +- `NON_2D_OBJECT` +- `SOURCE_LANGUAGE_MISMATCH` + +Flags describe observations; they should not automatically exclude records unless a named downstream profile says so. diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000..d3274e3 --- /dev/null +++ b/docs/IMPLEMENTATION_PLAN.md @@ -0,0 +1,133 @@ +# Implementation Plan + +## Milestone 0 — Source reconnaissance + repository foundation + +- verify Python package/CLI setup +- detect both expected source files: + - `data/raw/smk_all_da.json` + - `data/raw/smk_all_en.json` +- inspect actual top-level JSON structures without guessing +- compute bytes + SHA-256 +- count/summarize records with bounded memory +- compare DA/EN identity sets +- write `source-manifest.json` +- document any structural surprises + +Done when we understand the actual dumps well enough to implement the parser from evidence rather than assumptions. + +## Milestone 1 — Streaming source loading + canonical schema + +- implement bounded-memory loader for actual observed structure +- join DA/EN by `object_number`, never by row position +- preserve localized text where useful +- map stable non-language fields into canonical columns +- preserve source provenance/disagreements +- parse physical dimensions/units defensively + +Done when a representative fixture/sample becomes deterministic canonical rows. + +## Milestone 2 — Weighted palette + colour conversion + +- locate ordinary/weighted palette fields in the actual snapshot/enrichment data +- parse `colors_palette_weighted_s` when present +- normalize explicit weights +- implement/verify sRGB -> OKLab -> OKLCH +- preserve malformed input as quality warnings rather than crashing the run + +Done when known palette fixtures produce stable colour rows and valid weights. + +## Milestone 3 — Feature extraction + +Implement: + +- perceptual centroid +- palette entropy +- effective palette size +- palette spread +- weighted lightness/chroma variation +- hue concentration +- SMK visual fields + provenance +- physical/image geometry + logs + +Done when one artwork can produce a complete feature record independently of the corpus. + +## Milestone 4 — Quality layer + +Implement documented flags and source/language/aspect checks. + +Produce `quality-report.json` with: + +- DA/EN raw record counts +- unique IDs and language overlap +- public-domain count +- usable-image/visual count +- valid feature count +- null counts by feature +- flag counts +- parser warnings/errors +- palette-size distribution +- obvious extreme values + +## Milestone 5 — Pilot build + +Run 100–500 representative artworks. + +Review distributions and manually inspect selected records. Fix parser/source/math errors only; do not artistically tune features. + +## Milestone 6 — Corpus definition + normalization + +Do not hardcode 39,508. + +- define the exact inclusion predicate for `smk-public-visual-v1` +- record included/excluded counts and reasons +- compute count/null/min/max/quantiles +- median + MAD +- implement `robust_01` +- implement `percentile_01` +- implement robust z-score +- freeze profile with source-manifest reference + +## Milestone 7 — Full dataset build + +Run the complete local source snapshots. + +Definition of done: + +Every canonical artwork has valid applicable features or explicit nulls/flags explaining missing/invalid prerequisites. Every normalization result references a frozen profile and exact source manifest. + +## Milestone 8 — Similarity + +Implement separate profiles rather than one magic similarity score: + +- `color` +- `visual` +- `composition` + +Use fast feature-vector candidate retrieval followed optionally by weighted-palette optimal-transport reranking. Do not precompute all pairs. + +## Milestone 9 — Art Tone v1 + +Implement versioned, explainable sonification mapping separate from objective features. + +Initial mapping: + +- centroid hue -> chromatic pitch class +- physical area percentile -> register +- chroma -> velocity/richness +- lightness -> timbral brightness +- palette entropy -> complexity +- contrast -> attack +- hue concentration -> tonal purity + +Emit exact inputs alongside every mapped result. + +## Milestone 10 — Exploration + +Only after Dataset v1 is trustworthy, start experimental interfaces/notebooks/apps: + +- nearest-neighbour browser +- colour-space explorer +- timelines +- artist palette fingerprints +- mosaics +- sonification player diff --git a/docs/PIPELINE.md b/docs/PIPELINE.md new file mode 100644 index 0000000..5932398 --- /dev/null +++ b/docs/PIPELINE.md @@ -0,0 +1,92 @@ +# Pipeline + +## Normal path + +```text +smk_all_da.json + smk_all_en.json + ↓ +source reconnaissance + SHA-256 manifest + ↓ +streaming ingest + ↓ +language-aware canonical merge + ↓ +weighted palette parsing + colour conversion + ↓ +per-artwork feature extraction + ↓ +quality checks + ↓ +pilot diagnostics + ↓ +full feature build + ↓ +select/freeze normalization corpus + ↓ +normalized features + ↓ +similarity / sonification experiments +``` + +## Recommended CLI stages + +The exact CLI can evolve, but aim for commands conceptually like: + +```bash +smk inspect-source +smk ingest +smk canonicalize +smk features +smk quality +smk normalize --profile smk-public-visual-v1 +smk sonify --profile art-tone-v1 +smk build-all +``` + +Each stage should expose explicit input/output paths and be independently rerunnable. + +## Stage 0 — source reconnaissance + +Before writing the final record parser: + +- inspect the actual DA and EN top-level structure, +- count records without fully materializing the files in RAM, +- compute SHA-256 + byte sizes, +- sample representative records, +- compare identity sets between languages, +- report duplicates/missing IDs, +- create `source-manifest.json`. + +## Ingest performance rule + +The source files are large. Use bounded-memory streaming where practical. A one-time conversion to Parquet/Arrow as an interim representation is encouraged if it simplifies all later stages. + +Avoid repeatedly reparsing both ~900 MB JSON snapshots for every downstream command when a manifest-bound interim artifact can be reused safely. + +## Pilot + +Before the full normalization build, process 100–500 representative records and inspect distributions and quality flags. + +Include examples that are: + +- colourful / near-monochrome, +- dark / bright, +- small / large, +- missing dimensions, +- 2D / non-2D, +- odd/missing palette data, +- present in only one language snapshot if such cases exist. + +The pilot checks correctness; it is not artistic tuning. + +## Full build + +After a successful pilot: + +1. canonicalize every valid identity, +2. emit deterministic features or explicit nulls/flags, +3. report actual corpus counts instead of expecting a hardcoded number, +4. define exact membership of `smk-public-visual-v1`, +5. compute and freeze corpus statistics, +6. emit normalized features, +7. generate summary counts/distributions and exclusions. diff --git a/docs/SOURCE_DATA.md b/docs/SOURCE_DATA.md new file mode 100644 index 0000000..940c517 --- /dev/null +++ b/docs/SOURCE_DATA.md @@ -0,0 +1,89 @@ +# Source Data + +## Primary v1 snapshots + +```text +data/raw/smk_all_da.json +data/raw/smk_all_en.json +``` + +Treat both files as immutable. + +## Source reconnaissance comes first + +Before implementing a production parser, inspect the real files and record: + +- file size, +- SHA-256, +- top-level JSON type, +- whether records are a top-level array or nested under an envelope key, +- first/last representative record shape, +- record count, +- identity field availability, +- duplicate `object_number` values, +- major field presence/null rates, +- whether Danish/English snapshots contain the same identity set, +- whether non-language fields differ between snapshots. + +Do not assume record order is stable or aligned between languages. + +## Large-file rule + +The source files are roughly 900 MB each. Prefer streaming/token-based ingestion (`ijson`) or another bounded-memory strategy. It is acceptable to convert each snapshot once into a local columnar/intermediate representation after the source manifest has been recorded. + +Do not use `json.load()` on the whole file as the default implementation. + +## Source manifest + +Create `data/processed/source-manifest.json` containing at least: + +```json +{ + "manifest_version": "1.0.0", + "sources": [ + { + "language": "da", + "path": "data/raw/smk_all_da.json", + "bytes": 0, + "sha256": "...", + "record_count": 0, + "observed_top_level": "..." + }, + { + "language": "en", + "path": "data/raw/smk_all_en.json", + "bytes": 0, + "sha256": "...", + "record_count": 0, + "observed_top_level": "..." + } + ] +} +``` + +Include generation timestamp and parser/tool version where useful. + +## Language merge policy + +Use `object_number` as the preferred artwork join key when present and valid. + +Rules: + +- Never merge Danish and English rows by positional index. +- Preserve language-specific textual metadata separately (`title_da`, `title_en`, etc.) when included in the canonical dataset. +- Do not duplicate language-independent image, physical or palette features simply because they occur in both snapshots. +- If language-independent fields disagree, preserve the discrepancy/provenance and choose a documented deterministic precedence rule only where a single canonical value is required. +- Missing EN must not discard an otherwise valid DA artwork, and vice versa. + +## Live SMK API + +The public API and enrichment API are secondary sources in v1. Use them for: + +- manual/automated spot-checks of selected records, +- understanding fields not present in the bulk snapshot, +- enrichment data retrieval when it is not already represented locally, +- future incremental refresh tooling. + +Do not make tens of thousands of live requests merely to recreate data already present in the snapshots. + +SMK states that its full JSON dataset download is updated nightly. Therefore every local dump is a time-specific snapshot; reproducibility depends on hashing and manifesting the exact files used. diff --git a/docs/SOURCE_RECONNAISSANCE.md b/docs/SOURCE_RECONNAISSANCE.md new file mode 100644 index 0000000..c988893 --- /dev/null +++ b/docs/SOURCE_RECONNAISSANCE.md @@ -0,0 +1,125 @@ +# Source reconnaissance — Milestone 0 + +Generated from the local snapshots on 2026-10-08 (Europe/Copenhagen). The raw files were +read sequentially and were not modified. The detailed machine-readable scan is written locally +to `data/reports/source-reconnaissance.json`; it is intentionally ignored as generated output. + +## Files and exact structure + +| Language | Bytes | SHA-256 | Records | Unique `object_number` | +|---|---:|---|---:|---:| +| DA | 910,209,972 | `8b47b7af20e0450d6a4bbc1f507edbc6f7479f28a8a100e627c6d533db8d9856` | 202,728 | 202,394 | +| EN | 860,820,598 | `b8343ee9b12034f5e0aea5ac31c3eed5d3822e576cedcbe87e8234726d34e374` | 203,023 | 202,688 | + +Both files are a JSON top-level array with no envelope. Every item observed was an object, so +artwork records are reached at `$[]`. The files are not interchangeable snapshots: their row +counts and identity sets differ, and their last records differ. Row position must not be used to +join them. + +The scan used `tools/source-reconnaissance.mjs` under Node.js v24.12.0. It streams 1 MiB chunks, +materializes only the current array element, and retains compact identity/fingerprint indexes. +It does not call `json.load()` or materialize either source file as one object. + +## Record and identity summary + +| Measure | DA | EN | +|---|---:|---:| +| Raw records | 202,728 | 203,023 | +| Unique nonblank `object_number` | 202,394 | 202,688 | +| Missing/blank `object_number` | 0 | 0 | +| IDs occurring more than once | 305 | 306 | +| Extra rows beyond one per ID | 334 | 335 | +| `public_domain == true` | 152,216 | 152,511 | +| `has_image == true` | 54,423 | 54,426 | +| Non-empty `dimensions` | 199,141 | 199,442 | +| Non-empty `colors` | 52,249 | 52,249 | +| All five SMK visual fields present | 45,024 | 45,024 | + +The unique-ID union contains 202,689 identities: + +- 202,393 occur in both snapshots; +- 1 is DA-only: `KKSgb81877`; +- 295 are EN-only (the first include `KKS2026-53`, `KKS2026-54`, `KKS2026-55`, + `KKS2026-56`, and `KKS2026-57`). + +The full duplicate and language-only lists are in the local machine report. Duplicate IDs must +be resolved explicitly from record provenance in Milestone 1; selecting the first row silently +would be arbitrary. + +## Dataset v1 field locations + +All paths below are relative to one `$[]` record. + +| Dataset concern | Observed source path/shape | DA presence | EN presence | Notes | +|---|---|---:|---:|---| +| Identity | `object_number: string` | 202,728 | 203,023 | Present and nonblank in every row, but not unique. | +| Public domain | `public_domain: boolean` | 202,728 | 203,023 | Values agree on all overlapping unique IDs in the selected-field comparison. | +| Titles | `titles[].title`, plus `type` and often `language` | 202,724 arrays | 203,015 arrays | The EN snapshot is not guaranteed to contain English titles; the first EN record still has a Danish title/language tag. | +| Object type/name | `object_names[].name` | 202,699 arrays | 202,933 arrays | Localized (`Grafik` vs `Print` in the first record). | +| Physical dimensions | `dimensions[].{part,type,unit,value}` | 199,141 arrays | 199,442 arrays | Values are strings; labels/units are localized (`højde`/`millimeter` vs `height`/`mm`). Select meaningful parts such as Netto only after vocabulary inspection. | +| Image geometry | `image_width`, `image_height` | 54,272 each | 54,275 each | Numeric pixels; not physical dimensions. | +| Image availability | `has_image` | every row | every row | Seven overlapping IDs disagree between snapshots. | +| Unweighted palette | `colors: array` | 52,249 | 52,249 | Selected-field fingerprints agree across overlap. | +| Suggested background | `suggested_bg_color: array` | 52,248 | 52,248 | Selected-field fingerprints agree across overlap. | +| SMK visual values | top-level `brightness`, `contrast`, `saturation`, `colortemp`, `entropy` | see below | see below | `colortemp` has mixed numeric/string types. | + +Visual-field presence is identical between languages: `brightness` 52,171, `contrast` 52,171, +`saturation` 45,024, `colortemp` 52,249, and `entropy` 52,249. `colortemp` is numeric in +52,237 records and a string in 12 records in each file. No field whose name contains `palette` +was observed at any depth. In particular, `colors_palette_weighted_s` is absent from both bulk +snapshots; Milestone 2 cannot claim source weights from these files. + +Representative record `KKS5261` demonstrates the localization boundary: identity, image +geometry, colours and visual numbers match, while dimensions use DA `højde`/`bredde` with +`millimeter` versus EN `height`/`width` with `mm`, and object names differ. Record `KKS370` +demonstrates a normal record with physical dimensions but `has_image == false` and no palette or +visual values. + +## Cross-language discrepancies + +For the 202,393 overlapping unique IDs, compact fingerprints of selected nominally +language-independent top-level fields found these mismatch counts: + +- `rights`: 50,468; `modified`: 334; +- `image_width` and `image_height`: 15 each; +- `image_iiif_id` and `image_iiif_info`: 11 each; `image_size`: 9; +- `has_image`: 7; `image_mime_type`: 6; `image_hq`: 6; +- `image_orientation`: 8; `image_cropped`: 1; `on_display`: 2. + +Selected `colors`, `suggested_bg_color`, the five visual values, `public_domain`, `id`, creation +date, acquisition date and the stable URLs had zero mismatches in this comparison. Differences +on duplicate IDs require duplicate-aware reinspection because the compact comparison retains one +fingerprint per identity; they must not be interpreted as a final canonical precedence decision. + +## Ambiguities carried into Milestones 1–3 + +1. `object_number` is complete but non-unique. Duplicate records need comparison by source `id`, + timestamps and content before a deterministic keep/merge/error policy can be selected. +2. The snapshots are temporally asymmetric (295 EN-only versus one DA-only identity) and some + nominally language-independent image metadata differs. Missing one language must be retained, + and DA precedence alone is not safe for every image field. +3. The exact physical-dimension vocabulary and choice of meaningful Netto/Brutto/part variants + still need a bounded vocabulary profile before unit canonicalization. Values are strings and + labels/units are localized. +4. The bulk files have no explicit weighted palette. A later enrichment input, separately + manifested and provenance-labelled, is needed for source weights; unweighted `colors` must not + be presented as weighted SMK data. +5. `colortemp` needs explicit validation/coercion for its 12 string-valued cases. Titles in the EN + file cannot be treated as English merely because the file is the EN snapshot. +6. Corpus membership remains undefined. Counts above describe raw snapshots and observable field + availability, not membership in `smk-public-visual-v1`. + +## Recommended next parser/canonicalization approach + +For Milestone 1, use `ijson.items(stream, "item")` against the proven `$[]` path and write a +manifest-bound interim Parquet dataset per language in batches. Preserve source row number, +source `id`, `object_number`, snapshot language and source-manifest hash. Build a duplicate audit +grouped by `object_number` before canonical merge; do not silently deduplicate. + +Canonicalize DA and EN independently first, then full-outer-join by resolved `object_number`. +Keep localized fields separately and choose each language-independent field through an explicit +field-level rule that records agreement/disagreement and provenance. Profile dimension +`part`/`type`/`unit` vocabularies before mapping Netto width/height/depth to centimetres. Preserve +mixed or unknown values as null plus diagnostics. Treat top-level `colors` as an unweighted source +palette only; defer weighted palette parsing until an actual manifested weight-bearing source has +been established. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..1f609d9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,33 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "smk-playground" +version = "0.1.0" +description = "Reproducible visual feature engineering and experimental analysis for SMK artworks" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "duckdb>=1", + "ijson>=3.3", + "numpy>=2", + "polars>=1", + "pyarrow>=18", + "typer>=0.12", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8", + "ruff>=0.8", +] + +[project.scripts] +smk = "smk_playground.cli:app" + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.ruff] +line-length = 100 diff --git a/src/smk_playground/__init__.py b/src/smk_playground/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/smk_playground/canonical/__init__.py b/src/smk_playground/canonical/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/smk_playground/cli.py b/src/smk_playground/cli.py new file mode 100644 index 0000000..d09fc64 --- /dev/null +++ b/src/smk_playground/cli.py @@ -0,0 +1,23 @@ +"""Command-line entrypoint for the SMK Playground pipeline.""" + +from pathlib import Path + +import typer + +app = typer.Typer(no_args_is_help=True, help="SMK visual-feature pipeline") + + +@app.command() +def status() -> None: + """Show expected source paths and repository readiness.""" + expected = [ + Path("data/raw/smk_all_da.json"), + Path("data/raw/smk_all_en.json"), + ] + typer.echo("SMK Playground pipeline skeleton is ready.") + for path in expected: + typer.echo(f"{'FOUND' if path.exists() else 'MISSING'} {path}") + + +if __name__ == "__main__": + app() diff --git a/src/smk_playground/features/__init__.py b/src/smk_playground/features/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/smk_playground/ingest/__init__.py b/src/smk_playground/ingest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/smk_playground/normalize/__init__.py b/src/smk_playground/normalize/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/smk_playground/quality/__init__.py b/src/smk_playground/quality/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/smk_playground/similarity/__init__.py b/src/smk_playground/similarity/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/smk_playground/sonification/__init__.py b/src/smk_playground/sonification/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_smoke.py b/tests/test_smoke.py new file mode 100644 index 0000000..6203dde --- /dev/null +++ b/tests/test_smoke.py @@ -0,0 +1,12 @@ +from typer.testing import CliRunner + +from smk_playground.cli import app + + +def test_status_command() -> None: + runner = CliRunner() + result = runner.invoke(app, ["status"]) + assert result.exit_code == 0 + assert "smk playground pipeline skeleton is ready" in result.stdout.lower() + assert "smk_all_da.json" in result.stdout + assert "smk_all_en.json" in result.stdout diff --git a/tools/source-reconnaissance.mjs b/tools/source-reconnaissance.mjs new file mode 100644 index 0000000..fe46cfd --- /dev/null +++ b/tools/source-reconnaissance.mjs @@ -0,0 +1,267 @@ +#!/usr/bin/env node + +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + +const SOURCES = [ + { language: "da", file: "data/raw/smk_all_da.json" }, + { language: "en", file: "data/raw/smk_all_en.json" }, +]; + +const VISUAL_FIELDS = ["brightness", "contrast", "saturation", "colortemp", "entropy"]; +const COMPARE_FIELDS = [ + "id", "created", "modified", "acquisition_date", "acquisition_date_precision", + "object_number", "object_url", "frontend_url", "iiif_manifest", "enrichment_url", + "similar_images_url", "public_domain", "rights", "on_display", "image_mime_type", + "image_iiif_id", "image_iiif_info", "image_width", "image_height", "image_size", + "image_thumbnail", "image_native", "image_cropped", "image_orientation", "image_hq", + "has_3d_file", "has_image", "colors", "suggested_bg_color", ...VISUAL_FIELDS, +]; +const RELEVANT_KEYS = new Set([ + "object_number", "public_domain", "titles", "title", "object_names", "name", + "classification", "classifications", "dimensions", "part", "type", "unit", "value", + "image_width", "image_height", "has_image", "colors", "colors_palette", + "colors_palette_weighted", "colors_palette_weighted_s", "suggested_bg_color", + ...VISUAL_FIELDS, +]); + +function valueType(value) { + if (value === null) return "null"; + if (Array.isArray(value)) return "array"; + return typeof value; +} + +function stable(value) { + if (Array.isArray(value)) return `[${value.map(stable).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stable(value[key])}`).join(",")}}`; + } + return JSON.stringify(value); +} + +function digestValue(value) { + return createHash("sha256").update(stable(value)).digest("hex").slice(0, 16); +} + +function increment(map, key) { + map.set(key, (map.get(key) ?? 0) + 1); +} + +function inspectRelevant(value, currentPath, stats) { + if (Array.isArray(value)) { + for (const item of value) inspectRelevant(item, `${currentPath}[]`, stats); + return; + } + if (!value || typeof value !== "object") return; + for (const [key, child] of Object.entries(value)) { + const childPath = currentPath ? `${currentPath}.${key}` : key; + if (RELEVANT_KEYS.has(key) || /palette|color|brightness|contrast|saturation|entropy|dimension/i.test(key)) { + const statKey = `${childPath}\u0000${valueType(child)}`; + increment(stats, statKey); + } + inspectRelevant(child, childPath, stats); + } +} + +function sampleRecord(record) { + const keys = [ + "id", "object_number", "public_domain", "has_image", "image_width", "image_height", + "dimensions", "titles", "object_names", "colors", "suggested_bg_color", ...VISUAL_FIELDS, + ]; + return Object.fromEntries(keys.filter((key) => Object.hasOwn(record, key)).map((key) => [key, record[key]])); +} + +async function inspectSource(source) { + const hash = createHash("sha256"); + const ids = new Map(); + const comparison = new Map(); + const topLevelFields = new Map(); + const relevantPaths = new Map(); + const counts = { + records: 0, missing_object_number: 0, blank_object_number: 0, public_domain_true: 0, + has_image_true: 0, dimensions_present: 0, colors_present: 0, + all_visual_fields_present: 0, + }; + const visualFieldPresence = Object.fromEntries(VISUAL_FIELDS.map((key) => [key, 0])); + const samples = { first: null, last: null, first_without_image: null, first_without_object_number: null }; + + let bytes = 0; + let rootStarted = false; + let rootEnded = false; + let collecting = false; + let depth = 0; + let inString = false; + let escaped = false; + let recordText = ""; + + function accept(record) { + counts.records += 1; + const sampled = sampleRecord(record); + samples.first ??= sampled; + samples.last = sampled; + if (record.has_image !== true) samples.first_without_image ??= sampled; + + const rawId = record.object_number; + const validId = typeof rawId === "string" && rawId.trim() !== ""; + if (!Object.hasOwn(record, "object_number")) counts.missing_object_number += 1; + else if (!validId) counts.blank_object_number += 1; + if (!validId) samples.first_without_object_number ??= sampled; + if (validId) { + const id = rawId.trim(); + increment(ids, id); + const fingerprints = {}; + for (const key of COMPARE_FIELDS) { + fingerprints[key] = Object.hasOwn(record, key) ? digestValue(record[key]) : null; + } + if (!comparison.has(id)) comparison.set(id, fingerprints); + } + + if (record.public_domain === true) counts.public_domain_true += 1; + if (record.has_image === true) counts.has_image_true += 1; + if (Array.isArray(record.dimensions) && record.dimensions.length > 0) counts.dimensions_present += 1; + if (Array.isArray(record.colors) && record.colors.length > 0) counts.colors_present += 1; + if (VISUAL_FIELDS.every((key) => record[key] !== null && record[key] !== undefined)) { + counts.all_visual_fields_present += 1; + } + for (const key of VISUAL_FIELDS) { + if (record[key] !== null && record[key] !== undefined) visualFieldPresence[key] += 1; + } + for (const [key, value] of Object.entries(record)) { + increment(topLevelFields, `${key}\u0000${valueType(value)}`); + } + inspectRelevant(record, "", relevantPaths); + } + + const stream = createReadStream(source.file, { highWaterMark: 1024 * 1024 }); + for await (const chunk of stream) { + bytes += chunk.length; + hash.update(chunk); + const text = chunk.toString("utf8"); + for (const char of text) { + if (!rootStarted) { + if (/\s/.test(char)) continue; + if (char !== "[") throw new Error(`${source.file}: expected top-level array`); + rootStarted = true; + continue; + } + if (!collecting) { + if (/\s/.test(char) || char === ",") continue; + if (char === "]") { rootEnded = true; continue; } + if (char !== "{") throw new Error(`${source.file}: expected object array element`); + collecting = true; + depth = 1; + inString = false; + escaped = false; + recordText = "{"; + continue; + } + + recordText += char; + if (inString) { + if (escaped) escaped = false; + else if (char === "\\") escaped = true; + else if (char === "\"") inString = false; + continue; + } + if (char === "\"") inString = true; + else if (char === "{" || char === "[") depth += 1; + else if (char === "}" || char === "]") depth -= 1; + if (depth === 0) { + accept(JSON.parse(recordText)); + collecting = false; + recordText = ""; + } + } + } + if (!rootStarted || !rootEnded || collecting) throw new Error(`${source.file}: incomplete JSON array`); + + const duplicates = [...ids.entries()].filter(([, count]) => count > 1).sort((a, b) => a[0].localeCompare(b[0])); + return { + language: source.language, + path: source.file.replaceAll("\\", "/"), + bytes, + sha256: hash.digest("hex"), + observed_structure: { + top_level_type: "array", + record_path: "$[]", + element_type: "object", + envelope: null, + }, + counts: { + ...counts, + unique_object_numbers: ids.size, + duplicate_object_numbers: duplicates.length, + duplicate_rows_beyond_first: duplicates.reduce((sum, [, count]) => sum + count - 1, 0), + visual_field_presence: visualFieldPresence, + }, + duplicate_details: duplicates.map(([object_number, count]) => ({ object_number, count })), + top_level_fields: [...topLevelFields.entries()].map(([key, count]) => { + const [field, type] = key.split("\u0000"); + return { field, type, count }; + }).sort((a, b) => a.field.localeCompare(b.field) || a.type.localeCompare(b.type)), + relevant_field_locations: [...relevantPaths.entries()].map(([key, count]) => { + const [field_path, type] = key.split("\u0000"); + return { field_path, type, count }; + }).sort((a, b) => a.field_path.localeCompare(b.field_path) || a.type.localeCompare(b.type)), + representative_records: samples, + _ids: ids, + _comparison: comparison, + }; +} + +function compareSources(da, en) { + const daIds = new Set(da._ids.keys()); + const enIds = new Set(en._ids.keys()); + const daOnly = [...daIds].filter((id) => !enIds.has(id)).sort(); + const enOnly = [...enIds].filter((id) => !daIds.has(id)).sort(); + const overlap = [...daIds].filter((id) => enIds.has(id)); + const fieldDifferences = Object.fromEntries(COMPARE_FIELDS.map((field) => [field, 0])); + const differenceExamples = Object.fromEntries(COMPARE_FIELDS.map((field) => [field, []])); + for (const id of overlap) { + const left = da._comparison.get(id); + const right = en._comparison.get(id); + for (const field of COMPARE_FIELDS) { + if (left[field] !== right[field]) { + fieldDifferences[field] += 1; + if (differenceExamples[field].length < 5) differenceExamples[field].push(id); + } + } + } + return { + overlap_unique_object_numbers: overlap.length, + da_only_count: daOnly.length, + en_only_count: enOnly.length, + da_only_object_numbers: daOnly, + en_only_object_numbers: enOnly, + selected_field_difference_counts_on_overlap: fieldDifferences, + selected_field_difference_examples: Object.fromEntries( + Object.entries(differenceExamples).filter(([, ids]) => ids.length > 0), + ), + }; +} + +const startedAt = new Date().toISOString(); +const results = []; +for (const source of SOURCES) { + process.stderr.write(`Inspecting ${source.file}...\n`); + results.push(await inspectSource(source)); +} +const comparison = compareSources(results[0], results[1]); +for (const result of results) { + delete result._ids; + delete result._comparison; +} + +const output = { + reconnaissance_version: "1.0.0", + tool: { name: "tools/source-reconnaissance.mjs", version: "1.0.0", runtime: process.version }, + started_at_utc: startedAt, + generated_at_utc: new Date().toISOString(), + sources: results, + identity_comparison: comparison, +}; +await mkdir("data/reports", { recursive: true }); +await writeFile("data/reports/source-reconnaissance.json", `${JSON.stringify(output, null, 2)}\n`); +process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);