From 4e01c2e9fe2fd919a92301894a9da2339855e148 Mon Sep 17 00:00:00 2001 From: Christian Johannesen Date: Thu, 8 Oct 2026 10:24:35 +0200 Subject: [PATCH] Complete Milestone 5 exact similarity --- AGENTS.md | 5 + README.md | 9 +- docs/DATA_MODEL.md | 6 + docs/FEATURE_VECTOR_V1.md | 3 + docs/IMPLEMENTATION_PLAN.md | 7 +- docs/PIPELINE.md | 7 + docs/SIMILARITY_EXAMPLES.md | 278 ++++++++++++++++ docs/SIMILARITY_V1.md | 102 ++++++ src/smk_playground/cli.py | 36 +++ src/smk_playground/similarity/engine.py | 155 +++++++++ src/smk_playground/similarity/milestone5.py | 342 ++++++++++++++++++++ tests/test_similarity.py | 71 ++++ 12 files changed, 1016 insertions(+), 5 deletions(-) create mode 100644 docs/SIMILARITY_EXAMPLES.md create mode 100644 docs/SIMILARITY_V1.md create mode 100644 src/smk_playground/similarity/engine.py create mode 100644 src/smk_playground/similarity/milestone5.py create mode 100644 tests/test_similarity.py diff --git a/AGENTS.md b/AGENTS.md index 4793103..55a885c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,6 +40,11 @@ normalized table preserves nulls, and its compact vector table contains separate `color_core_v1`, `visual_character_v1` and optional `geometry_v1` families. Do not treat these families as a similarity profile or independently normalized OKLab a/b as perceptual geometry. +Milestone 5 adds versioned exact-distance profiles for color, visual character and optional +geometry. Keep their questions separate, return component contributions, omit and renormalize +missing optional components symmetrically, and tie-break by distance then canonical entity ID. +Do not call distance a percentage similarity or add ANN before measured need. + Never overwrite source data. Never silently invent missing values. ## Primary bulk source diff --git a/README.md b/README.md index a1e74c3..836e29e 100644 --- a/README.md +++ b/README.md @@ -121,11 +121,12 @@ See: ## Current status -Milestones 0–4 complete: source structure and audits are manifested, the language snapshots have +Milestones 0–5 complete: source structure and audits are manifested, the language snapshots have bounded-memory source Parquets, DA/EN rows are paired into stable SMK entities, and the exact `smk-public-visual-v1` membership rule is frozen. Deterministic raw visual features and analytical palettes now exist. Versioned corpus statistics, normalized scalar features and three unweighted -candidate feature families are materialized. Similarity has not started. +candidate feature families are materialized. Three deterministic, explainable exact-distance +profiles now support color, visual-character and specialist geometry queries; ANN is not used. Audit results are documented in: @@ -140,6 +141,8 @@ Audit results are documented in: - [`docs/FEATURE_EXAMPLES.md`](docs/FEATURE_EXAMPLES.md) - [`docs/NORMALIZATION_V1.md`](docs/NORMALIZATION_V1.md) - [`docs/FEATURE_VECTOR_V1.md`](docs/FEATURE_VECTOR_V1.md) +- [`docs/SIMILARITY_V1.md`](docs/SIMILARITY_V1.md) +- [`docs/SIMILARITY_EXAMPLES.md`](docs/SIMILARITY_EXAMPLES.md) Run the audited ingest with: @@ -148,4 +151,6 @@ smk milestone1 smk milestone2 smk milestone3 smk milestone4 +smk milestone5 +smk similar KMS8892 --profile color --limit 10 ``` diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index 2b30efa..6d9e6ec 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -66,6 +66,12 @@ robust-01, percentile and robust-z columns; `artwork-feature-vectors-v1.parquet` fixed-order candidate families. `normalization-profile-v1.json` is the authoritative definition and binds both outputs to the exact source-feature Parquet SHA-256. +### `similarity-profiles-v1.json` + +Defines exact transforms, sampled-pair scales, weights, required/optional inputs, missingness rules, +tie-breaking and bound input hashes for the three Milestone 5 profiles. Similarity results are +computed exactly on demand and expose per-component contributions; there is no full pair matrix. + ### `quality-report.json` Corpus-wide counts, missingness, exclusions, warnings and distribution diagnostics. diff --git a/docs/FEATURE_VECTOR_V1.md b/docs/FEATURE_VECTOR_V1.md index edbe953..5081952 100644 --- a/docs/FEATURE_VECTOR_V1.md +++ b/docs/FEATURE_VECTOR_V1.md @@ -38,3 +38,6 @@ components. No family has weights, and the three families are not concatenated i distance space. Later comparison may combine raw OKLab distance, normalized scalar distance, palette-level distance and optional geometry under separately versioned similarity profiles. +Milestone 5 consumes explicit scalar columns rather than treating these lists as an opaque +universal vector. See `SIMILARITY_V1.md` for frozen component selection, scales and weights. + diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index 464934c..92d4da1 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -69,9 +69,10 @@ aspect and circular-hue diagnostics, and publish separate unweighted candidate f ## Milestone 5 — Pilot build -Run 100–500 representative artworks. - -Review distributions and manually inspect selected records. Fix parser/source/math errors only; do not artistically tune features. +Status: complete as deterministic exact similarity. Three separately versioned profiles use +sample-calibrated component scales, explicit weights, shared-component missingness and explainable +contributions. QA examples, stability diagnostics and performance measurements are published in +`SIMILARITY_V1.md` and `SIMILARITY_EXAMPLES.md`. ## Milestone 6 — Similarity preparation diff --git a/docs/PIPELINE.md b/docs/PIPELINE.md index c473915..6d0f187 100644 --- a/docs/PIPELINE.md +++ b/docs/PIPELINE.md @@ -85,6 +85,13 @@ and emit raw plus robust-01, empirical-midrank percentile and robust-z represent aspect ratios and circular hue diagnostics, preserve nulls, and publish separate unweighted color, visual-character and optional geometry families. Similarity and final weights remain later stages. +## Stage 5 — deterministic exact similarity + +Calibrate component scales from a fixed 500,000-pair sample, freeze separate color, +visual-character and geometry profiles, and perform exact full-corpus ranking with explainable +contributions. Missing optional fields renormalize shared weights; required fields determine +eligibility. No ANN index, universal score or palette transport is introduced. + ## Ingest performance rule The source files are large. Use bounded-memory streaming where practical. A one-time conversion to Parquet/Arrow as an interim representation is encouraged if it simplifies all later stages. diff --git a/docs/SIMILARITY_EXAMPLES.md b/docs/SIMILARITY_EXAMPLES.md new file mode 100644 index 0000000..41bca0e --- /dev/null +++ b/docs/SIMILARITY_EXAMPLES.md @@ -0,0 +1,278 @@ +# Similarity examples + +All statements below are deterministic feature comparisons, not aesthetic judgments. + +## KMS8892 + +Entity: `1170009810_object`. Title: Udsigt fra en vej i udkanten af Næstved. + +Query features: + +`{"centroid_a": -0.011783688836877545, "centroid_b": 0.04060318013487916, "centroid_l": 0.7417738706964976, "chroma_stddev": 0.04160784030178029, "hue_concentration_01": 0.7039520108269886, "log_net_area_cm2": 8.298290634359283, "log_net_aspect_ratio": 0.4151270190199013, "log_net_height_cm": 3.9415818076696905, "log_net_width_cm": 4.356708826689592, "palette_spread_oklab": 0.16540494792170796, "smk_brightness": 7.1343875, "smk_contrast": 2.9756057, "smk_entropy": 9.686463, "smk_saturation": 2.0800343}` + +### color-similarity-v1 + +| Rank | Object | Title | Distance | Contributions | Factual explanation | +|---:|---|---|---:|---|---| +| 1 | KKSgb12500 | Næstved | 0.045643 | centroid_oklab=0.0274, palette_spread=0.0094, chroma_stddev=0.0020, hue_concentration=0.0069 | similar centroid color | +| 2 | KKS8511j | Uglspils testamente | 0.052996 | centroid_oklab=0.0393, palette_spread=0.0050, chroma_stddev=0.0066, hue_concentration=0.0021 | similar centroid color | +| 3 | KKSgb4181 | Telemachos møder Odysseus forklædt som tigger hos hyrden Eumaios - Odysseus; fragment af hele kompositionen. | 0.057937 | centroid_oklab=0.0105, palette_spread=0.0142, chroma_stddev=0.0021, hue_concentration=0.0311 | similar centroid color | +| 4 | KKS9507 | Landskab med markvej | 0.061763 | centroid_oklab=0.0212, palette_spread=0.0075, chroma_stddev=0.0001, hue_concentration=0.0330 | similar centroid color | +| 5 | KKS2015-61 | Forskellige skitser, bl.a. farvelagt skitse til "Diogenes" | 0.061972 | centroid_oklab=0.0314, palette_spread=0.0098, chroma_stddev=0.0036, hue_concentration=0.0171 | similar centroid color | +| 6 | KKS6798 | Freia | 0.062060 | centroid_oklab=0.0100, palette_spread=0.0001, chroma_stddev=0.0024, hue_concentration=0.0495 | similar centroid color | +| 7 | KKS8009 | Profilportræt af J. F. Struense | 0.062098 | centroid_oklab=0.0078, palette_spread=0.0073, chroma_stddev=0.0012, hue_concentration=0.0457 | similar centroid color | +| 8 | KKS1975-885 | Den fejrede Pajuelera's mandige mod (i arenaen) i Saragossa | 0.062098 | centroid_oklab=0.0078, palette_spread=0.0073, chroma_stddev=0.0012, hue_concentration=0.0457 | similar centroid color | +| 9 | KKSgb8331 | Sørup Kirke, Angel | 0.062699 | centroid_oklab=0.0458, palette_spread=0.0118, chroma_stddev=0.0000, hue_concentration=0.0051 | similar centroid color | +| 10 | KKSgb7286 | "Collegium politicum" (Den politiske kandestøber) | 0.063881 | centroid_oklab=0.0191, palette_spread=0.0048, chroma_stddev=0.0004, hue_concentration=0.0396 | similar centroid color | + +### visual-character-similarity-v1 + +| Rank | Object | Title | Distance | Contributions | Factual explanation | +|---:|---|---|---:|---|---| +| 1 | KKS1975-885 | Den fejrede Pajuelera's mandige mod (i arenaen) i Saragossa | 0.047052 | centroid_oklab=0.0054, palette_spread=0.0044, chroma_stddev=0.0010, hue_concentration=0.0274, brightness=0.0009, contrast=0.0027, saturation=0.0019, entropy=0.0034 | similar centroid color | +| 2 | KKS1640 | Aften på havet | 0.058309 | centroid_oklab=0.0117, palette_spread=0.0091, chroma_stddev=0.0032, hue_concentration=0.0187, brightness=0.0117, contrast=0.0002, saturation=0.0013, entropy=0.0025 | similar centroid color | +| 3 | KKSgb10716 | Søkamp mellem danske fartøjer og et tyrkisk sørøverskib | 0.058842 | centroid_oklab=0.0334, palette_spread=0.0015, chroma_stddev=0.0012, hue_concentration=0.0139, brightness=0.0027, contrast=0.0007, saturation=0.0053, entropy=0.0002 | similar centroid color | +| 4 | KKS8511j | Uglspils testamente | 0.061473 | centroid_oklab=0.0272, palette_spread=0.0030, chroma_stddev=0.0053, hue_concentration=0.0013, brightness=0.0112, contrast=0.0003, saturation=0.0112, entropy=0.0020 | similar centroid color | +| 5 | KKSgb14069 | En siddende kvinde vendt mod højre, i en bønligt ydmyg stilling | 0.064192 | centroid_oklab=0.0357, palette_spread=0.0006, chroma_stddev=0.0100, hue_concentration=0.0065, brightness=0.0014, contrast=0.0026, saturation=0.0029, entropy=0.0045 | similar centroid color | +| 6 | KMS1989 | Trompe l'oeil. Bagsiden af et indrammet maleri | 0.065277 | centroid_oklab=0.0078, palette_spread=0.0047, chroma_stddev=0.0049, hue_concentration=0.0297, brightness=0.0079, contrast=0.0001, saturation=0.0078, entropy=0.0023 | similar centroid color | +| 7 | KKSgb11028 | En elefant | 0.067431 | centroid_oklab=0.0349, palette_spread=0.0033, chroma_stddev=0.0116, hue_concentration=0.0028, brightness=0.0054, contrast=0.0036, saturation=0.0014, entropy=0.0044 | similar centroid color | +| 8 | KMS8825 | Åens udløb | 0.068133 | centroid_oklab=0.0248, palette_spread=0.0054, chroma_stddev=0.0087, hue_concentration=0.0138, brightness=0.0073, contrast=0.0041, saturation=0.0015, entropy=0.0025 | similar centroid color | +| 9 | KKS12774 | Molen ved Dyrnæs. Gråvejr | 0.068331 | centroid_oklab=0.0332, palette_spread=0.0114, chroma_stddev=0.0064, hue_concentration=0.0008, brightness=0.0006, contrast=0.0124, saturation=0.0030, entropy=0.0004 | similar centroid color | +| 10 | KKS8009 | Profilportræt af J. F. Struense | 0.070208 | centroid_oklab=0.0054, palette_spread=0.0044, chroma_stddev=0.0010, hue_concentration=0.0274, brightness=0.0013, contrast=0.0069, saturation=0.0224, entropy=0.0014 | similar centroid color | + +### geometry-similarity-v1 + +| Rank | Object | Title | Distance | Contributions | Factual explanation | +|---:|---|---|---:|---|---| +| 1 | KMSsp213 | Landskab med vej forbi et rettersted | 0.002788 | log_width=0.0006, log_height=0.0000, log_area=0.0006, log_aspect=0.0016 | shared normalized feature values | +| 2 | KMSst82 | En bjerghule | 0.011366 | log_width=0.0018, log_height=0.0029, log_area=0.0043, log_aspect=0.0024 | shared normalized feature values | +| 3 | KMS1700 | Parti af bugten ved Kalkbrænderiet set fra Strandpromenaden mod nord. Stille sommereftermiddag | 0.012536 | log_width=0.0030, log_height=0.0019, log_area=0.0045, log_aspect=0.0031 | shared normalized feature values | +| 4 | KMSsp295 | Jagtstilleben med døde fugle og en jagthund | 0.013413 | log_width=0.0018, log_height=0.0019, log_area=0.0000, log_aspect=0.0096 | shared normalized feature values | +| 5 | KMS448 | Parti af Sjællands nordkyst ved Dronningmølle | 0.017326 | log_width=0.0042, log_height=0.0029, log_area=0.0065, log_aspect=0.0038 | shared normalized feature values | +| 6 | KMS4458 | Skovlandskab med Johannes Døberen i ødemarken, Johannes Døberens prædiken og Jesu dåb | 0.019421 | log_width=0.0048, log_height=0.0038, log_area=0.0079, log_aspect=0.0030 | shared normalized feature values | +| 7 | KMSsp175 | Selskab i en park (Den fortabte Søn hos skøgerne?) | 0.021563 | log_width=0.0042, log_height=0.0010, log_area=0.0031, log_aspect=0.0134 | shared normalized feature values | +| 8 | KMS7105 | Landevejen ved Mogenstrup. Efterår | 0.025227 | log_width=0.0053, log_height=0.0066, log_area=0.0109, log_aspect=0.0024 | shared normalized feature values | +| 9 | KMS7601 | Fra Den Kongelige Malerisamling på Christiansborg Slot | 0.025619 | log_width=0.0048, log_height=0.0066, log_area=0.0104, log_aspect=0.0039 | shared normalized feature values | +| 10 | KMS3040 | Nymfer i en grotte | 0.028021 | log_width=0.0012, log_height=0.0066, log_area=0.0070, log_aspect=0.0132 | shared normalized feature values | + +## KMS1571 + +Entity: `1170013658_object`. Title: Interiør. + +Query features: + +`{"centroid_a": 1.7388777917570764e-11, "centroid_b": 8.006482379863122e-09, "centroid_l": 0.21480126006860106, "chroma_stddev": 1.3852316851121162e-09, "hue_concentration_01": null, "log_net_area_cm2": 7.890956716138919, "log_net_aspect_ratio": 0.488352767913932, "log_net_height_cm": 3.7013019741124933, "log_net_width_cm": 4.189654742026425, "palette_spread_oklab": 0.03716348815137517, "smk_brightness": 4.250505, "smk_contrast": 1.2342123, "smk_entropy": 9.539091, "smk_saturation": 1.3943743}` + +### color-similarity-v1 + +| Rank | Object | Title | Distance | Contributions | Factual explanation | +|---:|---|---|---:|---|---| +| 1 | KMSsp405 | Kristus i Emmaus | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 2 | DEP7 | René Descartes (1596-1650) | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 3 | DEP697 | Roslagsgatan, Stockholm | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 4 | KMSsp289 | Stenkartouche med guirlande af frugt og blomster | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 5 | KMS3885 | En gammel kone | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 6 | KMS3986 | Fingal ser sine forfædres ånder ved månens skin | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 7 | KMS1307 | Portræt af fru Gumperd | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 8 | KMS3735 | Jagthunde og vildt | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 9 | KMS3886 | Hjemkomsten fra en fuglejagt | 0.054877 | centroid_oklab=0.0163, palette_spread=0.0044, chroma_stddev=0.0342 | similar centroid color; hue concentration unavailable | +| 10 | KMS594 | Potuanerne undres ved at se Niels Klims knæfald for den vise fyrste | 0.062067 | centroid_oklab=0.0270, palette_spread=0.0039, chroma_stddev=0.0312 | similar centroid color; hue concentration unavailable | + +### visual-character-similarity-v1 + +| Rank | Object | Title | Distance | Contributions | Factual explanation | +|---:|---|---|---:|---|---| +| 1 | KMSsp405 | Kristus i Emmaus | 0.050142 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0401, contrast=0.0026, saturation=0.0032, entropy=0.0042 | similar centroid color; hue concentration unavailable | +| 2 | KMS3885 | En gammel kone | 0.050661 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0303, contrast=0.0087, saturation=0.0109, entropy=0.0007 | similar centroid color; hue concentration unavailable | +| 3 | DEP7 | René Descartes (1596-1650) | 0.051446 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0299, contrast=0.0023, saturation=0.0168, entropy=0.0024 | similar centroid color; hue concentration unavailable | +| 4 | KMS3986 | Fingal ser sine forfædres ånder ved månens skin | 0.060012 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0233, contrast=0.0025, saturation=0.0280, entropy=0.0063 | similar centroid color; hue concentration unavailable | +| 5 | KMS3735 | Jagthunde og vildt | 0.065779 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0248, contrast=0.0020, saturation=0.0356, entropy=0.0034 | similar centroid color; hue concentration unavailable | +| 6 | KMS1307 | Portræt af fru Gumperd | 0.075435 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0401, contrast=0.0026, saturation=0.0146, entropy=0.0181 | similar centroid color; hue concentration unavailable | +| 7 | DEP697 | Roslagsgatan, Stockholm | 0.079666 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0279, contrast=0.0044, saturation=0.0416, entropy=0.0058 | similar centroid color; hue concentration unavailable | +| 8 | KMSsp289 | Stenkartouche med guirlande af frugt og blomster | 0.079940 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0401, contrast=0.0077, saturation=0.0237, entropy=0.0085 | similar centroid color; hue concentration unavailable | +| 9 | KMS1497 | Begyndende aftenskumring i en skov. Motiv fra Iselingen | 0.085520 | centroid_oklab=0.0422, palette_spread=0.0188, chroma_stddev=0.0000, brightness=0.0142, contrast=0.0042, saturation=0.0026, entropy=0.0033 | similar centroid color; hue concentration unavailable | +| 10 | KMSsp868 | Et måneskinsstykke. Overfarten over Lillebælt ved Snoghøj | 0.087103 | centroid_oklab=0.0403, palette_spread=0.0057, chroma_stddev=0.0000, brightness=0.0135, contrast=0.0002, saturation=0.0028, entropy=0.0246 | similar centroid color; hue concentration unavailable | + +### geometry-similarity-v1 + +| Rank | Object | Title | Distance | Contributions | Factual explanation | +|---:|---|---|---:|---|---| +| 1 | KMSsp373 | Måneskinslandskab med mytologisk (?) motiv | 0.008385 | log_width=0.0003, log_height=0.0020, log_area=0.0020, log_aspect=0.0041 | shared normalized feature values | +| 2 | KMS6072 | Stående tyr | 0.015935 | log_width=0.0038, log_height=0.0024, log_area=0.0058, log_aspect=0.0039 | shared normalized feature values | +| 3 | KMS3682 | En bro ved Sorrento | 0.016740 | log_width=0.0037, log_height=0.0000, log_area=0.0035, log_aspect=0.0096 | shared normalized feature values | +| 4 | KMS3809 | Hedestudie | 0.019840 | log_width=0.0015, log_height=0.0036, log_area=0.0019, log_aspect=0.0129 | shared normalized feature values | +| 5 | KMS7214 | Bakkedrag bag Møns Klint (Jydeleje) | 0.021231 | log_width=0.0015, log_height=0.0051, log_area=0.0059, log_aspect=0.0088 | shared normalized feature values | +| 6 | DEP297 | Dansk landskab | 0.024781 | log_width=0.0056, log_height=0.0012, log_area=0.0064, log_aspect=0.0116 | shared normalized feature values | +| 7 | KMSsp277 | Venus begræder den døde Adonis | 0.028679 | log_width=0.0067, log_height=0.0033, log_area=0.0092, log_aspect=0.0095 | shared normalized feature values | +| 8 | KMS6155 | Ruinerne af det antikke teater ved Tusculum | 0.029898 | log_width=0.0021, log_height=0.0072, log_area=0.0084, log_aspect=0.0122 | shared normalized feature values | +| 9 | KMS4167 | Et hus på Alheden | 0.031200 | log_width=0.0076, log_height=0.0083, log_area=0.0146, log_aspect=0.0007 | shared normalized feature values | +| 10 | KMS7709 | Tøbrud | 0.033416 | log_width=0.0075, log_height=0.0012, log_area=0.0082, log_aspect=0.0165 | shared normalized feature values | + +## KKS10234 + +Entity: `1170013295_object`. Title: Kunstnerens datter båret af hendes amme. + +Query features: + +`{"centroid_a": 0.1292024592463376, "centroid_b": 0.06973442719350281, "centroid_l": 0.7071157211187032, "chroma_stddev": 0.0, "hue_concentration_01": 1.0, "log_net_area_cm2": null, "log_net_aspect_ratio": null, "log_net_height_cm": null, "log_net_width_cm": null, "palette_spread_oklab": 0.0, "smk_brightness": 8.002823, "smk_contrast": 1.353646, "smk_entropy": 9.792471, "smk_saturation": 4.476761}` + +### color-similarity-v1 + +| Rank | Object | Title | Distance | Contributions | Factual explanation | +|---:|---|---|---:|---|---| +| 1 | KKSgb3856 | Jordemoderen. Studie til Pigen fra Andros. [Jordemoderen forlader Androspigens hus] | 0.076569 | centroid_oklab=0.0766, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000 | similar centroid color | +| 2 | KKS2615 | Bagside af omslag til M. Hammerichs oversættelse af Kalidasas, "Sakuntala", København 1845 | 0.076569 | centroid_oklab=0.0766, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000 | similar centroid color | +| 3 | KMS6604 | Udkast til en portrætgruppe. Regine Sophie Tutein, f. Wraatz med sønnen Peter | 0.076569 | centroid_oklab=0.0766, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000 | similar centroid color | +| 4 | KKS1977-473 | Motiv fra et østgrønlandsk sagn (Et fuglevæsen med barnehoved. Den ene af Andreassen-familiens to familietupilakker) | 0.076569 | centroid_oklab=0.0766, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000 | similar centroid color | +| 5 | KKSgb7520 | Løve, vendt mod højre, med højre forpote på en kugle | 0.076569 | centroid_oklab=0.0766, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000 | similar centroid color | +| 6 | KKSgb7665 | Mandlig modelstudie, halvtliggende | 0.076569 | centroid_oklab=0.0766, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000 | similar centroid color | +| 7 | KKS8072 | Middelfart | 0.076569 | centroid_oklab=0.0766, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000 | similar centroid color | +| 8 | KKSgb6160 | Et ophængt dødt egern | 0.076569 | centroid_oklab=0.0766, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000 | similar centroid color | +| 9 | KKS13801 | Sappho og Mytilenerinden | 0.076569 | centroid_oklab=0.0766, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000 | similar centroid color | +| 10 | KKSgb3837 | Udkast til en mandsdragt. Han bærer en lang stav med en fugl på toppen | 0.076569 | centroid_oklab=0.0766, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000 | similar centroid color | + +### visual-character-similarity-v1 + +| Rank | Object | Title | Distance | Contributions | Factual explanation | +|---:|---|---|---:|---|---| +| 1 | KKSgb209 | Stående rødbroget ko | 0.091565 | centroid_oklab=0.0530, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000, brightness=0.0048, contrast=0.0022, saturation=0.0308, entropy=0.0007 | similar centroid color | +| 2 | KKSgb210 | Liggende rød- og sortbroget ko | 0.092230 | centroid_oklab=0.0530, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000, brightness=0.0000, contrast=0.0076, saturation=0.0307, entropy=0.0009 | similar centroid color | +| 3 | KKS1977-473 | Motiv fra et østgrønlandsk sagn (Et fuglevæsen med barnehoved. Den ene af Andreassen-familiens to familietupilakker) | 0.097519 | centroid_oklab=0.0530, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000, brightness=0.0041, contrast=0.0004, saturation=0.0400, entropy=0.0000 | similar centroid color | +| 4 | KKSgb13565 | Studie af apostlen Markus skrivende, samt detailstudie af hånden, der holder bogen | 0.105894 | centroid_oklab=0.0265, palette_spread=0.0118, chroma_stddev=0.0207, hue_concentration=0.0000, brightness=0.0001, contrast=0.0096, saturation=0.0369, entropy=0.0002 | similar centroid color | +| 5 | KKSgb96 | Vendet Mølle ved Dortheaslyst | 0.106721 | centroid_oklab=0.0265, palette_spread=0.0118, chroma_stddev=0.0207, hue_concentration=0.0000, brightness=0.0083, contrast=0.0095, saturation=0.0291, entropy=0.0008 | similar centroid color | +| 6 | KKS1974-43 | Studier af englebørns hoveder i en sky | 0.107220 | centroid_oklab=0.0265, palette_spread=0.0118, chroma_stddev=0.0207, hue_concentration=0.0000, brightness=0.0013, contrast=0.0000, saturation=0.0464, entropy=0.0004 | similar centroid color | +| 7 | KKSgb7710 | Studie efter Guido Renis freske i Cappella dell' Annunciazione, Palazzo Quirinale, Rom. (Marias fødsel, venstre side) | 0.109088 | centroid_oklab=0.0265, palette_spread=0.0118, chroma_stddev=0.0207, hue_concentration=0.0000, brightness=0.0106, contrast=0.0099, saturation=0.0256, entropy=0.0039 | similar centroid color | +| 8 | KKS13801 | Sappho og Mytilenerinden | 0.109548 | centroid_oklab=0.0530, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000, brightness=0.0013, contrast=0.0048, saturation=0.0499, entropy=0.0006 | similar centroid color | +| 9 | KKSgb7665 | Mandlig modelstudie, halvtliggende | 0.110903 | centroid_oklab=0.0530, palette_spread=0.0000, chroma_stddev=0.0000, hue_concentration=0.0000, brightness=0.0070, contrast=0.0039, saturation=0.0461, entropy=0.0009 | similar centroid color | +| 10 | KKS1993-69 | Portræt af Antoinette Christiane Købke | 0.115676 | centroid_oklab=0.0594, palette_spread=0.0260, chroma_stddev=0.0037, hue_concentration=0.0032, brightness=0.0158, contrast=0.0042, saturation=0.0033, entropy=0.0000 | similar centroid color | + +### geometry-similarity-v1 + +Not eligible: a required feature is null. + +## KKSgb8124 + +Entity: `1170063127_object`. Title: Skitsebog nr. 29. Dresden 1845-1846. + +Query features: + +`{"centroid_a": 4.047642776505711e-11, "centroid_b": 1.8636953813544466e-08, "centroid_l": 0.4999999967367731, "chroma_stddev": 1.8636997767596426e-08, "hue_concentration_01": null, "log_net_area_cm2": null, "log_net_aspect_ratio": null, "log_net_height_cm": null, "log_net_width_cm": null, "palette_spread_oklab": 0.4999999967367734, "smk_brightness": 8.459217, "smk_contrast": 4.6236353, "smk_entropy": 7.400999, "smk_saturation": 1.0932152}` + +### color-similarity-v1 + +| Rank | Object | Title | Distance | Contributions | Factual explanation | +|---:|---|---|---:|---|---| +| 1 | KKS12968 | Vinter vandløb ved en skov | 0.002207 | centroid_oklab=0.0022, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 2 | KKS11612 | C. P. Hansen | 0.002207 | centroid_oklab=0.0022, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 3 | KKSgb13111/27 | "Tiden afklædt" Nr. 3 | 0.033356 | centroid_oklab=0.0334, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 4 | KKSgb12947 | Frederik Adolph Mørck | 0.033356 | centroid_oklab=0.0334, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 5 | KKS10780 | To hekse slås om et kosteskaft | 0.034496 | centroid_oklab=0.0057, palette_spread=0.0288, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 6 | KMS6004 | Venus med æblet | 0.038755 | centroid_oklab=0.0088, palette_spread=0.0000, chroma_stddev=0.0299 | similar centroid color; hue concentration unavailable | +| 7 | KKSgb13111/15 | "Nøgle til adskillige Kobbere" | 0.039402 | centroid_oklab=0.0035, palette_spread=0.0359, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 8 | KKS3981 | Karl Mantzius | 0.041335 | centroid_oklab=0.0413, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 9 | KKS12012 | Vandkalv | 0.041767 | centroid_oklab=0.0168, palette_spread=0.0250, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 10 | KKS5374 | Henrik Ibsen | 0.041767 | centroid_oklab=0.0168, palette_spread=0.0250, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | + +### visual-character-similarity-v1 + +| Rank | Object | Title | Distance | Contributions | Factual explanation | +|---:|---|---|---:|---|---| +| 1 | KKS12968 | Vinter vandløb ved en skov | 0.054169 | centroid_oklab=0.0015, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0071, contrast=0.0164, saturation=0.0077, entropy=0.0215 | similar centroid color; hue concentration unavailable | +| 2 | KKS11612 | C. P. Hansen | 0.074956 | centroid_oklab=0.0015, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0115, contrast=0.0147, saturation=0.0014, entropy=0.0459 | similar centroid color; hue concentration unavailable | +| 3 | KKS2514 | Abbaten og den spindende pige | 0.075098 | centroid_oklab=0.0016, palette_spread=0.0418, chroma_stddev=0.0000, brightness=0.0098, contrast=0.0008, saturation=0.0191, entropy=0.0021 | similar centroid color; hue concentration unavailable | +| 4 | KKSgb8340/36 | Figurkomposition | 0.086247 | centroid_oklab=0.0128, palette_spread=0.0055, chroma_stddev=0.0328, brightness=0.0118, contrast=0.0046, saturation=0.0077, entropy=0.0109 | similar centroid color; hue concentration unavailable | +| 5 | KMS5888 | Skuespillerinden Fru Johanne Dybwad | 0.090488 | centroid_oklab=0.0159, palette_spread=0.0480, chroma_stddev=0.0000, entropy=0.0266 | similar centroid color; hue concentration unavailable | +| 6 | KKSgb8340/30 verso | Kompositionsrids med figurer samlet omkring en himmelseng, forrest kvinder, der holder et draperi til side | 0.093108 | centroid_oklab=0.0083, palette_spread=0.0000, chroma_stddev=0.0270, brightness=0.0171, contrast=0.0049, saturation=0.0005, entropy=0.0353 | similar centroid color; hue concentration unavailable | +| 7 | KKSgb8353/20 verso | Sejlbåd med muntert selskab | 0.094134 | centroid_oklab=0.0153, palette_spread=0.0012, chroma_stddev=0.0380, brightness=0.0139, contrast=0.0048, saturation=0.0025, entropy=0.0185 | similar centroid color; hue concentration unavailable | +| 8 | KKS12129 | E. P. Bruhn | 0.096996 | centroid_oklab=0.0126, palette_spread=0.0218, chroma_stddev=0.0000, brightness=0.0002, contrast=0.0267, saturation=0.0128, entropy=0.0230 | similar centroid color; hue concentration unavailable | +| 9 | KKSgb11875/69 | Philoktet | 0.099265 | centroid_oklab=0.0257, palette_spread=0.0113, chroma_stddev=0.0302, brightness=0.0109, contrast=0.0035, saturation=0.0032, entropy=0.0144 | similar centroid color; hue concentration unavailable | +| 10 | KKSgb9874/35 | Stor marmorvase i gården ved S. Cecilia i Trastevere | 0.099491 | centroid_oklab=0.0203, palette_spread=0.0180, chroma_stddev=0.0288, brightness=0.0036, contrast=0.0087, saturation=0.0194, entropy=0.0006 | similar centroid color; hue concentration unavailable | + +### geometry-similarity-v1 + +Not eligible: a required feature is null. + +## KKSgb22323 + +Entity: `1170000125_object`. Title: Den besejrede Syphax stilles for Scipio. + +Query features: + +`{"centroid_a": 6.412081976492345e-11, "centroid_b": 2.952370570241669e-08, "centroid_l": 0.7920743356340977, "chroma_stddev": 0.0, "hue_concentration_01": null, "log_net_area_cm2": null, "log_net_aspect_ratio": null, "log_net_height_cm": null, "log_net_width_cm": null, "palette_spread_oklab": 0.0, "smk_brightness": 6.372257, "smk_contrast": 3.7947528, "smk_entropy": 9.792436, "smk_saturation": 1.6356592}` + +### color-similarity-v1 + +| Rank | Object | Title | Distance | Contributions | Factual explanation | +|---:|---|---|---:|---|---| +| 1 | DEP971 | Første studie til billedet med kvinderne ved fontænen i A. | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 2 | KKSgb17345 | Studie til et portræt, formentlig af A. G. Moltke; til højre på bladet studie til Elefantridderordenens kæde | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 3 | KKSgb17357 | Diana og Aktæon (Ovids Metamorfoser) | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 4 | KKS7398 | En nøgen rytter i profil t.h. med et sværd i sin fremstrakte højre hånd | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 5 | KKSgb15957 | Tegning af en soldater statue | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 6 | KKSgb13571 | En kvinde med et skriftbånd i venstre og en murske i højre hånd, lænende sig til en himmelglobus | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 7 | KKSgb5608 | Hyrdernes tilbedelse | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 8 | KKSgb5695 | Syndefaldet | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 9 | KKS2507a | Jægeren | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 10 | KKS16441 | "Hververen" | 0.000000 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | + +### visual-character-similarity-v1 + +| Rank | Object | Title | Distance | Contributions | Factual explanation | +|---:|---|---|---:|---|---| +| 1 | KKSgb22339 | Demetrius lider kvælningsdøden ved forgiftning | 0.031855 | centroid_oklab=0.0003, palette_spread=0.0182, chroma_stddev=0.0000, brightness=0.0072, contrast=0.0012, saturation=0.0049, entropy=0.0000 | similar centroid color; hue concentration unavailable | +| 2 | KKSgb22286 | Marcus Atilius Regulus dræber en serpent i Afrika | 0.032861 | centroid_oklab=0.0003, palette_spread=0.0182, chroma_stddev=0.0000, brightness=0.0074, contrast=0.0018, saturation=0.0051, entropy=0.0000 | similar centroid color; hue concentration unavailable | +| 3 | KKS524 | Foran en kirkedør | 0.038461 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0120, contrast=0.0006, saturation=0.0258, entropy=0.0000 | similar centroid color; hue concentration unavailable | +| 4 | KKSgb9916/1 | Syndfloden | 0.059172 | centroid_oklab=0.0003, palette_spread=0.0182, chroma_stddev=0.0000, brightness=0.0148, contrast=0.0060, saturation=0.0153, entropy=0.0046 | similar centroid color; hue concentration unavailable | +| 5 | KKSgb22377 | En edsaflæggelse (In dem Namen der heilig göttlichen untheilbaren Dreyfaltigheit) | 0.064404 | centroid_oklab=0.0307, palette_spread=0.0137, chroma_stddev=0.0000, brightness=0.0036, contrast=0.0055, saturation=0.0109, entropy=0.0000 | similar centroid color; hue concentration unavailable | +| 6 | KKSgb12083 | Stille vejr på havet | 0.064719 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0377, contrast=0.0264, saturation=0.0003, entropy=0.0003 | similar centroid color; hue concentration unavailable | +| 7 | KKSgb11683 | Genius med bue i venstre hånd og højre hånd hævet mod en kronet fugl | 0.065258 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0425, contrast=0.0195, saturation=0.0031, entropy=0.0002 | similar centroid color; hue concentration unavailable | +| 8 | KKSgb22243 | Kort over Rufach | 0.068672 | centroid_oklab=0.0003, palette_spread=0.0182, chroma_stddev=0.0000, brightness=0.0283, contrast=0.0190, saturation=0.0028, entropy=0.0000 | similar centroid color; hue concentration unavailable | +| 9 | KKS6235 | Kristus velsignende | 0.068706 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0439, contrast=0.0163, saturation=0.0085, entropy=0.0000 | similar centroid color; hue concentration unavailable | +| 10 | KKSgb13097 | Kunstnernes Karneval 1845 | 0.068901 | centroid_oklab=0.0000, palette_spread=0.0000, chroma_stddev=0.0000, brightness=0.0371, contrast=0.0181, saturation=0.0135, entropy=0.0003 | similar centroid color; hue concentration unavailable | + +### geometry-similarity-v1 + +Not eligible: a required feature is null. + +## KKS4443 + +Entity: `1170001489_object`. Title: Landskab med et slot på en klippe. + +Query features: + +`{"centroid_a": 4.904935485550155e-11, "centroid_b": 2.2584269431432286e-08, "centroid_l": 0.6059002314916988, "chroma_stddev": 1.1584399951969442e-08, "hue_concentration_01": null, "log_net_area_cm2": null, "log_net_aspect_ratio": null, "log_net_height_cm": null, "log_net_width_cm": null, "palette_spread_oklab": 0.2898124065098517, "smk_brightness": 6.3518715, "smk_contrast": 5.8650985, "smk_entropy": 8.599682, "smk_saturation": null}` + +### color-similarity-v1 + +| Rank | Object | Title | Distance | Contributions | Factual explanation | +|---:|---|---|---:|---|---| +| 1 | KAS1333 | Kekrops og datter, Parthenons vestgavl, formindsket version | 0.004813 | centroid_oklab=0.0003, palette_spread=0.0045, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 2 | KMS5928 | Pigebarn, der piller tæer. Siddende nøgen kvinde | 0.004813 | centroid_oklab=0.0003, palette_spread=0.0045, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 3 | KMSsp325 | Dansen om majstangen | 0.004888 | centroid_oklab=0.0032, palette_spread=0.0017, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 4 | KKS12165 | Hans Egede | 0.005791 | centroid_oklab=0.0015, palette_spread=0.0042, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 5 | KKS9619e | Illustration til H. Drachmann, "Marts" | 0.007100 | centroid_oklab=0.0057, palette_spread=0.0014, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 6 | KAS66 | Galler dræber sin hustru og sig selv, Paetus og Arria | 0.012158 | centroid_oklab=0.0011, palette_spread=0.0111, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 7 | KAS440 | Buste fra profetstatue, Il Zucchone | 0.012158 | centroid_oklab=0.0011, palette_spread=0.0111, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 8 | DEP454 | Portræt af Antinoos med vedbendkrans | 0.012158 | centroid_oklab=0.0011, palette_spread=0.0111, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 9 | KAS2272 | Fra statuen af Athena Parthenos | 0.012158 | centroid_oklab=0.0011, palette_spread=0.0111, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | +| 10 | KAS2232 | Davids hoved | 0.012158 | centroid_oklab=0.0011, palette_spread=0.0111, chroma_stddev=0.0000 | similar centroid color; hue concentration unavailable | + +### visual-character-similarity-v1 + +| Rank | Object | Title | Distance | Contributions | Factual explanation | +|---:|---|---|---:|---|---| +| 1 | KMSr123 | Siddende nøgen kvinde. Leda | 0.021815 | centroid_oklab=0.0109, palette_spread=0.0018, chroma_stddev=0.0000, brightness=0.0045, contrast=0.0017, entropy=0.0030 | similar centroid color; hue concentration unavailable | +| 2 | KMSsp325 | Dansen om majstangen | 0.022439 | centroid_oklab=0.0024, palette_spread=0.0011, chroma_stddev=0.0000, brightness=0.0044, contrast=0.0050, entropy=0.0095 | similar centroid color; hue concentration unavailable | +| 3 | KKS11053 | Peter R. Saabye | 0.026205 | centroid_oklab=0.0097, palette_spread=0.0046, chroma_stddev=0.0000, entropy=0.0118 | similar centroid color; hue concentration unavailable | +| 4 | KKSgb5490 | Udsigt af den offentlige promenade af Hr. Conferenceråd Classen anlagt uden Københavns Østerport | 0.039989 | centroid_oklab=0.0186, palette_spread=0.0189, chroma_stddev=0.0000, entropy=0.0025 | similar centroid color; hue concentration unavailable | +| 5 | KMS959 | Ruiner i Den romerske Campagne | 0.042034 | centroid_oklab=0.0018, palette_spread=0.0221, chroma_stddev=0.0000, brightness=0.0062, contrast=0.0026, entropy=0.0093 | similar centroid color; hue concentration unavailable | +| 6 | KMS6195 | Dameportræt | 0.042190 | centroid_oklab=0.0023, palette_spread=0.0212, chroma_stddev=0.0000, brightness=0.0085, contrast=0.0062, entropy=0.0041 | similar centroid color; hue concentration unavailable | +| 7 | KKS19998/5 | Blank | 0.047098 | centroid_oklab=0.0078, palette_spread=0.0176, chroma_stddev=0.0000, brightness=0.0023, contrast=0.0002, entropy=0.0191 | similar centroid color; hue concentration unavailable | +| 8 | KMS1036 | Stående brun hest | 0.047324 | centroid_oklab=0.0166, palette_spread=0.0007, chroma_stddev=0.0000, brightness=0.0030, contrast=0.0160, entropy=0.0110 | similar centroid color; hue concentration unavailable | +| 9 | KKSgb13265 | Palmeofring. En ægyptisk kvinde på vej til hendes mands grav med en palmegren, efter østerlandsk skik | 0.051326 | centroid_oklab=0.0018, palette_spread=0.0218, chroma_stddev=0.0000, brightness=0.0137, contrast=0.0103, entropy=0.0037 | similar centroid color; hue concentration unavailable | +| 10 | KMS759 | Eftermiddag ved Himmelbjerget | 0.052000 | centroid_oklab=0.0312, palette_spread=0.0186, chroma_stddev=0.0000, entropy=0.0022 | similar centroid color; hue concentration unavailable | + +### geometry-similarity-v1 + +Not eligible: a required feature is null. + diff --git a/docs/SIMILARITY_V1.md b/docs/SIMILARITY_V1.md new file mode 100644 index 0000000..d2047e3 --- /dev/null +++ b/docs/SIMILARITY_V1.md @@ -0,0 +1,102 @@ +# Similarity v1 + +Milestone 5 defines three deterministic exact-distance profiles over the frozen 27,254-entity +corpus. Lower distance means more similar. Distances are not percentages or scientific certainty, +and every result exposes its active components and contribution sum. + +## Shared rules + +- Exact full-corpus search; no ANN index or precomputed distance matrix. +- Each component distance is divided by its deterministic sampled-pair P95 and clipped to `[0,1]`. +- Total distance is the weighted mean of active component distances. +- A missing required feature makes an entity ineligible for that profile. +- A missing optional feature omits that component and symmetrically renormalizes remaining weights; + no value is imputed. +- Ties sort by total distance and then `smk_entity_id`, both ascending. +- Results preserve canonical entity ID, non-unique object number and quality flags. + +The profile JSON binds the feature/normalized/profile input hashes, seed `20261008`, 500,000 +distinct calibration pairs, exact P95 scales, transforms, weights and output semantics. + +## `color-similarity-v1` + +| Component | Weight | Input | +|---|---:|---| +| centroid OKLab | 0.65 | raw Euclidean L/a/b | +| palette spread | 0.20 | robust-01 absolute difference | +| chroma standard deviation | 0.10 | robust-01 absolute difference | +| hue concentration | 0.05 | natural `[0,1]` absolute difference; optional | + +Centroid color is deliberately dominant. Raw OKLab geometry is preserved. Lightness standard +deviation is excluded because its corpus correlation with palette spread is approximately 0.99. +Achromatic works remain eligible because hue concentration is optional. + +## `visual-character-similarity-v1` + +| Component | Weight | +|---|---:| +| centroid OKLab | 0.45 | +| palette spread | 0.12 | +| chroma standard deviation | 0.08 | +| hue concentration (optional) | 0.03 | +| SMK brightness (optional) | 0.10 | +| SMK contrast (optional) | 0.05 | +| SMK saturation (optional) | 0.10 | +| SMK entropy (optional) | 0.07 | + +Contrast receives modest weight because it overlaps palette spread/lightness variation. Brightness, +saturation and entropy receive more room because Milestone 4 showed only moderate overlap with the +palette-derived features. Non-centroid inputs use their stored robust/natural 0–1 values. + +## `geometry-similarity-v1` + +| Component | Weight | +|---|---:| +| log width | 0.20 | +| log height | 0.20 | +| log area | 0.35 | +| log aspect ratio | 0.25 | + +All four normalized log-geometry inputs are required. Exactly 2,189 entities are eligible. This +specialist profile is not mixed into color or visual character. + +## Pair calibration + +| Component | Median | P95 | Sample n | +|---|---:|---:|---:| +| centroid OKLab | 0.153 | 0.409 | 500,000 | +| palette spread | 0.238 | 0.659 | 500,000 | +| chroma variation | 0.268 | 0.717 | 500,000 | +| hue concentration | 0.041 | 0.302 | 311,204 | +| brightness | 0.171 | 0.666 | 497,724 | +| contrast | 0.219 | 0.624 | 497,724 | +| saturation | 0.152 | 0.591 | 468,485 | +| entropy | 0.062 | 0.582 | 499,974 | + +Geometry received 3,195 jointly eligible sampled pairs; component P95 values range from 0.519 to +0.552. Full quantiles are stored in the profile and summary report. + +## Eligibility, stability, and performance + +- color: 27,254 entities; +- visual character: 27,254 entities, using shared available optional dimensions; +- geometry: 2,189 entities. + +For the deterministic QA set, each component weight was independently varied to 85% and 115% of +its frozen value. Median top-10 overlap is 10/10 and the minimum is 8/10, indicating no radical +ranking brittleness under these modest perturbations. + +On the recorded local run, normalized Parquet loading took about 0.35 seconds and engine loading +including localized titles about 1.05 seconds. Exact calculation and ranking timings are recorded +separately in `similarity-summary-v1.json`. These measurements do not justify ANN. + +## Interface + +```text +smk similar KMS8892 --profile color --limit 10 +smk similar --id 1170000000_object --profile visual --limit 10 +``` + +Ambiguous object numbers produce an error and list canonical IDs; they are never silently +collapsed. `--include-self` is available for debugging and yields distance zero. + diff --git a/src/smk_playground/cli.py b/src/smk_playground/cli.py index 873f9f5..93a0db0 100644 --- a/src/smk_playground/cli.py +++ b/src/smk_playground/cli.py @@ -1,5 +1,6 @@ """Command-line entrypoint for the SMK Playground pipeline.""" +import json from pathlib import Path import typer @@ -55,5 +56,40 @@ def milestone4(root: Path | None = None) -> None: typer.echo(f"Milestone 4 complete: {summary['row_count']} normalized rows") +@app.command("milestone5") +def milestone5(root: Path | None = None) -> None: + """Build exact deterministic similarity profiles and diagnostics.""" + from smk_playground.similarity.milestone5 import run + + summary = run((root or Path.cwd()).resolve()) + typer.echo(f"Milestone 5 complete: {summary['eligible_counts']}") + + +@app.command("similar") +def similar( + object_number: str | None = typer.Argument(None, help="SMK object number"), + smk_entity_id: str | None = typer.Option(None, "--id", help="Canonical SMK entity ID"), + profile: str = typer.Option("color", help="color, visual, or geometry"), + limit: int = typer.Option(10, min=1, max=100), + include_self: bool = typer.Option(False, help="Include the query entity for debugging"), + root: Path | None = None, +) -> None: + """Run an exact similarity query over the frozen corpus.""" + from smk_playground.similarity.engine import SimilarityEngine + + try: + query, results = SimilarityEngine((root or Path.cwd()).resolve()).search( + profile_name=profile, limit=limit, smk_entity_id=smk_entity_id, + object_number=object_number, include_self=include_self, + ) + except ValueError as error: + raise typer.BadParameter(str(error)) from error + typer.echo(json.dumps({ + "query": {"smk_entity_id": query["smk_entity_id"], + "object_number": query["object_number"]}, + "profile": profile, "results": results, + }, ensure_ascii=False, indent=2)) + + if __name__ == "__main__": app() diff --git a/src/smk_playground/similarity/engine.py b/src/smk_playground/similarity/engine.py new file mode 100644 index 0000000..2730d7e --- /dev/null +++ b/src/smk_playground/similarity/engine.py @@ -0,0 +1,155 @@ +"""Deterministic exact similarity search with explainable component distances.""" + +from __future__ import annotations + +import json +import math +from pathlib import Path +from typing import Any + +import pyarrow.parquet as pq + +PROFILE_ALIASES = { + "color": "color-similarity-v1", + "visual": "visual-character-similarity-v1", + "geometry": "geometry-similarity-v1", +} + + +def _component_raw(left: dict[str, Any], right: dict[str, Any], component: dict[str, Any]) -> float | None: + fields = component["fields"] + values = [(left.get(field), right.get(field)) for field in fields] + if any(x is None or y is None for x, y in values): + return None + if component["metric"] == "euclidean": + return math.sqrt(math.fsum((float(x) - float(y)) ** 2 for x, y in values)) + return abs(float(values[0][0]) - float(values[0][1])) + + +def compare(left: dict[str, Any], right: dict[str, Any], profile: dict[str, Any], + weight_overrides: dict[str, float] | None = None) -> dict[str, Any] | None: + """Compare two rows, returning normalized contributions that sum to total distance.""" + required = profile["required_features"] + if any(left.get(field) is None or right.get(field) is None for field in required): + return None + active: list[tuple[str, float, float, float]] = [] + overrides = weight_overrides or {} + for component in profile["components"]: + raw = _component_raw(left, right, component) + if raw is None: + continue + scale = float(component["scale"]) + scaled = min(raw / scale, 1.0) if scale > 0 else 0.0 + weight = float(overrides.get(component["name"], component["weight"])) + active.append((component["name"], raw, scaled, weight)) + active_weight_sum = math.fsum(item[3] for item in active) + if active_weight_sum <= 0: + return None + components = { + name: { + "raw_distance": raw, + "scaled_distance": scaled, + "weight": weight, + "contribution": weight * scaled / active_weight_sum, + } + for name, raw, scaled, weight in active + } + return { + "total_distance": math.fsum(item["contribution"] for item in components.values()), + "active_weight_sum": active_weight_sum, + "active_components": list(components), + "components": components, + } + + +def _first_title(value: str | None) -> str | None: + if not value: + return None + try: + items = json.loads(value) + except json.JSONDecodeError: + return None + for item in items or []: + title = item.get("title") if isinstance(item, dict) else None + if title: + return str(title) + return None + + +class SimilarityEngine: + """In-memory exact engine for the 27k-row frozen corpus.""" + + def __init__(self, root: Path): + self.root = root + profile_path = root / "data" / "processed" / "similarity-profiles-v1.json" + payload = json.loads(profile_path.read_text(encoding="utf-8")) + self.profiles = payload["profiles"] + normalized_path = root / "data" / "processed" / "artwork-normalized-features-v1.parquet" + self.rows = pq.read_table(normalized_path).to_pylist() + self.by_id = {row["smk_entity_id"]: row for row in self.rows} + self.by_object_number: dict[str, list[dict[str, Any]]] = {} + for row in self.rows: + self.by_object_number.setdefault(row["object_number"], []).append(row) + canonical_path = root / "data" / "processed" / "artworks.parquet" + title_table = pq.read_table( + canonical_path, columns=["smk_entity_id", "titles_da_json", "titles_en_json"] + ) + self.titles = {} + for row in title_table.to_pylist(): + self.titles[row["smk_entity_id"]] = ( + _first_title(row["titles_da_json"]) or _first_title(row["titles_en_json"]) + ) + + def profile(self, name: str) -> dict[str, Any]: + canonical = PROFILE_ALIASES.get(name, name) + if canonical not in self.profiles: + raise ValueError(f"Unknown similarity profile: {name}") + return self.profiles[canonical] + + def resolve(self, *, smk_entity_id: str | None = None, + object_number: str | None = None) -> dict[str, Any]: + if smk_entity_id: + if smk_entity_id not in self.by_id: + raise ValueError(f"Unknown corpus entity ID: {smk_entity_id}") + return self.by_id[smk_entity_id] + if not object_number: + raise ValueError("Provide an object number or --id") + matches = self.by_object_number.get(object_number, []) + if not matches: + raise ValueError(f"Object number is not in the frozen corpus: {object_number}") + if len(matches) > 1: + ids = ", ".join(sorted(row["smk_entity_id"] for row in matches)) + raise ValueError(f"Ambiguous object number {object_number}; use --id. Candidates: {ids}") + return matches[0] + + def search_row(self, query: dict[str, Any], profile_name: str, limit: int = 10, + include_self: bool = False, + weight_overrides: dict[str, float] | None = None) -> list[dict[str, Any]]: + profile = self.profile(profile_name) + if any(query.get(field) is None for field in profile["required_features"]): + raise ValueError(f"Query is not eligible for {profile['name']}") + matches = [] + for candidate in self.rows: + if not include_self and candidate["smk_entity_id"] == query["smk_entity_id"]: + continue + result = compare(query, candidate, profile, weight_overrides) + if result is None: + continue + matches.append({ + "smk_entity_id": candidate["smk_entity_id"], + "object_number": candidate["object_number"], + "title": self.titles.get(candidate["smk_entity_id"]), + "quality_flags": candidate["quality_flags"], + **result, + }) + matches.sort(key=lambda item: (item["total_distance"], item["smk_entity_id"])) + for rank, item in enumerate(matches[:limit], 1): + item["rank"] = rank + return matches[:limit] + + def search(self, *, profile_name: str, limit: int = 10, + smk_entity_id: str | None = None, object_number: str | None = None, + include_self: bool = False) -> tuple[dict[str, Any], list[dict[str, Any]]]: + query = self.resolve(smk_entity_id=smk_entity_id, object_number=object_number) + return query, self.search_row(query, profile_name, limit, include_self) + diff --git a/src/smk_playground/similarity/milestone5.py b/src/smk_playground/similarity/milestone5.py new file mode 100644 index 0000000..f8e2a19 --- /dev/null +++ b/src/smk_playground/similarity/milestone5.py @@ -0,0 +1,342 @@ +"""Build Milestone 5 similarity profiles, diagnostics, and QA examples.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import random +import statistics +import time +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pyarrow.parquet as pq + +from smk_playground.normalize.milestone4 import linear_quantile, sha256_file +from smk_playground.similarity.engine import SimilarityEngine, compare + +PROFILE_VERSION = "1.0.0" +PAIR_SEED = 20261008 +CALIBRATION_PAIR_COUNT = 500_000 +PERSISTED_PAIRS_PER_PROFILE = 50_000 + +COMPONENT_FIELDS = { + "centroid_oklab": (["centroid_l", "centroid_a", "centroid_b"], "euclidean"), + "palette_spread": (["palette_spread_oklab_robust_01"], "absolute"), + "chroma_stddev": (["chroma_stddev_robust_01"], "absolute"), + "hue_concentration": (["hue_concentration_01"], "absolute"), + "brightness": (["smk_brightness_robust_01"], "absolute"), + "contrast": (["smk_contrast_robust_01"], "absolute"), + "saturation": (["smk_saturation_robust_01"], "absolute"), + "entropy": (["smk_entropy_robust_01"], "absolute"), + "log_width": (["log_net_width_cm_robust_01"], "absolute"), + "log_height": (["log_net_height_cm_robust_01"], "absolute"), + "log_area": (["log_net_area_cm2_robust_01"], "absolute"), + "log_aspect": (["log_net_aspect_ratio_robust_01"], "absolute"), +} + +WEIGHTS = { + "color-similarity-v1": { + "centroid_oklab": 0.65, "palette_spread": 0.20, + "chroma_stddev": 0.10, "hue_concentration": 0.05, + }, + "visual-character-similarity-v1": { + "centroid_oklab": 0.45, "palette_spread": 0.12, "chroma_stddev": 0.08, + "hue_concentration": 0.03, "brightness": 0.10, "contrast": 0.05, + "saturation": 0.10, "entropy": 0.07, + }, + "geometry-similarity-v1": { + "log_width": 0.20, "log_height": 0.20, "log_area": 0.35, "log_aspect": 0.25, + }, +} + + +def _sample_pairs(size: int, count: int, seed: int) -> list[tuple[int, int]]: + rng = random.Random(seed) + pairs: set[tuple[int, int]] = set() + while len(pairs) < count: + left = rng.randrange(size) + right = rng.randrange(size - 1) + if right >= left: + right += 1 + pairs.add((min(left, right), max(left, right))) + return sorted(pairs) + + +def _raw_component(left: dict[str, Any], right: dict[str, Any], name: str) -> float | None: + fields, metric = COMPONENT_FIELDS[name] + values = [(left.get(field), right.get(field)) for field in fields] + if any(x is None or y is None for x, y in values): + return None + if metric == "euclidean": + return math.sqrt(math.fsum((float(x) - float(y)) ** 2 for x, y in values)) + return abs(float(values[0][0]) - float(values[0][1])) + + +def _distribution(values: list[float]) -> dict[str, float | int]: + return { + "count": len(values), "median": statistics.median(values), + "p75": linear_quantile(values, 0.75), "p90": linear_quantile(values, 0.90), + "p95": linear_quantile(values, 0.95), "p99": linear_quantile(values, 0.99), + } + + +def _profiles(root: Path, rows: list[dict[str, Any]], pairs: list[tuple[int, int]]) -> tuple[dict[str, Any], dict[str, Any]]: + samples = {name: [] for name in COMPONENT_FIELDS} + for left_index, right_index in pairs: + left, right = rows[left_index], rows[right_index] + for name, sample_values in samples.items(): + value = _raw_component(left, right, name) + if value is not None: + sample_values.append(value) + distributions = {name: _distribution(values) for name, values in samples.items()} + scales = {name: values["p95"] for name, values in distributions.items()} + normalized_path = root / "data" / "processed" / "artwork-normalized-features-v1.parquet" + features_path = root / "data" / "processed" / "artwork-visual-features-v1.parquet" + normalization_path = root / "data" / "processed" / "normalization-profile-v1.json" + normalization = json.loads(normalization_path.read_text(encoding="utf-8")) + + required = { + "color-similarity-v1": ["centroid_l", "centroid_a", "centroid_b", + "palette_spread_oklab_robust_01", "chroma_stddev_robust_01"], + "visual-character-similarity-v1": ["centroid_l", "centroid_a", "centroid_b", + "palette_spread_oklab_robust_01", "chroma_stddev_robust_01"], + "geometry-similarity-v1": ["log_net_width_cm_robust_01", "log_net_height_cm_robust_01", + "log_net_area_cm2_robust_01", "log_net_aspect_ratio_robust_01"], + } + optional = { + "color-similarity-v1": ["hue_concentration_01"], + "visual-character-similarity-v1": ["hue_concentration_01", "smk_brightness_robust_01", + "smk_contrast_robust_01", "smk_saturation_robust_01", + "smk_entropy_robust_01"], + "geometry-similarity-v1": [], + } + profiles = {} + for name, weights in WEIGHTS.items(): + profiles[name] = { + "name": name, "profile_version": PROFILE_VERSION, + "required_features": required[name], "optional_features": optional[name], + "components": [ + {"name": component, "fields": COMPONENT_FIELDS[component][0], + "metric": COMPONENT_FIELDS[component][1], "transform": "raw OKLab" if component == "centroid_oklab" else "robust_01/natural_01 absolute difference", + "scale": scales[component], "scale_rule": "deterministic pair-sample P95", + "weight": weight} + for component, weight in weights.items() + ], + "missing_value_behavior": "required missing => ineligible; optional missing on either side => omit and renormalize active weights", + "distance_semantics": "weighted mean of P95-scaled component distances clipped to [0,1]; lower is more similar", + "tie_breaking": ["total_distance ascending", "smk_entity_id ascending"], + } + payload = { + "profile_set_version": PROFILE_VERSION, + "generated_at_utc": normalization["generated_at_utc"], + "corpus_version": normalization["corpus_version"], + "normalization_profile": normalization["normalization_profile"], + "normalization_version": normalization["normalization_version"], + "input_hashes": { + "artwork_visual_features_v1_sha256": sha256_file(features_path), + "artwork_normalized_features_v1_sha256": sha256_file(normalized_path), + "normalization_profile_v1_sha256": sha256_file(normalization_path), + }, + "calibration": {"seed": PAIR_SEED, "distinct_pair_count": len(pairs), + "component_distributions": distributions}, + "profiles": profiles, + } + return payload, distributions + + +def _eligible_count(rows: list[dict[str, Any]], profile: dict[str, Any]) -> int: + return sum(all(row.get(field) is not None for field in profile["required_features"]) for row in rows) + + +def _profile_pair_rows(rows: list[dict[str, Any]], pairs: list[tuple[int, int]], + profiles: dict[str, Any]) -> list[dict[str, Any]]: + output = [] + for profile_name, profile in profiles.items(): + written = 0 + for left_index, right_index in pairs: + result = compare(rows[left_index], rows[right_index], profile) + if result is None: + continue + left, right = rows[left_index], rows[right_index] + output.append({ + "left_smk_entity_id": left["smk_entity_id"], + "right_smk_entity_id": right["smk_entity_id"], "profile": profile_name, + "total_distance": result["total_distance"], + "active_weight_sum": result["active_weight_sum"], + "active_components": result["active_components"], + "component_distances_json": json.dumps(result["components"], sort_keys=True, separators=(",", ":")), + }) + written += 1 + if written >= PERSISTED_PAIRS_PER_PROFILE: + break + return output + + +def _select_qa(rows: list[dict[str, Any]]) -> list[str]: + def entity_for(key: str, *, maximum: bool) -> str: + valid = [row for row in rows if row.get(key) is not None] + selected = sorted(valid, key=lambda row: (float(row[key]), row["smk_entity_id"]), reverse=maximum)[0] + return selected["smk_entity_id"] + incomplete = min((row for row in rows if row["smk_saturation"] is None), key=lambda row: row["smk_entity_id"]) + ids = [ + next(row["smk_entity_id"] for row in rows if row["object_number"] == "KMS8892"), + next(row["smk_entity_id"] for row in rows if row["object_number"] == "KMS1571"), + entity_for("centroid_chroma", maximum=True), entity_for("palette_spread_oklab", maximum=True), + entity_for("palette_spread_oklab", maximum=False), incomplete["smk_entity_id"], + ] + return list(dict.fromkeys(ids)) + + +def _stability(engine: SimilarityEngine, query_ids: list[str]) -> list[dict[str, Any]]: + output = [] + for query_id in query_ids: + query = engine.by_id[query_id] + for profile_name in engine.profiles: + profile = engine.profile(profile_name) + if any(query.get(field) is None for field in profile["required_features"]): + continue + baseline = engine.search_row(query, profile_name, 10) + baseline_ids = {item["smk_entity_id"] for item in baseline} + for component in profile["components"]: + for factor in (0.85, 1.15): + varied = engine.search_row( + query, profile_name, 10, + weight_overrides={component["name"]: component["weight"] * factor}, + ) + overlap = len(baseline_ids & {item["smk_entity_id"] for item in varied}) + output.append({"query_smk_entity_id": query_id, "profile": profile_name, + "component": component["name"], "factor": factor, + "top10_overlap": overlap}) + return output + + +def _examples_markdown(engine: SimilarityEngine, qa_ids: list[str]) -> tuple[str, dict[str, Any]]: + lines = ["# Similarity examples", "", "All statements below are deterministic feature comparisons, not aesthetic judgments.", ""] + payload = {} + for query_id in qa_ids: + query = engine.by_id[query_id] + key = query["object_number"] or query_id + payload[key] = {} + lines.extend([f"## {key}", "", f"Entity: `{query_id}`. Title: {engine.titles.get(query_id) or 'unavailable'}.", ""]) + query_features = { + name: query.get(name) for name in ( + "centroid_l", "centroid_a", "centroid_b", "palette_spread_oklab", + "chroma_stddev", "hue_concentration_01", "smk_brightness", + "smk_contrast", "smk_saturation", "smk_entropy", "log_net_width_cm", + "log_net_height_cm", "log_net_area_cm2", "log_net_aspect_ratio", + ) + } + lines.extend(["Query features:", "", f"`{json.dumps(query_features, sort_keys=True)}`", ""]) + for profile_name in engine.profiles: + profile = engine.profile(profile_name) + if any(query.get(field) is None for field in profile["required_features"]): + lines.extend([f"### {profile_name}", "", "Not eligible: a required feature is null.", ""]) + continue + results = engine.search_row(query, profile_name, 10) + payload[key][profile_name] = results + lines.extend([f"### {profile_name}", "", "| Rank | Object | Title | Distance | Contributions | Factual explanation |", "|---:|---|---|---:|---|---|"]) + for result in results: + contributions = ", ".join( + f"{name}={part['contribution']:.4f}" + for name, part in result["components"].items() + ) + centroid = result["components"].get("centroid_oklab") + explanation = "shared normalized feature values" + if centroid: + explanation = "similar centroid color" if centroid["scaled_distance"] < 0.25 else "centroid color contributes substantially" + if "hue_concentration" not in result["active_components"] and profile_name != "geometry-similarity-v1": + explanation += "; hue concentration unavailable" + title = (result["title"] or "unavailable").replace("|", "\\|") + lines.append(f"| {result['rank']} | {result['object_number']} | {title} | {result['total_distance']:.6f} | {contributions} | {explanation} |") + lines.append("") + return "\n".join(lines) + "\n", payload + + +def run(root: Path) -> dict[str, Any]: + normalized_path = root / "data" / "processed" / "artwork-normalized-features-v1.parquet" + load_start = time.perf_counter() + rows = pq.read_table(normalized_path).to_pylist() + load_seconds = time.perf_counter() - load_start + pairs = _sample_pairs(len(rows), CALIBRATION_PAIR_COUNT, PAIR_SEED) + profile_payload, distributions = _profiles(root, rows, pairs) + profile_path = root / "data" / "processed" / "similarity-profiles-v1.json" + profile_path.write_text(json.dumps(profile_payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + metadata = { + b"smk.similarity_profile_set_version": PROFILE_VERSION.encode(), + b"smk.similarity_profiles_sha256": hashlib.sha256(profile_path.read_bytes()).hexdigest().encode(), + } + pair_rows = _profile_pair_rows(rows, pairs, profile_payload["profiles"]) + pair_table = pa.Table.from_pylist(pair_rows).replace_schema_metadata(metadata) + pq.write_table(pair_table, root / "data" / "reports" / "similarity-pair-sample-v1.parquet", + compression="zstd", version="2.6", write_statistics=True) + + engine_start = time.perf_counter() + engine = SimilarityEngine(root) + engine_load_seconds = time.perf_counter() - engine_start + qa_ids = _select_qa(rows) + markdown, examples = _examples_markdown(engine, qa_ids) + (root / "docs" / "SIMILARITY_EXAMPLES.md").write_text(markdown, encoding="utf-8") + kms = engine.resolve(object_number="KMS8892") + calculation_times = [] + ranking_times = [] + for _ in range(5): + start = time.perf_counter() + unsorted = [] + profile = engine.profile("color") + for candidate in engine.rows: + if candidate["smk_entity_id"] == kms["smk_entity_id"]: + continue + result = compare(kms, candidate, profile) + if result is not None: + unsorted.append((result["total_distance"], candidate["smk_entity_id"])) + calculation_times.append(time.perf_counter() - start) + rank_start = time.perf_counter() + unsorted.sort() + unsorted[:10] + ranking_times.append(time.perf_counter() - rank_start) + stability = _stability(engine, qa_ids) + summary = { + "profile_set_version": PROFILE_VERSION, + "input_hashes": profile_payload["input_hashes"], + "eligible_counts": {name: _eligible_count(rows, profile) + for name, profile in profile_payload["profiles"].items()}, + "pair_calibration": {"seed": PAIR_SEED, "distinct_pair_count": len(pairs), + "distributions": distributions}, + "persisted_pair_rows": len(pair_rows), "qa_query_ids": qa_ids, + "kms8892": examples["KMS8892"], + "ranking_stability": { + "variation": "each component weight independently multiplied by 0.85 and 1.15", + "minimum_top10_overlap": min(item["top10_overlap"] for item in stability), + "median_top10_overlap": statistics.median(item["top10_overlap"] for item in stability), + "cases": stability, + }, + "performance": { + "direct_normalized_parquet_load_seconds": load_seconds, + "engine_load_seconds_including_titles": engine_load_seconds, + "exact_color_distance_calculation_seconds_median_of_5": statistics.median(calculation_times), + "full_sort_and_top10_seconds_median_of_5": statistics.median(ranking_times), + "exact_color_query_total_seconds_median_of_5": statistics.median( + [calculation + ranking for calculation, ranking in zip(calculation_times, ranking_times)] + ), + "corpus_rows": len(rows), + }, + } + summary_path = root / "data" / "reports" / "similarity-summary-v1.json" + summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return summary + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path.cwd()) + args = parser.parse_args() + print(json.dumps(run(args.root.resolve()), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_similarity.py b/tests/test_similarity.py new file mode 100644 index 0000000..ef11174 --- /dev/null +++ b/tests/test_similarity.py @@ -0,0 +1,71 @@ +"""Focused invariants for exact similarity.""" + +import pytest + +from smk_playground.similarity.engine import SimilarityEngine, compare + +PROFILE = { + "name": "test", + "required_features": ["centroid_l", "centroid_a", "centroid_b", "spread"], + "components": [ + {"name": "centroid", "fields": ["centroid_l", "centroid_a", "centroid_b"], + "metric": "euclidean", "scale": 0.5, "weight": 0.8}, + {"name": "spread", "fields": ["spread"], "metric": "absolute", + "scale": 1.0, "weight": 0.15}, + {"name": "hue", "fields": ["hue"], "metric": "absolute", + "scale": 1.0, "weight": 0.05}, + ], +} + + +def row(entity_id, l, spread, hue=0.5, geometry=0.2): + return {"smk_entity_id": entity_id, "object_number": entity_id, "centroid_l": l, + "centroid_a": 0.0, "centroid_b": 0.0, "spread": spread, "hue": hue, + "geometry": geometry, "quality_flags": []} + + +def test_self_distance_and_contribution_sum(): + item = row("a", 0.5, 0.2) + result = compare(item, item, PROFILE) + assert result["total_distance"] == 0.0 + assert sum(part["contribution"] for part in result["components"].values()) == 0.0 + + +def test_symmetry_and_optional_component_renormalization(): + left, right = row("a", 0.5, 0.2, None), row("b", 0.6, 0.4, 0.8) + forward, backward = compare(left, right, PROFILE), compare(right, left, PROFILE) + assert forward["total_distance"] == pytest.approx(backward["total_distance"]) + assert forward["active_components"] == ["centroid", "spread"] + assert forward["active_weight_sum"] == pytest.approx(0.95) + assert sum(part["contribution"] for part in forward["components"].values()) == pytest.approx(forward["total_distance"]) + + +def test_required_feature_and_geometry_eligibility(): + missing = row("a", 0.5, None) + assert compare(missing, row("b", 0.6, 0.2), PROFILE) is None + geometry_profile = {"name": "geometry", "required_features": ["geometry"], + "components": [{"name": "geometry", "fields": ["geometry"], + "metric": "absolute", "scale": 1.0, "weight": 1.0}]} + assert compare(row("a", 0.5, 0.2, geometry=None), row("b", 0.6, 0.2), geometry_profile) is None + + +def test_exact_topk_stable_tie_breaking_and_achromatic_eligibility(): + engine = SimilarityEngine.__new__(SimilarityEngine) + engine.rows = [row("q", 0.5, 0.2, None), row("b", 0.6, 0.2), row("a", 0.4, 0.2), row("z", 0.9, 0.9)] + engine.by_id = {item["smk_entity_id"]: item for item in engine.rows} + engine.by_object_number = {item["object_number"]: [item] for item in engine.rows} + engine.titles = {} + engine.profiles = {"color-similarity-v1": PROFILE} + first = engine.search_row(engine.rows[0], "color", 2) + second = engine.search_row(engine.rows[0], "color", 2) + assert [item["smk_entity_id"] for item in first] == ["a", "b"] + assert first == second + + +def test_object_number_ambiguity_requires_entity_id(): + engine = SimilarityEngine.__new__(SimilarityEngine) + engine.by_id = {"a": row("a", 0.1, 0.1), "b": row("b", 0.2, 0.2)} + engine.by_object_number = {"shared": [engine.by_id["a"], engine.by_id["b"]]} + with pytest.raises(ValueError, match="Ambiguous"): + engine.resolve(object_number="shared") + assert engine.resolve(smk_entity_id="b")["smk_entity_id"] == "b"