From 1503cbca8cd80c246c9161ecd31652102d169005 Mon Sep 17 00:00:00 2001 From: Christian Johannesen Date: Thu, 8 Oct 2026 11:22:37 +0200 Subject: [PATCH] Complete Milestone 6 weighted enrichment --- AGENTS.md | 4 + README.md | 9 +- docs/DATA_MODEL.md | 7 + docs/ENRICHMENT_V1.md | 32 ++ docs/FEATURE_SPEC_V1.md | 3 + docs/IMPLEMENTATION_PLAN.md | 6 +- docs/PIPELINE.md | 7 + docs/SIMILARITY_V1.md | 3 + docs/WEIGHTED_PALETTES_V1.md | 52 +++ docs/WEIGHTED_SIMILARITY_V1.md | 73 +++ src/smk_playground/cli.py | 66 ++- src/smk_playground/enrichment/__init__.py | 1 + src/smk_playground/enrichment/fetch.py | 187 ++++++++ src/smk_playground/enrichment/milestone6.py | 481 ++++++++++++++++++++ src/smk_playground/enrichment/weighted.py | 136 ++++++ tests/test_enrichment.py | 102 +++++ 16 files changed, 1165 insertions(+), 4 deletions(-) create mode 100644 docs/ENRICHMENT_V1.md create mode 100644 docs/WEIGHTED_PALETTES_V1.md create mode 100644 docs/WEIGHTED_SIMILARITY_V1.md create mode 100644 src/smk_playground/enrichment/__init__.py create mode 100644 src/smk_playground/enrichment/fetch.py create mode 100644 src/smk_playground/enrichment/milestone6.py create mode 100644 src/smk_playground/enrichment/weighted.py create mode 100644 tests/test_enrichment.py diff --git a/AGENTS.md b/AGENTS.md index 55a885c..dd7f7e9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,6 +45,10 @@ geometry. Keep their questions separate, return component contributions, omit an missing optional components symmetrically, and tie-break by distance then canonical entity ID. Do not call distance a percentage similarity or add ANN before measured need. +Milestone 6 keeps enrichment as a separate append-only source/cache. Weighted palette/features v2 +must never overwrite bulk uniform v1. Corpus membership remains unchanged when enrichment is +missing. Palette transport uses exact OKLab min-cost flow, and weighted similarity is a new profile. + Never overwrite source data. Never silently invent missing values. ## Primary bulk source diff --git a/README.md b/README.md index 836e29e..a4ad430 100644 --- a/README.md +++ b/README.md @@ -121,12 +121,14 @@ See: ## Current status -Milestones 0–5 complete: source structure and audits are manifested, the language snapshots have +Milestones 0–6 complete: source structure and audits are manifested, the language snapshots have bounded-memory source Parquets, DA/EN rows are paired into stable SMK entities, and the exact `smk-public-visual-v1` membership rule is frozen. Deterministic raw visual features and analytical palettes now exist. Versioned corpus statistics, normalized scalar features and three unweighted candidate feature families are materialized. Three deterministic, explainable exact-distance profiles now support color, visual-character and specialist geometry queries; ANN is not used. +Separately cached SMK enrichment responses add weighted palettes, v2 weighted features and an +exact palette-transport similarity profile without mutating the bulk-derived v1 artifacts. Audit results are documented in: @@ -143,6 +145,9 @@ Audit results are documented in: - [`docs/FEATURE_VECTOR_V1.md`](docs/FEATURE_VECTOR_V1.md) - [`docs/SIMILARITY_V1.md`](docs/SIMILARITY_V1.md) - [`docs/SIMILARITY_EXAMPLES.md`](docs/SIMILARITY_EXAMPLES.md) +- [`docs/ENRICHMENT_V1.md`](docs/ENRICHMENT_V1.md) +- [`docs/WEIGHTED_PALETTES_V1.md`](docs/WEIGHTED_PALETTES_V1.md) +- [`docs/WEIGHTED_SIMILARITY_V1.md`](docs/WEIGHTED_SIMILARITY_V1.md) Run the audited ingest with: @@ -153,4 +158,6 @@ smk milestone3 smk milestone4 smk milestone5 smk similar KMS8892 --profile color --limit 10 +smk enrichment status +smk similar KMS8892 --profile weighted-color --limit 10 ``` diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index 6d9e6ec..697b463 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -72,6 +72,13 @@ Defines exact transforms, sampled-pair scales, weights, required/optional inputs tie-breaking and bound input hashes for the three Milestone 5 profiles. Similarity results are computed exactly on demand and expose per-component contributions; there is no full pair matrix. +### Enrichment-backed weighted layer + +`enrichment-source-manifest-v1.json` freezes separately cached object-number responses. +`artwork-weighted-palettes-v1.parquet` stores token-frequency weights, and +`artwork-visual-features-v2.parquet` stores weighted derivatives plus availability/status for every +base-corpus entity. These additive artifacts never redefine `smk-public-visual-v1` or overwrite v1. + ### `quality-report.json` Corpus-wide counts, missingness, exclusions, warnings and distribution diagnostics. diff --git a/docs/ENRICHMENT_V1.md b/docs/ENRICHMENT_V1.md new file mode 100644 index 0000000..ecfb216 --- /dev/null +++ b/docs/ENRICHMENT_V1.md @@ -0,0 +1,32 @@ +# SMK enrichment source v1 + +Milestone 6 treats `https://enrichment.api.smk.dk/api/enrichment/` as a separate +source. It never writes into or reinterprets the immutable DA/EN bulk snapshots. + +The frozen corpus has 27,254 entities but only 27,238 unique usable object numbers. Sixteen object +numbers each identify two corpus entities. Retrieval therefore occurs once per unique object +number, while the response manifest explicitly records every corresponding `smk_entity_id`. + +## Cache and provenance + +Each attempt is stored under `data/raw/enrichment//.json` with +request/response timestamps, endpoint, HTTP status or transport failure, attempt number and the +lossless parsed JSON response. `fetch-state-v1.json` is an atomic resumable checkpoint pointing to +the latest attempt. Previous timestamped responses are never overwritten. + +The defaults are two workers, 0.2 seconds delay per request and conservative exponential retry. +Successful cached records are skipped under `--resume`; failures remain visible and retryable. +`--refresh` creates a new timestamped cache response. A refreshed analysis must be frozen into a +new manifest/version rather than silently replacing an already published manifest. + +```text +smk enrichment status +smk enrichment fetch --resume --concurrency 2 +smk enrichment fetch --limit 100 +smk enrichment inspect KMS8892 +``` + +`enrichment-source-manifest-v1.json` binds selected cache responses, outcomes, component metadata, +weighted-palette status and canonical-entity mappings. Missing, malformed, 404 and transport/server +outcomes remain explicit. + diff --git a/docs/FEATURE_SPEC_V1.md b/docs/FEATURE_SPEC_V1.md index 7f98c9e..be2e8cd 100644 --- a/docs/FEATURE_SPEC_V1.md +++ b/docs/FEATURE_SPEC_V1.md @@ -22,6 +22,9 @@ For `colors_palette_weighted_s`, the observed format may contain repeated space- Uniform v1 weights are `palette_weighting = "uniform"`, never original SMK weights. Future enrichment weights must retain separate provenance. +Milestone 6 implements those enrichment weights in separately versioned palette schema v1 and +feature schema v2. The uniform bulk feature v1 definition in this document remains unchanged. + Canonical weights must satisfy approximately: ```text diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index 92d4da1..8ba0592 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -76,8 +76,10 @@ contributions. QA examples, stability diagnostics and performance measurements a ## Milestone 6 — Similarity preparation -Review Milestone 4 correlations and feature families, then define separately versioned distance -profiles and weights. Do not reinterpret normalized OKLab axes as perceptual OKLab geometry. +Status: enrichment-backed weighted color layer implemented separately from bulk v1. It provides a +resumable append-only source cache, weighted palettes/features v2, bulk/enrichment diagnostics and +an exact palette-transport similarity profile. See `ENRICHMENT_V1.md`, `WEIGHTED_PALETTES_V1.md` +and `WEIGHTED_SIMILARITY_V1.md`. ## Milestone 7 — Full dataset build diff --git a/docs/PIPELINE.md b/docs/PIPELINE.md index 6d0f187..7170688 100644 --- a/docs/PIPELINE.md +++ b/docs/PIPELINE.md @@ -92,6 +92,13 @@ visual-character and geometry profiles, and perform exact full-corpus ranking wi contributions. Missing optional fields renormalize shared weights; required fields determine eligibility. No ANN index, universal score or palette transport is introduced. +## Stage 6 — enrichment-backed weighted color + +Fetch only unique frozen-corpus object numbers into an append-only resumable cache, freeze a +separate source manifest, parse repeated palette tokens into explicit weights, emit weighted +palette/features v2, and add exact OKLab palette transport under a new similarity profile. Existing +bulk v1 corpus, features, normalization and profiles remain byte-independent inputs. + ## Ingest performance rule The source files are large. Use bounded-memory streaming where practical. A one-time conversion to Parquet/Arrow as an interim representation is encouraged if it simplifies all later stages. diff --git a/docs/SIMILARITY_V1.md b/docs/SIMILARITY_V1.md index d2047e3..e5e4248 100644 --- a/docs/SIMILARITY_V1.md +++ b/docs/SIMILARITY_V1.md @@ -100,3 +100,6 @@ smk similar --id 1170000000_object --profile visual --limit 10 Ambiguous object numbers produce an error and list canonical IDs; they are never silently collapsed. `--include-self` is available for debugging and yields distance zero. +Milestone 6 adds `weighted-color-similarity-v1` in a separate v2 profile set. It does not alter +the scales, weights or outputs documented here; see `WEIGHTED_SIMILARITY_V1.md`. + diff --git a/docs/WEIGHTED_PALETTES_V1.md b/docs/WEIGHTED_PALETTES_V1.md new file mode 100644 index 0000000..048f8cd --- /dev/null +++ b/docs/WEIGHTED_PALETTES_V1.md @@ -0,0 +1,52 @@ +# Weighted enrichment palettes v1 + +The weighted palette source is `colorextractor.data.colors_palette_weighted_s` from separately +cached SMK enrichment responses. It is not present in the bulk snapshots and does not replace the +uniform bulk palette v1. + +Observed values are whitespace-separated `#RRGGBB` tokens with repeated colors. Parsing is: + +1. tokenize non-whitespace values; +2. validate complete hex tokens and normalize case; +3. retain first-occurrence order; +4. count each color; +5. assign `weight = token_count / valid_token_count`. + +Invalid tokens and partial/missing values are flagged rather than silently discarded. The source +record, raw field fingerprint, token count, unique-color count, component and `_version_` remain +traceable through the cache and enrichment manifest. + +`artwork-weighted-palettes-v1.parquet` uses palette source `smk_enrichment`, schema `1.0.0` and +weighting `enrichment_token_frequency`. Every entity palette sums to one within tolerance and +stores sRGB, OKLab and OKLCH coordinates. + +`artwork-visual-features-v2.parquet` retains one row for every base-corpus entity. Availability is +represented by `has_weighted_palette_v1` and `weighted_palette_status`; it does not change corpus +membership. Weighted rows recompute centroid, variation, spread, Shannon entropy, normalized +entropy, effective palette size, hue concentration, dominant color and dominant weight. Explicit +lineage distinguishes them from uniform bulk features v1. + +Final coverage/distributions and bulk-versus-enrichment comparison are recorded in +`enrichment-summary-v1.json` after the frozen cache build. + +## Frozen-corpus findings + +All 27,238 unique object-number responses succeeded and contained a valid token palette, covering +all 27,254 entities. Token counts range from 437 to 506 (median 499); unique colors range from 1 to +15 (median 4, P95 7). No palette has one token per color. There are 2,420 single-color palettes and +3,906 entities with dominant weight at least 0.9, so strongly concentrated palettes are real but +not universal. + +Compared with bulk palette color sets, 27,243 entities match exactly, nine partially overlap and +two have no overlap. Six differ in palette size and 12 differ in order. This supports shared color +extraction lineage for nearly all records while still requiring separate provenance. + +Dominant weight has median 0.50. Weighted entropy has median 1.08 and effective palette size median +2.96. Weighted palette spread has median 0.10. Median v1→v2 centroid shift is 0.08 OKLab (P95 0.26, +maximum 0.59), demonstrating that token-frequency weights materially change color summaries. + +Every observed component `_version_` is record-specific in this snapshot (the distinct count per +component equals its record count). It behaves as a per-document update/version token, not a small +algorithm-version vocabulary suitable for distribution grouping. Component type and exact token +are therefore retained per record; no incompatible version buckets were silently merged. + diff --git a/docs/WEIGHTED_SIMILARITY_V1.md b/docs/WEIGHTED_SIMILARITY_V1.md new file mode 100644 index 0000000..c7bfd8a --- /dev/null +++ b/docs/WEIGHTED_SIMILARITY_V1.md @@ -0,0 +1,73 @@ +# Weighted color similarity v1 + +`weighted-color-similarity-v1` is additive. It does not modify `color-similarity-v1`. + +## Palette transport + +For weighted palettes A and B, the transport component minimizes: + +```text +sum(i,j) flow(i,j) * EuclideanOKLab(colorA_i, colorB_j) +``` + +subject to every source color shipping its full weight and every target color receiving its full +weight. The implementation uses deterministic exact min-cost flow on the tiny complete bipartite +palette graph. It is symmetric, order-independent and zero for identical weighted palettes. + +## Profile components + +The frozen profile prioritizes palette transport and weighted centroid, with smaller spread, +chroma-variation, hue-concentration and dominant-weight modifiers. Each raw component is divided +by its P95 from a fixed 10,000-pair eligible sample and clipped to `[0,1]`. Optional achromatic hue +is omitted symmetrically and active weights are renormalized. + +The exact weights, scales, input hashes and calibration distributions live in +`similarity-profiles-v2.json`. Search is exact across every weighted-eligible entity; no ANN or +two-stage candidate heuristic is used unless later measurement justifies it. + +```text +smk similar KMS8892 --profile weighted-color --limit 10 +``` + +The final KMS8892 palette, neighbors, comparison with bulk color v1 and measured query performance +are stored in `weighted-similarity-summary-v1.json`. + +## Frozen profile and calibration + +| Component | Weight | P95 scale | +|---|---:|---:| +| weighted centroid OKLab | 0.30 | 0.5000 | +| palette transport OKLab | 0.40 | 0.5092 | +| weighted palette spread | 0.12 | 0.6770 | +| weighted chroma variation | 0.08 | 0.7015 | +| weighted hue concentration | 0.05 | 0.2470 | +| dominant-weight difference | 0.05 | 0.8083 | + +Transport distance over 10,000 deterministic pairs has median 0.1738, P90 0.3968, P95 0.5092 and +P99 0.6614. Exact full-corpus KMS8892 search takes about 12.3 seconds locally. A scalar prefilter is +not adopted: only five weighted top-10 results occur in the bulk color top 200, six in the top 500 +or 1,000, and one has bulk rank 6,295. Candidate reranking would therefore miss genuine weighted +neighbors. + +## KMS8892 + +The enrichment palette contains 500 tokens: + +| Color | Tokens | Weight | +|---|---:|---:| +| `#908d41` | 120 | 0.240 | +| `#fff1dc` | 103 | 0.206 | +| `#aed7ea` | 94 | 0.188 | +| `#ceeae5` | 89 | 0.178 | +| `#b7ac3e` | 33 | 0.066 | +| `#888888` | 31 | 0.062 | +| `#795025` | 26 | 0.052 | + +Weighted centroid OKLab is `(0.7914, -0.0176, 0.0340)`. Dominant color is `#908d41`; entropy is +1.8014, effective palette size 6.0579, spread 0.1549 and hue concentration 0.6577. + +Its weighted top 10 are KMS5549, KMS1851, KKS11245/58, KKS5936d, KKS16225a, KKSgb18373, +KKSgb8443, KKSgb7517, KKSgb13592 and KMS4209, with distances 0.1060–0.1274. Overlap with the bulk +color-v1 top 10 is zero. The ranking change follows the newly informative token-frequency mass and +palette transport, rather than a changed corpus or overwritten v1 feature. + diff --git a/src/smk_playground/cli.py b/src/smk_playground/cli.py index 93a0db0..23ebe5e 100644 --- a/src/smk_playground/cli.py +++ b/src/smk_playground/cli.py @@ -6,6 +6,8 @@ from pathlib import Path import typer app = typer.Typer(no_args_is_help=True, help="SMK visual-feature pipeline") +enrichment_app = typer.Typer(no_args_is_help=True, help="SMK enrichment cache") +app.add_typer(enrichment_app, name="enrichment") @app.command() @@ -65,16 +67,78 @@ def milestone5(root: Path | None = None) -> None: typer.echo(f"Milestone 5 complete: {summary['eligible_counts']}") +@app.command("milestone6") +def milestone6(root: Path | None = None) -> None: + """Build enrichment-backed weighted palettes, v2 features, and similarity.""" + from smk_playground.enrichment.milestone6 import run + + summary = run((root or Path.cwd()).resolve()) + typer.echo(f"Milestone 6 complete: {summary['weighted_palette_entity_count']} weighted entities") + + +@enrichment_app.command("status") +def enrichment_status(root: Path | None = None) -> None: + """Report resumable enrichment-cache status.""" + from smk_playground.enrichment.fetch import status + + typer.echo(json.dumps(status((root or Path.cwd()).resolve()), ensure_ascii=False, indent=2)) + + +@enrichment_app.command("fetch") +def enrichment_fetch( + limit: int | None = typer.Option(None, min=1), + resume: bool = typer.Option(True, "--resume/--no-resume"), + refresh: bool = typer.Option(False), + concurrency: int = typer.Option(2, min=1, max=16), + delay: float = typer.Option(0.2, min=0.0), + root: Path | None = None, +) -> None: + """Fetch corpus enrichment responses with cache/checkpoints.""" + from smk_playground.enrichment.fetch import fetch + + result = fetch((root or Path.cwd()).resolve(), limit=limit, resume=resume, + refresh=refresh, concurrency=concurrency, delay=delay) + typer.echo(json.dumps(result, ensure_ascii=False, indent=2)) + + +@enrichment_app.command("inspect") +def enrichment_inspect(object_number: str, root: Path | None = None) -> None: + """Print one cached enrichment response.""" + from smk_playground.enrichment.fetch import cached_record + + try: + result = cached_record((root or Path.cwd()).resolve(), object_number) + except ValueError as error: + raise typer.BadParameter(str(error)) from error + typer.echo(json.dumps(result, ensure_ascii=False, indent=2)) + + @app.command("similar") def similar( object_number: str | None = typer.Argument(None, help="SMK object number"), smk_entity_id: str | None = typer.Option(None, "--id", help="Canonical SMK entity ID"), - profile: str = typer.Option("color", help="color, visual, or geometry"), + profile: str = typer.Option("color", help="color, visual, geometry, or weighted-color"), limit: int = typer.Option(10, min=1, max=100), include_self: bool = typer.Option(False, help="Include the query entity for debugging"), root: Path | None = None, ) -> None: """Run an exact similarity query over the frozen corpus.""" + if profile == "weighted-color": + from smk_playground.enrichment.milestone6 import WeightedSimilarityEngine + + try: + engine = WeightedSimilarityEngine((root or Path.cwd()).resolve()) + query = engine.resolve(object_number=object_number, smk_entity_id=smk_entity_id) + results = engine.search_row(query, limit, include_self) + except ValueError as error: + raise typer.BadParameter(str(error)) from error + typer.echo(json.dumps({ + "query": {"smk_entity_id": query["smk_entity_id"], + "object_number": query["object_number"]}, + "profile": profile, "results": results, + }, ensure_ascii=False, indent=2)) + return + from smk_playground.similarity.engine import SimilarityEngine try: diff --git a/src/smk_playground/enrichment/__init__.py b/src/smk_playground/enrichment/__init__.py new file mode 100644 index 0000000..3609c5c --- /dev/null +++ b/src/smk_playground/enrichment/__init__.py @@ -0,0 +1 @@ +"""Separately versioned SMK enrichment source support.""" diff --git a/src/smk_playground/enrichment/fetch.py b/src/smk_playground/enrichment/fetch.py new file mode 100644 index 0000000..1cd4f1d --- /dev/null +++ b/src/smk_playground/enrichment/fetch.py @@ -0,0 +1,187 @@ +"""Polite, resumable enrichment retrieval for the frozen corpus.""" + +from __future__ import annotations + +import hashlib +import json +import time +import urllib.error +import urllib.parse +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import UTC, datetime +from pathlib import Path +from threading import Lock +from typing import Any + +import pyarrow.parquet as pq + +ENDPOINT = "https://enrichment.api.smk.dk/api/enrichment/{object_number}" +USER_AGENT = "smk-playground/0.1 enrichment-research" +STATE_VERSION = "1.0.0" + + +def _timestamp() -> str: + return datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _safe_key(object_number: str) -> str: + return hashlib.sha256(object_number.encode("utf-8")).hexdigest() + + +def corpus_targets(root: Path) -> tuple[list[str], dict[str, list[str]]]: + path = root / "data" / "processed" / "smk-public-visual-v1.parquet" + rows = pq.read_table(path, columns=["smk_entity_id", "object_number"]).to_pylist() + mapping: dict[str, list[str]] = {} + for row in rows: + number = row["object_number"] + if not number: + raise ValueError(f"Corpus entity lacks object_number: {row['smk_entity_id']}") + mapping.setdefault(number, []).append(row["smk_entity_id"]) + return sorted(mapping), {key: sorted(value) for key, value in mapping.items()} + + +def _load_state(path: Path) -> dict[str, Any]: + if not path.exists(): + return {"state_version": STATE_VERSION, "records": {}} + return json.loads(path.read_text(encoding="utf-8")) + + +def _save_state(path: Path, state: dict[str, Any]) -> None: + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(state, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + temporary.replace(path) + + +def _fetch_one(object_number: str, cache_dir: Path, delay: float, retries: int) -> dict[str, Any]: + endpoint = ENDPOINT.format(object_number=urllib.parse.quote(object_number, safe="")) + outcome: dict[str, Any] | None = None + for attempt in range(retries + 1): + if delay: + time.sleep(delay) + requested_at = _timestamp() + try: + request = urllib.request.Request(endpoint, headers={"User-Agent": USER_AGENT}) + with urllib.request.urlopen(request, timeout=30) as response: + body = response.read().decode("utf-8") + status_code = response.status + parsed = json.loads(body) + outcome = { + "object_number": object_number, "requested_at_utc": requested_at, + "response_at_utc": _timestamp(), "endpoint": endpoint, + "http_status": status_code, "outcome": "success", + "attempt": attempt + 1, "response_json": parsed, + } + break + except urllib.error.HTTPError as error: + body = error.read().decode("utf-8", errors="replace") + category = "not_found" if error.code == 404 else "http_failure" + outcome = { + "object_number": object_number, "requested_at_utc": requested_at, + "response_at_utc": _timestamp(), "endpoint": endpoint, + "http_status": error.code, "outcome": category, + "attempt": attempt + 1, "response_body": body, + } + if error.code < 500: + break + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as error: + outcome = { + "object_number": object_number, "requested_at_utc": requested_at, + "response_at_utc": _timestamp(), "endpoint": endpoint, + "http_status": None, "outcome": "transport_failure", + "attempt": attempt + 1, "error": repr(error), + } + if attempt < retries: + time.sleep(0.5 * (2 ** attempt)) + assert outcome is not None + stamp = outcome["response_at_utc"].replace(":", "").replace("-", "") + record_dir = cache_dir / _safe_key(object_number) + record_dir.mkdir(parents=True, exist_ok=True) + path = record_dir / f"{stamp}.json" + path.write_text(json.dumps(outcome, ensure_ascii=False, separators=(",", ":")) + "\n", encoding="utf-8") + outcome["cache_path"] = path.as_posix() + return outcome + + +def fetch( + root: Path, + *, + limit: int | None = None, + resume: bool = True, + refresh: bool = False, + concurrency: int = 2, + delay: float = 0.2, + retries: int = 2, +) -> dict[str, Any]: + if concurrency < 1 or concurrency > 16: + raise ValueError("concurrency must be between 1 and 16") + targets, entity_mapping = corpus_targets(root) + cache_dir = root / "data" / "raw" / "enrichment" + cache_dir.mkdir(parents=True, exist_ok=True) + state_path = cache_dir / "fetch-state-v1.json" + state = _load_state(state_path) + state["corpus_entity_count"] = sum(len(value) for value in entity_mapping.values()) + state["unique_object_number_count"] = len(targets) + state["duplicate_object_number_groups"] = sum(len(value) > 1 for value in entity_mapping.values()) + state["entity_mapping"] = entity_mapping + records = state.setdefault("records", {}) + selected = [] + for object_number in targets: + prior = records.get(object_number) + if resume and not refresh and prior and prior.get("outcome") == "success": + continue + selected.append(object_number) + if limit is not None: + selected = selected[:limit] + lock = Lock() + completed = 0 + with ThreadPoolExecutor(max_workers=concurrency) as executor: + futures = { + executor.submit(_fetch_one, number, cache_dir, delay, retries): number + for number in selected + } + for future in as_completed(futures): + result = future.result() + with lock: + records[result["object_number"]] = { + key: result.get(key) for key in ( + "outcome", "http_status", "response_at_utc", "attempt", "cache_path" + ) + } + completed += 1 + if completed % 25 == 0 or completed == len(selected): + state["updated_at_utc"] = _timestamp() + _save_state(state_path, state) + state["updated_at_utc"] = _timestamp() + _save_state(state_path, state) + return status(root) | {"fetched_this_run": completed} + + +def status(root: Path) -> dict[str, Any]: + targets, mapping = corpus_targets(root) + state_path = root / "data" / "raw" / "enrichment" / "fetch-state-v1.json" + state = _load_state(state_path) + counts: dict[str, int] = {} + for record in state.get("records", {}).values(): + outcome = record.get("outcome", "unknown") + counts[outcome] = counts.get(outcome, 0) + 1 + return { + "corpus_entities": sum(len(value) for value in mapping.values()), + "unique_object_numbers": len(targets), + "duplicate_object_number_groups": sum(len(value) > 1 for value in mapping.values()), + "cached_records": len(state.get("records", {})), "outcomes": dict(sorted(counts.items())), + "remaining": len(targets) - sum( + record.get("outcome") == "success" for record in state.get("records", {}).values() + ), + } + + +def cached_record(root: Path, object_number: str) -> dict[str, Any]: + state = _load_state(root / "data" / "raw" / "enrichment" / "fetch-state-v1.json") + record = state.get("records", {}).get(object_number) + if not record: + raise ValueError(f"No cached enrichment response for {object_number}") + path = Path(record["cache_path"]) + if not path.is_absolute(): + path = root / path + return json.loads(path.read_text(encoding="utf-8")) diff --git a/src/smk_playground/enrichment/milestone6.py b/src/smk_playground/enrichment/milestone6.py new file mode 100644 index 0000000..15d3038 --- /dev/null +++ b/src/smk_playground/enrichment/milestone6.py @@ -0,0 +1,481 @@ +"""Build frozen enrichment-backed weighted palettes, v2 features, and similarity.""" + +from __future__ import annotations + +import hashlib +import json +import math +import random +import statistics +import time +from collections import Counter +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pyarrow.parquet as pq + +from smk_playground.enrichment.fetch import ENDPOINT, corpus_targets, status +from smk_playground.enrichment.weighted import ( + palette_transport_distance, + parse_weighted_palette, + weighted_features, +) +from smk_playground.normalize.milestone4 import linear_quantile, sha256_file +from smk_playground.similarity.engine import SimilarityEngine + +ENRICHMENT_SCHEMA_VERSION = "1.0.0" +WEIGHTED_FEATURE_SCHEMA_VERSION = "2.0.0" +WEIGHTED_EXTRACTOR_VERSION = "1.0.0" +WEIGHTED_PROFILE = "weighted-color-similarity-v1" +PAIR_SEED = 20261009 +PAIR_COUNT = 10_000 +WEIGHTS = { + "weighted_centroid_oklab": 0.30, + "palette_transport_distance_oklab": 0.40, + "weighted_palette_spread": 0.12, + "weighted_chroma_stddev": 0.08, + "weighted_hue_concentration": 0.05, + "dominant_color_weight": 0.05, +} + + +def _colorextractor(response: Any) -> dict[str, Any] | None: + if not isinstance(response, list): + return None + matches = [item for item in response if isinstance(item, dict) and item.get("type") == "colorextractor"] + if not matches: + return None + return max(matches, key=lambda item: str(item.get("data", {}).get("_version_", ""))) + + +def _profile(values: list[float]) -> dict[str, Any]: + if not values: + return {key: None for key in ("min", "median", "p75", "p90", "p95", "p99", "max")} | {"count": 0} + return {"count": len(values), "min": min(values), "median": statistics.median(values), + "p75": linear_quantile(values, 0.75), "p90": linear_quantile(values, 0.90), + "p95": linear_quantile(values, 0.95), "p99": linear_quantile(values, 0.99), + "max": max(values)} + + +def _robust(value: float | None, bounds: tuple[float, float] | None) -> float | None: + if value is None or bounds is None: + return None + low, high = bounds + if low == high: + return None + return min(1.0, max(0.0, (value - low) / (high - low))) + + +def _sample_pairs(count: int, size: int) -> list[tuple[int, int]]: + total = min(count, size * (size - 1) // 2) + rng = random.Random(PAIR_SEED) + pairs: set[tuple[int, int]] = set() + while len(pairs) < total: + left, right = rng.randrange(size), rng.randrange(size) + if left != right: + pairs.add((min(left, right), max(left, right))) + return sorted(pairs) + + +def _centroid_distance(left: dict[str, Any], right: dict[str, Any]) -> float: + return math.sqrt(math.fsum( + (left[field] - right[field]) ** 2 for field in ("centroid_l", "centroid_a", "centroid_b") + )) + + +def compare_weighted( + left: dict[str, Any], right: dict[str, Any], left_palette: list[dict[str, Any]], + right_palette: list[dict[str, Any]], profile: dict[str, Any], +) -> dict[str, Any]: + raw = { + "weighted_centroid_oklab": _centroid_distance(left, right), + "palette_transport_distance_oklab": palette_transport_distance(left_palette, right_palette), + "weighted_palette_spread": abs(left["palette_spread_robust_01"] - right["palette_spread_robust_01"]), + "weighted_chroma_stddev": abs(left["chroma_stddev_robust_01"] - right["chroma_stddev_robust_01"]), + "dominant_color_weight": abs(left["dominant_color_weight_robust_01"] - right["dominant_color_weight_robust_01"]), + } + if left["hue_concentration_01"] is not None and right["hue_concentration_01"] is not None: + raw["weighted_hue_concentration"] = abs( + left["hue_concentration_01"] - right["hue_concentration_01"] + ) + components = {} + active_weight = math.fsum(profile["weights"][name] for name in raw) + for name, value in raw.items(): + scaled = min(value / profile["scales"][name], 1.0) + contribution = profile["weights"][name] * scaled / active_weight + components[name] = {"raw_distance": value, "scaled_distance": scaled, + "weight": profile["weights"][name], "contribution": contribution} + return {"total_distance": math.fsum(item["contribution"] for item in components.values()), + "active_weight_sum": active_weight, "active_components": list(components), + "components": components} + + +class WeightedSimilarityEngine: + def __init__(self, root: Path): + self.root = root + payload = json.loads((root / "data" / "processed" / "similarity-profiles-v2.json").read_text(encoding="utf-8")) + self.profile = payload["profiles"][WEIGHTED_PROFILE] + self.rows = [row for row in pq.read_table( + root / "data" / "processed" / "artwork-visual-features-v2.parquet" + ).to_pylist() if row["has_weighted_palette_v1"]] + palette_rows = pq.read_table( + root / "data" / "processed" / "artwork-weighted-palettes-v1.parquet" + ).to_pylist() + self.palettes: dict[str, list[dict[str, Any]]] = {} + for row in palette_rows: + self.palettes.setdefault(row["smk_entity_id"], []).append(row) + for values in self.palettes.values(): + values.sort(key=lambda row: row["palette_rank"]) + self.by_id = {row["smk_entity_id"]: row for row in self.rows} + self.by_number: dict[str, list[dict[str, Any]]] = {} + for row in self.rows: + self.by_number.setdefault(row["object_number"], []).append(row) + base = SimilarityEngine(root) + self.titles = base.titles + + def resolve(self, object_number: str | None = None, smk_entity_id: str | None = None) -> dict[str, Any]: + if smk_entity_id: + if smk_entity_id not in self.by_id: + raise ValueError(f"Entity is not weighted-palette eligible: {smk_entity_id}") + return self.by_id[smk_entity_id] + matches = self.by_number.get(object_number or "", []) + if not matches: + raise ValueError(f"Object number is not weighted-palette eligible: {object_number}") + if len(matches) > 1: + raise ValueError(f"Ambiguous object number {object_number}; use --id") + return matches[0] + + def search_row(self, query: dict[str, Any], limit: int = 10, + include_self: bool = False) -> list[dict[str, Any]]: + output = [] + query_palette = self.palettes[query["smk_entity_id"]] + for candidate in self.rows: + if not include_self and candidate["smk_entity_id"] == query["smk_entity_id"]: + continue + result = compare_weighted( + query, candidate, query_palette, self.palettes[candidate["smk_entity_id"]], self.profile + ) + output.append({"smk_entity_id": candidate["smk_entity_id"], + "object_number": candidate["object_number"], + "title": self.titles.get(candidate["smk_entity_id"]), **result}) + output.sort(key=lambda row: (row["total_distance"], row["smk_entity_id"])) + for rank, row in enumerate(output[:limit], 1): + row["rank"] = rank + return output[:limit] + + +def run(root: Path) -> dict[str, Any]: + fetch_status = status(root) + targets, entity_mapping = corpus_targets(root) + state = json.loads((root / "data" / "raw" / "enrichment" / "fetch-state-v1.json").read_text(encoding="utf-8")) + v1_path = root / "data" / "processed" / "artwork-visual-features-v1.parquet" + palette_v1_path = root / "data" / "processed" / "artwork-palettes-v1.parquet" + normalized_v1_path = root / "data" / "processed" / "artwork-normalized-features-v1.parquet" + v1_hashes = {path.name: sha256_file(path) for path in (v1_path, palette_v1_path, normalized_v1_path)} + v1_rows = pq.read_table(v1_path).to_pylist() + v1_by_id = {row["smk_entity_id"]: row for row in v1_rows} + bulk_palettes: dict[str, set[str]] = {} + bulk_palette_order: dict[str, list[tuple[int, str]]] = {} + for row in pq.read_table( + palette_v1_path, columns=["smk_entity_id", "palette_rank", "hex"] + ).to_pylist(): + bulk_palettes.setdefault(row["smk_entity_id"], set()).add(row["hex"]) + bulk_palette_order.setdefault(row["smk_entity_id"], []).append( + (row["palette_rank"], row["hex"]) + ) + + manifest_entries = [] + palettes_by_entity: dict[str, list[dict[str, Any]]] = {} + palette_rows = [] + comparison_rows = [] + feature_rows = [] + outcome_counts = Counter() + status_counts = Counter() + component_counts = Counter() + version_counts = Counter() + versions_by_component: dict[str, set[str]] = {} + token_counts = [] + unique_counts = [] + for target_index, object_number in enumerate(targets, 1): + state_record = state.get("records", {}).get(object_number) + if not state_record: + outcome_counts["not_requested"] += 1 + continue + cache_path = Path(state_record["cache_path"]) + if not cache_path.is_absolute(): + cache_path = root / cache_path + source = json.loads(cache_path.read_text(encoding="utf-8")) + outcome_counts[source["outcome"]] += 1 + response = source.get("response_json") + if isinstance(response, list): + for component in response: + if isinstance(component, dict): + component_type = str(component.get("type")) + version = str((component.get("data") or {}).get("_version_")) + component_counts[component_type] += 1 + version_counts[f"{component_type}:{version}"] += 1 + versions_by_component.setdefault(component_type, set()).add(version) + extractor = _colorextractor(response) + data = extractor.get("data", {}) if extractor else {} + raw = data.get("colors_palette_weighted_s") + parsed, audit = parse_weighted_palette(raw) + status_counts[audit["status"]] += 1 + if audit["valid_token_count"]: + token_counts.append(audit["valid_token_count"]) + unique_counts.append(audit["unique_color_count"]) + raw_hash = hashlib.sha256(raw.encode("utf-8")).hexdigest() if isinstance(raw, str) else None + manifest_entries.append({ + "object_number": object_number, "smk_entity_ids": entity_mapping[object_number], + "response_timestamp": source.get("response_at_utc"), "http_status": source.get("http_status"), + "outcome": source["outcome"], "source_endpoint": source["endpoint"], + "cache_path": state_record["cache_path"], "colorextractor_version": data.get("_version_"), + "weighted_palette_raw_sha256": raw_hash, "weighted_palette_status": audit["status"], + "token_count": audit["valid_token_count"], "unique_color_count": audit["unique_color_count"], + }) + for entity_id in entity_mapping[object_number]: + base = v1_by_id[entity_id] + status_value = audit["status"] if source["outcome"] == "success" else source["outcome"] + if parsed: + entity_palette = [] + for item in parsed: + record = {"smk_entity_id": entity_id, "object_number": object_number, + "palette_source": "smk_enrichment", "palette_schema_version": "1.0.0", + "palette_weighting": "enrichment_token_frequency", + "enrichment_component": "colorextractor", + "enrichment_version": str(data.get("_version_")), **item} + palette_rows.append(record) + entity_palette.append(record) + palettes_by_entity[entity_id] = entity_palette + calculated = weighted_features(entity_palette) + bulk_set = bulk_palettes[entity_id] + enrichment_set = {row["hex"] for row in entity_palette} + intersection = bulk_set & enrichment_set + comparison_rows.append({ + "smk_entity_id": entity_id, "object_number": object_number, + "exact_same_color_set": bulk_set == enrichment_set, + "partial_overlap": bool(intersection) and bulk_set != enrichment_set, + "no_overlap": not intersection, "bulk_palette_size": len(bulk_set), + "enrichment_palette_size": len(enrichment_set), + "different_palette_size": len(bulk_set) != len(enrichment_set), + "enrichment_only_colors": sorted(enrichment_set - bulk_set), + "bulk_only_colors": sorted(bulk_set - enrichment_set), + "order_differs": [row["hex"] for row in entity_palette] + != [value[1] for value in sorted(bulk_palette_order[entity_id])], + }) + else: + calculated = {key: None for key in ( + "centroid_l", "centroid_a", "centroid_b", "centroid_chroma", "centroid_hue_deg", + "palette_entropy", "palette_entropy_01", "effective_palette_size", + "palette_spread_oklab", "lightness_stddev", "chroma_stddev", + "hue_concentration_01", "dominant_color_weight", "dominant_color_hex")} + calculated["feature_quality_flags"] = ["WEIGHTED_PALETTE_UNAVAILABLE"] + feature_rows.append({ + "smk_entity_id": entity_id, "object_number": object_number, + "feature_schema": "artart-visual-features", "feature_schema_version": WEIGHTED_FEATURE_SCHEMA_VERSION, + "extractor_version": WEIGHTED_EXTRACTOR_VERSION, "bulk_feature_lineage_sha256": v1_hashes[v1_path.name], + "palette_source": "smk_enrichment", "palette_weighting": "enrichment_token_frequency", + "enrichment_component": "colorextractor" if extractor else None, + "enrichment_version": str(data.get("_version_")) if extractor else None, + "has_weighted_palette_v1": bool(parsed), "weighted_palette_status": status_value, + "weighted_palette_token_count": audit["valid_token_count"], + "weighted_palette_unique_color_count": audit["unique_color_count"], + **calculated, + "centroid_shift_from_v1_oklab": math.sqrt(math.fsum( + (calculated[field] - base[field]) ** 2 + for field in ("centroid_l", "centroid_a", "centroid_b") + )) if parsed else None, + "hue_concentration_shift_from_v1": ( + calculated["hue_concentration_01"] - base["hue_concentration_01"] + if parsed and calculated["hue_concentration_01"] is not None + and base["hue_concentration_01"] is not None else None + ), + }) + if target_index % 1_000 == 0: + print(f"Loaded enrichment cache: {target_index}/{len(targets)} object numbers", flush=True) + + # Ensure every corpus entity remains represented even when an object number was not fetched. + represented = {row["smk_entity_id"] for row in feature_rows} + for base in v1_rows: + if base["smk_entity_id"] not in represented: + feature_rows.append({ + "smk_entity_id": base["smk_entity_id"], "object_number": base["object_number"], + "feature_schema": "artart-visual-features", "feature_schema_version": WEIGHTED_FEATURE_SCHEMA_VERSION, + "extractor_version": WEIGHTED_EXTRACTOR_VERSION, "bulk_feature_lineage_sha256": v1_hashes[v1_path.name], + "palette_source": "smk_enrichment", "palette_weighting": "enrichment_token_frequency", + "enrichment_component": None, "enrichment_version": None, + "has_weighted_palette_v1": False, "weighted_palette_status": "not_requested", + "weighted_palette_token_count": 0, "weighted_palette_unique_color_count": 0, + **{key: None for key in ( + "centroid_l", "centroid_a", "centroid_b", "centroid_chroma", "centroid_hue_deg", + "palette_entropy", "palette_entropy_01", "effective_palette_size", + "palette_spread_oklab", "lightness_stddev", "chroma_stddev", "hue_concentration_01", + "dominant_color_weight", "dominant_color_hex", "centroid_shift_from_v1_oklab", + "hue_concentration_shift_from_v1")}, + "feature_quality_flags": ["WEIGHTED_PALETTE_UNAVAILABLE"], + }) + feature_rows.sort(key=lambda row: row["smk_entity_id"]) + palette_rows.sort(key=lambda row: (row["smk_entity_id"], row["palette_rank"])) + comparison_rows.sort(key=lambda row: row["smk_entity_id"]) + + valid_features = [row for row in feature_rows if row["has_weighted_palette_v1"]] + normalization_values = { + "palette_spread_oklab": [row["palette_spread_oklab"] for row in valid_features], + "chroma_stddev": [row["chroma_stddev"] for row in valid_features], + "dominant_color_weight": [row["dominant_color_weight"] for row in valid_features], + } + normalization_bounds = { + key: (linear_quantile(values, 0.01), linear_quantile(values, 0.99)) + if values else None + for key, values in normalization_values.items() + } + for row in feature_rows: + row["palette_spread_robust_01"] = _robust( + row["palette_spread_oklab"], normalization_bounds["palette_spread_oklab"] + ) + row["chroma_stddev_robust_01"] = _robust( + row["chroma_stddev"], normalization_bounds["chroma_stddev"] + ) + row["dominant_color_weight_robust_01"] = _robust( + row["dominant_color_weight"], normalization_bounds["dominant_color_weight"] + ) + + manifest = { + "manifest_version": ENRICHMENT_SCHEMA_VERSION, "source_endpoint": ENDPOINT, + "generated_at_utc": max( + (entry["response_timestamp"] for entry in manifest_entries if entry["response_timestamp"]), + default=None, + ), + "corpus_version": "smk-public-visual-v1", "corpus_entity_count": len(v1_rows), + "unique_object_number_count": len(targets), + "duplicate_object_number_groups": sum(len(ids) > 1 for ids in entity_mapping.values()), + "fetch_status": fetch_status, "records": manifest_entries, + } + manifest_path = root / "data" / "processed" / "enrichment-source-manifest-v1.json" + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + manifest_hash = sha256_file(manifest_path) + for row in feature_rows: + row["enrichment_source_manifest_sha256"] = manifest_hash + metadata = {b"smk.enrichment_manifest_sha256": manifest_hash.encode(), + b"smk.palette_weighting": b"enrichment_token_frequency"} + def write(rows: list[dict[str, Any]], path: Path) -> None: + table = pa.Table.from_pylist(rows).replace_schema_metadata(metadata) + pq.write_table(table, path, compression="zstd", version="2.6", write_statistics=True) + weighted_palette_path = root / "data" / "processed" / "artwork-weighted-palettes-v1.parquet" + feature_v2_path = root / "data" / "processed" / "artwork-visual-features-v2.parquet" + comparison_path = root / "data" / "reports" / "enrichment-palette-comparison-v1.parquet" + write(palette_rows, weighted_palette_path) + write(feature_rows, feature_v2_path) + write(comparison_rows, comparison_path) + + pairs = _sample_pairs(PAIR_COUNT, len(valid_features)) + component_values = {name: [] for name in WEIGHTS} + for left_index, right_index in pairs: + left, right = valid_features[left_index], valid_features[right_index] + component_values["weighted_centroid_oklab"].append(_centroid_distance(left, right)) + component_values["palette_transport_distance_oklab"].append(palette_transport_distance( + palettes_by_entity[left["smk_entity_id"]], palettes_by_entity[right["smk_entity_id"]] + )) + component_values["weighted_palette_spread"].append(abs(left["palette_spread_robust_01"] - right["palette_spread_robust_01"])) + component_values["weighted_chroma_stddev"].append(abs(left["chroma_stddev_robust_01"] - right["chroma_stddev_robust_01"])) + component_values["dominant_color_weight"].append(abs(left["dominant_color_weight_robust_01"] - right["dominant_color_weight_robust_01"])) + if left["hue_concentration_01"] is not None and right["hue_concentration_01"] is not None: + component_values["weighted_hue_concentration"].append(abs(left["hue_concentration_01"] - right["hue_concentration_01"])) + distributions = {name: _profile(values) for name, values in component_values.items()} + profile = { + "name": WEIGHTED_PROFILE, "profile_version": "1.0.0", + "required_features": ["weighted palette", "weighted centroid", "weighted spread", "weighted chroma stddev", "dominant weight"], + "optional_features": ["weighted hue concentration"], "weights": WEIGHTS, + "scales": {name: values["p95"] for name, values in distributions.items()}, + "scale_rule": f"P95 over {len(pairs)} deterministic eligible pairs, seed {PAIR_SEED}", + "missing_value_behavior": "weighted palette required; hue concentration omitted and active weights renormalized", + "distance_semantics": "weighted mean of P95-scaled component distances clipped to [0,1]; lower is more similar", + "tie_breaking": ["total_distance ascending", "smk_entity_id ascending"], + } + profiles_payload = { + "profile_set_version": "2.0.0", "additive_to_similarity_profiles_v1": True, + "input_hashes": {"enrichment_manifest": manifest_hash, + "weighted_palettes": sha256_file(weighted_palette_path), + "weighted_features_v2": sha256_file(feature_v2_path), **v1_hashes}, + "calibration_distributions": distributions, "profiles": {WEIGHTED_PROFILE: profile}, + } + profiles_path = root / "data" / "processed" / "similarity-profiles-v2.json" + profiles_path.write_text(json.dumps(profiles_payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + engine = WeightedSimilarityEngine(root) + kms = engine.resolve(object_number="KMS8892") + started = time.perf_counter() + weighted_matches = engine.search_row(kms, 10) + query_seconds = time.perf_counter() - started + base_engine = SimilarityEngine(root) + base_query = base_engine.resolve(object_number="KMS8892") + base_full = base_engine.search_row(base_query, "color", len(base_engine.rows)) + base_matches = base_full[:10] + base_ranks = {row["smk_entity_id"]: row["rank"] for row in base_full} + base_ids = [row["smk_entity_id"] for row in base_matches] + weighted_ids = [row["smk_entity_id"] for row in weighted_matches] + kms_palette = palettes_by_entity[kms["smk_entity_id"]] + summary = { + "fetch": fetch_status, "outcome_counts": dict(sorted(outcome_counts.items())), + "weighted_palette_status_counts": dict(sorted(status_counts.items())), + "component_distribution": dict(sorted(component_counts.items())), + "component_version_distinct_count": len(version_counts), + "distinct_versions_by_component": { + key: len(value) for key, value in sorted(versions_by_component.items()) + }, + "weighted_palette_entity_count": len(valid_features), + "suspicious_pattern_counts": { + "single_unique_color": sum(row["weighted_palette_unique_color_count"] == 1 for row in valid_features), + "dominant_weight_gte_0_9": sum(row["dominant_color_weight"] >= 0.9 for row in valid_features), + "one_token_per_color": sum( + row["weighted_palette_token_count"] == row["weighted_palette_unique_color_count"] + for row in valid_features + ), + }, + "token_count_distribution": _profile(token_counts), + "unique_color_count_distribution": _profile(unique_counts), + "bulk_palette_comparison": { + "exact_same_color_set": sum(row["exact_same_color_set"] for row in comparison_rows), + "partial_overlap": sum(row["partial_overlap"] for row in comparison_rows), + "no_overlap": sum(row["no_overlap"] for row in comparison_rows), + "different_palette_size": sum(row["different_palette_size"] for row in comparison_rows), + "order_differs": sum(row["order_differs"] for row in comparison_rows), + }, + "feature_distributions": {name: _profile([row[name] for row in valid_features if row[name] is not None]) + for name in ("dominant_color_weight", "palette_entropy", "palette_entropy_01", + "effective_palette_size", "palette_spread_oklab", + "centroid_shift_from_v1_oklab", "hue_concentration_shift_from_v1")}, + "palette_transport_calibration": distributions["palette_transport_distance_oklab"], + "weighted_profile": profile, "kms8892": { + "bulk_palette": v1_by_id[kms["smk_entity_id"]]["source_palette"], + "weighted_palette": [{key: row[key] for key in ("hex", "token_count", "weight")} + for row in kms_palette], + "weighted_features": {key: kms[key] for key in ( + "centroid_l", "centroid_a", "centroid_b", "centroid_chroma", "centroid_hue_deg", + "dominant_color_hex", "dominant_color_weight", "palette_entropy", + "palette_entropy_01", "effective_palette_size", "palette_spread_oklab", + "hue_concentration_01")}, + "weighted_top10": weighted_matches, "bulk_color_top10": base_matches, + "top10_overlap_count": len(set(base_ids) & set(weighted_ids)), + "newly_promoted_ids": [value for value in weighted_ids if value not in base_ids], + "demoted_ids": [value for value in base_ids if value not in weighted_ids], + "weighted_top10_ranks_under_bulk_color": [ + {"smk_entity_id": value, "bulk_color_rank": base_ranks[value]} + for value in weighted_ids + ], + "weighted_top10_retained_by_bulk_candidate_cutoffs": { + str(cutoff): sum(base_ranks[value] <= cutoff for value in weighted_ids) + for cutoff in (200, 500, 1000) + }, + }, + "performance": {"weighted_exact_full_corpus_query_seconds": query_seconds, + "eligible_entities": len(valid_features)}, + "v1_hashes_verified_unchanged": v1_hashes, + } + report = root / "data" / "reports" / "enrichment-summary-v1.json" + report.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + weighted_report = root / "data" / "reports" / "weighted-similarity-summary-v1.json" + weighted_report.write_text(json.dumps({key: summary[key] for key in ( + "palette_transport_calibration", "weighted_profile", "kms8892", "performance")}, + ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return summary diff --git a/src/smk_playground/enrichment/weighted.py b/src/smk_playground/enrichment/weighted.py new file mode 100644 index 0000000..cf0d006 --- /dev/null +++ b/src/smk_playground/enrichment/weighted.py @@ -0,0 +1,136 @@ +"""Weighted enrichment palette parsing, features, and exact palette transport.""" + +from __future__ import annotations + +import math +import re +from collections import Counter +from typing import Any + +from smk_playground.features.color import HEX_RE, hex_to_oklab +from smk_playground.features.extract import palette_features + +TOKEN_RE = re.compile(r"\S+") + + +def parse_weighted_palette(raw: Any) -> tuple[list[dict[str, Any]], dict[str, Any]]: + if not isinstance(raw, str) or not raw.strip(): + return [], {"status": "missing", "token_count": 0, "valid_token_count": 0, + "invalid_tokens": [], "unique_color_count": 0} + tokens = TOKEN_RE.findall(raw) + valid = [token.casefold() for token in tokens if HEX_RE.fullmatch(token)] + invalid = [token for token in tokens if not HEX_RE.fullmatch(token)] + if not valid: + return [], {"status": "malformed", "token_count": len(tokens), + "valid_token_count": 0, "invalid_tokens": invalid, + "unique_color_count": 0} + counts = Counter(valid) + first_positions = {color: valid.index(color) for color in counts} + ordered = sorted(counts, key=lambda color: first_positions[color]) + total = len(valid) + rows = [] + for rank, color in enumerate(ordered, 1): + converted = hex_to_oklab(color) + rows.append({ + "palette_rank": rank, "hex": color, "token_count": counts[color], + "weight": counts[color] / total, "srgb_r": converted.srgb_r, + "srgb_g": converted.srgb_g, "srgb_b": converted.srgb_b, + "oklab_l": converted.oklab_l, "oklab_a": converted.oklab_a, + "oklab_b": converted.oklab_b, "oklch_l": converted.oklch_l, + "oklch_c": converted.oklch_c, "oklch_h_deg": converted.oklch_h_deg, + }) + status = "valid" if not invalid else "partial_invalid" + return rows, {"status": status, "token_count": len(tokens), + "valid_token_count": total, "invalid_tokens": invalid, + "unique_color_count": len(rows)} + + +def weighted_features(palette: list[dict[str, Any]]) -> dict[str, Any]: + features, flags = palette_features(palette) + entropy = -math.fsum(row["weight"] * math.log(row["weight"]) + for row in palette if row["weight"] > 0) + dominant = max(palette, key=lambda row: (row["weight"], -row["palette_rank"])) if palette else None + return { + **features, "palette_entropy": entropy if palette else None, + "dominant_color_weight": dominant["weight"] if dominant else None, + "dominant_color_hex": dominant["hex"] if dominant else None, + "feature_quality_flags": flags, + } + + +def palette_transport_distance(left: list[dict[str, Any]], right: list[dict[str, Any]]) -> float: + """Exact min-cost flow between two tiny weighted palettes in OKLab.""" + if not left or not right: + raise ValueError("Palette transport requires two non-empty palettes") + left_weights = [float(row["weight"]) for row in left] + right_weights = [float(row["weight"]) for row in right] + if not math.isclose(math.fsum(left_weights), 1.0, abs_tol=1e-9): + raise ValueError("Left palette weights must sum to one") + if not math.isclose(math.fsum(right_weights), 1.0, abs_tol=1e-9): + raise ValueError("Right palette weights must sum to one") + costs = [[math.sqrt( + (a["oklab_l"] - b["oklab_l"]) ** 2 + + (a["oklab_a"] - b["oklab_a"]) ** 2 + + (a["oklab_b"] - b["oklab_b"]) ** 2 + ) for b in right] for a in left] + n_left, n_right = len(left), len(right) + source = n_left + n_right + sink = source + 1 + size = sink + 1 + graph: list[list[dict[str, float | int]]] = [[] for _ in range(size)] + + def add_edge(start: int, end: int, capacity: float, cost: float) -> None: + forward = {"to": end, "reverse": len(graph[end]), "capacity": capacity, "cost": cost} + reverse = {"to": start, "reverse": len(graph[start]), "capacity": 0.0, "cost": -cost} + graph[start].append(forward) + graph[end].append(reverse) + + for i, weight in enumerate(left_weights): + add_edge(source, i, weight, 0.0) + for i in range(n_left): + for j in range(n_right): + add_edge(i, n_left + j, 1.0, costs[i][j]) + for j, weight in enumerate(right_weights): + add_edge(n_left + j, sink, weight, 0.0) + + total_cost = 0.0 + remaining = 1.0 + while remaining > 1e-12: + distance = [math.inf] * size + parent: list[tuple[int, int] | None] = [None] * size + distance[source] = 0.0 + for _ in range(size - 1): + changed = False + for node, edges in enumerate(graph): + if distance[node] == math.inf: + continue + for edge_index, edge in enumerate(edges): + if float(edge["capacity"]) <= 1e-12: + continue + target = int(edge["to"]) + candidate = distance[node] + float(edge["cost"]) + if candidate < distance[target] - 1e-15: + distance[target] = candidate + parent[target] = (node, edge_index) + changed = True + if not changed: + break + if parent[sink] is None: + raise RuntimeError("No augmenting palette-transport path") + amount = remaining + node = sink + while node != source: + previous, edge_index = parent[node] + amount = min(amount, float(graph[previous][edge_index]["capacity"])) + node = previous + node = sink + while node != source: + previous, edge_index = parent[node] + edge = graph[previous][edge_index] + reverse = graph[node][int(edge["reverse"])] + edge["capacity"] = float(edge["capacity"]) - amount + reverse["capacity"] = float(reverse["capacity"]) + amount + total_cost += amount * float(edge["cost"]) + node = previous + remaining -= amount + return max(0.0, total_cost) diff --git a/tests/test_enrichment.py b/tests/test_enrichment.py new file mode 100644 index 0000000..23de4cb --- /dev/null +++ b/tests/test_enrichment.py @@ -0,0 +1,102 @@ +"""Focused enrichment palette and transport tests.""" + +import math +from pathlib import Path + +import pytest + +from smk_playground.enrichment import fetch as fetch_module +from smk_playground.enrichment.milestone6 import compare_weighted +from smk_playground.enrichment.weighted import ( + palette_transport_distance, + parse_weighted_palette, + weighted_features, +) + + +def palette(raw): + return parse_weighted_palette(raw)[0] + + +def test_weighted_parser_and_invalid_tokens(): + rows, audit = parse_weighted_palette("#FF0000 #00ff00 #ff0000 bad") + assert [row["hex"] for row in rows] == ["#ff0000", "#00ff00"] + assert [row["token_count"] for row in rows] == [2, 1] + assert math.fsum(row["weight"] for row in rows) == pytest.approx(1.0) + assert audit == {"status": "partial_invalid", "token_count": 4, + "valid_token_count": 3, "invalid_tokens": ["bad"], + "unique_color_count": 2} + assert parse_weighted_palette(None)[1]["status"] == "missing" + + +def test_weighted_feature_entropy_effective_size_and_dominant(): + features = weighted_features(palette("#ff0000 #ff0000 #ff0000 #000000")) + expected_entropy = -(0.75 * math.log(0.75) + 0.25 * math.log(0.25)) + assert features["palette_entropy"] == pytest.approx(expected_entropy) + assert features["effective_palette_size"] == pytest.approx(math.exp(expected_entropy)) + assert features["dominant_color_weight"] == 0.75 + assert features["dominant_color_hex"] == "#ff0000" + assert features["centroid_l"] is not None + + +def test_transport_invariants_and_order(): + first = palette("#ff0000 #ff0000 #0000ff") + reordered = list(reversed(first)) + assert palette_transport_distance(first, first) == pytest.approx(0.0, abs=1e-12) + assert palette_transport_distance(first, reordered) == pytest.approx(0.0, abs=1e-12) + other = palette("#ff0000 #0000ff #0000ff") + forward = palette_transport_distance(first, other) + assert forward > 0 + assert forward == pytest.approx(palette_transport_distance(other, first)) + assert forward == pytest.approx(palette_transport_distance(first, other)) + + +def test_one_color_transport_is_oklab_distance(): + red, blue = palette("#ff0000"), palette("#0000ff") + expected = math.sqrt(sum( + (red[0][field] - blue[0][field]) ** 2 + for field in ("oklab_l", "oklab_a", "oklab_b") + )) + assert palette_transport_distance(red, blue) == pytest.approx(expected) + + +def test_weighted_similarity_symmetry_and_contributions(): + left_palette = palette("#ff0000 #ff0000 #0000ff") + right_palette = palette("#ff0000 #0000ff #0000ff") + left = weighted_features(left_palette) | { + "palette_spread_robust_01": 0.2, "chroma_stddev_robust_01": 0.3, + "dominant_color_weight_robust_01": 0.5, + } + right = weighted_features(right_palette) | { + "palette_spread_robust_01": 0.4, "chroma_stddev_robust_01": 0.2, + "dominant_color_weight_robust_01": 0.5, + } + weights = {"weighted_centroid_oklab": 0.3, "palette_transport_distance_oklab": 0.4, + "weighted_palette_spread": 0.12, "weighted_chroma_stddev": 0.08, + "weighted_hue_concentration": 0.05, "dominant_color_weight": 0.05} + profile = {"weights": weights, "scales": {key: 1.0 for key in weights}} + forward = compare_weighted(left, right, left_palette, right_palette, profile) + backward = compare_weighted(right, left, right_palette, left_palette, profile) + assert forward["total_distance"] == pytest.approx(backward["total_distance"]) + assert sum(item["contribution"] for item in forward["components"].values()) == pytest.approx( + forward["total_distance"] + ) + + +def test_fetch_resume_skips_successful_cache(tmp_path, monkeypatch): + root = Path(tmp_path) + monkeypatch.setattr(fetch_module, "corpus_targets", lambda unused: (["A", "B"], {"A": ["1"], "B": ["2"]})) + calls = [] + + def fake_fetch(number, cache_dir, delay, retries): + calls.append(number) + path = cache_dir / f"{number}.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("{}", encoding="utf-8") + return {"object_number": number, "outcome": "success", "http_status": 200, + "response_at_utc": "x", "attempt": 1, "cache_path": path.as_posix()} + + monkeypatch.setattr(fetch_module, "_fetch_one", fake_fetch) + fetch_module.fetch(root, concurrency=1) + fetch_module.fetch(root, concurrency=1) + assert calls == ["A", "B"]