From b4a98feae96af86e746a1c47e4d57e6c89e5465f Mon Sep 17 00:00:00 2001 From: Christian Johannesen Date: Thu, 8 Oct 2026 10:10:33 +0200 Subject: [PATCH] Complete Milestone 4 normalization --- AGENTS.md | 5 + README.md | 8 +- docs/DATA_MODEL.md | 5 + docs/FEATURE_SPEC_V1.md | 7 + docs/FEATURE_VECTOR_V1.md | 40 +++ docs/IMPLEMENTATION_PLAN.md | 34 +- docs/NORMALIZATION_V1.md | 88 +++++ docs/PIPELINE.md | 7 + src/smk_playground/cli.py | 9 + src/smk_playground/normalize/milestone4.py | 375 +++++++++++++++++++++ tests/test_normalization.py | 61 ++++ 11 files changed, 611 insertions(+), 28 deletions(-) create mode 100644 docs/FEATURE_VECTOR_V1.md create mode 100644 docs/NORMALIZATION_V1.md create mode 100644 src/smk_playground/normalize/milestone4.py create mode 100644 tests/test_normalization.py diff --git a/AGENTS.md b/AGENTS.md index bf9aa38..4793103 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,6 +35,11 @@ Treat the project as four analytical layers: 3. `features` — deterministic per-artwork derived features. These MUST NOT depend on other artworks. 4. `normalized` — corpus-relative values based on a named, versioned normalization profile. +Milestone 4 freezes normalization profile `smk-public-visual-v1` version `1.0.0`. Its wide +normalized table preserves nulls, and its compact vector table contains separate unweighted +`color_core_v1`, `visual_character_v1` and optional `geometry_v1` families. Do not treat these +families as a similarity profile or independently normalized OKLab a/b as perceptual geometry. + Never overwrite source data. Never silently invent missing values. ## Primary bulk source diff --git a/README.md b/README.md index a9c0c5e..a1e74c3 100644 --- a/README.md +++ b/README.md @@ -121,10 +121,11 @@ See: ## Current status -Milestones 0–3 complete: source structure and audits are manifested, the language snapshots have +Milestones 0–4 complete: source structure and audits are manifested, the language snapshots have bounded-memory source Parquets, DA/EN rows are paired into stable SMK entities, and the exact `smk-public-visual-v1` membership rule is frozen. Deterministic raw visual features and analytical -palettes now exist; corpus-wide feature normalization and similarity have not started. +palettes now exist. Versioned corpus statistics, normalized scalar features and three unweighted +candidate feature families are materialized. Similarity has not started. Audit results are documented in: @@ -137,6 +138,8 @@ Audit results are documented in: - [`docs/CORPUS_V1.md`](docs/CORPUS_V1.md) - [`docs/FEATURE_EXTRACTION_V1.md`](docs/FEATURE_EXTRACTION_V1.md) - [`docs/FEATURE_EXAMPLES.md`](docs/FEATURE_EXAMPLES.md) +- [`docs/NORMALIZATION_V1.md`](docs/NORMALIZATION_V1.md) +- [`docs/FEATURE_VECTOR_V1.md`](docs/FEATURE_VECTOR_V1.md) Run the audited ingest with: @@ -144,4 +147,5 @@ Run the audited ingest with: smk milestone1 smk milestone2 smk milestone3 +smk milestone4 ``` diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index 96a5dfd..2b30efa 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -61,6 +61,11 @@ bound to exact canonical/corpus SHA-256 values and extractor/schema versions. ### `normalized-features.parquet` One row per artwork per normalization profile/version, or a wide equivalent if simpler to query. +Milestone 4 materializes the wide form as `artwork-normalized-features-v1.parquet`, with raw, +robust-01, percentile and robust-z columns; `artwork-feature-vectors-v1.parquet` carries separate +fixed-order candidate families. `normalization-profile-v1.json` is the authoritative definition +and binds both outputs to the exact source-feature Parquet SHA-256. + ### `quality-report.json` Corpus-wide counts, missingness, exclusions, warnings and distribution diagnostics. diff --git a/docs/FEATURE_SPEC_V1.md b/docs/FEATURE_SPEC_V1.md index 408ec82..7f98c9e 100644 --- a/docs/FEATURE_SPEC_V1.md +++ b/docs/FEATURE_SPEC_V1.md @@ -167,3 +167,10 @@ Initial quality-flag vocabulary may include: - `SOURCE_LANGUAGE_MISMATCH` Flags describe observations; they should not automatically exclude records unless a named downstream profile says so. + +## Corpus-relative derivatives + +Normalization is specified separately in `NORMALIZATION_V1.md`. Raw OKLab coordinates remain the +perceptual source of truth; normalized a/b variants do not preserve OKLab Euclidean distance. +Circular hue x/y are alternative interpretation fields and are not combined with a/b in the +primary color family. diff --git a/docs/FEATURE_VECTOR_V1.md b/docs/FEATURE_VECTOR_V1.md new file mode 100644 index 0000000..edbe953 --- /dev/null +++ b/docs/FEATURE_VECTOR_V1.md @@ -0,0 +1,40 @@ +# Candidate feature vectors v1 + +`artwork-feature-vectors-v1.parquet` is a compact candidate dataset, not a similarity definition. +It contains one row per corpus entity, completeness flags and three fixed-size list columns. Exact +component order is embedded in the normalization profile and repeated below. + +## `color_core_v1` + +1. raw `centroid_l` +2. raw `centroid_a` +3. raw `centroid_b` +4. `palette_spread_oklab_robust_01` +5. `lightness_stddev_robust_01` +6. `chroma_stddev_robust_01` +7. natural `hue_concentration_01` + +Raw OKLab coordinates deliberately preserve perceptual meaning. Centroid hue x/y are excluded to +avoid double-counting hue already represented by a/b. + +## `visual_character_v1` + +1. `smk_brightness_robust_01` +2. `smk_contrast_robust_01` +3. `smk_saturation_robust_01` +4. `smk_entropy_robust_01` + +`smk_colortemp` remains available in the wide normalized table but is excluded here. + +## `geometry_v1` + +1. `log_net_width_cm_robust_01` +2. `log_net_height_cm_robust_01` +3. `log_net_area_cm2_robust_01` +4. `log_net_aspect_ratio_robust_01` + +Geometry is optional and missing elements remain null. Raw area and raw aspect ratio are not vector +components. No family has weights, and the three families are not concatenated into a final +distance space. Later comparison may combine raw OKLab distance, normalized scalar distance, +palette-level distance and optional geometry under separately versioned similarity profiles. + diff --git a/docs/IMPLEMENTATION_PLAN.md b/docs/IMPLEMENTATION_PLAN.md index b1b380c..464934c 100644 --- a/docs/IMPLEMENTATION_PLAN.md +++ b/docs/IMPLEMENTATION_PLAN.md @@ -60,22 +60,12 @@ Implemented for frozen `smk-public-visual-v1`: Status: complete. Raw features remain corpus-independent; no corpus normalization or similarity is implemented. See `FEATURE_EXTRACTION_V1.md` and `FEATURE_EXAMPLES.md`. -## Milestone 4 — Quality layer +## Milestone 4 — Corpus statistics + normalization -Implement documented flags and source/language/aspect checks. - -Produce `quality-report.json` with: - -- DA/EN raw record counts -- unique IDs and language overlap -- public-domain count -- usable-image/visual count -- valid feature count -- null counts by feature -- flag counts -- parser warnings/errors -- palette-size distribution -- obvious extreme values +Status: complete. Profile every candidate scalar over frozen `smk-public-visual-v1`, preserve raw +OKLab semantics and missingness, generate robust-01/midrank-percentile/robust-z variants, add log +aspect and circular-hue diagnostics, and publish separate unweighted candidate families. See +`NORMALIZATION_V1.md` and `FEATURE_VECTOR_V1.md`. ## Milestone 5 — Pilot build @@ -83,18 +73,10 @@ Run 100–500 representative artworks. Review distributions and manually inspect selected records. Fix parser/source/math errors only; do not artistically tune features. -## Milestone 6 — Corpus-relative normalization +## Milestone 6 — Similarity preparation -Use the frozen `smk-public-visual-v1` entity membership from Milestone 2. Do not redefine corpus -membership from an expected count. - -- reference included/excluded counts and reasons from the frozen corpus artifact -- compute count/null/min/max/quantiles -- median + MAD -- implement `robust_01` -- implement `percentile_01` -- implement robust z-score -- freeze profile with source-manifest reference +Review Milestone 4 correlations and feature families, then define separately versioned distance +profiles and weights. Do not reinterpret normalized OKLab axes as perceptual OKLab geometry. ## Milestone 7 — Full dataset build diff --git a/docs/NORMALIZATION_V1.md b/docs/NORMALIZATION_V1.md new file mode 100644 index 0000000..b404cc3 --- /dev/null +++ b/docs/NORMALIZATION_V1.md @@ -0,0 +1,88 @@ +# Normalization v1 + +Milestone 4 freezes normalization profile `smk-public-visual-v1`, version `1.0.0`, over all +27,254 rows in `artwork-visual-features-v1.parquet`. The profile records the input SHA-256, +feature/extractor versions, generation timestamp, exact rules and per-feature corpus statistics. +Nulls remain null; no value is imputed. + +## Rules + +- Corpus quantiles use linear interpolation over finite, non-null values. +- `robust_01 = clamp((x - P01) / (P99 - P01), 0, 1)`. Values below/above the cutoffs are + clipped. A P01=P99 feature returns null and is marked non-informative. +- Natural `[0,1]` semantics are retained for hue concentration and palette entropy instead of + stretching their corpus ranges. +- Empirical percentile uses deterministic midranks: tied ranks are averaged and divided by + `n-1`; a singleton receives 0.5. +- `robust_z = (x - median) / (1.4826 * MAD)`, without clipping. MAD=0 returns null. +- Positive physical and image ratios also receive `ln(width/height)`, where square is zero, + landscape positive and portrait negative. +- Hue is circular. Valid centroid hue receives cosine/sine fields; null hue produces nulls. + These fields are diagnostic and are not added to `color_core_v1`, because OKLab a/b already + encode hue direction. + +Raw OKLab L/a/b remain the canonical perceptual coordinates. Their normalized variants are useful +for generic plots, but Euclidean distance between those variants is not OKLab distance. + +## Coverage and distributions + +| Feature family | Complete rows | +|---|---:| +| `color_core_v1` | 21,488 | +| `visual_character_v1` | 26,383 | +| `geometry_v1` | 2,189 | +| all three | 1,801 | + +Color-core incompleteness is entirely the 5,766 null hue-concentration values for achromatic +palettes. Visual-character incompleteness follows the 871 missing saturation values (including 62 +rows also missing brightness/contrast). Geometry remains optional: 25,065 rows lack a complete +Netto width/height pair, and the corpus is not reduced for that reason. + +Selected distribution findings: + +- centroid L has median 0.7207 and P01/P99 0.3496/0.9900; +- palette spread has median 0.1685 and P01/P99 0/0.3689; +- log area has median 8.2680 and P01/P99 3.8960/11.1684 among 2,189 valid rows; +- `smk_colortemp` retains every finite extreme (range -1.2760 to 15.7201), but is excluded from + primary vectors because of its extremes and uncertain semantics; +- each ordinary robust scaling clips approximately the bottom and top one percent of finite + observations; exact counts are stored per feature in the profile/statistics table. + +Palette entropy has MAD zero and therefore no robust z-score. Under uniform Milestone 3 weights, +entropy is overwhelmingly 1 for multi-colour palettes, while effective palette size primarily +restates analytical palette size. Neither belongs to the v1 primary machine families. + +## Correlations + +Correlations use pairwise complete observations and report sample count plus Pearson and Spearman. +No feature is removed automatically. The only pair above the diagnostic absolute 0.90 threshold is +palette spread versus lightness standard deviation (Pearson 0.99, Spearman 0.99; n=27,254). +SMK contrast also tracks lightness standard deviation (0.83/0.82) and palette spread (0.80/0.79). +Brightness and centroid L are related but not interchangeable (0.67/0.67); saturation and centroid +chroma similarly correlate only moderately (0.64/0.61). These results indicate that the SMK fields +partly overlap the palette-derived measures while retaining additional information. + +## Deterministic examples + +Representative rows are stored in `normalization-summary.json`. Selected values are shown as +raw → robust 0–1 → percentile → robust z: + +| Artwork | Feature | Values | +|---|---|---| +| KMS8892 | centroid L | 0.7418 → 0.6124 → 0.5571 → 0.1381 | +| KMS8892 | palette spread | 0.1654 → 0.4483 → 0.4825 → -0.0383 | + +KMS8892's raw centroid is `(0.7418, -0.0118, 0.0406)`, palette spread is 0.1654 and hue +concentration is 0.7040. Its SMK brightness/contrast/saturation/entropy values are +7.1344/2.9756/2.0800/9.6865. Netto geometry is 78 × 51.5 cm (4,017 cm²), and log aspect ratio is +0.4151. These remain features and normalized diagnostics only; they are not notes, similarity +scores or final weights. + +## Artifacts + +- `data/processed/normalization-profile-v1.json` +- `data/processed/artwork-normalized-features-v1.parquet` +- `data/processed/artwork-feature-vectors-v1.parquet` +- `data/reports/normalization-statistics.parquet` +- `data/reports/normalization-summary.json` + diff --git a/docs/PIPELINE.md b/docs/PIPELINE.md index a3bb220..c473915 100644 --- a/docs/PIPELINE.md +++ b/docs/PIPELINE.md @@ -78,6 +78,13 @@ analytical weights, convert sRGB to OKLab/OKLCH, calculate compact palette stati visual values and derive eligible geometry. Emit scalar feature and long-form palette Parquets plus diagnostics. Do not perform corpus-relative normalization or similarity here. +## Stage 4 — versioned corpus normalization + +Profile all candidate scalars over the frozen 27,254-row corpus, freeze P01/P99, median and MAD, +and emit raw plus robust-01, empirical-midrank percentile and robust-z representations. Derive log +aspect ratios and circular hue diagnostics, preserve nulls, and publish separate unweighted color, +visual-character and optional geometry families. Similarity and final weights remain later stages. + ## Ingest performance rule The source files are large. Use bounded-memory streaming where practical. A one-time conversion to Parquet/Arrow as an interim representation is encouraged if it simplifies all later stages. diff --git a/src/smk_playground/cli.py b/src/smk_playground/cli.py index c92a066..873f9f5 100644 --- a/src/smk_playground/cli.py +++ b/src/smk_playground/cli.py @@ -46,5 +46,14 @@ def milestone3(root: Path | None = None) -> None: typer.echo(f"Milestone 3 complete: {summary['feature_rows']} feature rows") +@app.command("milestone4") +def milestone4(root: Path | None = None) -> None: + """Build corpus-relative normalized features and candidate feature families.""" + from smk_playground.normalize.milestone4 import run + + summary = run((root or Path.cwd()).resolve()) + typer.echo(f"Milestone 4 complete: {summary['row_count']} normalized rows") + + if __name__ == "__main__": app() diff --git a/src/smk_playground/normalize/milestone4.py b/src/smk_playground/normalize/milestone4.py new file mode 100644 index 0000000..eb43015 --- /dev/null +++ b/src/smk_playground/normalize/milestone4.py @@ -0,0 +1,375 @@ +"""Versioned corpus statistics and normalized features for Milestone 4.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import statistics +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +import pyarrow as pa +import pyarrow.parquet as pq + +PROFILE_NAME = "smk-public-visual-v1" +NORMALIZATION_VERSION = "1.0.0" + +PROFILE_FEATURES = ( + "centroid_l", "centroid_a", "centroid_b", "centroid_chroma", + "palette_spread_oklab", "lightness_stddev", "chroma_stddev", + "hue_concentration_01", "palette_entropy_01", "effective_palette_size", + "smk_brightness", "smk_contrast", "smk_saturation", "smk_entropy", + "smk_colortemp", "net_width_cm", "net_height_cm", "net_area_cm2", + "net_aspect_ratio", "log_net_width_cm", "log_net_height_cm", + "log_net_area_cm2", "image_aspect_ratio", "aspect_mismatch_log", + "log_net_aspect_ratio", "log_image_aspect_ratio", +) +NATURAL_01 = frozenset({"hue_concentration_01", "palette_entropy_01"}) +PRIMARY_EXCLUSIONS = { + "smk_colortemp": "extreme finite values and uncertain semantics", + "palette_entropy_01": "uniform v1 palette weights make entropy non-informative", + "effective_palette_size": "mostly duplicates analytical palette size under uniform weights", +} +FAMILIES = { + "color_core_v1": ( + "centroid_l", "centroid_a", "centroid_b", "palette_spread_oklab_robust_01", + "lightness_stddev_robust_01", "chroma_stddev_robust_01", + "hue_concentration_01", + ), + "visual_character_v1": ( + "smk_brightness_robust_01", "smk_contrast_robust_01", + "smk_saturation_robust_01", "smk_entropy_robust_01", + ), + "geometry_v1": ( + "log_net_width_cm_robust_01", "log_net_height_cm_robust_01", + "log_net_area_cm2_robust_01", "log_net_aspect_ratio_robust_01", + ), +} + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def linear_quantile(values: list[float], probability: float) -> float: + ordered = sorted(values) + position = (len(ordered) - 1) * probability + low = math.floor(position) + high = math.ceil(position) + if low == high: + return ordered[low] + fraction = position - low + return ordered[low] * (1 - fraction) + ordered[high] * fraction + + +def midrank_percentiles(values: Iterable[float | None]) -> list[float | None]: + """Return deterministic empirical midranks, scaled to [0, 1].""" + source = list(values) + valid = [(index, float(value)) for index, value in enumerate(source) + if value is not None and math.isfinite(float(value))] + result: list[float | None] = [None] * len(source) + if not valid: + return result + ordered = sorted(valid, key=lambda item: (item[1], item[0])) + denominator = max(len(ordered) - 1, 1) + cursor = 0 + while cursor < len(ordered): + end = cursor + 1 + while end < len(ordered) and ordered[end][1] == ordered[cursor][1]: + end += 1 + rank = ((cursor + end - 1) / 2) / denominator if len(ordered) > 1 else 0.5 + for position in range(cursor, end): + result[ordered[position][0]] = rank + cursor = end + return result + + +def robust_01(value: float | None, p01: float, p99: float, *, natural: bool = False) -> float | None: + if value is None or not math.isfinite(float(value)) or p01 == p99: + return None + if natural: + return min(1.0, max(0.0, float(value))) + return min(1.0, max(0.0, (float(value) - p01) / (p99 - p01))) + + +def robust_z(value: float | None, median: float, mad: float) -> float | None: + if value is None or not math.isfinite(float(value)) or mad == 0: + return None + return (float(value) - median) / (1.4826 * mad) + + +def log_aspect_ratio(width: float | None, height: float | None) -> float | None: + if width is None or height is None or width <= 0 or height <= 0: + return None + return math.log(width / height) + + +def hue_xy(hue_degrees: float | None) -> tuple[float | None, float | None]: + if hue_degrees is None or not math.isfinite(float(hue_degrees)): + return None, None + angle = math.radians(float(hue_degrees)) + return math.cos(angle), math.sin(angle) + + +def feature_profile(name: str, values: list[float | None]) -> dict[str, Any]: + finite = [float(value) for value in values + if value is not None and math.isfinite(float(value))] + count = len(values) + null_count = count - len(finite) + if len(finite) == 0: + return {"feature": name, "count": count, "null_count": null_count, + "finite_count": 0, **{key: None for key in ( + "min", "max", "mean", "stddev", "median", "mad", "p01", "p05", + "p25", "p50", "p75", "p95", "p99", "skewness")}, + "unique_count": 0, "non_informative": True, "mad_zero": True, + "robust_01_clipped_low_count": 0, "robust_01_clipped_high_count": 0} + median = statistics.median(finite) + mad = statistics.median(abs(value - median) for value in finite) + mean = statistics.fmean(finite) + stddev = statistics.stdev(finite) if len(finite) > 1 else 0.0 + skewness = None + if len(finite) > 2 and stddev > 0: + population_stddev = statistics.pstdev(finite) + skewness = statistics.fmean(((value - mean) / population_stddev) ** 3 for value in finite) + p01 = linear_quantile(finite, 0.01) + p99 = linear_quantile(finite, 0.99) + return { + "feature": name, "count": count, "null_count": null_count, + "finite_count": len(finite), "min": min(finite), "max": max(finite), + "mean": mean, "stddev": stddev, "median": median, "mad": mad, + "p01": p01, "p05": linear_quantile(finite, 0.05), + "p25": linear_quantile(finite, 0.25), "p50": median, + "p75": linear_quantile(finite, 0.75), "p95": linear_quantile(finite, 0.95), + "p99": p99, "skewness": skewness, "unique_count": len(set(finite)), + "non_informative": p01 == p99, "mad_zero": mad == 0, + "robust_01_clipped_low_count": sum(value < p01 for value in finite), + "robust_01_clipped_high_count": sum(value > p99 for value in finite), + } + + +def _pearson(left: list[float], right: list[float]) -> float | None: + left_mean = statistics.fmean(left) + right_mean = statistics.fmean(right) + left_delta = [value - left_mean for value in left] + right_delta = [value - right_mean for value in right] + denominator = math.sqrt( + math.fsum(value * value for value in left_delta) + * math.fsum(value * value for value in right_delta) + ) + if denominator == 0: + return None + return math.fsum(x * y for x, y in zip(left_delta, right_delta)) / denominator + + +def correlations(columns: dict[str, list[float | None]]) -> list[dict[str, Any]]: + output = [] + names = list(columns) + for left_index, left in enumerate(names): + for right in names[left_index + 1:]: + pairs = [(x, y) for x, y in zip(columns[left], columns[right]) + if x is not None and y is not None + and math.isfinite(float(x)) and math.isfinite(float(y))] + if len(pairs) < 3: + continue + x = [float(pair[0]) for pair in pairs] + y = [float(pair[1]) for pair in pairs] + pearson = _pearson(x, y) + rx = [float(value) for value in midrank_percentiles(x)] + ry = [float(value) for value in midrank_percentiles(y)] + spearman = _pearson(rx, ry) + output.append({"left": left, "right": right, "sample_count": len(pairs), + "pearson": pearson, "spearman": spearman}) + return output + + +def build_profile(feature_hash: str, generated_at: str, row_count: int, + feature_schema_version: str, extractor_version: str, + profiles: list[dict[str, Any]]) -> dict[str, Any]: + return { + "normalization_profile": PROFILE_NAME, + "normalization_version": NORMALIZATION_VERSION, + "corpus_version": PROFILE_NAME, + "corpus_row_count": row_count, + "feature_schema_version": feature_schema_version, + "feature_extractor_version": extractor_version, + "source_feature_parquet_sha256": feature_hash, + "generated_at_utc": generated_at, + "rules": { + "quantiles": "NumPy linear interpolation over finite non-null corpus values", + "robust_01": "natural [0,1] values retained; otherwise clamp((x-P01)/(P99-P01),0,1)", + "percentile_01": "empirical midrank with ties averaged, rank/(n-1); singleton=0.5", + "robust_z": "(x-median)/(1.4826*MAD), null when MAD=0", + "missing": "null input remains null; no imputation", + "aspect": "ln(width/height) for positive dimensions", + "hue": "circular cosine/sine diagnostics; excluded from color_core when a/b present", + "color_axes": "raw OKLab L/a/b retained in color_core; normalized variants are not perceptual geometry", + }, + "natural_01_features": sorted(NATURAL_01), + "primary_vector_exclusions": PRIMARY_EXCLUSIONS, + "feature_families": {name: list(fields) for name, fields in FAMILIES.items()}, + "features": profiles, + } + + +def _metadata(profile: dict[str, Any]) -> dict[bytes, bytes]: + compact = json.dumps(profile, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return { + b"smk.normalization_profile": PROFILE_NAME.encode(), + b"smk.normalization_version": NORMALIZATION_VERSION.encode(), + b"smk.profile_json": compact.encode(), + b"smk.source_feature_parquet_sha256": profile["source_feature_parquet_sha256"].encode(), + } + + +def _write_parquet(path: Path, rows: list[dict[str, Any]], metadata: dict[bytes, bytes], + schema: pa.Schema | None = None) -> None: + table = pa.Table.from_pylist(rows, schema=schema) + table = table.replace_schema_metadata(metadata) + pq.write_table(table, path, compression="zstd", version="2.6", write_statistics=True) + + +def run(root: Path) -> dict[str, Any]: + feature_path = root / "data" / "processed" / "artwork-visual-features-v1.parquet" + profile_path = root / "data" / "processed" / "normalization-profile-v1.json" + normalized_path = root / "data" / "processed" / "artwork-normalized-features-v1.parquet" + vectors_path = root / "data" / "processed" / "artwork-feature-vectors-v1.parquet" + statistics_path = root / "data" / "reports" / "normalization-statistics.parquet" + summary_path = root / "data" / "reports" / "normalization-summary.json" + + parquet = pq.ParquetFile(feature_path) + source_metadata = parquet.schema_arrow.metadata or {} + rows = parquet.read().to_pylist() + generated_at = source_metadata[b"smk.generated_at_utc"].decode() + feature_schema_version = source_metadata[b"smk.feature_schema_version"].decode() + extractor_version = source_metadata[b"smk.extractor_version"].decode() + + columns: dict[str, list[float | None]] = {name: [] for name in PROFILE_FEATURES} + for row in rows: + row["log_net_aspect_ratio"] = log_aspect_ratio(row["net_width_cm"], row["net_height_cm"]) + row["log_image_aspect_ratio"] = log_aspect_ratio(row["image_width_px"], row["image_height_px"]) + row["centroid_hue_x"], row["centroid_hue_y"] = hue_xy(row["centroid_hue_deg"]) + for name in PROFILE_FEATURES: + columns[name].append(row[name]) + + profiles = [feature_profile(name, columns[name]) for name in PROFILE_FEATURES] + profile_by_name = {item["feature"]: item for item in profiles} + profile = build_profile(sha256_file(feature_path), generated_at, len(rows), + feature_schema_version, extractor_version, profiles) + profile_path.write_text(json.dumps(profile, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + metadata = _metadata(profile) + + percentile_columns = {name: midrank_percentiles(values) for name, values in columns.items()} + normalized_rows = [] + vector_rows = [] + family_coverage = {name: 0 for name in FAMILIES} + all_family_count = 0 + for index, source in enumerate(rows): + normalized: dict[str, Any] = { + "smk_entity_id": source["smk_entity_id"], "object_number": source["object_number"], + "normalization_profile": PROFILE_NAME, "normalization_version": NORMALIZATION_VERSION, + "quality_flags": source["quality_flags"], + "centroid_hue_deg": source["centroid_hue_deg"], + "centroid_hue_x": source["centroid_hue_x"], "centroid_hue_y": source["centroid_hue_y"], + } + for name in PROFILE_FEATURES: + item = profile_by_name[name] + value = source[name] + normalized[name] = value + normalized[f"{name}_robust_01"] = robust_01( + value, item["p01"], item["p99"], natural=name in NATURAL_01) + normalized[f"{name}_percentile_01"] = percentile_columns[name][index] + normalized[f"{name}_robust_z"] = robust_z(value, item["median"], item["mad"]) + normalized_rows.append(normalized) + family_values = {name: [normalized[field] for field in fields] + for name, fields in FAMILIES.items()} + complete = {name: all(value is not None for value in values) + for name, values in family_values.items()} + for name, is_complete in complete.items(): + family_coverage[name] += int(is_complete) + all_family_count += int(all(complete.values())) + vector_rows.append({ + "smk_entity_id": source["smk_entity_id"], "object_number": source["object_number"], + "normalization_profile": PROFILE_NAME, "normalization_version": NORMALIZATION_VERSION, + "color_core": family_values["color_core_v1"], + "visual_character": family_values["visual_character_v1"], + "geometry": family_values["geometry_v1"], + "color_core_complete": complete["color_core_v1"], + "visual_character_complete": complete["visual_character_v1"], + "geometry_complete": complete["geometry_v1"], + }) + + normalized_fields = [ + ("smk_entity_id", pa.string()), ("object_number", pa.string()), + ("normalization_profile", pa.string()), ("normalization_version", pa.string()), + ("quality_flags", pa.list_(pa.string())), ("centroid_hue_deg", pa.float64()), + ("centroid_hue_x", pa.float64()), ("centroid_hue_y", pa.float64()), + ] + for name in PROFILE_FEATURES: + normalized_fields.extend(( + (name, pa.float64()), (f"{name}_robust_01", pa.float64()), + (f"{name}_percentile_01", pa.float64()), (f"{name}_robust_z", pa.float64()), + )) + vector_schema = pa.schema([ + ("smk_entity_id", pa.string()), ("object_number", pa.string()), + ("normalization_profile", pa.string()), ("normalization_version", pa.string()), + ("color_core", pa.list_(pa.float64(), len(FAMILIES["color_core_v1"]))), + ("visual_character", pa.list_(pa.float64(), len(FAMILIES["visual_character_v1"]))), + ("geometry", pa.list_(pa.float64(), len(FAMILIES["geometry_v1"]))), + ("color_core_complete", pa.bool_()), ("visual_character_complete", pa.bool_()), + ("geometry_complete", pa.bool_()), + ]) + _write_parquet(normalized_path, normalized_rows, metadata, pa.schema(normalized_fields)) + _write_parquet(vectors_path, vector_rows, metadata, vector_schema) + _write_parquet(statistics_path, profiles, metadata) + + corr_names = ( + "smk_brightness", "centroid_l", "smk_saturation", "centroid_chroma", + "smk_contrast", "palette_spread_oklab", "smk_entropy", "lightness_stddev", + "chroma_stddev", + ) + correlation_rows = correlations({name: columns[name] for name in corr_names}) + redundant = [item for item in correlation_rows if + (item["pearson"] is not None and abs(item["pearson"]) >= 0.9) + or (item["spearman"] is not None and abs(item["spearman"]) >= 0.9)] + examples = {} + for object_number in ("KMS8892", "KMS1571", "KAS782", "KKS5261"): + match = next((row for row in normalized_rows if row["object_number"] == object_number), None) + if match: + examples[object_number] = {name: match.get(name) for name in ( + "centroid_l", "centroid_l_robust_01", "centroid_l_percentile_01", "centroid_l_robust_z", + "centroid_a", "centroid_b", "palette_spread_oklab", + "palette_spread_oklab_robust_01", "palette_spread_oklab_percentile_01", + "palette_spread_oklab_robust_z", "hue_concentration_01", + "smk_brightness", "smk_contrast", "smk_saturation", "smk_entropy", + "net_width_cm", "net_height_cm", "net_area_cm2", "log_net_aspect_ratio")} + summary = { + "normalization_profile": PROFILE_NAME, "normalization_version": NORMALIZATION_VERSION, + "source_feature_parquet_sha256": profile["source_feature_parquet_sha256"], + "row_count": len(rows), "feature_statistics": profiles, + "high_null_features": [p["feature"] for p in profiles if p["null_count"] / p["count"] >= 0.5], + "near_zero_variance_features": [p["feature"] for p in profiles if p["stddev"] is not None and p["stddev"] <= 1e-12], + "extreme_skew_features": [p["feature"] for p in profiles if p["skewness"] is not None and abs(p["skewness"]) >= 2], + "correlations": correlation_rows, "redundant_pairs_abs_correlation_gte_0_9": redundant, + "family_coverage": {**family_coverage, "all_three": all_family_count}, + "examples": examples, + } + summary_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return summary + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--root", type=Path, default=Path.cwd()) + args = parser.parse_args() + print(json.dumps(run(args.root.resolve()), indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/test_normalization.py b/tests/test_normalization.py new file mode 100644 index 0000000..4cdd123 --- /dev/null +++ b/tests/test_normalization.py @@ -0,0 +1,61 @@ +import json +import math + +import pyarrow as pa +import pyarrow.parquet as pq +import pytest + +from smk_playground.normalize.milestone4 import ( + FAMILIES, + build_profile, + hue_xy, + log_aspect_ratio, + midrank_percentiles, + robust_01, + robust_z, +) + + +def test_robust_01_and_clipping(): + assert robust_01(5.0, 0.0, 10.0) == 0.5 + assert robust_01(-1.0, 0.0, 10.0) == 0.0 + assert robust_01(11.0, 0.0, 10.0) == 1.0 + assert robust_01(0.4, 0.2, 0.8, natural=True) == 0.4 + assert robust_01(None, 0.0, 1.0) is None + assert robust_01(1.0, 1.0, 1.0) is None + + +def test_midrank_percentile_ties_and_nulls(): + assert midrank_percentiles([1.0, 2.0, 2.0, 4.0, None]) == [0.0, 0.5, 0.5, 1.0, None] + assert midrank_percentiles([7.0]) == [0.5] + + +def test_robust_z_and_zero_mad(): + assert robust_z(2.4826, 1.0, 1.0) == pytest.approx(1.0) + assert robust_z(None, 1.0, 1.0) is None + assert robust_z(1.0, 1.0, 0.0) is None + + +def test_log_aspect_and_circular_hue(): + assert log_aspect_ratio(20.0, 10.0) == math.log(2.0) + assert log_aspect_ratio(None, 10.0) is None + assert log_aspect_ratio(0.0, 10.0) is None + x, y = hue_xy(90.0) + assert abs(x) < 1e-12 and y == 1.0 + assert hue_xy(None) == (None, None) + + +def test_family_order_is_explicit(): + assert FAMILIES["color_core_v1"][:3] == ("centroid_l", "centroid_a", "centroid_b") + assert FAMILIES["geometry_v1"][-1] == "log_net_aspect_ratio_robust_01" + + +def test_profile_generation_and_parquet_are_deterministic(tmp_path): + profile_a = build_profile("abc", "2026-01-01T00:00:00Z", 2, "1.0.0", "1.0.0", []) + profile_b = build_profile("abc", "2026-01-01T00:00:00Z", 2, "1.0.0", "1.0.0", []) + assert json.dumps(profile_a, sort_keys=True) == json.dumps(profile_b, sort_keys=True) + table = pa.table({"id": ["a", "b"], "value": [1.0, None]}) + first, second = tmp_path / "a.parquet", tmp_path / "b.parquet" + pq.write_table(table, first, compression="zstd", version="2.6") + pq.write_table(table, second, compression="zstd", version="2.6") + assert first.read_bytes() == second.read_bytes()