feat(versions): e2e fixtures, check_versions, docs
Fixtures gain an XMP-linked version (renamed file), a name-linked version and an edit with a reference photo; e2e.sh counts with include_versions and runs check_versions.py. CLAUDE.md documents the feature. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Cameron CordesandClaude Opus 5.5 committed
1 parent
fa3554c411
commit
6135034d09
5 files changed
+394
-6
No files matched your search
+198
-3
@@ -24,8 +24,10 @@ import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import zlib
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
@@ -115,6 +117,14 @@ class Item:
|
||||
tags: list[str] = field(default_factory=list)
|
||||
scenario: str = ""
|
||||
copy_of: str | None = None # byte-identical duplicate of another item
|
||||
# Photo versions (ImageApi #191): this PNG is a restoration / upscale / edit
|
||||
# of another item. `xmp` embeds the producer's XMP link (DerivedFrom = the
|
||||
# original's blake3, Ingredients = `references`), as the ComfyUI node does;
|
||||
# without it only the `<name>.<kind>.png` file-name convention links it.
|
||||
version_of: str | None = None
|
||||
version_kind: str = "restored"
|
||||
xmp: bool = False
|
||||
references: list[str] = field(default_factory=list) # item paths (provenance only)
|
||||
|
||||
def label_date(self) -> str:
|
||||
if self.taken:
|
||||
@@ -215,6 +225,45 @@ def build_items(today: dt.date) -> list[Item]:
|
||||
"Duplicates/IMG_1000 copy.jpg", "image", copy_of="2021/Paris Trip/IMG_1000.jpg",
|
||||
scenario="exact duplicate of 2021/Paris Trip/IMG_1000.jpg",
|
||||
))
|
||||
|
||||
# Photo versions (#191). Originals are dated, GPS-tagged JPEGs so the
|
||||
# version's copied date / GPS is checkable; every case lives in its own
|
||||
# folder with the producer's `aiedits/` subfolder.
|
||||
canon_v = ("Canon", "Canon EOS R5")
|
||||
for folder, stem, when in (
|
||||
("Xmp Linked", "scan_0042", dt.datetime(2004, 6, 12, 14, 30)),
|
||||
("Name Linked", "family_shot", dt.datetime(2008, 12, 24, 9, 15)),
|
||||
("Reference Photo", "subject", dt.datetime(2010, 3, 7, 16, 45)),
|
||||
):
|
||||
items.append(Item(
|
||||
f"Versions/{folder}/{stem}.jpg", "image", taken=when, gps="paris",
|
||||
camera=canon_v, tags=["versions"], scenario=f"versions: original ({folder})",
|
||||
))
|
||||
# Linked by the XMP chunk alone: the name says nothing about the original.
|
||||
items.append(Item(
|
||||
"Versions/Xmp Linked/aiedits/enhanced_final.png", "image", version_of=
|
||||
"Versions/Xmp Linked/scan_0042.jpg", version_kind="restored", xmp=True,
|
||||
scenario="version linked by XMP only (renamed file)",
|
||||
))
|
||||
# Linked by the file-name convention alone: no XMP in the file.
|
||||
items.append(Item(
|
||||
"Versions/Name Linked/aiedits/family_shot.jpg.upscale.png", "image", version_of=
|
||||
"Versions/Name Linked/family_shot.jpg", version_kind="upscale",
|
||||
scenario="version linked by file name only (no XMP)",
|
||||
))
|
||||
# An edit that also fed a second photo to the model: the other photo is a
|
||||
# reference (provenance), never hidden or grouped.
|
||||
items.append(Item(
|
||||
"Versions/Reference Photo/other_view.jpg", "image",
|
||||
taken=dt.datetime(2010, 3, 7, 16, 50), camera=canon_v,
|
||||
scenario="reference photo of a version: stays an ordinary photo",
|
||||
))
|
||||
items.append(Item(
|
||||
"Versions/Reference Photo/aiedits/subject.jpg.edit.png", "image", version_of=
|
||||
"Versions/Reference Photo/subject.jpg", version_kind="edit", xmp=True,
|
||||
references=["Versions/Reference Photo/other_view.jpg"],
|
||||
scenario="version with a reference photo (XMP Ingredients)",
|
||||
))
|
||||
return items
|
||||
|
||||
|
||||
@@ -250,6 +299,139 @@ def _same_day(today: dt.date, year: int, hour: int) -> dt.datetime:
|
||||
return dt.datetime(year, today.month, 28, hour, 0)
|
||||
|
||||
|
||||
# ── photo versions (#191) ───────────────────────────────────────────────
|
||||
|
||||
try:
|
||||
import blake3 as _blake3
|
||||
|
||||
def blake3_hex(data: bytes) -> str:
|
||||
return _blake3.blake3(data).hexdigest()
|
||||
except ImportError: # keep the generator dependency-free: reference BLAKE3
|
||||
_IV = [0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A,
|
||||
0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19]
|
||||
_PERM = [2, 6, 3, 10, 7, 0, 4, 13, 1, 11, 12, 5, 9, 14, 15, 8]
|
||||
_CHUNK_START, _CHUNK_END, _PARENT, _ROOT = 1, 2, 4, 8
|
||||
_M = 0xFFFFFFFF
|
||||
|
||||
|
||||
def _compress(cv, block, counter, block_len, flags):
|
||||
v = list(cv) + _IV[:4] + [counter & _M, counter >> 32, block_len, flags]
|
||||
m = list(block)
|
||||
|
||||
def g(a, b, c, d, mx, my):
|
||||
v[a] = (v[a] + v[b] + mx) & _M
|
||||
v[d] = ((v[d] ^ v[a]) >> 16 | (v[d] ^ v[a]) << 16) & _M
|
||||
v[c] = (v[c] + v[d]) & _M
|
||||
v[b] = ((v[b] ^ v[c]) >> 12 | (v[b] ^ v[c]) << 20) & _M
|
||||
v[a] = (v[a] + v[b] + my) & _M
|
||||
v[d] = ((v[d] ^ v[a]) >> 8 | (v[d] ^ v[a]) << 24) & _M
|
||||
v[c] = (v[c] + v[d]) & _M
|
||||
v[b] = ((v[b] ^ v[c]) >> 7 | (v[b] ^ v[c]) << 25) & _M
|
||||
|
||||
for r in range(7):
|
||||
g(0, 4, 8, 12, m[0], m[1]); g(1, 5, 9, 13, m[2], m[3])
|
||||
g(2, 6, 10, 14, m[4], m[5]); g(3, 7, 11, 15, m[6], m[7])
|
||||
g(0, 5, 10, 15, m[8], m[9]); g(1, 6, 11, 12, m[10], m[11])
|
||||
g(2, 7, 8, 13, m[12], m[13]); g(3, 4, 9, 14, m[14], m[15])
|
||||
if r < 6:
|
||||
m = [m[i] for i in _PERM]
|
||||
return [v[i] ^ v[i + 8] for i in range(8)] + [v[i + 8] ^ cv[i] for i in range(8)]
|
||||
|
||||
|
||||
def _words(block):
|
||||
return struct.unpack("<16I", block.ljust(64, b"\0"))
|
||||
|
||||
|
||||
def _chunk_output(chunk, counter):
|
||||
"""(cv_in, block_words, counter, block_len, flags) of the chunk's last block,
|
||||
plus the chaining value entering it."""
|
||||
cv = _IV
|
||||
blocks = [chunk[i:i + 64] for i in range(0, len(chunk), 64)] or [b""]
|
||||
for n, blk in enumerate(blocks[:-1]):
|
||||
flags = _CHUNK_START if n == 0 else 0
|
||||
cv = _compress(cv, _words(blk), counter, 64, flags)[:8]
|
||||
last = blocks[-1]
|
||||
flags = _CHUNK_END | (_CHUNK_START if len(blocks) == 1 else 0)
|
||||
return cv, _words(last), counter, len(last), flags
|
||||
|
||||
|
||||
def blake3_hex(data: bytes) -> str:
|
||||
"""Plain 32-byte BLAKE3 of `data`, lowercase hex (the reference algorithm)."""
|
||||
chunks = [data[i:i + 1024] for i in range(0, len(data), 1024)] or [b""]
|
||||
|
||||
def subtree(lo, hi, root):
|
||||
n = hi - lo
|
||||
if n == 1:
|
||||
cv, w, ctr, ln, fl = _chunk_output(chunks[lo], lo)
|
||||
return _compress(cv, w, ctr, ln, fl | (_ROOT if root else 0))[:8]
|
||||
split = 1 << ((n - 1).bit_length() - 1) # largest power of two below n
|
||||
left = subtree(lo, lo + split, False)
|
||||
right = subtree(lo + split, hi, False)
|
||||
flags = _PARENT | (_ROOT if root else 0)
|
||||
return _compress(_IV, left + right, 0, 64, flags)[:8]
|
||||
|
||||
return struct.pack("<8I", *subtree(0, len(chunks), True)).hex()
|
||||
|
||||
XMP_KEY = "XML:com.adobe.xmp"
|
||||
IMAGEAPI_NS = "https://git.crkc.xyz/Apps/ImageApi/ns/versions/1.0/"
|
||||
|
||||
|
||||
def _resource_ref(file_hash: str, name: str) -> str:
|
||||
return (f"<stRef:documentID>blake3:{file_hash}</stRef:documentID>"
|
||||
f"<stRef:filePath>{name}</stRef:filePath>")
|
||||
|
||||
|
||||
def build_xmp(source_hash: str, source_name: str, kind: str,
|
||||
references: list[tuple[str, str]]) -> str:
|
||||
"""The packet comfyui-imageapi-versions writes (versioning.build_xmp)."""
|
||||
ingredients = ""
|
||||
if references:
|
||||
lis = "".join(
|
||||
f'<rdf:li rdf:parseType="Resource">{_resource_ref(h, n)}</rdf:li>'
|
||||
for h, n in references
|
||||
)
|
||||
ingredients = f"<xmpMM:Ingredients><rdf:Bag>{lis}</rdf:Bag></xmpMM:Ingredients>"
|
||||
return (
|
||||
'<?xpacket begin="\ufeff" id="W5M0MpCehiHzreSzNTczkc9d"?>'
|
||||
'<x:xmpmeta xmlns:x="adobe:ns:meta/">'
|
||||
'<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">'
|
||||
'<rdf:Description rdf:about=""'
|
||||
' xmlns:xmpMM="http://ns.adobe.com/xap/1.0/mm/"'
|
||||
' xmlns:stRef="http://ns.adobe.com/xap/1.0/sType/ResourceRef#"'
|
||||
f' xmlns:imageapi="{IMAGEAPI_NS}">'
|
||||
f'<xmpMM:DerivedFrom rdf:parseType="Resource">{_resource_ref(source_hash, source_name)}</xmpMM:DerivedFrom>'
|
||||
f"{ingredients}<imageapi:versionKind>{kind}</imageapi:versionKind>"
|
||||
"</rdf:Description></rdf:RDF></x:xmpmeta>"
|
||||
'<?xpacket end="w"?>'
|
||||
)
|
||||
|
||||
|
||||
def embed_xmp(png: Path, xmp: str) -> None:
|
||||
"""Insert the XMP as an uncompressed iTXt chunk right after IHDR."""
|
||||
data = png.read_bytes()
|
||||
assert data[:8] == b"\x89PNG\r\n\x1a\n", f"{png} is not a PNG"
|
||||
body = XMP_KEY.encode() + b"\0\0\0\0\0" + xmp.encode("utf-8")
|
||||
chunk = struct.pack(">I", len(body)) + b"iTXt" + body
|
||||
chunk += struct.pack(">I", zlib.crc32(b"iTXt" + body) & 0xFFFFFFFF)
|
||||
ihdr_end = 8 + 4 + 4 + 13 + 4
|
||||
png.write_bytes(data[:ihdr_end] + chunk + data[ihdr_end:])
|
||||
|
||||
|
||||
def link_versions(items: list["Item"], media: Path) -> None:
|
||||
"""Write the XMP link into each `xmp` version. Runs after EXIF stamping:
|
||||
the link is the original's hash over its final bytes."""
|
||||
by_path = {it.rel: it for it in items}
|
||||
for it in items:
|
||||
if not (it.version_of and it.xmp):
|
||||
continue
|
||||
original = media / it.version_of
|
||||
refs = [(blake3_hex((media / r).read_bytes()), Path(r).name) for r in it.references]
|
||||
embed_xmp(media / it.rel,
|
||||
build_xmp(blake3_hex(original.read_bytes()), original.name,
|
||||
it.version_kind, refs))
|
||||
assert by_path[it.version_of].kind == "image"
|
||||
|
||||
|
||||
def render(item: Item, idx: int, media: Path) -> None:
|
||||
out = media / item.rel
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
@@ -355,6 +537,7 @@ def main() -> int:
|
||||
if it.copy_of:
|
||||
(media / it.rel).parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(media / it.copy_of, media / it.rel)
|
||||
link_versions(items, media)
|
||||
# Pin mtimes last: exiftool rewrites (and so touches) the files.
|
||||
for it in items:
|
||||
when = it.mtime or it.taken or dt.datetime(2024, 1, 1)
|
||||
@@ -387,6 +570,15 @@ def main() -> int:
|
||||
"excluded_dirs": ["/private"],
|
||||
"places": {k: {"lat": v[0], "lon": v[1]} for k, v in PLACES.items()},
|
||||
"tags": tags,
|
||||
# Photo versions (#191): hidden behind `original` in listings; the
|
||||
# reference photos stay visible. `source` is how ImageApi links it.
|
||||
"versions": [
|
||||
{
|
||||
"original": it.version_of, "version": it.rel, "kind": it.version_kind,
|
||||
"source": "xmp" if it.xmp else "name", "references": it.references,
|
||||
}
|
||||
for it in items if it.version_of
|
||||
],
|
||||
"items": [
|
||||
{
|
||||
"path": it.rel, "kind": it.kind, "scenario": it.scenario,
|
||||
@@ -394,9 +586,12 @@ def main() -> int:
|
||||
"taken": it.taken.isoformat() if it.taken else None,
|
||||
"gps": it.gps, "camera": it.camera[1] if it.camera else None,
|
||||
"tags": it.tags, "duplicate_of": it.copy_of,
|
||||
# A duplicate carries its source's bytes, EXIF included.
|
||||
"wall_clock": expected_wall_clock(by_path.get(it.copy_of, it)),
|
||||
"true_instant": expected_true_instant(by_path.get(it.copy_of, it)),
|
||||
"version_of": it.version_of,
|
||||
# A duplicate carries its source's bytes, EXIF included; a
|
||||
# version takes its original's date and GPS (ImageApi copies
|
||||
# them onto the version row).
|
||||
"wall_clock": expected_wall_clock(by_path.get(it.copy_of or it.version_of, it)),
|
||||
"true_instant": expected_true_instant(by_path.get(it.copy_of or it.version_of, it)),
|
||||
}
|
||||
for it in items
|
||||
],
|
||||
|
||||
Reference in new issue
Block a user