mirror of
https://github.com/jung-geun/PSO.git
synced 2026-09-20 14:11:48 +09:00
jung-geun
6fe31394b9
Keep objective construction on CPU and evaluate cached detection features in source-sized chunks. Adam gradients now accumulate per chunk, avoiding a monolithic 2,500-image CUDA graph without changing the full-objective mean. Constraint: Preserve 2,500-image objective and exact 41 Adam evaluations Rejected: Reduce objective sample count | changes the sealed protocol Confidence: high Scope-risk: moderate Not-tested: Full remote 2,500-image CUDA optimization
2435 lines
99 KiB
Python
2435 lines
99 KiB
Python
"""Pinned Ultralytics YOLO11n/VOC adapter for the convergence protocol.
|
|
|
|
The module deliberately keeps Ultralytics, torchvision and ensemble-boxes imports
|
|
inside the operations that need them. Importing this module is consequently safe
|
|
in the normal (non-detection) installation. All persistent writes go through the
|
|
common protocol's atomic helpers and every phase is guarded by the run state.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import csv
|
|
import dataclasses
|
|
import hashlib
|
|
import json
|
|
import math
|
|
import os
|
|
import random
|
|
import shutil
|
|
import time
|
|
import warnings
|
|
import xml.etree.ElementTree as ET
|
|
from pathlib import Path
|
|
from typing import Any, Callable, Iterable, Iterator, Mapping, Sequence
|
|
|
|
import numpy as np
|
|
import torch
|
|
from torch import nn
|
|
from pso.optimizer import _RandomSource
|
|
from pso.plugins import ConstrictionMovement, IterationContext, SwarmState
|
|
|
|
from test.post_training_model_convergence import (
|
|
BASE_SEEDS,
|
|
PROJECTION_SEED,
|
|
PSO_GENERATIONS,
|
|
PARTICLE_COUNT,
|
|
RESIDUAL_DIMENSION,
|
|
RESIDUAL_BOUND,
|
|
SWARM_SEEDS,
|
|
AuditResult,
|
|
ObjectiveResult,
|
|
PROTOCOL_VERSION,
|
|
ProtocolError,
|
|
ResourceCounters,
|
|
SealError,
|
|
SelectedResidualCodec,
|
|
StudyConfig,
|
|
StudyState,
|
|
StudyStateMachine,
|
|
atomic_write_bytes,
|
|
atomic_write_json,
|
|
begin_confirmation,
|
|
canonical_json,
|
|
fingerprint_file,
|
|
fingerprint_module,
|
|
fingerprint_nonselected_state,
|
|
finish_confirmation,
|
|
freeze_run,
|
|
load_frozen_manifest,
|
|
load_state,
|
|
prepare_run,
|
|
run_equal_budget_random,
|
|
run_residual_pso,
|
|
run_state_neutral_audit,
|
|
select_endpoint,
|
|
sha256_bytes,
|
|
verify_frozen_manifest,
|
|
)
|
|
|
|
WORKLOAD_ID = "voc_yolo11n"
|
|
FAMILY = "detection"
|
|
ULTRALYTICS_VERSION = "8.4.142"
|
|
ENSEMBLE_BOXES_VERSION = "1.0.9"
|
|
VOC_CLASSES = (
|
|
"aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car",
|
|
"cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike",
|
|
"person", "pottedplant", "sheep", "sofa", "train", "tvmonitor",
|
|
)
|
|
VOC_CLASS_TO_ID = {name: index for index, name in enumerate(VOC_CLASSES)}
|
|
BP_COUNT, REFINE_COUNT, SELECTION_COUNT, OBJECTIVE_COUNT = 11551, 2500, 2500, 512
|
|
IMG_SIZE = 640
|
|
EXPECTED_BLOCK_INDEX = 22
|
|
EXPECTED_DETECT_INDEX = 23
|
|
EXPECTED_HEAD_BIAS_COUNT = 252
|
|
VOC_YEARS = ("2007", "2012")
|
|
|
|
WBF_PARTICLE_COUNT = 12
|
|
WBF_GENERATIONS = 20
|
|
|
|
class YoloProtocolError(ProtocolError):
|
|
"""A detection-specific protocol violation."""
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class VOCRecord:
|
|
year: str
|
|
image_id: str
|
|
image_path: str
|
|
annotation_path: str
|
|
width: int
|
|
height: int
|
|
labels: tuple[tuple[int, float, float, float, float], ...]
|
|
difficult_excluded: int
|
|
fingerprint: str
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return dataclasses.asdict(self)
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class VOCManifest:
|
|
bp_train: tuple[VOCRecord, ...]
|
|
refine_search: tuple[VOCRecord, ...]
|
|
selection_val: tuple[VOCRecord, ...]
|
|
permutation_seed: int
|
|
duplicate_groups: Mapping[str, tuple[str, ...]]
|
|
counts: Mapping[str, int]
|
|
objective_keys: tuple[tuple[str, str], ...]
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"bp_train": [r.to_dict() for r in self.bp_train],
|
|
"refine_search": [r.to_dict() for r in self.refine_search],
|
|
"selection_val": [r.to_dict() for r in self.selection_val],
|
|
"permutation_seed": self.permutation_seed,
|
|
"duplicate_groups": {k: list(v) for k, v in self.duplicate_groups.items()},
|
|
"counts": dict(self.counts),
|
|
"objective_keys": [list(x) for x in self.objective_keys],
|
|
}
|
|
|
|
|
|
@dataclasses.dataclass(frozen=True)
|
|
class VOCTestGuard:
|
|
run_root: str
|
|
frozen_manifest_hash: str
|
|
confirmation_started: bool
|
|
|
|
def require_open(self) -> None:
|
|
if not self.confirmation_started or not self.frozen_manifest_hash:
|
|
raise SealError("VOC2007 test is sealed until frozen confirmation begins")
|
|
|
|
|
|
def _ultralytics() -> Any:
|
|
"""Import the pinned optional package only when a detection operation runs."""
|
|
try:
|
|
import ultralytics # type: ignore
|
|
except ImportError as exc:
|
|
raise YoloProtocolError(
|
|
"Ultralytics is required for voc_yolo11n; install ultralytics==8.4.142"
|
|
) from exc
|
|
version = str(getattr(ultralytics, "__version__", ""))
|
|
if version != ULTRALYTICS_VERSION:
|
|
raise YoloProtocolError(
|
|
f"Ultralytics version mismatch: expected {ULTRALYTICS_VERSION}, got {version or 'unknown'}"
|
|
)
|
|
return ultralytics
|
|
|
|
|
|
def _torchvision_voc() -> Any:
|
|
try:
|
|
from torchvision.datasets import VOCDetection # type: ignore
|
|
except ImportError as exc:
|
|
raise YoloProtocolError("torchvision with VOCDetection is required for VOC preparation") from exc
|
|
return VOCDetection
|
|
|
|
|
|
def _wbf() -> Callable[..., Any]:
|
|
try:
|
|
from ensemble_boxes import weighted_boxes_fusion # type: ignore
|
|
except ImportError as exc:
|
|
raise YoloProtocolError(
|
|
"ensemble-boxes is required for YOLO ensemble arms; install ensemble-boxes==1.0.9"
|
|
) from exc
|
|
module = __import__("ensemble_boxes")
|
|
version = str(getattr(module, "__version__", ""))
|
|
if version and version != ENSEMBLE_BOXES_VERSION:
|
|
raise YoloProtocolError(
|
|
f"ensemble-boxes version mismatch: expected {ENSEMBLE_BOXES_VERSION}, got {version}"
|
|
)
|
|
return weighted_boxes_fusion
|
|
|
|
|
|
def pinned_preflight() -> dict[str, str]:
|
|
"""Check optional package pins without importing them at module import time."""
|
|
ultra = _ultralytics()
|
|
# Importing ensemble-boxes here verifies the package before a run starts.
|
|
_wbf()
|
|
return {
|
|
"ultralytics": str(getattr(ultra, "__version__", "")),
|
|
"ensemble_boxes": ENSEMBLE_BOXES_VERSION,
|
|
"voc_classes": str(len(VOC_CLASSES)),
|
|
}
|
|
|
|
|
|
def _canonical_pixels(image: Any) -> tuple[int, int, bytes]:
|
|
"""Return the label-free duplicate key required by the protocol."""
|
|
try:
|
|
from PIL import Image
|
|
if not isinstance(image, Image.Image):
|
|
image = Image.open(image)
|
|
rgb = image.convert("RGB")
|
|
width, height = rgb.size
|
|
return width, height, np.asarray(rgb, dtype=np.uint8).tobytes(order="C")
|
|
except ImportError as exc:
|
|
raise YoloProtocolError("Pillow is required for VOC duplicate fingerprinting") from exc
|
|
|
|
|
|
def image_fingerprint(image: Any) -> str:
|
|
width, height, pixels = _canonical_pixels(image)
|
|
digest = hashlib.sha256()
|
|
digest.update(width.to_bytes(8, "little", signed=False))
|
|
digest.update(height.to_bytes(8, "little", signed=False))
|
|
digest.update(pixels)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _parse_int(node: ET.Element, tag: str) -> int:
|
|
child = node.find(tag)
|
|
if child is None or child.text is None:
|
|
raise YoloProtocolError(f"VOC annotation missing {tag}")
|
|
try:
|
|
return int(child.text)
|
|
except ValueError as exc:
|
|
raise YoloProtocolError(f"invalid integer in VOC annotation {tag}") from exc
|
|
|
|
|
|
def parse_voc_xml(annotation_path: str | os.PathLike[str], image_path: str | os.PathLike[str], *, year: str, image_id: str) -> VOCRecord:
|
|
"""Parse one XML and convert non-difficult objects to normalized xywh labels."""
|
|
path = Path(annotation_path)
|
|
root = ET.parse(path).getroot()
|
|
size = root.find("size")
|
|
if size is None:
|
|
raise YoloProtocolError(f"VOC annotation has no size: {path}")
|
|
width, height = _parse_int(size, "width"), _parse_int(size, "height")
|
|
if width <= 0 or height <= 0:
|
|
raise YoloProtocolError(f"invalid VOC dimensions in {path}")
|
|
labels: list[tuple[int, float, float, float, float]] = []
|
|
excluded = 0
|
|
for object_node in root.findall("object"):
|
|
name_node = object_node.find("name")
|
|
if name_node is None or not name_node.text:
|
|
raise YoloProtocolError(f"VOC object has no class in {path}")
|
|
class_name = name_node.text.strip().lower()
|
|
if class_name not in VOC_CLASS_TO_ID:
|
|
raise YoloProtocolError(f"unknown VOC class {class_name!r} in {path}")
|
|
difficult_node = object_node.find("difficult")
|
|
difficult = difficult_node is not None and (difficult_node.text or "0").strip() == "1"
|
|
if difficult:
|
|
excluded += 1
|
|
continue
|
|
box = object_node.find("bndbox")
|
|
if box is None:
|
|
raise YoloProtocolError(f"VOC object has no bndbox in {path}")
|
|
xmin, ymin = _parse_int(box, "xmin"), _parse_int(box, "ymin")
|
|
xmax, ymax = _parse_int(box, "xmax"), _parse_int(box, "ymax")
|
|
if xmax < xmin or ymax < ymin:
|
|
raise YoloProtocolError(f"inverted VOC box in {path}")
|
|
# This is the pinned Ultralytics VOC convention: center uses -1, while
|
|
# width and height are the XML extent without an additional correction.
|
|
center_x = ((xmin + xmax) / 2.0 - 1.0) / width
|
|
center_y = ((ymin + ymax) / 2.0 - 1.0) / height
|
|
box_width = (xmax - xmin) / width
|
|
box_height = (ymax - ymin) / height
|
|
values = (center_x, center_y, box_width, box_height)
|
|
if not all(math.isfinite(value) for value in values):
|
|
raise YoloProtocolError(f"non-finite VOC box in {path}")
|
|
labels.append((VOC_CLASS_TO_ID[class_name], *values))
|
|
try:
|
|
fingerprint = image_fingerprint(image_path)
|
|
except (OSError, ValueError) as exc:
|
|
raise YoloProtocolError(f"cannot fingerprint VOC image {image_path}") from exc
|
|
return VOCRecord(year, image_id, str(image_path), str(annotation_path), width, height, tuple(labels), excluded, fingerprint)
|
|
|
|
|
|
def write_yolo_label(record: VOCRecord, path: str | os.PathLike[str]) -> Path:
|
|
lines = ["%d %.10f %.10f %.10f %.10f" % label for label in record.labels]
|
|
return atomic_write_bytes(path, ("\n".join(lines) + ("\n" if lines else "")).encode("utf-8"))
|
|
|
|
|
|
def _voc_roots(data_root: Path, year: str, *, split: str = "trainval") -> tuple[Path, Path, Path]:
|
|
if split not in {"trainval", "test"}:
|
|
raise YoloProtocolError(f"unsupported VOC split: {split}")
|
|
root = data_root / "VOCdevkit" / f"VOC{year}"
|
|
return root / "JPEGImages", root / "Annotations", root / "ImageSets" / "Main" / f"{split}.txt"
|
|
def _records_from_voc(data_root: Path, *, allow_download: bool) -> list[VOCRecord]:
|
|
records: list[VOCRecord] = []
|
|
for year in VOC_YEARS:
|
|
image_root, annotation_root, split_path = _voc_roots(data_root, year)
|
|
if not split_path.is_file():
|
|
if not allow_download:
|
|
raise YoloProtocolError(f"VOC{year} is unavailable; rerun preparation with --allow-download")
|
|
VOCDetection = _torchvision_voc()
|
|
VOCDetection(root=str(data_root), year=year, image_set="trainval", download=True)
|
|
if not split_path.is_file():
|
|
raise YoloProtocolError(f"torchvision did not create VOC{year} trainval manifest")
|
|
ids = [line.strip() for line in split_path.read_text(encoding="utf-8").splitlines() if line.strip()]
|
|
for image_id in ids:
|
|
image_path = image_root / f"{image_id}.jpg"
|
|
annotation_path = annotation_root / f"{image_id}.xml"
|
|
if not image_path.is_file() or not annotation_path.is_file():
|
|
raise YoloProtocolError(f"incomplete VOC{year} item: {image_id}")
|
|
records.append(parse_voc_xml(annotation_path, image_path, year=year, image_id=image_id))
|
|
return records
|
|
|
|
|
|
def _assign_duplicate_groups(records: Sequence[VOCRecord], *, seed: int) -> tuple[list[VOCRecord], dict[str, tuple[str, ...]]]:
|
|
groups: dict[str, list[VOCRecord]] = {}
|
|
for record in records:
|
|
groups.setdefault(record.fingerprint, []).append(record)
|
|
rng = random.Random(seed)
|
|
order = list(records)
|
|
rng.shuffle(order)
|
|
order_position = {f"{record.year}:{record.image_id}": index for index, record in enumerate(order)}
|
|
group_map: dict[str, tuple[str, ...]] = {}
|
|
for fingerprint, members in groups.items():
|
|
members.sort(key=lambda record: order_position[f"{record.year}:{record.image_id}"])
|
|
keys = tuple(f"{record.year}:{record.image_id}" for record in members)
|
|
group_map[fingerprint] = keys
|
|
grouped_order: list[VOCRecord] = []
|
|
seen: set[str] = set()
|
|
for record in order:
|
|
if record.fingerprint in seen:
|
|
continue
|
|
seen.add(record.fingerprint)
|
|
grouped_order.extend(groups[record.fingerprint])
|
|
return grouped_order, group_map
|
|
|
|
|
|
def make_voc_manifests(records: Sequence[VOCRecord], *, seed: int = 20260908) -> VOCManifest:
|
|
if len(records) != 16551:
|
|
raise YoloProtocolError(f"expected 16,551 VOC trainval records, found {len(records)}")
|
|
ordered, groups = _assign_duplicate_groups(records, seed=seed)
|
|
by_key = {
|
|
f"{record.year}:{record.image_id}": record for record in ordered
|
|
}
|
|
ordered_groups: list[tuple[VOCRecord, ...]] = []
|
|
seen_fingerprints: set[str] = set()
|
|
for record in ordered:
|
|
if record.fingerprint in seen_fingerprints:
|
|
continue
|
|
seen_fingerprints.add(record.fingerprint)
|
|
ordered_groups.append(
|
|
tuple(by_key[key] for key in groups[record.fingerprint])
|
|
)
|
|
|
|
def take_groups(
|
|
available: Sequence[tuple[VOCRecord, ...]],
|
|
count: int,
|
|
) -> tuple[tuple[VOCRecord, ...], list[tuple[VOCRecord, ...]]]:
|
|
chosen: list[VOCRecord] = []
|
|
deferred: list[tuple[VOCRecord, ...]] = []
|
|
for group in available:
|
|
if len(chosen) + len(group) <= count:
|
|
chosen.extend(group)
|
|
else:
|
|
deferred.append(group)
|
|
if len(chosen) != count:
|
|
raise YoloProtocolError(
|
|
f"duplicate-safe partition cannot satisfy exact size {count}"
|
|
)
|
|
return tuple(chosen), deferred
|
|
|
|
bp, remaining = take_groups(ordered_groups, BP_COUNT)
|
|
refine, remaining = take_groups(remaining, REFINE_COUNT)
|
|
selection = tuple(record for group in remaining for record in group)
|
|
if len(bp) != BP_COUNT or len(refine) != REFINE_COUNT or len(selection) != SELECTION_COUNT:
|
|
raise YoloProtocolError("VOC duplicate grouping did not produce exact manifest sizes")
|
|
all_keys = {f"{r.year}:{r.image_id}" for r in bp + refine + selection}
|
|
if len(all_keys) != len(bp) + len(refine) + len(selection):
|
|
raise YoloProtocolError("VOC manifests overlap")
|
|
for split in (bp, refine, selection):
|
|
if not set(range(20)).issubset({label[0] for record in split for label in record.labels}):
|
|
raise YoloProtocolError("VOC manifest does not contain all 20 classes")
|
|
objective = tuple((r.year, r.image_id) for r in refine[:OBJECTIVE_COUNT])
|
|
return VOCManifest(bp, refine, selection, seed, groups, {
|
|
"bp_train": len(bp), "refine_search": len(refine), "selection_val": len(selection),
|
|
}, objective)
|
|
|
|
|
|
def prepare_voc(
|
|
data_root: str | os.PathLike[str],
|
|
run_root: str | os.PathLike[str],
|
|
*,
|
|
allow_download: bool,
|
|
seed: int = 20260908,
|
|
) -> VOCManifest:
|
|
data = Path(data_root)
|
|
root = Path(run_root) / "workloads" / WORKLOAD_ID
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
records = _records_from_voc(data, allow_download=allow_download)
|
|
manifest = make_voc_manifests(records, seed=seed)
|
|
atomic_write_json(root / "voc_manifest.json", manifest.to_dict())
|
|
labels_root = root / "labels"
|
|
images_root = root / "images"
|
|
labels_root.mkdir(parents=True, exist_ok=True)
|
|
for split_name, split in (
|
|
("bp_train", manifest.bp_train),
|
|
("refine_search", manifest.refine_search),
|
|
("selection_val", manifest.selection_val),
|
|
):
|
|
for record in split:
|
|
write_yolo_label(
|
|
record,
|
|
labels_root / split_name / f"{record.year}_{record.image_id}.txt",
|
|
)
|
|
destination = images_root / split_name / f"{record.year}_{record.image_id}.jpg"
|
|
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
try:
|
|
destination.symlink_to(Path(record.image_path).resolve())
|
|
except FileExistsError:
|
|
if not destination.exists():
|
|
raise YoloProtocolError(f"stale image link: {destination}")
|
|
except OSError:
|
|
shutil.copy2(record.image_path, destination)
|
|
return manifest
|
|
|
|
|
|
def guarded_voc_test_loader(
|
|
data_root: str | os.PathLike[str],
|
|
run_root: str | os.PathLike[str],
|
|
*,
|
|
confirmation: bool = False,
|
|
) -> Any:
|
|
"""Open VOC2007 test only after a valid frozen manifest and confirmation."""
|
|
root = Path(run_root)
|
|
state = load_state(root)
|
|
if state.state != StudyState.CONFIRMING or not confirmation:
|
|
raise SealError("VOC2007 official test is sealed until confirm phase")
|
|
frozen = verify_frozen_manifest(root)
|
|
image_root, annotation_root, split_path = _voc_roots(Path(data_root), "2007", split="test")
|
|
if not split_path.is_file():
|
|
raise YoloProtocolError("VOC2007 test manifest is unavailable")
|
|
records = []
|
|
for image_id in (
|
|
line.strip()
|
|
for line in split_path.read_text(encoding="utf-8").splitlines()
|
|
if line.strip()
|
|
):
|
|
image_path = image_root / f"{image_id}.jpg"
|
|
annotation_path = annotation_root / f"{image_id}.xml"
|
|
records.append(
|
|
parse_voc_xml(annotation_path, image_path, year="2007", image_id=image_id)
|
|
)
|
|
return records, VOCTestGuard(str(root), frozen.manifest_hash, True)
|
|
|
|
|
|
def write_study_yaml(
|
|
manifest: VOCManifest,
|
|
path: str | os.PathLike[str],
|
|
*,
|
|
labels_root: str | os.PathLike[str],
|
|
) -> Path:
|
|
"""Write a study-only YAML with train/selection images and no test/download."""
|
|
labels = Path(labels_root).resolve()
|
|
dataset_root = labels.parent
|
|
yaml = (
|
|
f"path: {dataset_root}\n"
|
|
f"train: {dataset_root / 'images' / 'bp_train'}\n"
|
|
f"val: {dataset_root / 'images' / 'selection_val'}\n"
|
|
"names:\n"
|
|
+ "\n".join(f" {i}: {name}" for i, name in enumerate(VOC_CLASSES))
|
|
+ "\n"
|
|
)
|
|
if "download:" in yaml or "\ntest:" in yaml:
|
|
raise YoloProtocolError("study YAML cannot contain download or test entries")
|
|
if manifest.counts.get("bp_train") != BP_COUNT:
|
|
raise YoloProtocolError("study YAML manifest is not the pinned bp_train split")
|
|
return atomic_write_bytes(path, yaml.encode("utf-8"))
|
|
|
|
|
|
def _model_yaml_path() -> str:
|
|
try:
|
|
import ultralytics
|
|
except ImportError as exc:
|
|
raise YoloProtocolError("Ultralytics is required to create YOLO11n") from exc
|
|
path = Path(ultralytics.__file__).resolve().parent / "cfg" / "models" / "11" / "yolo11.yaml"
|
|
if not path.is_file():
|
|
raise YoloProtocolError(f"pinned yolo11.yaml is missing: {path}")
|
|
return str(path)
|
|
|
|
|
|
def make_yolo11n(*, device: str | torch.device = "cpu", nc: int = 20) -> Any:
|
|
"""Create a scratch YOLO11n with a rebuilt 20-class Detect head."""
|
|
ultra = _ultralytics()
|
|
if nc != 20:
|
|
raise YoloProtocolError("VOC YOLO11n must have exactly 20 classes")
|
|
wrapper = ultra.YOLO(_model_yaml_path(), task="detect")
|
|
try:
|
|
from ultralytics.nn.tasks import DetectionModel # type: ignore
|
|
wrapper.model = DetectionModel(_model_yaml_path(), ch=3, nc=nc, verbose=False)
|
|
from ultralytics.cfg import get_cfg # type: ignore
|
|
wrapper.model.args = get_cfg()
|
|
except (ImportError, TypeError) as exc:
|
|
raise YoloProtocolError("pinned Ultralytics cannot construct 20-class DetectionModel") from exc
|
|
wrapper.model.to(device)
|
|
assert_yolo_topology(wrapper.model)
|
|
return wrapper
|
|
|
|
|
|
def assert_yolo_topology(model: nn.Module) -> None:
|
|
layers = getattr(model, "model", None)
|
|
if layers is None or len(layers) <= EXPECTED_DETECT_INDEX:
|
|
raise YoloProtocolError("YOLO11n graph is shorter than the pinned block22/Detect graph")
|
|
block = layers[EXPECTED_BLOCK_INDEX]
|
|
detect = layers[EXPECTED_DETECT_INDEX]
|
|
if block.__class__.__name__ != "C3k2":
|
|
raise YoloProtocolError(f"expected model.22 C3k2, found {block.__class__.__name__}")
|
|
if detect.__class__.__name__ != "Detect":
|
|
raise YoloProtocolError(f"expected model.23 Detect, found {detect.__class__.__name__}")
|
|
if int(getattr(detect, "nc", -1)) != 20:
|
|
raise YoloProtocolError(f"expected Detect.nc=20, found {getattr(detect, 'nc', None)}")
|
|
if not hasattr(detect, "cv2") or not hasattr(detect, "cv3") or len(detect.cv2) != 3 or len(detect.cv3) != 3:
|
|
raise YoloProtocolError("pinned Detect head must expose three cv2 and cv3 branches")
|
|
bias = [*list(detect.cv2[i][-1].bias for i in range(3)), *list(detect.cv3[i][-1].bias for i in range(3))]
|
|
if any(value is None for value in bias):
|
|
raise YoloProtocolError("all Detect terminal heads must expose biases")
|
|
if sum(int(value.numel()) for value in bias) != EXPECTED_HEAD_BIAS_COUNT:
|
|
raise YoloProtocolError("Detect output bias dimension does not equal pinned 252")
|
|
|
|
|
|
def selected_block_names(model: nn.Module) -> tuple[str, ...]:
|
|
names = tuple(name for name, _ in model.named_parameters() if name.startswith("model.22."))
|
|
if not names:
|
|
raise YoloProtocolError("no model.22 floating parameters found")
|
|
if any(not dict(model.named_parameters())[name].is_floating_point() for name in names):
|
|
raise YoloProtocolError("model.22 contains a non-floating selected parameter")
|
|
return names
|
|
|
|
|
|
def selected_head_bias_names(model: nn.Module) -> tuple[str, ...]:
|
|
names = tuple(
|
|
name
|
|
for name, _ in model.named_parameters()
|
|
if len(name.split(".")) == 6
|
|
and name.split(".")[0:2] == ["model", "23"]
|
|
and name.split(".")[2] in {"cv2", "cv3"}
|
|
and name.split(".")[3] in {"0", "1", "2"}
|
|
and name.split(".")[4:] == ["2", "bias"]
|
|
)
|
|
if len(names) != 6 or sum(dict(model.named_parameters())[name].numel() for name in names) != EXPECTED_HEAD_BIAS_COUNT:
|
|
raise YoloProtocolError("Detect bias selection does not match six tensors and 252 scalars")
|
|
return names
|
|
|
|
|
|
def _detect_inputs(model: nn.Module, images: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
"""Execute the frozen prefix and return the three Detect inputs."""
|
|
outputs: dict[int, torch.Tensor] = {}
|
|
x = images
|
|
layers = model.model
|
|
for index, module in enumerate(layers[:EXPECTED_DETECT_INDEX]):
|
|
source = getattr(module, "f", -1)
|
|
if isinstance(source, int):
|
|
x = x if source == -1 else outputs[source]
|
|
else:
|
|
x = [x if item == -1 else outputs[item] for item in source]
|
|
x = module(x)
|
|
outputs[index] = x
|
|
try:
|
|
return outputs[16].detach(), outputs[19].detach(), outputs[21].detach()
|
|
except KeyError as exc:
|
|
raise YoloProtocolError(
|
|
"pinned YOLO graph did not produce cached tensors 16,19,21"
|
|
) from exc
|
|
|
|
|
|
def _letterbox_record(record: VOCRecord, *, size: int = IMG_SIZE) -> tuple[torch.Tensor, tuple[float, tuple[float, float]]]:
|
|
"""Decode one VOC image and apply the pinned fixed 640 letterbox."""
|
|
try:
|
|
from PIL import Image
|
|
except ImportError as exc:
|
|
raise YoloProtocolError("Pillow is required for VOC image decoding") from exc
|
|
image = Image.open(record.image_path).convert("RGB")
|
|
width, height = image.size
|
|
gain = min(size / width, size / height)
|
|
resized = image.resize((max(1, round(width * gain)), max(1, round(height * gain))), Image.Resampling.BILINEAR)
|
|
canvas = Image.new("RGB", (size, size), (114, 114, 114))
|
|
pad_x = (size - resized.width) / 2
|
|
pad_y = (size - resized.height) / 2
|
|
canvas.paste(resized, (round(pad_x), round(pad_y)))
|
|
value = (
|
|
torch.from_numpy(np.asarray(canvas, dtype=np.uint8).copy())
|
|
.permute(2, 0, 1)
|
|
.float()
|
|
.div_(255.0)
|
|
)
|
|
return value, (gain, (pad_x, pad_y))
|
|
|
|
def _native_target(
|
|
record: VOCRecord,
|
|
*,
|
|
index: int = 0,
|
|
ratio_pad: tuple[float, tuple[float, float]],
|
|
size: int = IMG_SIZE,
|
|
) -> dict[str, torch.Tensor]:
|
|
gain, (pad_x, pad_y) = ratio_pad
|
|
transformed = []
|
|
for label, center_x, center_y, width, height in record.labels:
|
|
transformed.append(
|
|
(
|
|
(
|
|
center_x * record.width * gain + pad_x
|
|
) / size,
|
|
(
|
|
center_y * record.height * gain + pad_y
|
|
) / size,
|
|
width * record.width * gain / size,
|
|
height * record.height * gain / size,
|
|
)
|
|
)
|
|
cls = torch.tensor(
|
|
[label[0] for label in record.labels],
|
|
dtype=torch.float32,
|
|
)
|
|
boxes = torch.tensor(transformed, dtype=torch.float32)
|
|
return {
|
|
"batch_idx": torch.full(
|
|
(len(record.labels),),
|
|
index,
|
|
dtype=torch.int64,
|
|
),
|
|
"cls": cls.reshape(-1, 1),
|
|
"bboxes": boxes.reshape(-1, 4),
|
|
}
|
|
|
|
def native_batch(records: Sequence[VOCRecord], *, device: torch.device | str) -> tuple[torch.Tensor, dict[str, torch.Tensor], tuple[tuple[float, tuple[float, float]], ...]]:
|
|
images, targets, ratio_pad = [], [], []
|
|
for index, record in enumerate(records):
|
|
image, padding = _letterbox_record(record)
|
|
images.append(image)
|
|
targets.append(
|
|
_native_target(record, index=index, ratio_pad=padding)
|
|
)
|
|
ratio_pad.append(padding)
|
|
if not images:
|
|
raise YoloProtocolError("native batch cannot be empty")
|
|
batch = {key: torch.cat([target[key] for target in targets], dim=0).to(device) for key in ("batch_idx", "cls", "bboxes")}
|
|
batch["img"] = torch.stack(images).to(device)
|
|
return batch["img"], batch, tuple(ratio_pad)
|
|
|
|
@dataclasses.dataclass
|
|
class DetectionCache:
|
|
images: torch.Tensor
|
|
detect_inputs: tuple[torch.Tensor, torch.Tensor, torch.Tensor]
|
|
targets: tuple[Mapping[str, Any], ...]
|
|
provenance: Mapping[str, Any]
|
|
|
|
def __post_init__(self) -> None:
|
|
self.images = self.images.detach().clone()
|
|
self.detect_inputs = tuple(value.detach().clone() for value in self.detect_inputs) # type: ignore[assignment]
|
|
|
|
def to(self, device: torch.device | str) -> "DetectionCache":
|
|
return DetectionCache(self.images.to(device), tuple(value.to(device) for value in self.detect_inputs), self.targets, self.provenance)
|
|
|
|
|
|
def build_detection_cache(model: nn.Module, batches: Iterable[tuple[torch.Tensor, Mapping[str, Any]]], *, provenance: Mapping[str, Any], device: torch.device | str) -> DetectionCache:
|
|
images_out: list[torch.Tensor] = []
|
|
inputs_out = [[], [], []]
|
|
targets: list[Mapping[str, Any]] = []
|
|
model.eval()
|
|
with torch.no_grad():
|
|
for images, batch in batches:
|
|
images = images.to(device=device, dtype=torch.float32)
|
|
cached = _detect_inputs(model, images)
|
|
images_out.append(images.cpu())
|
|
for index, value in enumerate(cached):
|
|
inputs_out[index].append(value.cpu())
|
|
targets.append(
|
|
{
|
|
key: value.detach().cpu()
|
|
for key, value in batch.items()
|
|
if key != "img" and torch.is_tensor(value)
|
|
}
|
|
| {"_image_count": int(images.shape[0])}
|
|
)
|
|
if not images_out:
|
|
raise YoloProtocolError("cannot create a cache from zero batches")
|
|
return DetectionCache(
|
|
torch.cat(images_out),
|
|
tuple(torch.cat(values) for values in inputs_out),
|
|
tuple(targets),
|
|
dict(provenance),
|
|
)
|
|
|
|
|
|
def _loss_callable(model: nn.Module) -> Any:
|
|
try:
|
|
from ultralytics.utils.loss import v8DetectionLoss # type: ignore
|
|
except ImportError as exc:
|
|
raise YoloProtocolError(
|
|
"pinned v8DetectionLoss is unavailable"
|
|
) from exc
|
|
if not hasattr(model, "args") or not hasattr(model, "model"):
|
|
raise YoloProtocolError(
|
|
"native detection loss requires an Ultralytics DetectionModel"
|
|
)
|
|
return v8DetectionLoss(model)
|
|
|
|
|
|
def _loss_value(loss: Any) -> torch.Tensor:
|
|
value = loss[0] if isinstance(loss, tuple) else loss
|
|
if not torch.is_tensor(value):
|
|
value = torch.as_tensor(value)
|
|
return value.sum()
|
|
|
|
|
|
def _cache_target_batch(
|
|
target: Mapping[str, Any],
|
|
images: torch.Tensor,
|
|
device: torch.device,
|
|
) -> dict[str, Any]:
|
|
batch = {
|
|
key: torch.as_tensor(target[key], device=device)
|
|
for key in ("batch_idx", "cls", "bboxes")
|
|
if key in target
|
|
}
|
|
batch["img"] = images
|
|
return batch
|
|
|
|
|
|
def cached_detection_loss_tensor(
|
|
model: nn.Module,
|
|
cache: DetectionCache,
|
|
*,
|
|
model_device: torch.device | str = "cpu",
|
|
backward: bool = False,
|
|
) -> torch.Tensor:
|
|
"""Evaluate cached loss in source-sized chunks.
|
|
|
|
When ``backward`` is true, gradients are accumulated per chunk so the
|
|
complete 2,500-image objective never materializes one CUDA graph.
|
|
"""
|
|
model.eval()
|
|
device = torch.device(model_device)
|
|
loss_fn = _loss_callable(model)
|
|
total = torch.zeros((), device=device)
|
|
cursor = 0
|
|
for target in cache.targets:
|
|
image_count = int(target["_image_count"])
|
|
stop = cursor + image_count
|
|
images = cache.images[cursor:stop].to(device)
|
|
inputs = tuple(
|
|
value[cursor:stop].to(device)
|
|
for value in cache.detect_inputs
|
|
)
|
|
batch = _cache_target_batch(target, images, device)
|
|
with torch.set_grad_enabled(backward):
|
|
predictions = model.model[EXPECTED_BLOCK_INDEX](inputs[2])
|
|
outputs = model.model[EXPECTED_DETECT_INDEX](
|
|
[inputs[0], inputs[1], predictions]
|
|
)
|
|
chunk = _loss_value(loss_fn(outputs, batch))
|
|
if backward:
|
|
(chunk / int(cache.images.shape[0])).backward()
|
|
total = total + chunk.detach()
|
|
cursor = stop
|
|
if cursor != int(cache.images.shape[0]):
|
|
raise YoloProtocolError("cached target/image counts disagree")
|
|
return total / max(cursor, 1)
|
|
|
|
|
|
def cached_detection_objective(
|
|
model: nn.Module,
|
|
cache: DetectionCache,
|
|
*,
|
|
codec: SelectedResidualCodec | None = None,
|
|
residual: torch.Tensor | None = None,
|
|
model_device: torch.device | str = "cpu",
|
|
backward: bool = False,
|
|
) -> ObjectiveResult:
|
|
"""Evaluate cached block22+Detect outputs through native v8DetectionLoss."""
|
|
if codec is not None and residual is not None:
|
|
codec.apply_residual(model, residual)
|
|
value = cached_detection_loss_tensor(
|
|
model,
|
|
cache,
|
|
model_device=model_device,
|
|
backward=backward,
|
|
)
|
|
if backward:
|
|
value.backward()
|
|
count = int(cache.images.shape[0])
|
|
return ObjectiveResult(
|
|
float(value.detach().cpu()),
|
|
count,
|
|
1,
|
|
int(backward),
|
|
)
|
|
|
|
|
|
def cached_full_parity(model: nn.Module, cache: DetectionCache, *, model_device: torch.device | str = "cpu", atol: float = 1e-6, rtol: float = 1e-5) -> dict[str, Any]:
|
|
"""Compare full-prefix block22+Detect outputs against cached suffix execution."""
|
|
device = torch.device(model_device)
|
|
model.eval()
|
|
with torch.no_grad():
|
|
cached_inputs = tuple(value.to(device) for value in cache.detect_inputs)
|
|
suffix = model.model[EXPECTED_BLOCK_INDEX](cached_inputs[2])
|
|
cached = model.model[EXPECTED_DETECT_INDEX]([cached_inputs[0], cached_inputs[1], suffix])
|
|
full_inputs = _detect_inputs(model, cache.images.to(device))
|
|
full_suffix = model.model[EXPECTED_BLOCK_INDEX](full_inputs[2])
|
|
full = model.model[EXPECTED_DETECT_INDEX]([full_inputs[0], full_inputs[1], full_suffix])
|
|
def max_error(left: Any, right: Any) -> float:
|
|
if torch.is_tensor(left) and torch.is_tensor(right):
|
|
return float((left - right).abs().max().cpu())
|
|
if isinstance(left, (tuple, list)) and isinstance(right, (tuple, list)):
|
|
return max((max_error(a, b) for a, b in zip(left, right)), default=0.0)
|
|
return 0.0
|
|
error = max_error(cached, full)
|
|
reference = cached[0] if isinstance(cached, (tuple, list)) else cached
|
|
scale = float(reference.detach().abs().max().cpu()) if torch.is_tensor(reference) else 1.0
|
|
allowed = atol + rtol * max(scale, 1.0)
|
|
if error > allowed:
|
|
raise YoloProtocolError(f"cached/full parity failed: max_error={error}, allowed={allowed}")
|
|
return {"max_abs_error": error, "allowed": allowed, "passed": True}
|
|
|
|
|
|
def native_detection_metrics(predictions: Sequence[Mapping[str, Any]], targets: Sequence[Mapping[str, Any]], *, iou_thresholds: Sequence[float] = tuple(np.arange(0.5, 0.96, 0.05))) -> dict[str, Any]:
|
|
"""Compute dataset-level detection metrics from native boxes, not image means."""
|
|
if len(predictions) != len(targets):
|
|
raise YoloProtocolError("prediction/target image counts differ")
|
|
# The pinned validator is the authority for production metrics. This helper
|
|
# is intentionally strict about shape and delegates matching when available.
|
|
try:
|
|
from ultralytics.utils.metrics import ap_per_class # type: ignore
|
|
except ImportError as exc:
|
|
raise YoloProtocolError("pinned Ultralytics metric implementation unavailable") from exc
|
|
stats: list[tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]] = []
|
|
for prediction, target in zip(predictions, targets):
|
|
stats.append((
|
|
np.asarray(prediction.get("correct", []), dtype=bool),
|
|
np.asarray(prediction.get("conf", []), dtype=np.float32),
|
|
np.asarray(prediction.get("pred_cls", []), dtype=np.float32),
|
|
np.asarray(target.get("target_cls", []), dtype=np.float32),
|
|
))
|
|
if not stats:
|
|
return {"map50": 0.0, "map50_95": 0.0, "precision": 0.0, "recall": 0.0, "per_class_ap": []}
|
|
correct, conf, pred_cls, target_cls = (np.concatenate(parts) if any(parts) else np.empty((0,)) for parts in zip(*stats))
|
|
if correct.ndim == 1:
|
|
correct = correct[:, None]
|
|
if correct.size == 0:
|
|
return {"map50": 0.0, "map50_95": 0.0, "precision": 0.0, "recall": 0.0, "per_class_ap": [0.0] * 20}
|
|
result = ap_per_class(correct, conf, pred_cls, target_cls, plot=False, names={i: n for i, n in enumerate(VOC_CLASSES)})
|
|
# Ultralytics has changed tuple ordering across versions; pinned 8.4.142 is
|
|
# checked here rather than silently publishing an incorrectly labelled metric.
|
|
if len(result) < 4:
|
|
raise YoloProtocolError("unexpected pinned ap_per_class return shape")
|
|
tp, fp, p, r, f1, ap, unique = result[:7]
|
|
ap = np.asarray(ap)
|
|
if ap.ndim == 2:
|
|
per_class = ap.mean(axis=1)
|
|
map50 = float(ap[:, 0].mean()) if ap.shape[1] else 0.0
|
|
map5095 = float(ap.mean())
|
|
else:
|
|
per_class, map50, map5095 = ap, float(ap.mean()), float(ap.mean())
|
|
return {"map50": map50, "map50_95": map5095, "precision": float(np.asarray(p).mean()), "recall": float(np.asarray(r).mean()), "per_class_ap": per_class.tolist()}
|
|
|
|
|
|
def transform_boxes_to_original(boxes: np.ndarray, *, ratio_pad: tuple[float, tuple[float, float]], shape: tuple[int, int]) -> np.ndarray:
|
|
values = np.asarray(boxes, dtype=np.float64).copy()
|
|
if values.ndim != 2 or values.shape[1] < 4:
|
|
raise YoloProtocolError("boxes must have shape (N,4+) in letterbox pixels")
|
|
gain, pad = float(ratio_pad[0]), ratio_pad[1]
|
|
if gain <= 0:
|
|
raise YoloProtocolError("letterbox gain must be positive")
|
|
values[:, [0, 2]] = (values[:, [0, 2]] - float(pad[0])) / gain
|
|
values[:, [1, 3]] = (values[:, [1, 3]] - float(pad[1])) / gain
|
|
height, width = shape
|
|
values[:, [0, 2]] = np.clip(values[:, [0, 2]], 0, width)
|
|
values[:, [1, 3]] = np.clip(values[:, [1, 3]], 0, height)
|
|
return values
|
|
|
|
|
|
def transform_boxes_to_letterbox(boxes: np.ndarray, *, ratio_pad: tuple[float, tuple[float, float]]) -> np.ndarray:
|
|
values = np.asarray(boxes, dtype=np.float64).copy()
|
|
gain, pad = float(ratio_pad[0]), ratio_pad[1]
|
|
if gain <= 0:
|
|
raise YoloProtocolError("letterbox gain must be positive")
|
|
values[:, [0, 2]] = values[:, [0, 2]] * gain + float(pad[0])
|
|
values[:, [1, 3]] = values[:, [1, 3]] * gain + float(pad[1])
|
|
return values
|
|
|
|
|
|
def weighted_box_fusion(images: Sequence[Mapping[str, Any]], weights: Sequence[float]) -> Mapping[str, np.ndarray]:
|
|
if len(images) != len(weights) or not images:
|
|
raise YoloProtocolError("WBF requires one image prediction per model and one weight per model")
|
|
normalized_weights = np.asarray(weights, dtype=np.float64)
|
|
if (
|
|
not np.isfinite(normalized_weights).all()
|
|
or (normalized_weights < 0).any()
|
|
or float(normalized_weights.sum()) <= 0
|
|
):
|
|
raise YoloProtocolError(
|
|
"WBF weights must be finite, nonnegative, and have positive sum"
|
|
)
|
|
normalized_weights /= normalized_weights.sum()
|
|
fuse = _wbf()
|
|
boxes_list, scores_list, labels_list = [], [], []
|
|
for image in images:
|
|
boxes = np.asarray(image.get("boxes", []), dtype=np.float64)
|
|
scores = np.asarray(image.get("scores", []), dtype=np.float64)
|
|
labels = np.asarray(image.get("labels", []), dtype=np.int64)
|
|
if boxes.size:
|
|
boxes = boxes.reshape(-1, 4)
|
|
if (boxes < 0).any() or (boxes > 1).any():
|
|
raise YoloProtocolError("WBF boxes must be normalized original-coordinate xyxy")
|
|
boxes_list.append(boxes.tolist())
|
|
scores_list.append(scores.tolist())
|
|
labels_list.append(labels.tolist())
|
|
boxes, scores, labels = fuse(
|
|
boxes_list, scores_list, labels_list, weights=normalized_weights.tolist(),
|
|
iou_thr=.55, skip_box_thr=.001, conf_type="avg", allows_overflow=False,
|
|
)
|
|
order = np.argsort(-np.asarray(scores))[:300]
|
|
return {"boxes": np.asarray(boxes)[order], "scores": np.asarray(scores)[order], "labels": np.asarray(labels, dtype=np.int64)[order]}
|
|
|
|
|
|
|
|
def _wbf_dataset_metrics(
|
|
member_predictions: Sequence[Sequence[Mapping[str, Any]]],
|
|
targets: Sequence[Mapping[str, Any]],
|
|
weights: Sequence[float],
|
|
) -> dict[str, Any]:
|
|
from test.evaluate_post_training_model_convergence import detection_metrics
|
|
|
|
records: list[dict[str, Any]] = []
|
|
for image_index, target in enumerate(targets):
|
|
fused = weighted_box_fusion(
|
|
[member_predictions[member][image_index] for member in range(3)],
|
|
weights,
|
|
)
|
|
predictions = [
|
|
{
|
|
"box": [float(value) for value in box],
|
|
"class_id": int(label),
|
|
"score": float(score),
|
|
}
|
|
for box, score, label in zip(
|
|
fused["boxes"], fused["scores"], fused["labels"]
|
|
)
|
|
]
|
|
boxes = np.asarray(target.get("boxes", []), dtype=np.float64).reshape(-1, 4)
|
|
labels = np.asarray(target.get("labels", []), dtype=np.int64)
|
|
ground_truth = [
|
|
{
|
|
"box": [float(value) for value in box],
|
|
"class_id": int(label),
|
|
}
|
|
for box, label in zip(boxes, labels)
|
|
]
|
|
records.append(
|
|
{
|
|
"image_id": str(target.get("image_id", image_index)),
|
|
"predictions": predictions,
|
|
"ground_truth": ground_truth,
|
|
}
|
|
)
|
|
return detection_metrics(records, class_count=len(VOC_CLASSES))
|
|
|
|
|
|
def run_wbf_weight_search(
|
|
member_predictions: Sequence[Sequence[Mapping[str, Any]]],
|
|
targets: Sequence[Mapping[str, Any]],
|
|
*,
|
|
seed: int,
|
|
random_mode: bool,
|
|
) -> dict[str, Any]:
|
|
if (
|
|
len(member_predictions) != 3
|
|
or any(len(rows) != len(targets) for rows in member_predictions)
|
|
or seed not in SWARM_SEEDS
|
|
):
|
|
raise YoloProtocolError(
|
|
"WBF search requires three aligned member sets and a fixed swarm seed"
|
|
)
|
|
rng = _RandomSource(seed=seed, device="cpu")
|
|
positions = [torch.zeros(3, dtype=torch.float32)]
|
|
for _ in range(5):
|
|
value = rng.uniform((3,), -0.25, 0.25, device="cpu")
|
|
positions.extend((value, -value))
|
|
positions.append(rng.uniform((3,), -0.25, 0.25, device="cpu"))
|
|
velocities = [torch.zeros_like(position) for position in positions]
|
|
pbest = [position.clone() for position in positions]
|
|
pbest_scores = [math.inf] * len(positions)
|
|
best = positions[0].clone()
|
|
best_score = math.inf
|
|
best_metrics: dict[str, Any] | None = None
|
|
trajectory: list[dict[str, Any]] = []
|
|
movement = ConstrictionMovement(c0=2.05, c1=2.05)
|
|
evaluations = 0
|
|
|
|
for generation in range(1, WBF_GENERATIONS + 1):
|
|
for index, position in enumerate(positions):
|
|
weights = torch.softmax(position, dim=0).tolist()
|
|
metrics = _wbf_dataset_metrics(member_predictions, targets, weights)
|
|
score = -float(metrics["map50_95"])
|
|
evaluations += 1
|
|
if score < pbest_scores[index]:
|
|
pbest_scores[index] = score
|
|
pbest[index] = position.clone()
|
|
if score < best_score:
|
|
best_score = score
|
|
best = position.clone()
|
|
best_metrics = metrics
|
|
trajectory.append(
|
|
{
|
|
"generation": generation,
|
|
"objective": best_score,
|
|
"map50_95": -best_score,
|
|
}
|
|
)
|
|
if generation == 20:
|
|
break
|
|
if random_mode:
|
|
positions = [
|
|
rng.uniform((3,), -5.0, 5.0, device="cpu")
|
|
for _ in positions
|
|
]
|
|
velocities = [torch.zeros_like(position) for position in positions]
|
|
continue
|
|
state = SwarmState(
|
|
positions=tuple(position.clone() for position in positions),
|
|
velocities=tuple(velocity.clone() for velocity in velocities),
|
|
pbest_positions=tuple(position.clone() for position in pbest),
|
|
pbest_scores=tuple((score, 0.0, 0.0) for score in pbest_scores),
|
|
gbest_position=best.clone(),
|
|
gbest_score=(best_score, 0.0, 0.0),
|
|
pbest_improved=tuple(False for _ in positions),
|
|
)
|
|
next_positions: list[torch.Tensor] = []
|
|
next_velocities: list[torch.Tensor] = []
|
|
for index, position in enumerate(positions):
|
|
context = IterationContext(
|
|
epoch=generation + 1,
|
|
total_epochs=20,
|
|
w=1.0,
|
|
particle_idx=index,
|
|
is_negative=False,
|
|
rng=rng,
|
|
optimizer=None,
|
|
)
|
|
_, velocity = movement.propose(index, state, context)
|
|
candidate = position + velocity
|
|
outside = (candidate < -5.0) | (candidate > 5.0)
|
|
next_positions.append(torch.clamp(candidate, -5.0, 5.0))
|
|
next_velocities.append(
|
|
torch.where(outside, torch.zeros_like(velocity), velocity)
|
|
)
|
|
positions, velocities = next_positions, next_velocities
|
|
if (
|
|
best_metrics is None
|
|
or evaluations != WBF_PARTICLE_COUNT * WBF_GENERATIONS
|
|
):
|
|
raise YoloProtocolError("WBF search did not complete exactly 240 evaluations")
|
|
return {
|
|
"seed": seed,
|
|
"method": "ensemble_random" if random_mode else "ensemble_pso",
|
|
"queries": evaluations,
|
|
"sample_evaluations": evaluations * len(targets),
|
|
"logits": best.tolist(),
|
|
"weights": torch.softmax(best, dim=0).tolist(),
|
|
"metrics": best_metrics,
|
|
"trajectory": trajectory,
|
|
}
|
|
class StrictScratchTrainer:
|
|
"""Native training boundary: OOM, NaN and invalid checkpoints are fatal."""
|
|
def __init__(
|
|
self,
|
|
*,
|
|
device: str,
|
|
batch: int = 16,
|
|
epochs: int = 100,
|
|
) -> None:
|
|
if device not in {"cpu", "mps", "cuda"}:
|
|
raise YoloProtocolError(
|
|
"device must remain cpu, mps, or cuda for the "
|
|
"complete run"
|
|
)
|
|
if epochs not in {2, 100}:
|
|
raise YoloProtocolError("trainer epochs must be smoke 2 or production 100")
|
|
self.device, self.batch, self.epochs = device, batch, epochs
|
|
self.ema_capture: dict[str, torch.Tensor] | None = None
|
|
self.telemetry: list[dict[str, Any]] = []
|
|
|
|
@property
|
|
def overrides(self) -> dict[str, Any]:
|
|
return {
|
|
"epochs": self.epochs, "optimizer": "SGD", "lr0": .01, "lrf": .01,
|
|
"momentum": .937, "weight_decay": .0005, "cos_lr": True,
|
|
"warmup_epochs": 3, "batch": self.batch, "imgsz": IMG_SIZE,
|
|
"amp": False, "workers": 0, "deterministic": True, "patience": 0,
|
|
"pretrained": False, "close_mosaic": 10, "device": self.device,
|
|
"val": True, "plots": False, "save": True,
|
|
}
|
|
|
|
def capture_live_ema(self, trainer: Any, epoch: int) -> None:
|
|
metrics = getattr(trainer, "metrics", None)
|
|
if self.epochs == 2 or epoch >= self.epochs - 11:
|
|
row = {"epoch": epoch + 1}
|
|
if isinstance(metrics, Mapping):
|
|
row.update(
|
|
{
|
|
str(key): float(value)
|
|
for key, value in metrics.items()
|
|
if isinstance(value, (int, float))
|
|
}
|
|
)
|
|
self.telemetry.append(row)
|
|
if epoch != self.epochs - 1:
|
|
return
|
|
ema = getattr(getattr(trainer, "ema", None), "ema", None)
|
|
if ema is None:
|
|
raise YoloProtocolError("live fp32 EMA is unavailable at final epoch")
|
|
self.ema_capture = {name: value.detach().float().cpu().clone() for name, value in ema.state_dict().items()}
|
|
if not all(bool(torch.isfinite(value).all()) for value in self.ema_capture.values()):
|
|
raise YoloProtocolError("live EMA contains non-finite values")
|
|
|
|
def refusal(self, error: BaseException) -> None:
|
|
message = str(error).lower()
|
|
if "out of memory" in message or "nan" in message or "checkpoint" in message:
|
|
raise YoloProtocolError(f"native training failed without recovery: {error}") from error
|
|
raise error
|
|
|
|
|
|
def _reused_native_baseline(
|
|
run_path: Path,
|
|
trainer: StrictScratchTrainer,
|
|
base_seed: int,
|
|
) -> dict[str, Any] | None:
|
|
baseline_root = (
|
|
run_path
|
|
/ "workloads"
|
|
/ WORKLOAD_ID
|
|
/ "baselines"
|
|
/ str(base_seed)
|
|
)
|
|
marker_path = baseline_root / "baseline_reuse.json"
|
|
if not marker_path.is_file():
|
|
return None
|
|
marker = json.loads(marker_path.read_text(encoding="utf-8"))
|
|
output = baseline_root / "ema_fp32.pt"
|
|
results_path = (
|
|
run_path
|
|
/ "ultralytics"
|
|
/ f"base-{base_seed}-{trainer.epochs}e"
|
|
/ "results.csv"
|
|
)
|
|
if (
|
|
marker.get("protocol_version") != PROTOCOL_VERSION
|
|
or marker.get("checkpoint_hash") != fingerprint_file(output)
|
|
or marker.get("results_hash") != fingerprint_file(results_path)
|
|
):
|
|
raise YoloProtocolError(
|
|
f"invalid reused baseline marker: {marker_path}"
|
|
)
|
|
with results_path.open(newline="", encoding="utf-8") as stream:
|
|
rows = list(csv.DictReader(stream))
|
|
if (
|
|
len(rows) != trainer.epochs
|
|
or int(float(rows[-1]["epoch"])) != trainer.epochs
|
|
):
|
|
raise YoloProtocolError(
|
|
"reused native baseline does not contain every epoch"
|
|
)
|
|
state = torch.load(output, map_location="cpu", weights_only=True)
|
|
if not isinstance(state, Mapping) or not state or not all(
|
|
torch.is_tensor(value)
|
|
and bool(torch.isfinite(value).all())
|
|
for value in state.values()
|
|
):
|
|
raise YoloProtocolError(
|
|
"reused native baseline checkpoint is invalid"
|
|
)
|
|
telemetry = [
|
|
{
|
|
key.strip(): float(value)
|
|
for key, value in row.items()
|
|
if key is not None
|
|
and value is not None
|
|
and value.strip()
|
|
}
|
|
for row in rows[-11:]
|
|
]
|
|
return {
|
|
"seed": base_seed,
|
|
"checkpoint": str(output.relative_to(run_path)),
|
|
"telemetry": telemetry,
|
|
"resolved": trainer.overrides,
|
|
"checkpoint_hash": fingerprint_file(output),
|
|
"result": f"reused:{marker['source_run']}",
|
|
"reused": True,
|
|
}
|
|
|
|
|
|
def train_baseline(
|
|
*,
|
|
model: Any,
|
|
yaml_path: str,
|
|
trainer: StrictScratchTrainer,
|
|
run_root: str | os.PathLike[str],
|
|
base_seed: int,
|
|
) -> dict[str, Any]:
|
|
"""Run one native baseline, or reuse an explicitly hash-verified run."""
|
|
_ultralytics()
|
|
if not isinstance(base_seed, int) or base_seed not in BASE_SEEDS:
|
|
raise YoloProtocolError("base seed must be one of 501, 502, 503")
|
|
run_path = Path(run_root)
|
|
os.environ["YOLO_CONFIG_DIR"] = str(run_path / "yolo_config")
|
|
os.environ["ULTRALYTICS_HUB"] = "0"
|
|
os.environ["ULTRALYTICS_SETTINGS_YAML"] = str(run_path / "ultralytics_settings.yaml")
|
|
reused = _reused_native_baseline(
|
|
run_path,
|
|
trainer,
|
|
base_seed,
|
|
)
|
|
if reused is not None:
|
|
return reused
|
|
random.seed(base_seed)
|
|
np.random.seed(base_seed)
|
|
torch.manual_seed(base_seed)
|
|
callback = lambda tr: trainer.capture_live_ema(tr, int(getattr(tr, "epoch", -1)))
|
|
add_callback = getattr(model, "add_callback", None)
|
|
if not callable(add_callback):
|
|
raise YoloProtocolError("Ultralytics model does not expose add_callback")
|
|
add_callback("on_train_epoch_end", callback)
|
|
try:
|
|
results = model.train(
|
|
data=yaml_path,
|
|
seed=base_seed,
|
|
project=str(run_path.resolve() / "ultralytics"),
|
|
name=f"base-{base_seed}-{trainer.epochs}e",
|
|
exist_ok=False,
|
|
**trainer.overrides,
|
|
)
|
|
except BaseException as exc:
|
|
trainer.refusal(exc)
|
|
if trainer.ema_capture is None:
|
|
raise YoloProtocolError("training did not capture final live fp32 EMA")
|
|
output = (
|
|
run_path
|
|
/ "workloads"
|
|
/ WORKLOAD_ID
|
|
/ "baselines"
|
|
/ str(base_seed)
|
|
/ "ema_fp32.pt"
|
|
)
|
|
atomic_write_bytes(
|
|
output,
|
|
_torch_save_bytes(trainer.ema_capture),
|
|
)
|
|
return {
|
|
"seed": base_seed,
|
|
"checkpoint": str(output.relative_to(run_path)),
|
|
"checkpoint_hash": fingerprint_file(output),
|
|
"telemetry": trainer.telemetry,
|
|
"resolved": trainer.overrides,
|
|
"result": str(results),
|
|
}
|
|
|
|
|
|
def _resolve_run_path(
|
|
run_root: str | os.PathLike[str],
|
|
value: str | os.PathLike[str],
|
|
) -> Path:
|
|
path = Path(value)
|
|
if path.is_absolute():
|
|
return path
|
|
return Path(run_root) / path
|
|
|
|
def _checkpoint_record_valid(
|
|
run_root: str | os.PathLike[str],
|
|
record: Any,
|
|
) -> bool:
|
|
if not isinstance(record, Mapping):
|
|
return False
|
|
checkpoint = record.get("checkpoint")
|
|
expected = record.get("checkpoint_hash")
|
|
if not isinstance(checkpoint, str) or not isinstance(expected, str):
|
|
return False
|
|
path = _resolve_run_path(run_root, checkpoint)
|
|
return path.is_file() and fingerprint_file(path) == expected
|
|
|
|
|
|
def _torch_save_bytes(value: Any) -> bytes:
|
|
import io
|
|
stream = io.BytesIO(); torch.save(value, stream); return stream.getvalue()
|
|
|
|
|
|
def _manifest_from_json(path: str | os.PathLike[str]) -> VOCManifest:
|
|
value = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
def records(key: str) -> tuple[VOCRecord, ...]:
|
|
rows = []
|
|
for row in value[key]:
|
|
item = dict(row)
|
|
item["labels"] = tuple(tuple(label) for label in item["labels"])
|
|
rows.append(VOCRecord(**item))
|
|
return tuple(rows)
|
|
return VOCManifest(
|
|
records("bp_train"), records("refine_search"), records("selection_val"),
|
|
int(value["permutation_seed"]),
|
|
{str(k): tuple(v) for k, v in value["duplicate_groups"].items()},
|
|
{str(k): int(v) for k, v in value["counts"].items()},
|
|
tuple(tuple(item) for item in value["objective_keys"]),
|
|
)
|
|
|
|
|
|
def _native_loss_tensor(detector: nn.Module, images: torch.Tensor, batch: Mapping[str, Any]) -> torch.Tensor:
|
|
detector.train()
|
|
prediction = detector(images)
|
|
loss = detector.loss(dict(batch), prediction) if callable(getattr(detector, "loss", None)) else _loss_callable(detector)(prediction, dict(batch))
|
|
return _loss_value(loss) / max(int(images.shape[0]), 1)
|
|
|
|
|
|
def _accumulated_native_loss(detector: nn.Module, batches: Sequence[tuple[torch.Tensor, Mapping[str, Any]]]) -> torch.Tensor:
|
|
if not batches:
|
|
raise YoloProtocolError("native objective requires at least one batch")
|
|
total: torch.Tensor | None = None
|
|
count = 0
|
|
for images, batch in batches:
|
|
value = _native_loss_tensor(detector, images, batch)
|
|
weight = int(images.shape[0])
|
|
total = value * weight if total is None else total + value * weight
|
|
count += weight
|
|
if total is None or count == 0:
|
|
raise YoloProtocolError("native objective contains zero images")
|
|
return total / count
|
|
|
|
|
|
def _native_predictions(detector: nn.Module, images: torch.Tensor) -> list[Any]:
|
|
from ultralytics.utils.nms import non_max_suppression # type: ignore
|
|
detector.eval()
|
|
with torch.no_grad():
|
|
raw = detector(images)
|
|
return non_max_suppression(raw, conf_thres=.001, iou_thres=.7, max_det=300, multi_label=True, agnostic=False)
|
|
|
|
|
|
|
|
|
|
def evaluate_detection_records(
|
|
detector: nn.Module,
|
|
records: Sequence[VOCRecord],
|
|
*,
|
|
device: torch.device | str,
|
|
batch_size: int = 4,
|
|
) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
from test.evaluate_post_training_model_convergence import detection_metrics
|
|
|
|
rows: list[dict[str, Any]] = []
|
|
for start in range(0, len(records), batch_size):
|
|
subset = records[start : start + batch_size]
|
|
images, _, ratio_pad = native_batch(subset, device=device)
|
|
outputs = _native_predictions(detector, images)
|
|
for record, output, padding in zip(subset, outputs, ratio_pad):
|
|
boxes = (
|
|
output[:, :4].detach().cpu().numpy()
|
|
if output.numel()
|
|
else np.empty((0, 4))
|
|
)
|
|
boxes = transform_boxes_to_original(
|
|
boxes,
|
|
ratio_pad=padding,
|
|
shape=(record.height, record.width),
|
|
)
|
|
predictions = [
|
|
{
|
|
"box": [float(value) for value in box],
|
|
"class_id": int(label),
|
|
"score": float(score),
|
|
}
|
|
for box, score, label in zip(
|
|
boxes,
|
|
output[:, 4].detach().cpu().numpy()
|
|
if output.numel()
|
|
else np.empty((0,)),
|
|
output[:, 5].detach().cpu().numpy()
|
|
if output.numel()
|
|
else np.empty((0,)),
|
|
)
|
|
]
|
|
ground_truth = []
|
|
for label in record.labels:
|
|
_, cx, cy, width, height = label
|
|
ground_truth.append(
|
|
{
|
|
"box": [
|
|
float((cx - width / 2) * record.width),
|
|
float((cy - height / 2) * record.height),
|
|
float((cx + width / 2) * record.width),
|
|
float((cy + height / 2) * record.height),
|
|
],
|
|
"class_id": int(label[0]),
|
|
}
|
|
)
|
|
rows.append(
|
|
{
|
|
"image_id": f"{record.year}:{record.image_id}",
|
|
"predictions": predictions,
|
|
"ground_truth": ground_truth,
|
|
}
|
|
)
|
|
return detection_metrics(rows, class_count=len(VOC_CLASSES)), rows
|
|
|
|
def run_smoke_feature_pso(detector: nn.Module, cache: DetectionCache, *, device: torch.device | str) -> dict[str, Any]:
|
|
"""Run the real two-generation smoke PSO over cached native loss."""
|
|
codec = SelectedResidualCodec(detector, selected_block_names(detector), projection_seed=PROJECTION_SEED)
|
|
generator = torch.Generator(device="cpu").manual_seed(601)
|
|
positions = [torch.zeros(RESIDUAL_DIMENSION, device=device)]
|
|
for _ in range(11):
|
|
positions.append(torch.rand(RESIDUAL_DIMENSION, generator=generator).to(device).mul_(.5).sub_(.25))
|
|
velocities = [torch.zeros_like(position) for position in positions]
|
|
pbest = [position.clone() for position in positions]
|
|
scores: list[float | None] = [None] * len(positions)
|
|
gbest: torch.Tensor | None = None
|
|
gscore = math.inf
|
|
trajectory = []
|
|
for generation in range(1, 3):
|
|
for index, position in enumerate(positions):
|
|
result = cached_detection_objective(detector, cache, codec=codec, residual=position, model_device=device)
|
|
if scores[index] is None or result.loss < scores[index]:
|
|
scores[index] = result.loss
|
|
pbest[index] = position.clone()
|
|
if result.loss < gscore:
|
|
gscore, gbest = result.loss, position.clone()
|
|
if gbest is None:
|
|
raise YoloProtocolError("smoke PSO did not produce an incumbent")
|
|
trajectory.append({"generation": generation, "objective_best": gscore})
|
|
if generation == 2:
|
|
break
|
|
for index in range(len(positions)):
|
|
r1 = torch.rand(RESIDUAL_DIMENSION, generator=generator).to(device)
|
|
r2 = torch.rand(RESIDUAL_DIMENSION, generator=generator).to(device)
|
|
velocity = .7 * velocities[index] + 1.49445 * r1 * (pbest[index] - positions[index]) + 1.49445 * r2 * (gbest - positions[index])
|
|
proposal = torch.clamp(positions[index] + velocity, -1.0, 1.0)
|
|
velocities[index] = torch.where((proposal == -1.0) | (proposal == 1.0), torch.zeros_like(velocity), velocity)
|
|
positions[index] = proposal
|
|
return {"generations": 2, "queries": 24, "best_objective": gscore, "trajectory": trajectory}
|
|
def run_feature_search(
|
|
model: nn.Module,
|
|
objective: Callable[[torch.Tensor], ObjectiveResult | float],
|
|
*,
|
|
base_seed: int,
|
|
swarm_seed: int,
|
|
device: torch.device | str,
|
|
validation: Callable[[torch.Tensor], AuditResult] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Run fixed feature PSO and equal-query random arms for one base."""
|
|
names = selected_block_names(model)
|
|
codec = SelectedResidualCodec(
|
|
model,
|
|
names,
|
|
projection_seed=PROJECTION_SEED,
|
|
)
|
|
if base_seed not in BASE_SEEDS or swarm_seed not in SWARM_SEEDS:
|
|
raise YoloProtocolError(
|
|
"feature arms require the fixed base and swarm seed sets"
|
|
)
|
|
pso = run_residual_pso(
|
|
objective,
|
|
codec,
|
|
seed=swarm_seed,
|
|
device=device,
|
|
model=model,
|
|
validation=validation,
|
|
objective_samples=OBJECTIVE_COUNT,
|
|
)
|
|
random_result = run_equal_budget_random(
|
|
objective,
|
|
codec,
|
|
seed=swarm_seed,
|
|
device=device,
|
|
model=model,
|
|
validation=validation,
|
|
objective_samples=OBJECTIVE_COUNT,
|
|
)
|
|
return {
|
|
"base_seed": base_seed,
|
|
"swarm_seed": swarm_seed,
|
|
"selected_names": list(names),
|
|
"projection_seed": PROJECTION_SEED,
|
|
"feature_pso": pso.to_dict(include_vectors=True),
|
|
"feature_random": random_result.to_dict(include_vectors=True),
|
|
}
|
|
|
|
|
|
def run_bounded_adam(
|
|
model: nn.Module,
|
|
parameters: Sequence[str],
|
|
objective: Callable[[bool], torch.Tensor],
|
|
*,
|
|
updates: int = 40,
|
|
lr: float = 1e-3,
|
|
bounds: float | Mapping[str, float] = 0.25,
|
|
) -> dict[str, Any]:
|
|
"""Run the fixed 40-step full-objective AdamW arm from the base state."""
|
|
if updates != 40 or lr != 1e-3:
|
|
raise YoloProtocolError("AdamW arm requires exactly 40 updates at lr=1e-3")
|
|
named = dict(model.named_parameters())
|
|
selected = [named[name] for name in parameters if name in named]
|
|
if len(selected) != len(parameters):
|
|
raise YoloProtocolError("AdamW arm includes an unknown parameter")
|
|
base = {name: value.detach().clone() for name, value in zip(parameters, selected)}
|
|
optimizer = torch.optim.AdamW(selected, lr=lr, betas=(.9, .999), eps=1e-8, weight_decay=0.0)
|
|
trajectory: list[dict[str, float]] = []
|
|
try:
|
|
for update in range(updates + 1):
|
|
should_update = update < updates
|
|
if should_update:
|
|
optimizer.zero_grad(set_to_none=True)
|
|
value = objective(should_update)
|
|
if not torch.is_tensor(value) or value.ndim != 0 or not bool(torch.isfinite(value).item()):
|
|
raise YoloProtocolError("AdamW objective must return one finite scalar tensor")
|
|
trajectory.append({"update": update, "objective": float(value.detach().cpu())})
|
|
if not should_update:
|
|
break
|
|
optimizer.step()
|
|
with torch.no_grad():
|
|
for name, parameter in zip(parameters, selected):
|
|
limit = float(bounds[name] if isinstance(bounds, Mapping) else bounds)
|
|
parameter.copy_(torch.clamp(parameter, base[name] - limit, base[name] + limit))
|
|
finally:
|
|
optimizer.zero_grad(set_to_none=True)
|
|
return {
|
|
"method": "feature_adam" if any(name.startswith("model.22.") for name in parameters) else "head_adam",
|
|
"parameters": list(parameters),
|
|
"updates": updates,
|
|
"trajectory": trajectory,
|
|
"final_objective": trajectory[-1]["objective"],
|
|
}
|
|
|
|
|
|
def run_head_adam(
|
|
model: nn.Module,
|
|
objective: Callable[[bool], torch.Tensor],
|
|
) -> dict[str, Any]:
|
|
"""Run the six Detect-terminal-bias control with its declared ±0.25 box."""
|
|
return run_bounded_adam(
|
|
model,
|
|
selected_head_bias_names(model),
|
|
objective,
|
|
bounds=0.25,
|
|
)
|
|
|
|
|
|
class YoloConvergenceAdapter:
|
|
def __init__(self, *, workload_id: str, config: StudyConfig, run_root: str | os.PathLike[str], data_root: str | os.PathLike[str], device: str | torch.device, allow_download: bool) -> None:
|
|
if workload_id != WORKLOAD_ID:
|
|
raise YoloProtocolError(f"unsupported workload id: {workload_id}")
|
|
if str(device) not in {"cpu", "mps", "cuda"}:
|
|
raise YoloProtocolError(
|
|
"device must be cpu, mps, or cuda"
|
|
)
|
|
self.workload_id, self.config = workload_id, config
|
|
self.run_root, self.data_root, self.device = Path(run_root), Path(data_root), str(device)
|
|
self.allow_download = bool(allow_download)
|
|
self.root = self.run_root / "workloads" / WORKLOAD_ID
|
|
self.result_path = self.root / "result.json"
|
|
self.result: dict[str, Any] = {"workload_id": WORKLOAD_ID, "family": FAMILY, "config": config.to_dict(), "manifests": {}, "provenance": {}, "baselines": {}, "arms": {}, "ensemble": {}, "development_selection": {}, "confirmation": {}, "integrity": {}, "leakage_counters": {"official_test_data_loaded_before_freeze": False, "official_test_evaluations_before_freeze": 0, "official_test_construction": 0, "official_test_forward_passes": 0}, "resource_ledger": {}, "artifact_hashes": {}}
|
|
if self.result_path.is_file():
|
|
persisted = json.loads(self.result_path.read_text(encoding="utf-8"))
|
|
if persisted.get("workload_id") != WORKLOAD_ID:
|
|
raise YoloProtocolError("persisted workload id mismatch")
|
|
if StudyConfig.from_dict(persisted.get("config", {})) != config:
|
|
raise YoloProtocolError("persisted configuration mismatch")
|
|
self.result = persisted
|
|
|
|
def _save(self) -> None:
|
|
self.root.mkdir(parents=True, exist_ok=True)
|
|
atomic_write_json(self.result_path, self.result)
|
|
|
|
def prepare(self) -> dict[str, Any]:
|
|
if not (self.run_root / "state.json").is_file():
|
|
prepare_run(self.run_root, self.config)
|
|
package = pinned_preflight()
|
|
manifest = prepare_voc(self.data_root, self.run_root, allow_download=self.allow_download, seed=self.config.split_seed)
|
|
yaml_path = write_study_yaml(manifest, self.root / "study.yaml", labels_root=self.root / "labels")
|
|
self.result["manifests"] = {"voc": str(self.root / "voc_manifest.json"), "study_yaml": str(yaml_path), "counts": dict(manifest.counts), "objective_count": len(manifest.objective_keys)}
|
|
self.result["provenance"] = {"packages": package, "projection_seed": PROJECTION_SEED, "data_root": str(self.data_root)}
|
|
self.result["integrity"] = {"prepared": True, "classes": list(VOC_CLASSES), "test_sealed": True, "official_test_opened": False}
|
|
self._save()
|
|
return self.result
|
|
|
|
def smoke(self) -> dict[str, Any]:
|
|
if not (self.root / "voc_manifest.json").is_file():
|
|
self.prepare()
|
|
pinned_preflight()
|
|
manifest = _manifest_from_json(self.root / "voc_manifest.json")
|
|
smoke_train = manifest.bp_train[:32]
|
|
smoke_val = manifest.selection_val[:16]
|
|
train_list = self.root / "smoke_train.txt"
|
|
val_list = self.root / "smoke_val.txt"
|
|
atomic_write_bytes(
|
|
train_list,
|
|
(
|
|
"\n".join(
|
|
str(
|
|
(
|
|
self.root
|
|
/ "images"
|
|
/ "bp_train"
|
|
/ f"{record.year}_{record.image_id}.jpg"
|
|
).absolute()
|
|
)
|
|
for record in smoke_train
|
|
)
|
|
+ "\n"
|
|
).encode("utf-8"),
|
|
)
|
|
atomic_write_bytes(
|
|
val_list,
|
|
(
|
|
"\n".join(
|
|
str(
|
|
(
|
|
self.root
|
|
/ "images"
|
|
/ "selection_val"
|
|
/ f"{record.year}_{record.image_id}.jpg"
|
|
).absolute()
|
|
)
|
|
for record in smoke_val
|
|
)
|
|
+ "\n"
|
|
).encode("utf-8"),
|
|
)
|
|
smoke_yaml = self.root / "smoke.yaml"
|
|
atomic_write_bytes(
|
|
smoke_yaml,
|
|
(
|
|
f"path: {self.root.resolve()}\n"
|
|
f"train: {train_list.resolve()}\n"
|
|
f"val: {val_list.resolve()}\n"
|
|
"names:\n"
|
|
+ "\n".join(
|
|
f" {index}: {name}"
|
|
for index, name in enumerate(VOC_CLASSES)
|
|
)
|
|
+ "\n"
|
|
).encode("utf-8"),
|
|
)
|
|
smoke_wrapper = make_yolo11n(device=self.device)
|
|
smoke_training = train_baseline(
|
|
model=smoke_wrapper,
|
|
yaml_path=str(smoke_yaml),
|
|
trainer=StrictScratchTrainer(
|
|
device=self.device,
|
|
batch=4,
|
|
epochs=2,
|
|
),
|
|
run_root=self.run_root,
|
|
base_seed=BASE_SEEDS[0],
|
|
)
|
|
images, batch, _ = native_batch(
|
|
manifest.refine_search[:2],
|
|
device=self.device,
|
|
)
|
|
detector = make_yolo11n(device=self.device).model
|
|
detector.load_state_dict(
|
|
torch.load(
|
|
_resolve_run_path(
|
|
self.run_root,
|
|
smoke_training["checkpoint"],
|
|
),
|
|
map_location=self.device,
|
|
weights_only=True,
|
|
),
|
|
strict=True,
|
|
)
|
|
detector.train()
|
|
loss = _native_loss_tensor(detector, images, batch)
|
|
if not bool(torch.isfinite(loss).item()):
|
|
raise YoloProtocolError("smoke native detection loss is non-finite")
|
|
detector.zero_grad(set_to_none=True)
|
|
loss.backward()
|
|
detector.zero_grad(set_to_none=True)
|
|
detector.eval()
|
|
predictions = _native_predictions(detector, images)
|
|
cache = build_detection_cache(detector, [(images, batch)], provenance={"phase": "smoke"}, device=self.device)
|
|
parity_zero = cached_full_parity(detector, cache, model_device=self.device)
|
|
names = selected_block_names(detector)
|
|
codec = SelectedResidualCodec(detector, names, projection_seed=PROJECTION_SEED)
|
|
zero = codec.zero_residual(device=self.device)
|
|
zero_loss = cached_detection_objective(detector, cache, codec=codec, residual=zero, model_device=self.device)
|
|
smoke_pso = run_smoke_feature_pso(detector, cache, device=self.device)
|
|
nonzero = torch.full((RESIDUAL_DIMENSION,), .1, device=self.device)
|
|
nonzero_loss = cached_detection_objective(detector, cache, codec=codec, residual=nonzero, model_device=self.device)
|
|
nonzero_parity = cached_full_parity(detector, cache, model_device=self.device)
|
|
codec.restore_base(detector)
|
|
if not math.isfinite(nonzero_loss.loss):
|
|
raise YoloProtocolError("smoke cached native loss is non-finite")
|
|
checkpoint = self.root / "smoke_checkpoint.pt"
|
|
atomic_write_bytes(checkpoint, _torch_save_bytes(detector.state_dict()))
|
|
reloaded = make_yolo11n(device=self.device).model
|
|
reloaded.load_state_dict(torch.load(checkpoint, map_location=self.device, weights_only=True), strict=True)
|
|
self.result["integrity"].update({
|
|
"smoke": True,
|
|
"selected_block_names": list(names),
|
|
"selected_head_bias_names": list(selected_head_bias_names(detector)),
|
|
"topology": "model.22 C3k2 -> model.23 Detect",
|
|
"smoke_native_loss": zero_loss.to_dict(),
|
|
"smoke_nonzero_loss": nonzero_loss.to_dict(),
|
|
"smoke_cached_full_parity": parity_zero,
|
|
"smoke_nonzero_full_parity": nonzero_parity,
|
|
"smoke_pso": smoke_pso,
|
|
"smoke_training": {
|
|
"epochs": 2,
|
|
"batch": 4,
|
|
"telemetry": smoke_training["telemetry"],
|
|
"checkpoint": str(checkpoint.relative_to(self.run_root)),
|
|
},
|
|
"smoke_nms_images": len(predictions),
|
|
"smoke_checkpoint": str(checkpoint),
|
|
})
|
|
self._save()
|
|
return self.result
|
|
|
|
def develop(self) -> dict[str, Any]:
|
|
if not (self.root / "voc_manifest.json").is_file():
|
|
self.prepare()
|
|
pinned_preflight()
|
|
manifest = _manifest_from_json(self.root / "voc_manifest.json")
|
|
yaml_path = self.root / "study.yaml"
|
|
if not yaml_path.is_file():
|
|
write_study_yaml(manifest, yaml_path, labels_root=self.root / "labels")
|
|
objective_records = manifest.refine_search[:OBJECTIVE_COUNT]
|
|
objective_batches = []
|
|
for start in range(0, len(objective_records), 8):
|
|
subset = objective_records[start:start + 8]
|
|
images, batch, _ = native_batch(subset, device="cpu")
|
|
objective_batches.append((images, batch))
|
|
selection_records = manifest.selection_val
|
|
baselines: dict[str, Any] = dict(
|
|
self.result.get("baselines", {})
|
|
)
|
|
for base_seed in BASE_SEEDS:
|
|
model = make_yolo11n(device=self.device)
|
|
trainer = StrictScratchTrainer(device=self.device, batch=16)
|
|
baseline = baselines.get(str(base_seed))
|
|
if not _checkpoint_record_valid(self.run_root, baseline):
|
|
baseline = train_baseline(
|
|
model=model,
|
|
yaml_path=str(yaml_path),
|
|
trainer=trainer,
|
|
run_root=self.run_root,
|
|
base_seed=base_seed,
|
|
)
|
|
baselines[str(base_seed)] = baseline
|
|
self.result["baselines"] = baselines
|
|
self._save()
|
|
detector = model.model
|
|
state = torch.load(
|
|
_resolve_run_path(
|
|
self.run_root,
|
|
baseline["checkpoint"],
|
|
),
|
|
map_location=self.device,
|
|
weights_only=True,
|
|
)
|
|
detector.load_state_dict(state, strict=True)
|
|
detector.to(self.device)
|
|
detector.eval()
|
|
cache = build_detection_cache(
|
|
detector, objective_batches,
|
|
provenance={"base_seed": base_seed, "split": "refine_search", "count": OBJECTIVE_COUNT},
|
|
device=self.device,
|
|
)
|
|
objective = lambda residual, detector=detector, cache=cache: cached_detection_objective(detector, cache, codec=None, residual=None, model_device=self.device)
|
|
pristine_state = {
|
|
name: value.detach().cpu().clone()
|
|
for name, value in detector.state_dict().items()
|
|
}
|
|
self.result["arms"].setdefault("feature_pso", {}).setdefault(
|
|
str(base_seed), {}
|
|
)
|
|
self.result["arms"].setdefault("feature_random", {}).setdefault(
|
|
str(base_seed), {}
|
|
)
|
|
arm_record: dict[str, Any] = {}
|
|
for swarm_seed in SWARM_SEEDS:
|
|
existing = [
|
|
self.result["arms"][method][str(base_seed)].get(
|
|
str(swarm_seed)
|
|
)
|
|
for method in ("feature_pso", "feature_random")
|
|
]
|
|
if all(
|
|
_checkpoint_record_valid(self.run_root, record)
|
|
for record in existing
|
|
):
|
|
for method, record in zip(
|
|
("feature_pso", "feature_random"),
|
|
existing,
|
|
):
|
|
arm_record[f"{method}:{swarm_seed}"] = record
|
|
continue
|
|
codec = SelectedResidualCodec(
|
|
detector,
|
|
selected_block_names(detector),
|
|
projection_seed=PROJECTION_SEED,
|
|
)
|
|
arm_objective = (
|
|
lambda residual, detector=detector, cache=cache:
|
|
cached_detection_objective(
|
|
detector,
|
|
cache,
|
|
codec=None,
|
|
residual=None,
|
|
model_device=self.device,
|
|
)
|
|
)
|
|
def selection_audit(
|
|
residual: torch.Tensor,
|
|
detector: nn.Module = detector,
|
|
codec: SelectedResidualCodec = codec,
|
|
) -> AuditResult:
|
|
with codec.applied(detector, residual):
|
|
metrics, _ = evaluate_detection_records(
|
|
detector,
|
|
selection_records,
|
|
device=self.device,
|
|
)
|
|
return AuditResult(
|
|
loss=-float(metrics["map50_95"]),
|
|
primary_metric=float(metrics["map50_95"]),
|
|
samples=len(selection_records),
|
|
metadata=metrics,
|
|
)
|
|
|
|
combined = run_feature_search(
|
|
detector,
|
|
arm_objective,
|
|
base_seed=base_seed,
|
|
swarm_seed=swarm_seed,
|
|
device=self.device,
|
|
validation=selection_audit,
|
|
)
|
|
for method in ("feature_pso", "feature_random"):
|
|
record = dict(combined[method])
|
|
record.update(
|
|
{
|
|
"base_seed": base_seed,
|
|
"swarm_seed": swarm_seed,
|
|
"projection_seed": PROJECTION_SEED,
|
|
}
|
|
)
|
|
endpoint_by_generation = {
|
|
int(endpoint["generation"]): endpoint["residual"]
|
|
for endpoint in record.get("endpoints", [])
|
|
}
|
|
ranked_checkpoints: list[
|
|
tuple[float, int, list[float], Mapping[str, Any]]
|
|
] = []
|
|
trajectory = record.get("trajectory", [])
|
|
if trajectory:
|
|
initial = trajectory[0].get("initial_validation")
|
|
if isinstance(initial, Mapping):
|
|
ranked_checkpoints.append(
|
|
(
|
|
float(initial["loss"]),
|
|
0,
|
|
[0.0] * RESIDUAL_DIMENSION,
|
|
initial,
|
|
)
|
|
)
|
|
for row in trajectory:
|
|
audit = row.get("validation")
|
|
generation = int(row.get("generation", -1))
|
|
if isinstance(audit, Mapping):
|
|
ranked_checkpoints.append(
|
|
(
|
|
float(audit["loss"]),
|
|
generation,
|
|
list(endpoint_by_generation[generation]),
|
|
audit,
|
|
)
|
|
)
|
|
if not ranked_checkpoints:
|
|
raise YoloProtocolError(
|
|
"feature arm has no selection checkpoints"
|
|
)
|
|
ranked_checkpoints.sort(key=lambda item: item[:2])
|
|
_, selected_generation, selected_residual, selected_audit = (
|
|
ranked_checkpoints[0]
|
|
)
|
|
selection_metrics = dict(
|
|
selected_audit.get("metadata", {})
|
|
)
|
|
candidate_model = make_yolo11n(device=self.device).model
|
|
candidate_model.load_state_dict(pristine_state, strict=True)
|
|
candidate_codec = SelectedResidualCodec(
|
|
candidate_model,
|
|
selected_block_names(candidate_model),
|
|
projection_seed=PROJECTION_SEED,
|
|
)
|
|
residual = torch.as_tensor(
|
|
selected_residual,
|
|
dtype=torch.float32,
|
|
device=self.device,
|
|
)
|
|
candidate_codec.apply_residual(candidate_model, residual)
|
|
checkpoint = (
|
|
self.root
|
|
/ "arms"
|
|
/ str(base_seed)
|
|
/ f"{method}-{swarm_seed}.pt"
|
|
)
|
|
atomic_write_bytes(
|
|
checkpoint,
|
|
_torch_save_bytes(candidate_model.state_dict()),
|
|
)
|
|
record["selection_metrics"] = selection_metrics
|
|
record["selected_generation"] = selected_generation
|
|
record["selected_residual"] = residual.detach().cpu().tolist()
|
|
record["checkpoint"] = str(
|
|
checkpoint.relative_to(self.run_root)
|
|
)
|
|
record["checkpoint_hash"] = fingerprint_file(checkpoint)
|
|
self.result["arms"][method][str(base_seed)][
|
|
str(swarm_seed)
|
|
] = record
|
|
arm_record[f"{method}:{swarm_seed}"] = record
|
|
self._save()
|
|
feature_detector = make_yolo11n(device=self.device).model
|
|
feature_detector.load_state_dict(pristine_state, strict=True)
|
|
head_detector = make_yolo11n(device=self.device).model
|
|
head_detector.load_state_dict(pristine_state, strict=True)
|
|
feature_names = selected_block_names(feature_detector)
|
|
feature_codec = SelectedResidualCodec(
|
|
feature_detector,
|
|
feature_names,
|
|
projection_seed=PROJECTION_SEED,
|
|
)
|
|
feature_bounds = {
|
|
name: RESIDUAL_BOUND * scale
|
|
for name, scale in zip(
|
|
feature_codec.names,
|
|
feature_codec.scales,
|
|
)
|
|
}
|
|
feature_adam = run_bounded_adam(
|
|
feature_detector,
|
|
feature_names,
|
|
lambda backward, detector=feature_detector, cache=cache:
|
|
cached_detection_loss_tensor(
|
|
detector,
|
|
cache,
|
|
model_device=self.device,
|
|
backward=backward,
|
|
),
|
|
bounds=feature_bounds,
|
|
)
|
|
head_adam = run_head_adam(
|
|
head_detector,
|
|
lambda backward, detector=head_detector, cache=cache:
|
|
cached_detection_loss_tensor(
|
|
detector,
|
|
cache,
|
|
model_device=self.device,
|
|
backward=backward,
|
|
),
|
|
)
|
|
for method, control_model, record in (
|
|
("feature_adam", feature_detector, feature_adam),
|
|
("head_adam", head_detector, head_adam),
|
|
):
|
|
selection_metrics, _ = evaluate_detection_records(
|
|
control_model,
|
|
selection_records,
|
|
device=self.device,
|
|
)
|
|
checkpoint = (
|
|
self.root / "arms" / str(base_seed) / f"{method}.pt"
|
|
)
|
|
atomic_write_bytes(
|
|
checkpoint,
|
|
_torch_save_bytes(control_model.state_dict()),
|
|
)
|
|
record.update(
|
|
{
|
|
"base_seed": base_seed,
|
|
"selection_metrics": selection_metrics,
|
|
"checkpoint": str(checkpoint.relative_to(self.run_root)),
|
|
"checkpoint_hash": fingerprint_file(checkpoint),
|
|
}
|
|
)
|
|
self.result["arms"].setdefault(method, {})[
|
|
str(base_seed)
|
|
] = record
|
|
arm_record[method] = record
|
|
selected: dict[str, Any] = {}
|
|
for method in ("feature_pso", "feature_random"):
|
|
records = self.result["arms"][method][str(base_seed)]
|
|
winner = max(
|
|
records.values(),
|
|
key=lambda record: (
|
|
float(record["selection_metrics"]["map50_95"]),
|
|
-int(record["swarm_seed"]),
|
|
),
|
|
)
|
|
selected[method] = {
|
|
"swarm_seed": int(winner["swarm_seed"]),
|
|
"selection_map50_95": float(
|
|
winner["selection_metrics"]["map50_95"]
|
|
),
|
|
"best_residual": list(winner["selected_residual"]),
|
|
"generation": int(winner["selected_generation"]),
|
|
"checkpoint": winner["checkpoint"],
|
|
"checkpoint_hash": winner["checkpoint_hash"],
|
|
}
|
|
self.result["development_selection"][str(base_seed)] = selected
|
|
arm_path = self.root / "arms" / str(base_seed) / "record.json"
|
|
atomic_write_json(arm_path, arm_record)
|
|
pool_records = objective_records + selection_records
|
|
member_rows = [[] for _ in pool_records]
|
|
for baseline in baselines.values():
|
|
detector = make_yolo11n(device=self.device).model
|
|
detector.load_state_dict(
|
|
torch.load(
|
|
_resolve_run_path(
|
|
self.run_root,
|
|
baseline["checkpoint"],
|
|
),
|
|
map_location=self.device,
|
|
weights_only=True,
|
|
),
|
|
strict=True,
|
|
)
|
|
detector.eval()
|
|
for start in range(0, len(pool_records), 4):
|
|
subset = pool_records[start : start + 4]
|
|
images, _, ratio_pad = native_batch(
|
|
subset,
|
|
device=self.device,
|
|
)
|
|
outputs = _native_predictions(detector, images)
|
|
for offset, (record, output, padding) in enumerate(
|
|
zip(subset, outputs, ratio_pad)
|
|
):
|
|
boxes = (
|
|
output[:, :4].detach().cpu().numpy()
|
|
if output.numel()
|
|
else np.empty((0, 4))
|
|
)
|
|
boxes = transform_boxes_to_original(
|
|
boxes,
|
|
ratio_pad=padding,
|
|
shape=(record.height, record.width),
|
|
)
|
|
boxes[:, [0, 2]] /= record.width
|
|
boxes[:, [1, 3]] /= record.height
|
|
member_rows[start + offset].append(
|
|
{
|
|
"boxes": boxes,
|
|
"scores": (
|
|
output[:, 4].detach().cpu().numpy()
|
|
if output.numel()
|
|
else np.empty((0,))
|
|
),
|
|
"labels": (
|
|
output[:, 5]
|
|
.detach()
|
|
.cpu()
|
|
.numpy()
|
|
.astype(np.int64)
|
|
if output.numel()
|
|
else np.empty((0,), dtype=np.int64)
|
|
),
|
|
}
|
|
)
|
|
member_predictions = [
|
|
[member_rows[index][member] for index in range(len(pool_records))]
|
|
for member in range(len(baselines))
|
|
]
|
|
target_rows = [
|
|
{
|
|
"image_id": f"{record.year}:{record.image_id}",
|
|
"labels": [int(label[0]) for label in record.labels],
|
|
"boxes": [
|
|
[
|
|
float(label[1] - label[3] / 2),
|
|
float(label[2] - label[4] / 2),
|
|
float(label[1] + label[3] / 2),
|
|
float(label[2] + label[4] / 2),
|
|
]
|
|
for label in record.labels
|
|
],
|
|
}
|
|
for record in pool_records
|
|
]
|
|
objective_members = [
|
|
rows[:OBJECTIVE_COUNT] for rows in member_predictions
|
|
]
|
|
selection_members = [
|
|
rows[OBJECTIVE_COUNT:] for rows in member_predictions
|
|
]
|
|
objective_targets = target_rows[:OBJECTIVE_COUNT]
|
|
selection_targets = target_rows[OBJECTIVE_COUNT:]
|
|
uniform_weights = [1.0 / len(baselines)] * len(baselines)
|
|
uniform_objective = _wbf_dataset_metrics(
|
|
objective_members, objective_targets, uniform_weights
|
|
)
|
|
uniform_selection = _wbf_dataset_metrics(
|
|
selection_members, selection_targets, uniform_weights
|
|
)
|
|
ensemble_pso = [
|
|
run_wbf_weight_search(
|
|
objective_members,
|
|
objective_targets,
|
|
seed=seed,
|
|
random_mode=False,
|
|
)
|
|
for seed in SWARM_SEEDS
|
|
]
|
|
ensemble_random = [
|
|
run_wbf_weight_search(
|
|
objective_members,
|
|
objective_targets,
|
|
seed=seed,
|
|
random_mode=True,
|
|
)
|
|
for seed in SWARM_SEEDS
|
|
]
|
|
for record in ensemble_pso + ensemble_random:
|
|
record["selection_metrics"] = _wbf_dataset_metrics(
|
|
selection_members,
|
|
selection_targets,
|
|
record["weights"],
|
|
)
|
|
selected_pso = max(
|
|
ensemble_pso,
|
|
key=lambda record: (
|
|
float(record["selection_metrics"]["map50_95"]),
|
|
-int(record["seed"]),
|
|
),
|
|
)
|
|
selected_random = max(
|
|
ensemble_random,
|
|
key=lambda record: (
|
|
float(record["selection_metrics"]["map50_95"]),
|
|
-int(record["seed"]),
|
|
),
|
|
)
|
|
fused = [
|
|
weighted_box_fusion(rows, uniform_weights)
|
|
for rows in member_rows
|
|
]
|
|
ensemble_path = self.root / "ensemble_uniform_wbf.json"
|
|
atomic_write_json(
|
|
ensemble_path,
|
|
[{key: value.tolist() for key, value in row.items()} for row in fused],
|
|
)
|
|
self.result["ensemble"] = {
|
|
"uniform_wbf": {
|
|
"path": str(ensemble_path.relative_to(self.run_root)),
|
|
"weights": uniform_weights,
|
|
"objective_metrics": uniform_objective,
|
|
"selection_metrics": uniform_selection,
|
|
},
|
|
"ensemble_pso": ensemble_pso,
|
|
"ensemble_random": ensemble_random,
|
|
}
|
|
self.result.setdefault("development_selection", {})["ensemble"] = {
|
|
"ensemble_pso": {
|
|
"seed": int(selected_pso["seed"]),
|
|
"weights": list(selected_pso["weights"]),
|
|
"selection_map50_95": float(
|
|
selected_pso["selection_metrics"]["map50_95"]
|
|
),
|
|
},
|
|
"ensemble_random": {
|
|
"seed": int(selected_random["seed"]),
|
|
"weights": list(selected_random["weights"]),
|
|
"selection_map50_95": float(
|
|
selected_random["selection_metrics"]["map50_95"]
|
|
),
|
|
},
|
|
}
|
|
self.result["baselines"] = baselines
|
|
artifact_paths = [
|
|
_resolve_run_path(self.run_root, item["checkpoint"])
|
|
for item in baselines.values()
|
|
] + [
|
|
ensemble_path,
|
|
self.root / "voc_manifest.json",
|
|
self.root / "voc_study.yaml",
|
|
]
|
|
artifact_paths.extend(
|
|
path for path in (self.root / "arms").rglob("*") if path.is_file()
|
|
)
|
|
self.result["artifact_hashes"] = {
|
|
str(path.relative_to(self.run_root)): fingerprint_file(path)
|
|
for path in artifact_paths
|
|
if path.is_file()
|
|
}
|
|
primary_queries = (
|
|
len(BASE_SEEDS)
|
|
* len(SWARM_SEEDS)
|
|
* PARTICLE_COUNT
|
|
* PSO_GENERATIONS
|
|
)
|
|
ensemble_queries = (
|
|
len(SWARM_SEEDS)
|
|
* WBF_PARTICLE_COUNT
|
|
* WBF_GENERATIONS
|
|
)
|
|
self.result["resource_ledger"] = {
|
|
"baseline_count": len(baselines),
|
|
"objective_samples": len(objective_records),
|
|
"selection_samples": len(selection_records),
|
|
"primary_pso_queries": primary_queries,
|
|
"primary_random_queries": primary_queries,
|
|
"ensemble_pso_queries": ensemble_queries,
|
|
"ensemble_random_queries": ensemble_queries,
|
|
"pso_candidate_samples": (
|
|
primary_queries + ensemble_queries
|
|
) * len(objective_records),
|
|
"random_candidate_samples": (
|
|
primary_queries + ensemble_queries
|
|
) * len(objective_records),
|
|
"pso_cells": len(BASE_SEEDS) * len(SWARM_SEEDS),
|
|
}
|
|
self.result["integrity"].update(
|
|
{"developed": True, "official_test_opened": False}
|
|
)
|
|
self._save()
|
|
return self.result
|
|
def confirm(self) -> dict[str, Any]:
|
|
state = load_state(self.run_root)
|
|
if state.state != StudyState.FROZEN:
|
|
raise SealError("confirm requires a frozen complete development matrix")
|
|
begin_confirmation(self.run_root, state)
|
|
atomic_write_json(self.run_root / "state.json", state.to_dict())
|
|
records, guard = guarded_voc_test_loader(
|
|
self.data_root,
|
|
self.run_root,
|
|
confirmation=True,
|
|
)
|
|
self.result["leakage_counters"]["official_test_construction"] += 1
|
|
predictions_root = self.root / "confirmation_predictions"
|
|
predictions_root.mkdir(parents=True, exist_ok=True)
|
|
confirmation: dict[str, Any] = {
|
|
"test_records": len(records),
|
|
"manifest_hash": guard.frozen_manifest_hash,
|
|
}
|
|
base_rows: dict[str, list[dict[str, Any]]] = {}
|
|
|
|
def evaluate_checkpoint(
|
|
method: str,
|
|
base_seed: str,
|
|
checkpoint: str,
|
|
) -> list[dict[str, Any]]:
|
|
detector = make_yolo11n(device=self.device).model
|
|
detector.load_state_dict(
|
|
torch.load(
|
|
self.run_root / checkpoint
|
|
if not Path(checkpoint).is_absolute()
|
|
else checkpoint,
|
|
map_location=self.device,
|
|
weights_only=True,
|
|
),
|
|
strict=True,
|
|
)
|
|
metrics, rows = evaluate_detection_records(
|
|
detector,
|
|
records,
|
|
device=self.device,
|
|
)
|
|
self.result["leakage_counters"][
|
|
"official_test_forward_passes"
|
|
] += 1
|
|
destination = predictions_root / f"{method}_{base_seed}.json"
|
|
atomic_write_json(destination, rows)
|
|
confirmation.setdefault(method, {})[base_seed] = {
|
|
"predictions": str(destination.relative_to(self.run_root)),
|
|
"metrics": metrics,
|
|
"images": len(rows),
|
|
}
|
|
return rows
|
|
|
|
for base_seed, baseline in sorted(
|
|
self.result.get("baselines", {}).items()
|
|
):
|
|
base_rows[base_seed] = evaluate_checkpoint(
|
|
"base",
|
|
base_seed,
|
|
baseline["checkpoint"],
|
|
)
|
|
selected = self.result["development_selection"][base_seed]
|
|
for method in ("feature_pso", "feature_random"):
|
|
swarm_seed = str(selected[method]["swarm_seed"])
|
|
checkpoint = self.result["arms"][method][base_seed][
|
|
swarm_seed
|
|
]["checkpoint"]
|
|
evaluate_checkpoint(method, base_seed, checkpoint)
|
|
for method in ("feature_adam", "head_adam"):
|
|
checkpoint = self.result["arms"][method][base_seed][
|
|
"checkpoint"
|
|
]
|
|
evaluate_checkpoint(method, base_seed, checkpoint)
|
|
|
|
base_seed_order = [str(seed) for seed in BASE_SEEDS]
|
|
member_predictions: list[list[dict[str, np.ndarray]]] = []
|
|
for base_seed in base_seed_order:
|
|
member: list[dict[str, np.ndarray]] = []
|
|
for record, row in zip(records, base_rows[base_seed]):
|
|
boxes = np.asarray(
|
|
[item["box"] for item in row["predictions"]],
|
|
dtype=np.float64,
|
|
).reshape(-1, 4)
|
|
if boxes.size:
|
|
boxes[:, [0, 2]] /= record.width
|
|
boxes[:, [1, 3]] /= record.height
|
|
member.append(
|
|
{
|
|
"boxes": boxes,
|
|
"scores": np.asarray(
|
|
[item["score"] for item in row["predictions"]],
|
|
dtype=np.float64,
|
|
),
|
|
"labels": np.asarray(
|
|
[item["class_id"] for item in row["predictions"]],
|
|
dtype=np.int64,
|
|
),
|
|
}
|
|
)
|
|
member_predictions.append(member)
|
|
target_rows = []
|
|
for record, row in zip(records, base_rows[base_seed_order[0]]):
|
|
boxes = np.asarray(
|
|
[item["box"] for item in row["ground_truth"]],
|
|
dtype=np.float64,
|
|
).reshape(-1, 4)
|
|
if boxes.size:
|
|
boxes[:, [0, 2]] /= record.width
|
|
boxes[:, [1, 3]] /= record.height
|
|
target_rows.append(
|
|
{
|
|
"image_id": row["image_id"],
|
|
"boxes": boxes.tolist(),
|
|
"labels": [
|
|
int(item["class_id"])
|
|
for item in row["ground_truth"]
|
|
],
|
|
}
|
|
)
|
|
|
|
weights_by_method = {
|
|
"uniform_wbf": [1.0 / len(BASE_SEEDS)] * len(BASE_SEEDS),
|
|
"ensemble_pso": self.result["development_selection"]["ensemble"][
|
|
"ensemble_pso"
|
|
]["weights"],
|
|
"ensemble_random": self.result["development_selection"][
|
|
"ensemble"
|
|
]["ensemble_random"]["weights"],
|
|
}
|
|
for method, weights in weights_by_method.items():
|
|
fused_rows = []
|
|
for index, record in enumerate(records):
|
|
fused = weighted_box_fusion(
|
|
[member[index] for member in member_predictions],
|
|
weights,
|
|
)
|
|
boxes = fused["boxes"].copy()
|
|
if boxes.size:
|
|
boxes[:, [0, 2]] *= record.width
|
|
boxes[:, [1, 3]] *= record.height
|
|
fused_rows.append(
|
|
{
|
|
"image_id": target_rows[index]["image_id"],
|
|
"predictions": [
|
|
{
|
|
"box": [float(value) for value in box],
|
|
"class_id": int(label),
|
|
"score": float(score),
|
|
}
|
|
for box, label, score in zip(
|
|
boxes.tolist(),
|
|
fused["labels"].tolist(),
|
|
fused["scores"].tolist(),
|
|
)
|
|
],
|
|
"ground_truth": base_rows[
|
|
base_seed_order[0]
|
|
][index]["ground_truth"],
|
|
}
|
|
)
|
|
destination = predictions_root / f"{method}.json"
|
|
atomic_write_json(destination, fused_rows)
|
|
confirmation[method] = {
|
|
"predictions": str(destination.relative_to(self.run_root)),
|
|
"metrics": _wbf_dataset_metrics(
|
|
member_predictions,
|
|
target_rows,
|
|
weights,
|
|
),
|
|
"weights": [float(weight) for weight in weights],
|
|
"images": len(fused_rows),
|
|
}
|
|
self.result["confirmation"] = confirmation
|
|
self.result["artifact_hashes"].update(
|
|
{
|
|
str(path.relative_to(self.run_root)): fingerprint_file(path)
|
|
for path in predictions_root.glob("*.json")
|
|
}
|
|
)
|
|
finish_confirmation(state, success=True)
|
|
atomic_write_json(self.run_root / "state.json", state.to_dict())
|
|
self._save()
|
|
return self.result
|
|
def run_phase(self, phase: str) -> dict[str, Any]:
|
|
if phase == "prepare": return self.prepare()
|
|
if phase == "smoke": return self.smoke()
|
|
if phase == "develop": return self.develop()
|
|
if phase == "confirm": return self.confirm()
|
|
if phase == "publish":
|
|
if not self.result_path.is_file(): raise SealError("cannot publish without adapter result")
|
|
return json.loads(self.result_path.read_text(encoding="utf-8"))
|
|
raise YoloProtocolError(f"unsupported adapter phase: {phase}")
|
|
|
|
|
|
def create_adapter(*, workload_id: str = WORKLOAD_ID, config: StudyConfig, run_root: str | os.PathLike[str], data_root: str | os.PathLike[str], device: str | torch.device, allow_download: bool = False) -> YoloConvergenceAdapter:
|
|
return YoloConvergenceAdapter(workload_id=workload_id, config=config, run_root=run_root, data_root=data_root, device=device, allow_download=allow_download)
|
|
|
|
|
|
__all__ = [
|
|
"BASE_SEEDS", "DetectionCache", "FAMILY", "IMG_SIZE", "OBJECTIVE_COUNT",
|
|
"PROJECTION_SEED", "StrictScratchTrainer", "ULTRALYTICS_VERSION",
|
|
"VOC_CLASSES", "VOCManifest", "VOCRecord", "VOCTestGuard", "WORKLOAD_ID",
|
|
"YoloConvergenceAdapter", "YoloProtocolError", "assert_yolo_topology",
|
|
"build_detection_cache", "cached_detection_objective", "cached_full_parity",
|
|
"create_adapter", "guarded_voc_test_loader", "image_fingerprint",
|
|
"make_voc_manifests", "make_yolo11n", "native_detection_metrics",
|
|
"parse_voc_xml", "pinned_preflight", "prepare_voc", "run_bounded_adam",
|
|
"run_feature_search", "run_head_adam", "selected_block_names",
|
|
"selected_head_bias_names", "train_baseline", "transform_boxes_to_letterbox",
|
|
"transform_boxes_to_original", "weighted_box_fusion", "write_study_yaml",
|
|
"write_yolo_label",
|
|
]
|