mirror of
https://github.com/jung-geun/PSO.git
synced 2026-09-20 14:11:48 +09:00
feat: modernize PSO and add convergence research
Migrate the package and examples to the tensor-native PyTorch implementation, add benchmark evidence, and add the guarded post-training convergence protocol with TensorBoard progress monitoring and hash-verified recovery. Constraint: Preserve one-shot official-test sealing and auditable research artifacts Rejected: Commit local .omc runs and downloaded datasets | multi-gigabyte runtime state is machine-local Confidence: high Scope-risk: broad Not-tested: Production CUDA run on pieroot-server
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def xor_data():
|
||||
"""
|
||||
Returns deterministic XOR input features (4, 2) and labels (4, 1) as float32 torch tensors on CPU.
|
||||
"""
|
||||
x = torch.tensor([[0.0, 0.0], [0.0, 1.0], [1.0, 0.0], [1.0, 1.0]], dtype=torch.float32)
|
||||
y = torch.tensor([[0.0], [1.0], [1.0], [0.0]], dtype=torch.float32)
|
||||
return x, y
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def model_factory():
|
||||
"""
|
||||
Factory fixture producing deterministic, PyTorch nn.Module models.
|
||||
Supports units tuning, zero initialization, and input/output dimension changes.
|
||||
"""
|
||||
|
||||
def _create_model(
|
||||
units: int = 4,
|
||||
zero_init: bool = False,
|
||||
input_dim: int = 2,
|
||||
output_dim: int = 1,
|
||||
) -> nn.Module:
|
||||
torch.manual_seed(42)
|
||||
layers = [
|
||||
nn.Linear(input_dim, units),
|
||||
nn.ReLU(),
|
||||
nn.Linear(units, output_dim),
|
||||
]
|
||||
model = nn.Sequential(*layers)
|
||||
if zero_init:
|
||||
for m in model.modules():
|
||||
if isinstance(m, nn.Linear):
|
||||
nn.init.zeros_(m.weight)
|
||||
if m.bias is not None:
|
||||
nn.init.zeros_(m.bias)
|
||||
return model
|
||||
|
||||
return _create_model
|
||||
@@ -0,0 +1,235 @@
|
||||
"""
|
||||
Focused Offline Unit Tests for MNIST Deep PSO Methods (Protocol MNIST-PSO-RAW-V5 1.0.0)
|
||||
|
||||
Tests:
|
||||
1. Split balance and nested subset inclusion invariant (I_2k subset of I_10k subset of I_50k).
|
||||
2. Latent transform exactness (z_0 = 0 -> theta_0), antithetic symmetry (even swarm pairing), and subspace dimensions.
|
||||
3. Stage transition pbest re-evaluation, gbest rebuild, and exact query/sample accounting.
|
||||
4. Validation-only pilot and elite ensemble selection with greedy disagreement.
|
||||
5. CLI argument validation.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Insert repo test/ path so deep_pso_methods can be imported
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "test"))
|
||||
|
||||
import math
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from deep_pso_methods import (
|
||||
CompactCNN,
|
||||
LatentTransform,
|
||||
build_nested_stratified_subsets,
|
||||
evaluate_latent_batch,
|
||||
evaluate_probabilistic_metrics,
|
||||
make_compact_cnn,
|
||||
run_latent_pso,
|
||||
select_diverse_candidates,
|
||||
validate_cli_args,
|
||||
build_parser,
|
||||
)
|
||||
|
||||
|
||||
def test_split_balance_and_nesting():
|
||||
"""Verify stratified split balance and nested index inclusion: I_2k subset of I_10k subset of I_50k."""
|
||||
N = 50000
|
||||
num_classes = 10
|
||||
samples_per_class = N // num_classes
|
||||
y_search = torch.cat([torch.full((samples_per_class,), c, dtype=torch.long) for c in range(num_classes)])
|
||||
|
||||
split_seed = 20260902
|
||||
nested_subsets = build_nested_stratified_subsets(
|
||||
y_search=y_search,
|
||||
subset_sizes=[2000, 10000, 50000],
|
||||
subset_seed=split_seed,
|
||||
)
|
||||
|
||||
idx_2k = nested_subsets[2000]
|
||||
idx_10k = nested_subsets[10000]
|
||||
idx_50k = nested_subsets[50000]
|
||||
|
||||
assert len(idx_2k) == 2000
|
||||
assert len(idx_10k) == 10000
|
||||
assert len(idx_50k) == 50000
|
||||
|
||||
set_2k = set(idx_2k.numpy().tolist())
|
||||
set_10k = set(idx_10k.numpy().tolist())
|
||||
set_50k = set(idx_50k.numpy().tolist())
|
||||
|
||||
# Strict nesting: I_2k subset of I_10k subset of I_50k
|
||||
assert set_2k.issubset(set_10k), "I_2k must be a strict subset of I_10k"
|
||||
assert set_10k.issubset(set_50k), "I_10k must be a strict subset of I_50k"
|
||||
|
||||
# Exact stratification (equal counts per class)
|
||||
y_2k = y_search[idx_2k].numpy()
|
||||
y_10k = y_search[idx_10k].numpy()
|
||||
|
||||
counts_2k = np.bincount(y_2k, minlength=10)
|
||||
counts_10k = np.bincount(y_10k, minlength=10)
|
||||
|
||||
for c in range(num_classes):
|
||||
assert counts_2k[c] == 200, f"Class {c} in 2k subset must have 200 samples; got {counts_2k[c]}"
|
||||
assert counts_10k[c] == 1000, f"Class {c} in 10k subset must have 1000 samples; got {counts_10k[c]}"
|
||||
|
||||
|
||||
def test_latent_transform_exactness_and_antithetic_symmetry():
|
||||
"""Verify transform exactness (z_0 = 0 -> theta_0), even swarm pairing, zero centroid, and subspace shapes."""
|
||||
device = torch.device("cpu")
|
||||
base_model = make_compact_cnn(seed=41).to(device)
|
||||
base_vec = torch.cat([p.detach().view(-1) for p in base_model.parameters()])
|
||||
|
||||
dims = [290, 1024, 4096, "full"]
|
||||
swarm_size = 30 # Even swarm size
|
||||
|
||||
for d in dims:
|
||||
transform = LatentTransform(base_model, latent_dim=d, device=device)
|
||||
Z = transform.init_swarm(swarm_size=swarm_size, seed=91)
|
||||
|
||||
# 1. Particle 0 is exact zero (base model)
|
||||
decoded_p0 = transform.decode(Z[0:1]).squeeze(0)
|
||||
assert torch.allclose(decoded_p0, base_vec, atol=1e-6), f"Particle 0 for dim={d} must match exact base vector"
|
||||
|
||||
# 2. For even N=30: particle N-1 (index 29) is also exact zero
|
||||
decoded_plast = transform.decode(Z[29:30]).squeeze(0)
|
||||
assert torch.allclose(decoded_plast, base_vec, atol=1e-6), f"Particle N-1 for dim={d} must match exact zero"
|
||||
|
||||
# 3. Antithetic pairs (particles 1..28 in 14 exact pairs)
|
||||
z1 = Z[1:2]
|
||||
z2 = Z[2:3]
|
||||
assert torch.allclose(z1 + z2, torch.zeros_like(z1), atol=1e-6), "Antithetic pair latent sum must be zero"
|
||||
|
||||
delta1 = transform.decode(z1).squeeze(0) - transform.base_vec
|
||||
delta2 = transform.decode(z2).squeeze(0) - transform.base_vec
|
||||
assert torch.allclose(delta1 + delta2, torch.zeros_like(delta1), atol=1e-5), "Antithetic pair delta sum must be zero"
|
||||
|
||||
# 4. Latent swarm centroid is strictly zero
|
||||
centroid_z = Z.mean(dim=0)
|
||||
assert torch.allclose(centroid_z, torch.zeros_like(centroid_z), atol=1e-6), "Overall swarm centroid must be zero"
|
||||
|
||||
|
||||
def test_transition_reevaluation_and_exact_accounting():
|
||||
"""Verify objective size transition re-evaluates all pbests, rebuilds gbest, and asserts exact query/sample counts."""
|
||||
device = torch.device("cpu")
|
||||
base_model = make_compact_cnn(seed=41).to(device)
|
||||
|
||||
N_samples = 100
|
||||
x_synth = torch.randn(N_samples, 1, 28, 28)
|
||||
y_synth = torch.randint(0, 10, (N_samples,))
|
||||
|
||||
nested_subsets = {
|
||||
20: torch.arange(20, dtype=torch.long),
|
||||
50: torch.arange(50, dtype=torch.long),
|
||||
}
|
||||
|
||||
transform = LatentTransform(base_model, latent_dim=290, device=device)
|
||||
|
||||
# Run 2-stage PSO: stage 0 (20 samples, 2 epochs), stage 1 (50 samples, 2 epochs), swarm_size = 10
|
||||
# Stage 0: 2 * 10 = 20 queries, 20 * 20 = 400 sample evals. No transition re-eval.
|
||||
# Stage 1 transition: 1 * 10 = 10 queries, 10 * 50 = 500 sample evals. Transition count = 10.
|
||||
# Stage 1: 2 * 10 = 20 queries, 20 * 50 = 1000 sample evals.
|
||||
# Total queries = 20 + 10 + 20 = 50.
|
||||
# Total sample evals = 400 + 500 + 1000 = 1900.
|
||||
res = run_latent_pso(
|
||||
transform=transform,
|
||||
base_model=base_model,
|
||||
x_search=x_synth,
|
||||
y_search=y_synth,
|
||||
nested_subsets=nested_subsets,
|
||||
schedule_str="20:2,50:2",
|
||||
epochs=4,
|
||||
swarm_size=10,
|
||||
seed=42,
|
||||
device=device,
|
||||
)
|
||||
|
||||
assert res["transition_reevaluation_counts"] == 10, f"Expected 10 transition re-evaluations; got {res['transition_reevaluation_counts']}"
|
||||
assert res["total_queries"] == 50, f"Expected 50 total queries; got {res['total_queries']}"
|
||||
assert res["total_sample_evaluations"] == 1900, f"Expected 1900 sample evaluations; got {res['total_sample_evaluations']}"
|
||||
assert len(res["stage_histories"]) == 4
|
||||
|
||||
|
||||
def test_validation_only_elite_and_ensemble_selection():
|
||||
"""Verify validation metrics correctly rank candidates and metric routines compute expected values."""
|
||||
N = 100
|
||||
C = 10
|
||||
y_val = torch.randint(0, C, (N,))
|
||||
|
||||
# Candidate 1: Perfect predictions
|
||||
probs_perfect = torch.zeros((N, C), dtype=torch.float32)
|
||||
probs_perfect[torch.arange(N), y_val] = 1.0
|
||||
|
||||
# Candidate 2: Random noise
|
||||
probs_random = torch.full((N, C), 1.0 / C, dtype=torch.float32)
|
||||
|
||||
m1 = evaluate_probabilistic_metrics(probs_perfect, y_val)
|
||||
m2 = evaluate_probabilistic_metrics(probs_random, y_val)
|
||||
|
||||
assert m1["accuracy"] == 100.0
|
||||
assert m1["nll"] < m2["nll"]
|
||||
assert m1["brier"] < m2["brier"]
|
||||
assert m1["ece"] <= 0.01
|
||||
|
||||
candidates = [
|
||||
{"id": "cand2", "val_loss": m2["nll"], "val_acc": m2["accuracy"]},
|
||||
{"id": "cand1", "val_loss": m1["nll"], "val_acc": m1["accuracy"]},
|
||||
]
|
||||
candidates.sort(key=lambda c: (c["val_loss"], -c["val_acc"]))
|
||||
assert candidates[0]["id"] == "cand1"
|
||||
|
||||
diverse_candidates = [
|
||||
{
|
||||
"seed": 7,
|
||||
"particle_idx": particle_idx,
|
||||
"val_loss": 0.2 + 0.01 * particle_idx,
|
||||
"val_acc": 90.0 - 0.25 * particle_idx,
|
||||
"val_probs": torch.roll(probs_perfect, shifts=particle_idx, dims=1),
|
||||
"latent_z": torch.full((4,), float(particle_idx)),
|
||||
}
|
||||
for particle_idx in range(3)
|
||||
]
|
||||
selected = select_diverse_candidates(
|
||||
diverse_candidates, max_size=3, accuracy_window=2.0
|
||||
)
|
||||
assert len(selected) == 3
|
||||
assert len({
|
||||
(candidate["seed"], candidate["particle_idx"])
|
||||
for candidate in selected
|
||||
}) == 3
|
||||
|
||||
|
||||
def test_cli_argument_validation():
|
||||
"""Verify CLI argument validation rejects invalid parameters and accepts valid settings."""
|
||||
parser = build_parser()
|
||||
|
||||
# Valid args
|
||||
valid_args = parser.parse_args([
|
||||
"--pilot-epochs", "160",
|
||||
"--confirmation-epochs", "600",
|
||||
"--confirmation-schedule", "2000:420,10000:135,50000:45",
|
||||
"--seeds", "101", "102", "103",
|
||||
"--dimensions", "290", "1024", "4096", "full"
|
||||
])
|
||||
validate_cli_args(valid_args)
|
||||
|
||||
# Invalid schedule sum mismatch
|
||||
invalid_schedule = parser.parse_args([
|
||||
"--confirmation-epochs", "600",
|
||||
"--confirmation-schedule", "2000:400,10000:100,50000:50" # Sums to 550 != 600
|
||||
])
|
||||
with pytest.raises(ValueError, match="Schedule epoch sum"):
|
||||
validate_cli_args(invalid_schedule)
|
||||
|
||||
# Invalid negative seed
|
||||
invalid_seed = parser.parse_args(["--seeds", "-1"])
|
||||
with pytest.raises(ValueError, match="Seeds must be non-negative"):
|
||||
validate_cli_args(invalid_seed)
|
||||
|
||||
# Invalid duplicate seed
|
||||
duplicate_seed = parser.parse_args(["--seeds", "101", "101"])
|
||||
with pytest.raises(ValueError, match="Confirmation seeds must be unique"):
|
||||
validate_cli_args(duplicate_seed)
|
||||
@@ -0,0 +1,515 @@
|
||||
"""
|
||||
Unit tests for MNIST PSO V6 Root-Cause Isolation (Phases A & B).
|
||||
|
||||
Covers:
|
||||
1. Scale modes (per_tensor_sd, global_rms, identity)
|
||||
2. G0 decode & default movement parity with V5 full-D
|
||||
3. Exact antithetic & independent initialization invariants
|
||||
4. Deterministic projection seeds
|
||||
5. Equalized-dimension helper RMS behavior
|
||||
6. Mutation moment reset
|
||||
7. Transition full-pbest reevaluation & exact accounting
|
||||
8. Validation checkpoints state neutrality
|
||||
9. Geometry configuration table G0-G8
|
||||
10. Deterministic confirmation selection logic
|
||||
11. Guard proving Phase B loader never requests official test dataset (train=False)
|
||||
"""
|
||||
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
# Ensure test directory and repo root are in Python path
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent if Path(__file__).resolve().parent.name != "PSO" else Path(__file__).resolve().parent
|
||||
TEST_DIR = REPO_ROOT / "test"
|
||||
if str(TEST_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TEST_DIR))
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from deep_pso_methods import LatentTransform, make_compact_cnn
|
||||
from deep_pso_v6 import (
|
||||
V6GeometryConfig,
|
||||
V6LatentTransform,
|
||||
compute_equalized_subspace_radius,
|
||||
get_v6_geometry_table,
|
||||
prepare_mnist_v6_data,
|
||||
run_v6_pso,
|
||||
run_g8_optimizer,
|
||||
select_confirmation_configs,
|
||||
)
|
||||
|
||||
|
||||
class TinyModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.fc1 = nn.Linear(10, 5)
|
||||
self.fc2 = nn.Linear(5, 2)
|
||||
# Initialize deterministic weights
|
||||
nn.init.constant_(self.fc1.weight, 1.0)
|
||||
nn.init.constant_(self.fc1.bias, 0.5)
|
||||
nn.init.constant_(self.fc2.weight, -0.5)
|
||||
nn.init.constant_(self.fc2.bias, 0.0)
|
||||
|
||||
def forward(self, x):
|
||||
return self.fc2(torch.relu(self.fc1(x)))
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 1. Scale Modes Test
|
||||
# =====================================================================
|
||||
|
||||
def test_scale_modes():
|
||||
device = torch.device("cpu")
|
||||
model = TinyModel()
|
||||
|
||||
# Per-tensor SD
|
||||
cfg_per_tensor = V6GeometryConfig(config_id="T1", scale_type="per_tensor_sd")
|
||||
tf_per_tensor = V6LatentTransform(model, cfg_per_tensor, device)
|
||||
assert tf_per_tensor.scale_vec.shape[0] == tf_per_tensor.total_dim
|
||||
|
||||
# Global RMS
|
||||
cfg_global_rms = V6GeometryConfig(config_id="T2", scale_type="global_rms")
|
||||
tf_global_rms = V6LatentTransform(model, cfg_global_rms, device)
|
||||
expected_rms = max(float(torch.sqrt(torch.mean(tf_global_rms.base_vec ** 2))), 1e-4)
|
||||
assert torch.allclose(tf_global_rms.scale_vec, torch.full_like(tf_global_rms.scale_vec, expected_rms))
|
||||
|
||||
# Identity
|
||||
cfg_identity = V6GeometryConfig(config_id="T3", scale_type="identity")
|
||||
tf_identity = V6LatentTransform(model, cfg_identity, device)
|
||||
assert torch.allclose(tf_identity.scale_vec, torch.ones_like(tf_identity.scale_vec))
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 2. G0 Decode & Default Parity with V5 Full-D
|
||||
# =====================================================================
|
||||
|
||||
def test_G0_decode_default_parity():
|
||||
device = torch.device("cpu")
|
||||
|
||||
base_v5 = make_compact_cnn(seed=41).to(device)
|
||||
base_v6 = make_compact_cnn(seed=41).to(device)
|
||||
|
||||
tf_v5 = LatentTransform(base_v5, latent_dim="full", device=device)
|
||||
|
||||
cfg_g0 = get_v6_geometry_table()["G0"]
|
||||
tf_v6 = V6LatentTransform(base_v6, cfg_g0, device=device)
|
||||
|
||||
# Verify scale_vec equality
|
||||
assert torch.allclose(tf_v5.scale_vec, tf_v6.scale_vec, atol=1e-6)
|
||||
assert torch.allclose(tf_v5.base_vec, tf_v6.base_vec, atol=1e-6)
|
||||
|
||||
# Initial swarm parity with seed 91
|
||||
swarm_size = 30
|
||||
seed = 91
|
||||
Z_v5 = tf_v5.init_swarm(swarm_size=swarm_size, seed=seed)
|
||||
Z_v6 = tf_v6.init_swarm(swarm_size=swarm_size, seed=seed)
|
||||
|
||||
assert torch.allclose(Z_v5, Z_v6, atol=1e-6)
|
||||
|
||||
# Decode parity
|
||||
theta_v5 = tf_v5.decode(Z_v5)
|
||||
theta_v6 = tf_v6.decode(Z_v6)
|
||||
assert torch.allclose(theta_v5, theta_v6, atol=1e-6)
|
||||
|
||||
# Single movement step parity
|
||||
c0 = c1 = 1.49618
|
||||
w = 0.7298
|
||||
latent_dim = tf_v6.latent_dim
|
||||
|
||||
V_v5 = torch.zeros((swarm_size, latent_dim), dtype=torch.float32, device=device)
|
||||
V_v6 = torch.zeros((swarm_size, latent_dim), dtype=torch.float32, device=device)
|
||||
|
||||
P_v5 = Z_v5.clone()
|
||||
P_v6 = Z_v6.clone()
|
||||
|
||||
gbest_z_v5 = Z_v5[0].clone()
|
||||
gbest_z_v6 = Z_v6[0].clone()
|
||||
|
||||
move_rng_v5 = torch.Generator(device=device)
|
||||
move_rng_v5.manual_seed(seed)
|
||||
|
||||
move_rng_v6 = torch.Generator(device=device)
|
||||
move_rng_v6.manual_seed(seed)
|
||||
|
||||
r1_v5 = torch.rand((swarm_size, latent_dim), generator=move_rng_v5, device=device)
|
||||
r2_v5 = torch.rand((swarm_size, latent_dim), generator=move_rng_v5, device=device)
|
||||
|
||||
r1_v6 = torch.rand((swarm_size, latent_dim), generator=move_rng_v6, device=device)
|
||||
r2_v6 = torch.rand((swarm_size, latent_dim), generator=move_rng_v6, device=device)
|
||||
|
||||
assert torch.allclose(r1_v5, r1_v6)
|
||||
assert torch.allclose(r2_v5, r2_v6)
|
||||
|
||||
V_raw_v5 = w * V_v5 + c0 * r1_v5 * (P_v5 - Z_v5) + c1 * r2_v5 * (gbest_z_v5.unsqueeze(0) - Z_v5)
|
||||
V_raw_v6 = w * V_v6 + c0 * r1_v6 * (P_v6 - Z_v6) + c1 * r2_v6 * (gbest_z_v6.unsqueeze(0) - Z_v6)
|
||||
|
||||
assert torch.allclose(V_raw_v5, V_raw_v6, atol=1e-6)
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 3. Exact Antithetic & Independent Initialization Invariants
|
||||
# =====================================================================
|
||||
|
||||
def test_exact_independent_init_invariants():
|
||||
device = torch.device("cpu")
|
||||
model = TinyModel()
|
||||
swarm_size = 30
|
||||
seed = 123
|
||||
|
||||
# 1. Antithetic mode
|
||||
cfg_anti = V6GeometryConfig(config_id="T_anti", init_position_mode="antithetic", position_radius=0.5)
|
||||
tf_anti = V6LatentTransform(model, cfg_anti, device)
|
||||
Z_anti = tf_anti.init_swarm(swarm_size=swarm_size, seed=seed)
|
||||
|
||||
# Particle 0 is exact zero
|
||||
assert torch.norm(Z_anti[0]).item() == 0.0
|
||||
|
||||
# For even swarm_size=30, particles 1..28 form exact pairs: (1, 2), (3, 4), ..., (27, 28)
|
||||
for idx in range(1, 28, 2):
|
||||
assert torch.allclose(Z_anti[idx], -Z_anti[idx + 1], atol=1e-6)
|
||||
|
||||
# Particle 29 is zero filler
|
||||
assert torch.norm(Z_anti[29]).item() == 0.0
|
||||
|
||||
# 2. Independent mode
|
||||
cfg_indep = V6GeometryConfig(config_id="T_indep", init_position_mode="independent", position_radius=0.5)
|
||||
tf_indep = V6LatentTransform(model, cfg_indep, device)
|
||||
Z_indep = tf_indep.init_swarm(swarm_size=swarm_size, seed=seed)
|
||||
|
||||
# Particle 0 is exact zero
|
||||
assert torch.norm(Z_indep[0]).item() == 0.0
|
||||
|
||||
# Particles 1..29 are non-zero and independent
|
||||
assert torch.norm(Z_indep[1]).item() > 0.0
|
||||
assert not torch.allclose(Z_indep[1], -Z_indep[2])
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 4. Deterministic Projection Seeds Test
|
||||
# =====================================================================
|
||||
|
||||
def test_deterministic_projection_seeds():
|
||||
device = torch.device("cpu")
|
||||
model = TinyModel()
|
||||
|
||||
cfg1 = V6GeometryConfig(config_id="P1", latent_dim=8, projection_seed=42)
|
||||
cfg2 = V6GeometryConfig(config_id="P2", latent_dim=8, projection_seed=42)
|
||||
cfg3 = V6GeometryConfig(config_id="P3", latent_dim=8, projection_seed=99)
|
||||
|
||||
tf1 = V6LatentTransform(model, cfg1, device)
|
||||
tf2 = V6LatentTransform(model, cfg2, device)
|
||||
tf3 = V6LatentTransform(model, cfg3, device)
|
||||
|
||||
# Same seed yields identical mapping
|
||||
assert torch.equal(tf1.k_indices, tf2.k_indices)
|
||||
assert torch.equal(tf1.weights, tf2.weights)
|
||||
|
||||
# Different seed yields different mapping
|
||||
assert not torch.equal(tf1.k_indices, tf3.k_indices) or not torch.equal(tf1.weights, tf3.weights)
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 5. Equalized-Dimension Helper RMS Behavior Test
|
||||
# =====================================================================
|
||||
|
||||
def test_equalized_dimension_helper_rms_behavior():
|
||||
r_290 = compute_equalized_subspace_radius(latent_dim=290, total_dim=9098, base_radius=0.5)
|
||||
r_1024 = compute_equalized_subspace_radius(latent_dim=1024, total_dim=9098, base_radius=0.5)
|
||||
r_4096 = compute_equalized_subspace_radius(latent_dim=4096, total_dim=9098, base_radius=0.5)
|
||||
r_full = compute_equalized_subspace_radius(latent_dim=9098, total_dim=9098, base_radius=0.5)
|
||||
|
||||
assert r_290 == pytest.approx(0.5 * math.sqrt(9098 / 290), rel=1e-5)
|
||||
assert r_1024 == pytest.approx(0.5 * math.sqrt(9098 / 1024), rel=1e-5)
|
||||
assert r_4096 == pytest.approx(0.5 * math.sqrt(9098 / 4096), rel=1e-5)
|
||||
assert r_full == 0.5
|
||||
|
||||
# Radii decrease monotonically as latent dimension increases toward full-D
|
||||
assert r_290 > r_1024 > r_4096 > r_full
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 6. Mutation Moment Reset Test
|
||||
# =====================================================================
|
||||
|
||||
def test_mutation_moment_reset():
|
||||
device = torch.device("cpu")
|
||||
model = TinyModel()
|
||||
|
||||
x_search = torch.randn(20, 10)
|
||||
y_search = torch.randint(0, 2, (20,))
|
||||
x_val = torch.randn(10, 10)
|
||||
y_val = torch.randint(0, 2, (10,))
|
||||
nested_subsets = {10: torch.arange(10), 20: torch.arange(20)}
|
||||
|
||||
# Always-on mutation: mutation_prob = 1.0
|
||||
cfg = V6GeometryConfig(
|
||||
config_id="M1",
|
||||
mutation_prob=1.0,
|
||||
reset_velocity_radius=0.02,
|
||||
)
|
||||
transform = V6LatentTransform(model, cfg, device)
|
||||
|
||||
res = run_v6_pso(
|
||||
transform=transform,
|
||||
base_model=model,
|
||||
x_search=x_search,
|
||||
y_search=y_search,
|
||||
x_val=x_val,
|
||||
y_val=y_val,
|
||||
nested_subsets=nested_subsets,
|
||||
schedule_str="10:2",
|
||||
epochs=2,
|
||||
swarm_size=5,
|
||||
seed=42,
|
||||
device=device,
|
||||
geom_config=cfg,
|
||||
)
|
||||
|
||||
assert res["config_id"] == "M1"
|
||||
assert res["total_queries"] == 5 * 2
|
||||
assert res["mutation_events"] == 5 * 2
|
||||
assert res["final_moment_steps"] == [1] * 5
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 7. Transition Reevaluation & Exact Accounting Test
|
||||
# =====================================================================
|
||||
|
||||
def test_transition_full_pbest_reevaluation_plus_exact_accounting():
|
||||
device = torch.device("cpu")
|
||||
model = TinyModel()
|
||||
|
||||
x_search = torch.randn(50, 10)
|
||||
y_search = torch.randint(0, 2, (50,))
|
||||
x_val = torch.randn(10, 10)
|
||||
y_val = torch.randint(0, 2, (10,))
|
||||
|
||||
nested_subsets = {
|
||||
5: torch.arange(5),
|
||||
20: torch.arange(20),
|
||||
}
|
||||
|
||||
cfg = get_v6_geometry_table()["G0"]
|
||||
transform = V6LatentTransform(model, cfg, device)
|
||||
|
||||
# Run 2-stage schedule: 5 samples for 2 epochs, then 20 samples for 2 epochs
|
||||
res = run_v6_pso(
|
||||
transform=transform,
|
||||
base_model=model,
|
||||
x_search=x_search,
|
||||
y_search=y_search,
|
||||
x_val=x_val,
|
||||
y_val=y_val,
|
||||
nested_subsets=nested_subsets,
|
||||
schedule_str="5:2,20:2",
|
||||
epochs=4,
|
||||
swarm_size=10,
|
||||
seed=42,
|
||||
device=device,
|
||||
geom_config=cfg,
|
||||
transition_reset_policy="reset_vm",
|
||||
)
|
||||
|
||||
# Exact query accounting:
|
||||
# Stage 0: 10 particles x 2 epochs = 20 queries
|
||||
# Transition: 10 particles reevaluated on new subset = 10 queries
|
||||
# Stage 1: 10 particles x 2 epochs = 20 queries
|
||||
# Total queries = 50
|
||||
assert res["total_queries"] == 50
|
||||
assert res["transition_reevaluation_counts"] == 10
|
||||
|
||||
# Sample evaluations:
|
||||
# Stage 0: 20 queries x 5 samples = 100
|
||||
# Transition: 10 queries x 20 samples = 200
|
||||
# Stage 1: 20 queries x 20 samples = 400
|
||||
# Total sample evals = 700
|
||||
assert res["total_sample_evaluations"] == 700
|
||||
assert res["final_moment_steps"] == [2] * 10
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 8. Validation Checkpoints State Neutrality Test
|
||||
# =====================================================================
|
||||
|
||||
def test_validation_checkpoints_state_neutral():
|
||||
device = torch.device("cpu")
|
||||
model = TinyModel()
|
||||
|
||||
x_search = torch.randn(20, 10)
|
||||
y_search = torch.randint(0, 2, (20,))
|
||||
x_val = torch.randn(10, 10)
|
||||
y_val = torch.randint(0, 2, (10,))
|
||||
nested_subsets = {20: torch.arange(20)}
|
||||
|
||||
cfg = get_v6_geometry_table()["G0"]
|
||||
|
||||
# Run with validation checkpoints every epoch (val_check_interval=1)
|
||||
tf1 = V6LatentTransform(model, cfg, device)
|
||||
res_chk = run_v6_pso(
|
||||
transform=tf1,
|
||||
base_model=model,
|
||||
x_search=x_search,
|
||||
y_search=y_search,
|
||||
x_val=x_val,
|
||||
y_val=y_val,
|
||||
nested_subsets=nested_subsets,
|
||||
schedule_str="20:5",
|
||||
epochs=5,
|
||||
swarm_size=6,
|
||||
seed=99,
|
||||
device=device,
|
||||
geom_config=cfg,
|
||||
val_check_interval=1,
|
||||
)
|
||||
|
||||
# Run without intermediate validation checkpoints (val_check_interval=0)
|
||||
tf2 = V6LatentTransform(model, cfg, device)
|
||||
res_nochk = run_v6_pso(
|
||||
transform=tf2,
|
||||
base_model=model,
|
||||
x_search=x_search,
|
||||
y_search=y_search,
|
||||
x_val=x_val,
|
||||
y_val=y_val,
|
||||
nested_subsets=nested_subsets,
|
||||
schedule_str="20:5",
|
||||
epochs=5,
|
||||
swarm_size=6,
|
||||
seed=99,
|
||||
device=device,
|
||||
geom_config=cfg,
|
||||
val_check_interval=0,
|
||||
)
|
||||
|
||||
# Final gbest positions, loss, and training accuracy must be IDENTICAL
|
||||
assert torch.allclose(res_chk["gbest_z"], res_nochk["gbest_z"], atol=1e-6)
|
||||
assert res_chk["gbest_loss"] == res_nochk["gbest_loss"]
|
||||
assert res_chk["gbest_acc"] == res_nochk["gbest_acc"]
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 9. Configuration Table G0-G8 Test
|
||||
# =====================================================================
|
||||
|
||||
def test_configuration_table_G0_G8():
|
||||
table = get_v6_geometry_table()
|
||||
assert len(table) == 9
|
||||
for i in range(9):
|
||||
cid = f"G{i}"
|
||||
assert cid in table
|
||||
assert table[cid].config_id == cid
|
||||
|
||||
assert table["G0"].scale_type == "per_tensor_sd"
|
||||
assert table["G0"].init_position_mode == "antithetic"
|
||||
assert table["G0"].initial_velocity_radius == 0.0
|
||||
|
||||
assert table["G1"].scale_type == "global_rms"
|
||||
|
||||
assert table["G2"].initial_velocity_radius == 0.5
|
||||
|
||||
assert table["G3"].mutation_prob == 0.02
|
||||
|
||||
assert table["G5"].reflective_bound == 6.0
|
||||
|
||||
assert table["G6"].position_radius == 1.5
|
||||
|
||||
assert table["G7"].init_position_mode == "independent"
|
||||
|
||||
assert table["G8"].scale_type == "optimizer_default"
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 10. Deterministic Confirmation Selection Test
|
||||
# =====================================================================
|
||||
|
||||
def test_deterministic_confirmation_selection():
|
||||
mock_screen = {
|
||||
"G0": {"val_selected_loss": 0.70, "val_selected_acc": 78.0},
|
||||
"G1": {"val_selected_loss": 0.65, "val_selected_acc": 80.0},
|
||||
"G2": {"val_selected_loss": 0.60, "val_selected_acc": 82.0},
|
||||
"G3": {"val_selected_loss": 0.58, "val_selected_acc": 83.0}, # Top 1 eligible
|
||||
"G4": {"val_selected_loss": 0.55, "val_selected_acc": 84.0}, # Top 0 eligible (best)
|
||||
"G5": {"val_selected_loss": 0.62, "val_selected_acc": 81.0},
|
||||
"G6": {"val_selected_loss": 0.64, "val_selected_acc": 80.5},
|
||||
"G7": {"val_selected_loss": 0.61, "val_selected_acc": 81.5},
|
||||
"G8": {"val_selected_loss": 0.48, "val_selected_acc": 85.0},
|
||||
}
|
||||
|
||||
selected = select_confirmation_configs(mock_screen)
|
||||
assert len(selected) == 5
|
||||
assert selected[:3] == ["G0", "G1", "G8"]
|
||||
assert set(selected[3:]) == {"G4", "G3"}
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 11. Guard: Loader Never Requests Official Test Dataset (train=False)
|
||||
# =====================================================================
|
||||
|
||||
def test_guard_loader_never_requests_official_test_dataset(monkeypatch):
|
||||
import torchvision.datasets
|
||||
|
||||
called_train_flags = []
|
||||
|
||||
original_mnist_init = torchvision.datasets.MNIST.__init__
|
||||
|
||||
def mock_mnist_init(self, root, train=True, transform=None, target_transform=None, download=False):
|
||||
called_train_flags.append(train)
|
||||
if not train:
|
||||
raise AssertionError("CRITICAL VIOLATION: MNIST(train=False) requested during Phase B data loader!")
|
||||
# Perform mock initialization with synthetic data
|
||||
self.data = torch.randint(0, 256, (60000, 28, 28), dtype=torch.uint8)
|
||||
self.targets = torch.randint(0, 10, (60000,), dtype=torch.long)
|
||||
|
||||
monkeypatch.setattr(torchvision.datasets.MNIST, "__init__", mock_mnist_init)
|
||||
|
||||
x_search, y_search, x_val, y_val, nested_subsets, data_fp, provenance = prepare_mnist_v6_data()
|
||||
|
||||
assert len(called_train_flags) > 0
|
||||
assert all(flag is True for flag in called_train_flags)
|
||||
assert provenance["official_test_evaluations"] == 0
|
||||
assert provenance["test_samples"] == 0
|
||||
assert x_search.shape == (50000, 1, 28, 28)
|
||||
assert x_val.shape == (10000, 1, 28, 28)
|
||||
|
||||
|
||||
def test_g8_uses_exact_supplied_objective(monkeypatch):
|
||||
torch.manual_seed(7)
|
||||
device = torch.device("cpu")
|
||||
model = TinyModel()
|
||||
x_search = torch.randn(12, 10)
|
||||
y_search = torch.randint(0, 2, (12,))
|
||||
x_val = torch.randn(8, 10)
|
||||
y_val = torch.randint(0, 2, (8,))
|
||||
from pso.optimizer import Optimizer
|
||||
|
||||
fit_args = {}
|
||||
original_fit = Optimizer.fit
|
||||
|
||||
def recording_fit(self, *args, **kwargs):
|
||||
fit_args.update(kwargs)
|
||||
return original_fit(self, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(Optimizer, "fit", recording_fit)
|
||||
|
||||
result = run_g8_optimizer(
|
||||
base_model=model,
|
||||
x_2k=x_search,
|
||||
y_2k=y_search,
|
||||
x_val=x_val,
|
||||
y_val=y_val,
|
||||
epochs=2,
|
||||
swarm_size=4,
|
||||
seed=11,
|
||||
device=device,
|
||||
)
|
||||
|
||||
assert result["total_queries"] == 8
|
||||
assert result["total_sample_evaluations"] == 8 * len(y_search)
|
||||
assert result["validation_evaluations"] == 5
|
||||
assert result["official_test_evaluations"] == 0
|
||||
assert fit_args["renewal"] == "loss"
|
||||
@@ -0,0 +1,484 @@
|
||||
"""
|
||||
Unit tests for Strict Evaluator of Post-Training PSO Ensemble Study.
|
||||
|
||||
Covers:
|
||||
1. Evaluator version and constant exports.
|
||||
2. Complete valid study artifact evaluation (pass=True, 0 failed hard gates, valid score).
|
||||
3. Schema tampering (non-dict, missing top-level keys, missing workloads).
|
||||
4. Config tampering (wrong split seed, sample counts, pool seeds, PSO parameters).
|
||||
5. Non-finite value scan (NaN or Inf values in nested metrics or weights).
|
||||
6. Simplex weight validation failure (non-unit sum, negative elements).
|
||||
7. Query and sample accounting mismatch.
|
||||
8. Base-model forward count gate failure.
|
||||
9. Data leakage contradictions, frozen-policy drift, and post-test tuning.
|
||||
10. Duplicate or missing frozen swarm seeds.
|
||||
11. SLSQP gap, uniform ensemble accuracy/NLL regression, and baseline NLL gates.
|
||||
12. Per-workload wall-time ratio gate enforcement.
|
||||
13. Missing-confirmation failure and evaluator CLI output.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Ensure test directory and repo root are in sys.path
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent if Path(__file__).resolve().parent.name != "PSO" else Path(__file__).resolve().parent
|
||||
TEST_DIR = REPO_ROOT / "test"
|
||||
if str(TEST_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TEST_DIR))
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
import pytest
|
||||
|
||||
from evaluate_post_training_ensemble import (
|
||||
EVALUATOR_VERSION,
|
||||
EXPECTED_DATASETS,
|
||||
EXPECTED_SPLIT_SEED,
|
||||
evaluate_artifact,
|
||||
main,
|
||||
save_json_atomic,
|
||||
)
|
||||
|
||||
|
||||
def make_valid_metrics(nll: float = 0.35, accuracy: float = 90.0):
|
||||
return {
|
||||
"accuracy": accuracy,
|
||||
"nll": nll,
|
||||
"brier": 0.15,
|
||||
"ece": 0.02,
|
||||
"margin": 0.5,
|
||||
}
|
||||
|
||||
|
||||
def make_valid_method(
|
||||
nll: float = 0.35,
|
||||
accuracy: float = 90.0,
|
||||
weights: list = None,
|
||||
method_type: str = "base",
|
||||
):
|
||||
if weights is None:
|
||||
weights = [0.2, 0.2, 0.2, 0.2, 0.2]
|
||||
|
||||
metrics = make_valid_metrics(nll, accuracy)
|
||||
|
||||
if method_type == "pso":
|
||||
return {
|
||||
"selected_seed": 301,
|
||||
"selected_weights": weights,
|
||||
"weights": weights,
|
||||
"metrics": metrics,
|
||||
"queries_per_seed": 900,
|
||||
"sample_evaluations_per_seed": 9000000,
|
||||
"median_one_seed_wall_time_seconds": 2.0,
|
||||
"total_wall_time_seconds": 6.0,
|
||||
"per_seed_runs": [
|
||||
{
|
||||
"seed": 301,
|
||||
"queries": 900,
|
||||
"sample_evaluations": 9000000,
|
||||
"wall_time_seconds": 2.0,
|
||||
"metrics": metrics,
|
||||
"weights": weights,
|
||||
},
|
||||
{
|
||||
"seed": 302,
|
||||
"queries": 900,
|
||||
"sample_evaluations": 9000000,
|
||||
"wall_time_seconds": 2.0,
|
||||
"metrics": make_valid_metrics(nll + 0.01, accuracy),
|
||||
"weights": weights,
|
||||
},
|
||||
{
|
||||
"seed": 303,
|
||||
"queries": 900,
|
||||
"sample_evaluations": 9000000,
|
||||
"wall_time_seconds": 2.0,
|
||||
"metrics": make_valid_metrics(nll + 0.02, accuracy),
|
||||
"weights": weights,
|
||||
},
|
||||
],
|
||||
}
|
||||
elif method_type == "slsqp":
|
||||
return {
|
||||
"weights": weights,
|
||||
"success": True,
|
||||
"wall_time_seconds": 0.5,
|
||||
"metrics": metrics,
|
||||
}
|
||||
elif method_type == "temp":
|
||||
return {
|
||||
"weights": weights,
|
||||
"fitted_temperature": 1.0,
|
||||
"metrics": metrics,
|
||||
}
|
||||
|
||||
return metrics
|
||||
|
||||
|
||||
def make_valid_workload_entry():
|
||||
return {
|
||||
"provenance": {"dataset_name": "mnist", "split_seed": EXPECTED_SPLIT_SEED},
|
||||
"training": {
|
||||
"adam_pool_model_epochs": 50,
|
||||
"adam_pool_wall_time_seconds": 100.0,
|
||||
"equal_budget_50e_single_wall_time_seconds": 25.0,
|
||||
},
|
||||
"validation_cache": {
|
||||
"pool_forward_passes": 5,
|
||||
"long_single_forward_passes": 1,
|
||||
"base_cnn_forward_passes_during_optimization": 0,
|
||||
"size_bytes": 2000000,
|
||||
},
|
||||
"validation": {
|
||||
"methods": {
|
||||
"reference_single_10e": make_valid_method(nll=0.50, accuracy=85.0, method_type="base"),
|
||||
"best_single_10e": make_valid_method(nll=0.45, accuracy=87.0, method_type="base"),
|
||||
"single_50e": make_valid_method(nll=0.40, accuracy=89.0, method_type="base"),
|
||||
"uniform_ensemble": make_valid_method(nll=0.36, accuracy=89.9, method_type="base"),
|
||||
"uniform_temperature": make_valid_method(nll=0.355, accuracy=90.0, method_type="temp"),
|
||||
"slsqp_weights": make_valid_method(nll=0.35, accuracy=90.0, method_type="slsqp"),
|
||||
"pso_weights": make_valid_method(nll=0.35, accuracy=90.0, method_type="pso"),
|
||||
}
|
||||
},
|
||||
"official_test_data_loaded_before_freeze": False,
|
||||
"official_test_evaluations_before_freeze": 0,
|
||||
"confirmation": {
|
||||
"test_cache_counts": {
|
||||
"dataset_loads": 1,
|
||||
"pool_forward_passes": 5,
|
||||
"long_single_forward_passes": 1,
|
||||
},
|
||||
"frozen_methods": {
|
||||
"selected_pso_seed": 301,
|
||||
"selected_pso_weights": [0.2, 0.2, 0.2, 0.2, 0.2],
|
||||
"slsqp_weights": [0.2, 0.2, 0.2, 0.2, 0.2],
|
||||
"fitted_temperature": 1.0,
|
||||
},
|
||||
"methods": {
|
||||
"reference_single_10e": make_valid_metrics(nll=0.52, accuracy=84.5),
|
||||
"best_single_10e": make_valid_metrics(nll=0.47, accuracy=86.5),
|
||||
"single_50e": make_valid_metrics(nll=0.42, accuracy=88.5),
|
||||
"uniform_ensemble": make_valid_metrics(nll=0.37, accuracy=89.5),
|
||||
"uniform_temperature": make_valid_metrics(nll=0.365, accuracy=89.6),
|
||||
"slsqp_weights": make_valid_metrics(nll=0.36, accuracy=89.7),
|
||||
"pso_weights": make_valid_metrics(nll=0.36, accuracy=89.7),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def make_valid_study_artifact():
|
||||
return {
|
||||
"protocol_version": "POST-TRAINING-PSO-ENSEMBLE 1.1.0",
|
||||
"config": {
|
||||
"datasets": ["mnist", "fashion_mnist"],
|
||||
"split_seed": 20260904,
|
||||
"search_samples": 50000,
|
||||
"validation_samples": 10000,
|
||||
"pool_seeds": [201, 202, 203, 204, 205],
|
||||
"reference_single_seed": 201,
|
||||
"equal_budget_single_epochs": 50,
|
||||
"pso": {
|
||||
"method": "constriction",
|
||||
"evaluation": "full",
|
||||
"renewal": "loss",
|
||||
"particles": 30,
|
||||
"epochs": 30,
|
||||
"swarm_seeds": [301, 302, 303],
|
||||
"particle_bounds": [-4.0, 4.0],
|
||||
"boundary_strategy": "reflect",
|
||||
"velocity_limit_ratio": 0.1,
|
||||
"initial_position_noise": 0.0,
|
||||
"queries_per_seed": 900,
|
||||
"sample_evaluations_per_seed": 9000000,
|
||||
},
|
||||
},
|
||||
"development_pass": True,
|
||||
"policy_frozen": True,
|
||||
"official_test_data_loaded": True,
|
||||
"official_test_data_loaded_before_freeze": False,
|
||||
"official_test_evaluations_before_freeze": 0,
|
||||
"post_test_tuning_or_reruns": 0,
|
||||
"resource_totals": {
|
||||
"total_adam_pool_model_epochs": 100,
|
||||
"total_pso_queries": 5400,
|
||||
"total_pso_sample_evaluations": 54000000,
|
||||
"total_pso_wall_time_seconds": 12.0,
|
||||
"pso_to_pool_wall_ratio": 0.02,
|
||||
},
|
||||
"workloads": {
|
||||
"mnist": make_valid_workload_entry(),
|
||||
"fashion_mnist": make_valid_workload_entry(),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_evaluator_version_and_imports():
|
||||
"""Verify evaluator version identifier."""
|
||||
assert isinstance(EVALUATOR_VERSION, str)
|
||||
assert EVALUATOR_VERSION.startswith("POST-TRAINING-PSO-ENSEMBLE-EVALUATOR")
|
||||
|
||||
|
||||
def test_evaluate_artifact_valid_passing_study():
|
||||
"""Verify evaluator approves valid study artifact with zero hard gate failures."""
|
||||
artifact = make_valid_study_artifact()
|
||||
result = evaluate_artifact(artifact)
|
||||
|
||||
assert result["pass"] is True
|
||||
assert result["development_pass"] is True
|
||||
assert result["confirmation_pass"] is True
|
||||
assert result["failed_hard_gate_count"] == 0
|
||||
assert isinstance(result["score"], float)
|
||||
assert result["score"] > -100.0
|
||||
|
||||
|
||||
def test_evaluate_artifact_schema_tampering():
|
||||
"""Verify evaluator rejects non-dict, missing config, and missing workload structures."""
|
||||
# 1. Non-dict artifact
|
||||
res_non_dict = evaluate_artifact("invalid_string_artifact")
|
||||
assert res_non_dict["pass"] is False
|
||||
assert res_non_dict["failed_hard_gate_count"] >= 1
|
||||
assert "schema" in res_non_dict["issues"]
|
||||
assert len(res_non_dict["issues"]["schema"]) > 0
|
||||
|
||||
# 2. Missing config
|
||||
art_no_cfg = make_valid_study_artifact()
|
||||
del art_no_cfg["config"]
|
||||
res_no_cfg = evaluate_artifact(art_no_cfg)
|
||||
assert res_no_cfg["pass"] is False
|
||||
assert len(res_no_cfg["issues"]["schema"]) > 0
|
||||
|
||||
# 3. Missing dataset in workloads
|
||||
art_missing_ds = make_valid_study_artifact()
|
||||
del art_missing_ds["workloads"]["fashion_mnist"]
|
||||
res_missing_ds = evaluate_artifact(art_missing_ds)
|
||||
assert res_missing_ds["pass"] is False
|
||||
assert len(res_missing_ds["issues"]["schema"]) > 0
|
||||
|
||||
|
||||
def test_evaluate_artifact_config_tampering():
|
||||
"""Verify evaluator flags mismatched split seed, sample counts, or PSO parameters."""
|
||||
art = make_valid_study_artifact()
|
||||
art["config"]["split_seed"] = 99999999 # Mismatched seed
|
||||
art["config"]["pso"]["particles"] = 15 # Expected 30
|
||||
art["config"]["pso"]["epochs"] = 15 # Expected 30
|
||||
|
||||
res = evaluate_artifact(art)
|
||||
assert res["pass"] is False
|
||||
assert len(res["issues"]["config"]) >= 2
|
||||
|
||||
|
||||
def test_evaluate_artifact_non_finite_tampering():
|
||||
"""Verify evaluator detects non-finite values (NaN / Inf) in nested metrics or weights."""
|
||||
art = make_valid_study_artifact()
|
||||
# Inject NaN into validation NLL
|
||||
art["workloads"]["mnist"]["validation"]["methods"]["pso_weights"]["per_seed_runs"][0]["metrics"]["nll"] = float("nan")
|
||||
|
||||
res = evaluate_artifact(art)
|
||||
assert res["pass"] is False
|
||||
assert res["failed_hard_gate_count"] >= 1
|
||||
assert len(res["issues"]["finite"]) >= 1
|
||||
|
||||
|
||||
def test_evaluate_artifact_simplex_weights_tampering():
|
||||
"""Verify evaluator rejects weight vectors that do not sum to 1.0 within tolerance."""
|
||||
art = make_valid_study_artifact()
|
||||
# Set weights that sum to 1.5
|
||||
art["workloads"]["mnist"]["validation"]["methods"]["slsqp_weights"]["weights"] = [0.3, 0.3, 0.3, 0.3, 0.3]
|
||||
|
||||
res = evaluate_artifact(art)
|
||||
assert res["pass"] is False
|
||||
assert res["failed_hard_gate_count"] >= 1
|
||||
assert len(res["issues"]["weights"]) >= 1
|
||||
|
||||
|
||||
def test_evaluate_artifact_accounting_tampering():
|
||||
"""Verify evaluator flags invalid PSO queries or sample evaluations accounting."""
|
||||
art = make_valid_study_artifact()
|
||||
art["config"]["pso"]["queries_per_seed"] = 899 # Expected 900
|
||||
|
||||
res = evaluate_artifact(art)
|
||||
assert res["pass"] is False
|
||||
assert len(res["issues"]["accounting"]) >= 1
|
||||
|
||||
def test_evaluate_artifact_requires_each_frozen_swarm_seed_once():
|
||||
"""Duplicate seed records cannot stand in for independent replication."""
|
||||
art = make_valid_study_artifact()
|
||||
runs = art["workloads"]["mnist"]["validation"]["methods"]["pso_weights"][
|
||||
"per_seed_runs"
|
||||
]
|
||||
runs[1]["seed"] = 301
|
||||
|
||||
result = evaluate_artifact(art)
|
||||
|
||||
assert result["pass"] is False
|
||||
assert any(
|
||||
"each frozen seed exactly once" in issue
|
||||
for issue in result["issues"]["config"]
|
||||
)
|
||||
|
||||
|
||||
def test_evaluate_artifact_base_model_forward_count_tampering():
|
||||
"""Verify evaluator flags non-zero base model forward passes during optimization."""
|
||||
art = make_valid_study_artifact()
|
||||
art["workloads"]["mnist"]["validation_cache"]["base_cnn_forward_passes_during_optimization"] = 2
|
||||
|
||||
res = evaluate_artifact(art)
|
||||
assert res["pass"] is False
|
||||
assert res["failed_hard_gate_count"] >= 1
|
||||
assert len(res["issues"]["accounting"]) >= 1
|
||||
|
||||
|
||||
def test_evaluate_artifact_leakage_and_post_test_tuning_tampering():
|
||||
"""Global/local leakage contradictions and post-test tuning must fail."""
|
||||
art_loaded = make_valid_study_artifact()
|
||||
art_loaded["official_test_data_loaded_before_freeze"] = True
|
||||
res_loaded = evaluate_artifact(art_loaded)
|
||||
assert res_loaded["pass"] is False
|
||||
assert len(res_loaded["issues"]["leakage"]) >= 1
|
||||
|
||||
art_evals = make_valid_study_artifact()
|
||||
art_evals["official_test_evaluations_before_freeze"] = 1
|
||||
res_evals = evaluate_artifact(art_evals)
|
||||
assert res_evals["pass"] is False
|
||||
assert len(res_evals["issues"]["leakage"]) >= 1
|
||||
|
||||
art_tune = make_valid_study_artifact()
|
||||
art_tune["post_test_tuning_or_reruns"] = 1
|
||||
res_tune = evaluate_artifact(art_tune)
|
||||
assert res_tune["pass"] is False
|
||||
assert len(res_tune["issues"]["tuning"]) >= 1
|
||||
|
||||
def test_evaluate_artifact_rejects_confirmation_policy_drift():
|
||||
"""Confirmation must identify the exact validation-frozen method parameters."""
|
||||
mutations = [
|
||||
("policy_frozen", False),
|
||||
(
|
||||
"selected_pso_seed",
|
||||
302,
|
||||
),
|
||||
(
|
||||
"selected_pso_weights",
|
||||
[1.0, 0.0, 0.0, 0.0, 0.0],
|
||||
),
|
||||
(
|
||||
"slsqp_weights",
|
||||
[1.0, 0.0, 0.0, 0.0, 0.0],
|
||||
),
|
||||
("fitted_temperature", 2.0),
|
||||
]
|
||||
|
||||
for field, value in mutations:
|
||||
art = make_valid_study_artifact()
|
||||
if field == "policy_frozen":
|
||||
art[field] = value
|
||||
else:
|
||||
art["workloads"]["mnist"]["confirmation"]["frozen_methods"][
|
||||
field
|
||||
] = value
|
||||
|
||||
result = evaluate_artifact(art)
|
||||
|
||||
assert result["pass"] is False, field
|
||||
assert result["confirmation_gates"]["frozen_policy_consistency"] is False
|
||||
|
||||
|
||||
def test_evaluate_artifact_slsqp_gap_and_uniform_regression_tampering():
|
||||
"""Verify evaluator flags PSO NLL gap vs SLSQP > 0.5% or accuracy regression > 0.1 pp vs uniform."""
|
||||
# 1. SLSQP gap > 0.005
|
||||
art_slsqp = make_valid_study_artifact()
|
||||
# SLSQP NLL = 0.30, PSO NLL = 0.35 -> relative gap (0.35 - 0.30)/0.30 = 0.1667 > 0.005
|
||||
art_slsqp["workloads"]["mnist"]["validation"]["methods"]["slsqp_weights"]["metrics"]["nll"] = 0.30
|
||||
art_slsqp["workloads"]["mnist"]["validation"]["methods"]["pso_weights"]["metrics"]["nll"] = 0.35
|
||||
art_slsqp["workloads"]["mnist"]["validation"]["methods"]["pso_weights"]["per_seed_runs"][0]["metrics"]["nll"] = 0.35
|
||||
|
||||
res_slsqp = evaluate_artifact(art_slsqp)
|
||||
assert res_slsqp["pass"] is False
|
||||
assert len(res_slsqp["issues"]["gates"]) >= 1
|
||||
|
||||
# 2. PSO accuracy regression > 0.1 pp below uniform
|
||||
art_acc = make_valid_study_artifact()
|
||||
art_acc["workloads"]["mnist"]["validation"]["methods"]["uniform_ensemble"]["accuracy"] = 90.0
|
||||
# Set PSO accuracy to 89.5 (0.5 pp regression)
|
||||
art_acc["workloads"]["mnist"]["validation"]["methods"]["pso_weights"]["metrics"]["accuracy"] = 89.5
|
||||
art_acc["workloads"]["mnist"]["validation"]["methods"]["pso_weights"]["per_seed_runs"][0]["metrics"]["accuracy"] = 89.5
|
||||
|
||||
res_acc = evaluate_artifact(art_acc)
|
||||
assert res_acc["pass"] is False
|
||||
assert len(res_acc["issues"]["gates"]) >= 1
|
||||
|
||||
|
||||
def test_evaluate_artifact_reference_single_and_equal_budget_tampering():
|
||||
"""Verify evaluator flags PSO validation NLL >= reference single or > equal-budget single NLL + 1e-7."""
|
||||
# PSO NLL > reference single NLL
|
||||
art_ref = make_valid_study_artifact()
|
||||
art_ref["workloads"]["mnist"]["validation"]["methods"]["reference_single_10e"]["nll"] = 0.30
|
||||
art_ref["workloads"]["mnist"]["validation"]["methods"]["pso_weights"]["metrics"]["nll"] = 0.35
|
||||
art_ref["workloads"]["mnist"]["validation"]["methods"]["pso_weights"]["per_seed_runs"][0]["metrics"]["nll"] = 0.35
|
||||
|
||||
res_ref = evaluate_artifact(art_ref)
|
||||
assert res_ref["pass"] is False
|
||||
assert len(res_ref["issues"]["gates"]) >= 1
|
||||
|
||||
|
||||
def test_evaluate_artifact_wall_time_ratio_tampering():
|
||||
"""Each workload's recomputed median PSO/Adam ratio must stay at most 10%."""
|
||||
art_time = make_valid_study_artifact()
|
||||
art_time["workloads"]["mnist"]["training"]["adam_pool_wall_time_seconds"] = 10.0
|
||||
runs = art_time["workloads"]["mnist"]["validation"]["methods"][
|
||||
"pso_weights"
|
||||
]["per_seed_runs"]
|
||||
for run in runs:
|
||||
run["wall_time_seconds"] = 2.0
|
||||
|
||||
result = evaluate_artifact(art_time)
|
||||
|
||||
assert result["pass"] is False
|
||||
assert result["development_gates"][
|
||||
"maximum_median_one_seed_pso_to_pool_training_wall_ratio"
|
||||
] is False
|
||||
assert len(result["issues"]["gates"]) >= 1
|
||||
|
||||
|
||||
def test_evaluate_artifact_missing_confirmation_on_dev_pass():
|
||||
"""Verify missing confirmation on development pass fails overall study evaluation."""
|
||||
art_no_conf = make_valid_study_artifact()
|
||||
art_no_conf["official_test_data_loaded"] = False
|
||||
art_no_conf["workloads"]["mnist"]["confirmation"] = None
|
||||
art_no_conf["workloads"]["fashion_mnist"]["confirmation"] = None
|
||||
|
||||
res = evaluate_artifact(art_no_conf)
|
||||
assert res["pass"] is False
|
||||
assert res["confirmation_pass"] is False
|
||||
assert len(res["issues"]["gates"]) >= 1 or len(res["issues"]["leakage"]) >= 1
|
||||
|
||||
|
||||
def test_evaluator_cli(tmp_path, monkeypatch):
|
||||
"""Verify CLI main entrypoint writes evaluation payload atomically."""
|
||||
art = make_valid_study_artifact()
|
||||
art_path = tmp_path / "study_artifact.json"
|
||||
save_json_atomic(art, art_path)
|
||||
|
||||
out_path = tmp_path / "evaluation_output.json"
|
||||
|
||||
# Simulate command-line arguments: --artifact <art_path> --output <out_path>
|
||||
test_args = [
|
||||
"evaluate_post_training_ensemble.py",
|
||||
"--artifact",
|
||||
str(art_path),
|
||||
"--output",
|
||||
str(out_path),
|
||||
]
|
||||
monkeypatch.setattr(sys, "argv", test_args)
|
||||
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
main()
|
||||
|
||||
assert exc_info.value.code == 0
|
||||
assert out_path.exists()
|
||||
|
||||
eval_data = json.loads(out_path.read_text())
|
||||
assert eval_data["pass"] is True
|
||||
assert eval_data["failed_hard_gate_count"] == 0
|
||||
assert isinstance(eval_data["score"], float)
|
||||
@@ -0,0 +1,256 @@
|
||||
"""Behavioral tests for the independent model-convergence evaluator.
|
||||
|
||||
These fixtures intentionally stay in prediction/artifact space: no dataset, model,
|
||||
optional detection dependency, or network access is needed.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
TEST_DIR = REPO_ROOT / "test"
|
||||
if str(TEST_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TEST_DIR))
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
import evaluate_post_training_model_convergence as evaluator
|
||||
|
||||
|
||||
CLASSIFICATION_RECORDS = [
|
||||
{"image_id": "a", "probabilities": [0.80, 0.20], "target": 0},
|
||||
{"image_id": "b", "probabilities": [0.40, 0.60], "target": 1},
|
||||
{"image_id": "c", "probabilities": [0.70, 0.30], "target": 1},
|
||||
{"image_id": "d", "probabilities": [0.55, 0.45], "target": 0},
|
||||
]
|
||||
|
||||
|
||||
def _secondary_classification_metrics(records):
|
||||
brier_terms = []
|
||||
confidences = []
|
||||
correctness = []
|
||||
for record in records:
|
||||
probabilities = record["probabilities"]
|
||||
target = record["target"]
|
||||
brier_terms.append(
|
||||
sum((probability - (index == target)) ** 2 for index, probability in enumerate(probabilities))
|
||||
)
|
||||
prediction = max(range(len(probabilities)), key=probabilities.__getitem__)
|
||||
confidences.append(max(probabilities))
|
||||
correctness.append(prediction == target)
|
||||
|
||||
# Match the protocol's 15 equal-width confidence bins, including the
|
||||
# right-most endpoint in the final bin.
|
||||
ece = 0.0
|
||||
for bin_index in range(15):
|
||||
lower, upper = bin_index / 15.0, (bin_index + 1) / 15.0
|
||||
members = [
|
||||
index
|
||||
for index, confidence in enumerate(confidences)
|
||||
if (confidence >= lower and (confidence < upper or bin_index == 14 and confidence <= upper))
|
||||
]
|
||||
if members:
|
||||
accuracy = sum(correctness[index] for index in members) / len(members)
|
||||
confidence = sum(confidences[index] for index in members) / len(members)
|
||||
ece += abs(accuracy - confidence) * len(members) / len(records)
|
||||
return sum(brier_terms) / len(records), ece
|
||||
|
||||
|
||||
def test_classification_metrics_recompute_exact_nll_accuracy_brier_and_ece():
|
||||
"""Per-example probabilities determine all classification statistics without rounding."""
|
||||
metrics = evaluator.classification_metrics(CLASSIFICATION_RECORDS)
|
||||
expected_nll = -math.fsum(math.log(record["probabilities"][record["target"]]) for record in CLASSIFICATION_RECORDS) / 4
|
||||
expected_brier, expected_ece = _secondary_classification_metrics(CLASSIFICATION_RECORDS)
|
||||
|
||||
assert metrics["n"] == 4
|
||||
assert metrics["accuracy"] == pytest.approx(0.75)
|
||||
assert metrics["nll"] == pytest.approx(expected_nll, rel=0, abs=1e-15)
|
||||
assert metrics["brier"] == pytest.approx(expected_brier, rel=0, abs=1e-15)
|
||||
assert metrics["ece15"] == pytest.approx(expected_ece, rel=0, abs=1e-15)
|
||||
# The evaluator must retain the unrounded probability/target evidence used
|
||||
# for the secondary metrics rather than substituting aggregate values.
|
||||
assert metrics["probabilities"] == [record["probabilities"] for record in CLASSIFICATION_RECORDS]
|
||||
assert metrics["targets"] == [record["target"] for record in CLASSIFICATION_RECORDS]
|
||||
|
||||
|
||||
def test_classification_metrics_reject_invalid_probability_contracts():
|
||||
with pytest.raises(ValueError, match="sum to one"):
|
||||
evaluator.classification_metrics([{"probabilities": [0.8, 0.3], "target": 0}])
|
||||
with pytest.raises(ValueError, match="invalid classification"):
|
||||
evaluator.classification_metrics([{"probabilities": [1.0, 0.0], "target": True}])
|
||||
with pytest.raises(ZeroDivisionError):
|
||||
evaluator.classification_metrics([])
|
||||
|
||||
|
||||
def test_detection_metrics_deduplicates_predictions_and_counts_empty_images():
|
||||
"""One image may have duplicate detections while other images are empty."""
|
||||
box = [0.0, 0.0, 10.0, 10.0]
|
||||
records = [
|
||||
{
|
||||
"image_id": "duplicate",
|
||||
"ground_truth": [{"class_id": 0, "box": box}],
|
||||
# Deliberately preserve a low-score row first: matching is one-to-one,
|
||||
# then confidence ranking makes the duplicate a false positive.
|
||||
"predictions": [
|
||||
{"class_id": 0, "score": 0.10, "box": box},
|
||||
{"class_id": 0, "score": 0.90, "box": box},
|
||||
],
|
||||
},
|
||||
{"image_id": "empty-predictions", "ground_truth": [{"class_id": 1, "box": box}], "predictions": []},
|
||||
{"image_id": "empty-image", "ground_truth": [], "predictions": []},
|
||||
]
|
||||
|
||||
metrics = evaluator.detection_metrics(records, class_count=2)
|
||||
|
||||
assert metrics["n"] == 3
|
||||
assert metrics["ground_truth"] == 2
|
||||
assert metrics["predictions"] == 2
|
||||
expected_ap = 0.49750000000000033
|
||||
assert metrics["per_class_ap"]["0"] == pytest.approx(
|
||||
[expected_ap] * 10,
|
||||
rel=0,
|
||||
abs=1e-12,
|
||||
)
|
||||
assert metrics["per_class_ap"]["1"] == pytest.approx(
|
||||
[0.0] * 10,
|
||||
rel=0,
|
||||
abs=1e-12,
|
||||
)
|
||||
assert metrics["map50"] == pytest.approx(expected_ap / 2, abs=1e-12)
|
||||
assert metrics["map50_95"] == pytest.approx(expected_ap / 2, abs=1e-12)
|
||||
|
||||
|
||||
def test_detection_metrics_empty_dataset_is_a_finite_zero_result():
|
||||
metrics = evaluator.detection_metrics([], class_count=3)
|
||||
assert metrics["n"] == 0
|
||||
assert metrics["ground_truth"] == 0
|
||||
assert metrics["predictions"] == 0
|
||||
assert metrics["map50"] == 0.0
|
||||
assert metrics["map50_95"] == 0.0
|
||||
assert set(metrics["per_class_ap"]) == {"0", "1", "2"}
|
||||
assert all(value == [0.0] * 10 for value in metrics["per_class_ap"].values())
|
||||
|
||||
|
||||
def test_bootstrap_helper_is_deterministic_and_uses_improvement_orientation():
|
||||
base = [
|
||||
{"image_id": "0", "probabilities": [0.60, 0.40], "target": 0},
|
||||
{"image_id": "1", "probabilities": [0.40, 0.60], "target": 1},
|
||||
{"image_id": "2", "probabilities": [0.60, 0.40], "target": 0},
|
||||
{"image_id": "3", "probabilities": [0.40, 0.60], "target": 1},
|
||||
]
|
||||
improved = [
|
||||
{**record, "probabilities": [0.90, 0.10] if record["target"] == 0 else [0.10, 0.90]}
|
||||
for record in base
|
||||
]
|
||||
pairs = [(base, improved), (base, improved)]
|
||||
|
||||
first = evaluator._bootstrap_from_records(pairs, "classification")
|
||||
second = evaluator._bootstrap_from_records(pairs, "classification")
|
||||
|
||||
assert first == second
|
||||
assert first["available"] is True
|
||||
assert first["seed"] == evaluator.BOOTSTRAP_SEED
|
||||
assert first["resamples"] == evaluator.BOOTSTRAP_RESAMPLES
|
||||
assert first["alpha"] == evaluator.BOOTSTRAP_ALPHA
|
||||
assert first["statistic"] > 0.0
|
||||
assert first["lower"] > 0.0
|
||||
assert first["excludes_zero"] is True
|
||||
|
||||
|
||||
def test_bootstrap_helper_rejects_misaligned_image_identity():
|
||||
base = [{"image_id": "a", "probabilities": [1.0, 0.0], "target": 0}]
|
||||
reordered = [{"image_id": "b", "probabilities": [1.0, 0.0], "target": 0}]
|
||||
result = evaluator._bootstrap_from_records([(base, reordered)], "classification")
|
||||
assert result["available"] is False
|
||||
assert "image IDs/order differ" in result["reason"]
|
||||
|
||||
|
||||
def test_global_query_and_candidate_sample_constants_are_exact():
|
||||
expected_queries = (
|
||||
len(evaluator.WORKLOADS)
|
||||
* len(evaluator.BASE_SEEDS)
|
||||
* len(evaluator.SWARM_SEEDS)
|
||||
* evaluator.PRIMARY_QUERIES
|
||||
+ len(evaluator.WORKLOADS)
|
||||
* len(evaluator.SWARM_SEEDS)
|
||||
* evaluator.ENSEMBLE_QUERIES
|
||||
)
|
||||
expected_samples = sum(
|
||||
(
|
||||
len(evaluator.BASE_SEEDS) * len(evaluator.SWARM_SEEDS) * evaluator.PRIMARY_QUERIES
|
||||
+ len(evaluator.SWARM_SEEDS) * evaluator.ENSEMBLE_QUERIES
|
||||
)
|
||||
* evaluator.OBJECTIVE_SAMPLES[workload]
|
||||
for workload in evaluator.WORKLOADS
|
||||
)
|
||||
|
||||
assert evaluator.PRIMARY_QUERIES == 720
|
||||
assert evaluator.ENSEMBLE_QUERIES == 240
|
||||
assert evaluator.TOTAL_PSO_QUERIES == expected_queries == 21_600
|
||||
assert evaluator.TOTAL_CANDIDATE_SAMPLES == expected_samples == 18_432_000
|
||||
|
||||
|
||||
def test_pt_prediction_artifact_is_resolved_without_model_import(tmp_path):
|
||||
torch = pytest.importorskip("torch")
|
||||
records = [{"image_id": "one", "probabilities": [0.25, 0.75], "target": 1}]
|
||||
artifact = tmp_path / "predictions.pt"
|
||||
torch.save({"predictions": records}, artifact)
|
||||
|
||||
resolved = evaluator._prediction_records({"prediction_artifact": "predictions.pt"}, tmp_path)
|
||||
|
||||
assert resolved == records
|
||||
|
||||
|
||||
def _write_compact_results(root: Path, *, leakage_bad: bool, malformed_matrix: bool) -> None:
|
||||
leakage = {
|
||||
"official_test_data_loaded_before_freeze": True if leakage_bad else False,
|
||||
"official_test_evaluations_before_freeze": 1 if leakage_bad else 0,
|
||||
"official_test_construction": 0 if leakage_bad else 1,
|
||||
"official_test_forward_passes": 0 if leakage_bad else 1,
|
||||
}
|
||||
for workload in evaluator.WORKLOADS:
|
||||
workload_dir = root / "workloads" / workload
|
||||
workload_dir.mkdir(parents=True, exist_ok=True)
|
||||
result = {
|
||||
"workload_id": workload,
|
||||
"family": "detection" if workload == evaluator.DETECTION_WORKLOAD else "classification",
|
||||
"manifests": {},
|
||||
"provenance": {},
|
||||
"baselines": {},
|
||||
"arms": {} if malformed_matrix else {"feature_pso": []},
|
||||
"ensemble": {},
|
||||
"development_selection": {},
|
||||
"confirmation": {},
|
||||
"integrity": {},
|
||||
"leakage_counters": leakage,
|
||||
"resource_ledger": {},
|
||||
"artifact_hashes": {},
|
||||
}
|
||||
(workload_dir / "result.json").write_text(json.dumps(result), encoding="utf-8")
|
||||
|
||||
|
||||
def test_evaluate_run_rejects_incomplete_matrix_from_temp_fixture(tmp_path):
|
||||
_write_compact_results(tmp_path, leakage_bad=False, malformed_matrix=True)
|
||||
|
||||
result = evaluator.evaluate_run(tmp_path)
|
||||
|
||||
assert result["pass"] is False
|
||||
assert result["issue_counts"]["matrix"] >= len(evaluator.WORKLOADS)
|
||||
assert any("missing arms" in issue for issue in result["issues"]["matrix"])
|
||||
|
||||
|
||||
def test_evaluate_run_rejects_pre_freeze_test_leakage_from_temp_fixture(tmp_path):
|
||||
_write_compact_results(tmp_path, leakage_bad=True, malformed_matrix=False)
|
||||
|
||||
result = evaluator.evaluate_run(tmp_path)
|
||||
|
||||
assert result["pass"] is False
|
||||
assert result["issue_counts"]["leakage"] >= len(evaluator.WORKLOADS)
|
||||
assert any("must be explicitly marked not loaded" in issue for issue in result["issues"]["leakage"])
|
||||
assert any("exposure before freeze" in issue for issue in result["issues"]["leakage"])
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,670 @@
|
||||
"""
|
||||
Unit tests for Heavy PSO Cross-Split Runner and Evaluator.
|
||||
|
||||
Covers:
|
||||
1. Split seed propagation through data preparation, confirm runner, and autoresearch.
|
||||
2. Per-workload projection seed override validation, parsing, and effective seeds.
|
||||
3. Artifact accounting (exact queries/samples), fingerprint matching, state math, and test seals.
|
||||
4. Cross-split runner phase seed enforcement (development vs confirmation).
|
||||
5. All evaluator hard gates, missing confirmation rejection, non-finite rejection, leakage control,
|
||||
per-cell non-regression, development gates, confirmation gates, combined gates, and score formula.
|
||||
"""
|
||||
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
# Ensure test directory and repo root are in Python path
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent if Path(__file__).resolve().parent.name != "PSO" else Path(__file__).resolve().parent
|
||||
TEST_DIR = REPO_ROOT / "test"
|
||||
if str(TEST_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TEST_DIR))
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
import evaluate_heavy_cross_split as evaluator
|
||||
import heavy_pso_autoresearch as autoresearch
|
||||
import heavy_pso_cross_split as runner
|
||||
import heavy_task_feasibility as heavy_task
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_heavy_task_deps(monkeypatch):
|
||||
"""Mocks data preparation and execution functions for fast, deterministic unit testing."""
|
||||
split_records = {}
|
||||
|
||||
def fake_prepare_heavy_task_data(dataset_name: str, split_seed: int = 20260902, cache_dir=None):
|
||||
N_search, N_val = 20, 10
|
||||
x_search = torch.randn(N_search, 1, 28, 28)
|
||||
y_search = torch.randint(0, 10, (N_search,))
|
||||
x_val = torch.randn(N_val, 1, 28, 28)
|
||||
y_val = torch.randint(0, 10, (N_val,))
|
||||
nested_subsets = {2000: torch.arange(10), 10000: torch.arange(20), 50000: torch.arange(20)}
|
||||
data_fp = f"data-fp-{dataset_name}-{split_seed}"
|
||||
split_fp = f"split-fp-{dataset_name}-{split_seed}"
|
||||
split_records[dataset_name] = split_seed
|
||||
provenance = {
|
||||
"dataset_name": dataset_name,
|
||||
"official_test_data_loaded": False,
|
||||
"official_test_evaluations": 0,
|
||||
"test_samples": 0,
|
||||
"search_samples": 50000,
|
||||
"val_samples": 10000,
|
||||
"split_seed": split_seed,
|
||||
"split_fingerprint": split_fp,
|
||||
"data_fingerprint": data_fp,
|
||||
}
|
||||
return x_search, y_search, x_val, y_val, nested_subsets, data_fp, provenance
|
||||
|
||||
monkeypatch.setattr(heavy_task, "prepare_heavy_task_data", fake_prepare_heavy_task_data)
|
||||
monkeypatch.setattr(autoresearch, "prepare_heavy_task_data", fake_prepare_heavy_task_data)
|
||||
|
||||
def fake_run_v6_pso(
|
||||
transform, base_model, x_search, y_search, x_val, y_val, nested_subsets,
|
||||
schedule_str, epochs, swarm_size, seed, device, geom_config=None, val_check_interval=10
|
||||
):
|
||||
return {
|
||||
"val_selected_loss": 0.50,
|
||||
"val_selected_acc": 85.0,
|
||||
"gbest_loss": 0.48,
|
||||
"gbest_acc": 86.0,
|
||||
"wall_time_sec": 0.01,
|
||||
"optimization_wall_time_sec": 0.01,
|
||||
"validation_wall_time_sec": 0.001,
|
||||
"total_queries": swarm_size * epochs,
|
||||
"total_sample_evaluations": swarm_size * epochs * 10000,
|
||||
"validation_evaluations": 2,
|
||||
"val_metrics": {"brier": 0.1, "ece": 0.02},
|
||||
}
|
||||
|
||||
def fake_run_g8_optimizer(
|
||||
base_model, x_2k, y_2k, x_val, y_val, epochs, swarm_size, seed, device
|
||||
):
|
||||
return {
|
||||
"val_selected_loss": 0.55,
|
||||
"val_selected_acc": 83.0,
|
||||
"gbest_loss": 0.52,
|
||||
"gbest_acc": 84.0,
|
||||
"wall_time_sec": 0.01,
|
||||
"optimization_wall_time_sec": 0.01,
|
||||
"validation_wall_time_sec": 0.001,
|
||||
"total_queries": swarm_size * epochs,
|
||||
"total_sample_evaluations": swarm_size * epochs * 10000,
|
||||
"validation_evaluations": 2,
|
||||
"val_metrics": {"brier": 0.12, "ece": 0.03},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(heavy_task, "run_v6_pso", fake_run_v6_pso)
|
||||
monkeypatch.setattr(heavy_task, "run_g8_optimizer", fake_run_g8_optimizer)
|
||||
monkeypatch.setattr(autoresearch, "run_v6_pso", fake_run_v6_pso)
|
||||
|
||||
return split_records
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 1. Split Seed Propagation Tests
|
||||
# =====================================================================
|
||||
|
||||
def test_split_seed_propagation(mock_heavy_task_deps):
|
||||
"""Verify alternate split seeds reach prepare_heavy_task_data across runners."""
|
||||
res_confirm = heavy_task.run_heavy_task_confirm(
|
||||
workloads={"mnist_compact": heavy_task.WORKLOADS["mnist_compact"]},
|
||||
selected_methods={"mnist_compact": ["G8"]},
|
||||
particles=2,
|
||||
epochs=2,
|
||||
seeds=[101],
|
||||
split_seed=20260905,
|
||||
)
|
||||
assert res_confirm["mnist_compact"]["G8"]["split_seed"] == 20260905
|
||||
assert res_confirm["mnist_compact"]["G8"]["data_fingerprint"] == "data-fp-mnist-20260905"
|
||||
|
||||
res_auto = autoresearch.run_heavy_pso_autoresearch(
|
||||
ratios=[0.5],
|
||||
particles=2,
|
||||
epochs=2,
|
||||
seeds=[101],
|
||||
split_seed=20260906,
|
||||
projection_seed_mode="explicit",
|
||||
projection_seed=12345,
|
||||
)
|
||||
meta = res_auto["workloads"]["mnist_compact"]
|
||||
assert meta["data_fingerprint"] == "data-fp-mnist-20260906"
|
||||
assert meta["split_fingerprint"] == "split-fp-mnist-20260906"
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 2. Projection Seed Override Validation & Parsing Tests
|
||||
# =====================================================================
|
||||
|
||||
def test_projection_override_validation_and_parsing():
|
||||
"""Verify per-workload projection seed dictionary validation and CLI argument parsing."""
|
||||
# Valid dict validation
|
||||
valid_dict = {
|
||||
"mnist_compact": 1800044939,
|
||||
"mnist_wide": 592157828,
|
||||
"fashion_compact": 1363313651,
|
||||
"fashion_wide": 189641451,
|
||||
}
|
||||
autoresearch.validate_projection_seed_config("explicit", valid_dict)
|
||||
|
||||
# Valid CLI string parsing
|
||||
parsed_json = autoresearch.parse_projection_seed_arg(
|
||||
'{"mnist_compact": 1800044939, "mnist_wide": 592157828}'
|
||||
)
|
||||
assert parsed_json["mnist_compact"] == 1800044939
|
||||
|
||||
parsed_kv = autoresearch.parse_projection_seed_arg(
|
||||
"mnist_compact:1800044939,mnist_wide:592157828"
|
||||
)
|
||||
assert parsed_kv["mnist_compact"] == 1800044939
|
||||
assert parsed_kv["mnist_wide"] == 592157828
|
||||
|
||||
parsed_int = autoresearch.parse_projection_seed_arg("1800044939")
|
||||
assert parsed_int == 1800044939
|
||||
|
||||
# Effective projection seeds in derive_projection_seed
|
||||
s1 = autoresearch.derive_projection_seed("mnist_compact", 0.5, 101, mode="explicit", projection_seed=valid_dict)
|
||||
s2 = autoresearch.derive_projection_seed("mnist_wide", 0.5, 101, mode="explicit", projection_seed=valid_dict)
|
||||
assert s1 == 1800044939
|
||||
assert s2 == 592157828
|
||||
|
||||
# Invalid cases
|
||||
with pytest.raises(ValueError, match="Invalid projection_seed_mode"):
|
||||
autoresearch.validate_projection_seed_config("invalid_mode", None)
|
||||
|
||||
with pytest.raises(ValueError, match="projection_seed must be provided"):
|
||||
autoresearch.validate_projection_seed_config("explicit", None)
|
||||
|
||||
with pytest.raises(ValueError, match="Unknown workload_id"):
|
||||
autoresearch.validate_projection_seed_config("explicit", {"unknown_wl": 12345})
|
||||
|
||||
with pytest.raises(ValueError, match="non-negative integer"):
|
||||
autoresearch.validate_projection_seed_config(
|
||||
"explicit",
|
||||
{
|
||||
"mnist_compact": -5,
|
||||
"mnist_wide": 1800044939,
|
||||
"fashion_compact": 1363313651,
|
||||
"fashion_wide": 189641451,
|
||||
},
|
||||
)
|
||||
with pytest.raises(ValueError, match="projection_seed can only be provided"):
|
||||
autoresearch.validate_projection_seed_config("coupled", 12345)
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 3. Artifact Accounting & Provenance Tests
|
||||
# =====================================================================
|
||||
|
||||
def test_artifact_accounting_and_provenance(mock_heavy_task_deps):
|
||||
"""Verify artifact accounting, fingerprint matching, state ratio, and official test seals."""
|
||||
payload = runner.run_heavy_pso_cross_split(
|
||||
phase="development",
|
||||
particles=2,
|
||||
epochs=2,
|
||||
)
|
||||
assert payload["phase"] == "development"
|
||||
assert payload["official_test_data_loaded"] is False
|
||||
assert payload["official_test_evaluations"] == 0
|
||||
|
||||
res_totals = payload["resource_totals"]
|
||||
# 2 splits * 4 workloads * 3 seeds * 2 (baseline + candidate) = 48 runs
|
||||
assert res_totals["total_runs"] == 48
|
||||
# Each run has 2 particles * 2 epochs = 4 queries, 4 * 10000 = 40000 samples
|
||||
assert res_totals["total_queries"] == 48 * 4
|
||||
assert res_totals["total_samples_evaluated"] == 48 * 40000
|
||||
|
||||
# Fingerprint matching check in splits payload
|
||||
dev_split = payload["splits"]["20260905"]
|
||||
b_fp = dev_split["baselines"]["mnist_compact"]["data_fingerprint"]
|
||||
c_fp = dev_split["candidates"]["mnist_compact"]["data_fingerprint"]
|
||||
assert b_fp == c_fp, "Baseline and candidate data fingerprints must match"
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 4. Phase Seed Enforcement Tests
|
||||
# =====================================================================
|
||||
|
||||
def test_cross_split_runner_phase_enforcement(mock_heavy_task_deps):
|
||||
"""Verify runner enforces exact phase split and swarm seeds."""
|
||||
dev_payload = runner.run_heavy_pso_cross_split(phase="development", particles=2, epochs=2)
|
||||
assert dev_payload["split_seeds"] == [20260905, 20260906]
|
||||
assert dev_payload["swarm_seeds"] == [101, 102, 103]
|
||||
|
||||
conf_payload = runner.run_heavy_pso_cross_split(phase="confirmation", particles=2, epochs=2)
|
||||
assert conf_payload["split_seeds"] == [20260907]
|
||||
assert conf_payload["swarm_seeds"] == [111, 112, 113]
|
||||
|
||||
with pytest.raises(ValueError, match="Invalid phase"):
|
||||
runner.run_heavy_pso_cross_split(phase="invalid_phase")
|
||||
|
||||
|
||||
def test_cross_split_cli_defaults_to_frozen_policy():
|
||||
"""The CLI must execute the frozen policy when no method flags are supplied."""
|
||||
args = runner.build_parser().parse_args([])
|
||||
assert args.geometry_policy == "baseline_aligned"
|
||||
assert args.projection_scope == "global"
|
||||
assert args.projection_seed_mode == "explicit"
|
||||
assert args.projection_seed is None
|
||||
|
||||
|
||||
def test_cross_split_mixed_projection_scope(mock_heavy_task_deps):
|
||||
"""Verify cross-split runner accepts and serializes mixed projection_scope dictionary mapping."""
|
||||
mixed_scope = {
|
||||
"mnist_compact": "global",
|
||||
"mnist_wide": "balanced_global",
|
||||
"fashion_compact": "global",
|
||||
"fashion_wide": "balanced_global",
|
||||
}
|
||||
payload = runner.run_heavy_pso_cross_split(
|
||||
phase="development",
|
||||
projection_scope=mixed_scope,
|
||||
particles=2,
|
||||
epochs=2,
|
||||
)
|
||||
assert payload["candidate_config"]["projection_scope"] == mixed_scope
|
||||
assert payload["workloads"]["mnist_compact"]["projection_scope"] == "global"
|
||||
assert payload["workloads"]["mnist_wide"]["projection_scope"] == "balanced_global"
|
||||
|
||||
dev_split = payload["splits"]["20260905"]
|
||||
assert dev_split["candidates"]["mnist_compact"]["projection_scope"] == "global"
|
||||
assert dev_split["candidates"]["mnist_wide"]["projection_scope"] == "balanced_global"
|
||||
|
||||
|
||||
def test_cross_split_mixed_projection_scope_two_hash(mock_heavy_task_deps):
|
||||
"""Verify cross-split runner accepts and serializes mixed projection_scope dictionary mapping with two_hash_global on Wide."""
|
||||
mixed_scope = {
|
||||
"mnist_compact": "global",
|
||||
"mnist_wide": "two_hash_global",
|
||||
"fashion_compact": "global",
|
||||
"fashion_wide": "two_hash_global",
|
||||
}
|
||||
payload = runner.run_heavy_pso_cross_split(
|
||||
phase="development",
|
||||
projection_scope=mixed_scope,
|
||||
particles=2,
|
||||
epochs=2,
|
||||
)
|
||||
assert payload["candidate_config"]["projection_scope"] == mixed_scope
|
||||
assert payload["workloads"]["mnist_compact"]["projection_scope"] == "global"
|
||||
assert payload["workloads"]["mnist_wide"]["projection_scope"] == "two_hash_global"
|
||||
|
||||
dev_split = payload["splits"]["20260905"]
|
||||
assert dev_split["candidates"]["mnist_compact"]["projection_scope"] == "global"
|
||||
assert dev_split["candidates"]["mnist_wide"]["projection_scope"] == "two_hash_global"
|
||||
|
||||
|
||||
def test_cross_split_mixed_projection_scope_largest_tensor_hash(mock_heavy_task_deps):
|
||||
"""Verify cross-split runner accepts and serializes mixed projection_scope dictionary mapping with largest_tensor_hash on Wide."""
|
||||
mixed_scope = {
|
||||
"mnist_compact": "global",
|
||||
"mnist_wide": "largest_tensor_hash",
|
||||
"fashion_compact": "global",
|
||||
"fashion_wide": "largest_tensor_hash",
|
||||
}
|
||||
payload = runner.run_heavy_pso_cross_split(
|
||||
phase="development",
|
||||
projection_scope=mixed_scope,
|
||||
particles=2,
|
||||
epochs=2,
|
||||
)
|
||||
assert payload["candidate_config"]["projection_scope"] == mixed_scope
|
||||
assert payload["workloads"]["mnist_compact"]["projection_scope"] == "global"
|
||||
assert payload["workloads"]["mnist_wide"]["projection_scope"] == "largest_tensor_hash"
|
||||
|
||||
dev_split = payload["splits"]["20260905"]
|
||||
assert dev_split["candidates"]["mnist_compact"]["projection_scope"] == "global"
|
||||
assert dev_split["candidates"]["mnist_wide"]["projection_scope"] == "largest_tensor_hash"
|
||||
def test_cross_split_mixed_projection_scope_largest_tensor_row_hash(mock_heavy_task_deps):
|
||||
"""Verify cross-split runner accepts and serializes mixed projection_scope dictionary mapping with largest_tensor_row_hash on Wide."""
|
||||
mixed_scope = {
|
||||
"mnist_compact": "global",
|
||||
"mnist_wide": "largest_tensor_row_hash",
|
||||
"fashion_compact": "global",
|
||||
"fashion_wide": "largest_tensor_row_hash",
|
||||
}
|
||||
payload = runner.run_heavy_pso_cross_split(
|
||||
phase="development",
|
||||
projection_scope=mixed_scope,
|
||||
particles=2,
|
||||
epochs=2,
|
||||
)
|
||||
assert payload["candidate_config"]["projection_scope"] == mixed_scope
|
||||
assert payload["workloads"]["mnist_compact"]["projection_scope"] == "global"
|
||||
assert payload["workloads"]["mnist_wide"]["projection_scope"] == "largest_tensor_row_hash"
|
||||
|
||||
dev_split = payload["splits"]["20260905"]
|
||||
assert dev_split["candidates"]["mnist_compact"]["projection_scope"] == "global"
|
||||
assert dev_split["candidates"]["mnist_wide"]["projection_scope"] == "largest_tensor_row_hash"
|
||||
def test_cross_split_mixed_projection_scope_adjacent_pair(mock_heavy_task_deps):
|
||||
"""Verify cross-split runner accepts and serializes mixed projection_scope dictionary mapping with adjacent_pair on Wide."""
|
||||
mixed_scope = {
|
||||
"mnist_compact": "global",
|
||||
"mnist_wide": "adjacent_pair",
|
||||
"fashion_compact": "global",
|
||||
"fashion_wide": "adjacent_pair",
|
||||
}
|
||||
payload = runner.run_heavy_pso_cross_split(
|
||||
phase="development",
|
||||
projection_scope=mixed_scope,
|
||||
particles=2,
|
||||
epochs=2,
|
||||
)
|
||||
assert payload["candidate_config"]["projection_scope"] == mixed_scope
|
||||
assert payload["workloads"]["mnist_compact"]["projection_scope"] == "global"
|
||||
assert payload["workloads"]["mnist_wide"]["projection_scope"] == "adjacent_pair"
|
||||
|
||||
dev_split = payload["splits"]["20260905"]
|
||||
assert dev_split["candidates"]["mnist_compact"]["projection_scope"] == "global"
|
||||
assert dev_split["candidates"]["mnist_wide"]["projection_scope"] == "adjacent_pair"
|
||||
def test_cross_split_mixed_projection_scope_adjacent_difference(mock_heavy_task_deps):
|
||||
"""Verify cross-split runner accepts and serializes mixed projection_scope dictionary mapping with adjacent_difference on Wide."""
|
||||
mixed_scope = {
|
||||
"mnist_compact": "global",
|
||||
"mnist_wide": "adjacent_difference",
|
||||
"fashion_compact": "global",
|
||||
"fashion_wide": "adjacent_difference",
|
||||
}
|
||||
payload = runner.run_heavy_pso_cross_split(
|
||||
phase="development",
|
||||
projection_scope=mixed_scope,
|
||||
particles=2,
|
||||
epochs=2,
|
||||
)
|
||||
assert payload["candidate_config"]["projection_scope"] == mixed_scope
|
||||
assert payload["workloads"]["mnist_compact"]["projection_scope"] == "global"
|
||||
assert payload["workloads"]["mnist_wide"]["projection_scope"] == "adjacent_difference"
|
||||
|
||||
dev_split = payload["splits"]["20260905"]
|
||||
assert dev_split["candidates"]["mnist_compact"]["projection_scope"] == "global"
|
||||
assert dev_split["candidates"]["mnist_wide"]["projection_scope"] == "adjacent_difference"
|
||||
|
||||
# =====================================================================
|
||||
# 5. Evaluator Hard Gates & Score Calculation Tests
|
||||
# =====================================================================
|
||||
|
||||
def create_synthetic_artifact(
|
||||
phase: str,
|
||||
acc_delta: float = 2.0,
|
||||
nll_delta: float = -0.10,
|
||||
) -> Dict[str, Any]:
|
||||
split_seeds = [20260905, 20260906] if phase == "development" else [20260907]
|
||||
swarm_seeds = [101, 102, 103] if phase == "development" else [111, 112, 113]
|
||||
projection_seeds = dict(runner.FROZEN_PROJECTION_SEEDS)
|
||||
splits = {}
|
||||
for split_seed in split_seeds:
|
||||
baselines = {}
|
||||
candidates = {}
|
||||
for workload in evaluator.WORKLOADS:
|
||||
total_dim = evaluator.TOTAL_DIMS[workload]
|
||||
latent_dim = evaluator.compute_latent_dim(total_dim, 0.5)
|
||||
baseline_states = 5 * 12 + (
|
||||
1 if evaluator.BASELINE_METHODS[workload] == "G8" else 0
|
||||
)
|
||||
baseline_bytes = baseline_states * total_dim * 4
|
||||
candidate_bytes = evaluator.compute_core_swarm_state_bytes(12, latent_dim)
|
||||
baseline_runs = []
|
||||
candidate_runs = []
|
||||
for seed in swarm_seeds:
|
||||
common = {
|
||||
"seed": seed,
|
||||
"gbest_loss": 0.55,
|
||||
"gbest_acc": 79.0,
|
||||
"wall_time_sec": 1.0,
|
||||
"optimization_wall_time_sec": 0.9,
|
||||
"validation_wall_time_sec": 0.1,
|
||||
"total_queries": 960,
|
||||
"total_sample_evaluations": 9_600_000,
|
||||
"validation_evaluations": 8,
|
||||
"official_test_evaluations": 0,
|
||||
"throughput_samples_per_sec": 10_666_666.0,
|
||||
}
|
||||
baseline_runs.append(
|
||||
{
|
||||
**common,
|
||||
"val_selected_loss": 0.60,
|
||||
"val_selected_acc": 80.0,
|
||||
"val_metrics": {"nll": 0.60, "brier": 0.20, "ece": 0.05},
|
||||
"core_swarm_state_bytes": baseline_bytes,
|
||||
}
|
||||
)
|
||||
candidate_runs.append(
|
||||
{
|
||||
**common,
|
||||
"projection_seed": projection_seeds[workload],
|
||||
"projection_scope": "global",
|
||||
"projection_seed_mode": "explicit",
|
||||
"geometry_multiplier": 1.0,
|
||||
"val_selected_loss": 0.60 + nll_delta,
|
||||
"val_selected_acc": 80.0 + acc_delta,
|
||||
"val_metrics": {
|
||||
"nll": 0.60 + nll_delta,
|
||||
"brier": 0.18,
|
||||
"ece": 0.04,
|
||||
},
|
||||
"core_swarm_state_bytes": candidate_bytes,
|
||||
"is_finite": True,
|
||||
}
|
||||
)
|
||||
fingerprint = f"fp-{workload}-{split_seed}"
|
||||
shared_entry = {
|
||||
"workload_id": workload,
|
||||
"split_seed": split_seed,
|
||||
"data_fingerprint": fingerprint,
|
||||
"split_fingerprint": f"split-{workload}-{split_seed}",
|
||||
"subset_size": 10_000,
|
||||
"particles": 12,
|
||||
"epochs": 80,
|
||||
"seeds": list(swarm_seeds),
|
||||
}
|
||||
baselines[workload] = {
|
||||
**shared_entry,
|
||||
"method_id": evaluator.BASELINE_METHODS[workload],
|
||||
"stats": {
|
||||
"val_acc": {"mean": 80.0},
|
||||
"val_nll": {"mean": 0.60},
|
||||
},
|
||||
"per_seed_runs": baseline_runs,
|
||||
}
|
||||
candidates[workload] = {
|
||||
**shared_entry,
|
||||
"candidate_id": "pexplicit_aligned_r0.5",
|
||||
"ratio": 0.5,
|
||||
"geometry_policy": "baseline_aligned",
|
||||
"geometry_multiplier": 1.0,
|
||||
"projection_scope": "global",
|
||||
"projection_seed_mode": "explicit",
|
||||
"projection_seed": projection_seeds[workload],
|
||||
"total_dim": total_dim,
|
||||
"latent_dim": latent_dim,
|
||||
"state_ratio": candidate_bytes / baseline_bytes,
|
||||
"core_swarm_state_bytes": candidate_bytes,
|
||||
"baseline_core_swarm_state_bytes": baseline_bytes,
|
||||
"stats": {
|
||||
"val_acc": {"mean": 80.0 + acc_delta},
|
||||
"val_nll": {"mean": 0.60 + nll_delta},
|
||||
},
|
||||
"per_seed_runs": candidate_runs,
|
||||
}
|
||||
splits[str(split_seed)] = {
|
||||
"split_seed": split_seed,
|
||||
"baselines": baselines,
|
||||
"candidates": candidates,
|
||||
}
|
||||
total_runs = len(split_seeds) * 4 * len(swarm_seeds) * 2
|
||||
return {
|
||||
"version": runner.PROTOCOL_VERSION,
|
||||
"protocol_version": runner.PROTOCOL_VERSION,
|
||||
"phase": phase,
|
||||
"split_seeds": split_seeds,
|
||||
"swarm_seeds": swarm_seeds,
|
||||
"official_test_data_loaded": False,
|
||||
"official_test_evaluations": 0,
|
||||
"candidate_config": {
|
||||
"ratio": 0.5,
|
||||
"geometry_policy": "baseline_aligned",
|
||||
"projection_scope": "global",
|
||||
"projection_seed_mode": "explicit",
|
||||
"projection_seed": projection_seeds,
|
||||
"geometry_multiplier": 1.0,
|
||||
"particles": 12,
|
||||
"epochs": 80,
|
||||
"subset_size": 10_000,
|
||||
},
|
||||
"workloads": {
|
||||
workload: {
|
||||
"workload_id": workload,
|
||||
"dataset_name": (
|
||||
"mnist" if workload.startswith("mnist") else "fashion_mnist"
|
||||
),
|
||||
"model_name": (
|
||||
"compact_cnn" if workload.endswith("compact") else "wide_cnn"
|
||||
),
|
||||
"baseline_method": evaluator.BASELINE_METHODS[workload],
|
||||
"effective_projection_seed": projection_seeds[workload],
|
||||
}
|
||||
for workload in evaluator.WORKLOADS
|
||||
},
|
||||
"splits": splits,
|
||||
"resource_totals": {
|
||||
"total_runs": total_runs,
|
||||
"total_queries": total_runs * 960,
|
||||
"total_samples_evaluated": total_runs * 9_600_000,
|
||||
"official_test_evaluations": 0,
|
||||
"wall_time_sec": 1.0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_evaluator_all_gates_and_score():
|
||||
"""Verify evaluator passes valid dev + conf artifacts and rejects violations."""
|
||||
dev_art = create_synthetic_artifact("development", acc_delta=2.5, nll_delta=-0.05)
|
||||
conf_art = create_synthetic_artifact("confirmation", acc_delta=2.5, nll_delta=-0.05)
|
||||
|
||||
res_pass = evaluator.evaluate_heavy_cross_split(dev_art, conf_art)
|
||||
assert res_pass["pass"] is True
|
||||
assert res_pass["failed_hard_gate_count"] == 0
|
||||
assert res_pass["score"] > 0
|
||||
|
||||
# Test missing confirmation artifact rejection
|
||||
res_no_conf = evaluator.evaluate_heavy_cross_split(dev_art, None)
|
||||
assert res_no_conf["pass"] is False
|
||||
assert "confirmation_executed" in res_no_conf["failed_gates"]
|
||||
assert res_no_conf["failed_hard_gate_count"] > 0
|
||||
|
||||
# Test test leakage rejection
|
||||
dev_leak = create_synthetic_artifact("development")
|
||||
dev_leak["official_test_data_loaded"] = True
|
||||
res_leak = evaluator.evaluate_heavy_cross_split(dev_leak, conf_art)
|
||||
assert res_leak["pass"] is False
|
||||
assert "official_test_sealed" in res_leak["failed_gates"]
|
||||
|
||||
# Test non-finite metric rejection
|
||||
dev_inf = create_synthetic_artifact("development")
|
||||
dev_inf["splits"]["20260905"]["candidates"]["mnist_compact"]["per_seed_runs"][0]["val_selected_loss"] = float("nan")
|
||||
res_inf = evaluator.evaluate_heavy_cross_split(dev_inf, conf_art)
|
||||
assert res_inf["pass"] is False
|
||||
assert "all_runs_finite" in res_inf["failed_gates"]
|
||||
|
||||
# Test fingerprint mismatch rejection
|
||||
dev_fp_mismatch = create_synthetic_artifact("development")
|
||||
dev_fp_mismatch["splits"]["20260905"]["candidates"]["mnist_compact"]["split_fingerprint"] = "bad-fp"
|
||||
res_fp_mismatch = evaluator.evaluate_heavy_cross_split(dev_fp_mismatch, conf_art)
|
||||
assert res_fp_mismatch["pass"] is False
|
||||
assert "split_and_fingerprint_matched" in res_fp_mismatch["failed_gates"]
|
||||
|
||||
# Test per-cell accuracy regression violation (> 1.0 pp)
|
||||
dev_reg = create_synthetic_artifact("development", acc_delta=-1.5, nll_delta=0.0)
|
||||
res_reg = evaluator.evaluate_heavy_cross_split(dev_reg, conf_art)
|
||||
assert res_reg["pass"] is False
|
||||
assert "maximum_accuracy_regression_percentage_points_each_split_workload" in res_reg["failed_gates"]
|
||||
|
||||
|
||||
def test_evaluator_accepts_development_only_as_confirmation_eligible():
|
||||
development = create_synthetic_artifact(
|
||||
"development", acc_delta=2.5, nll_delta=-0.05
|
||||
)
|
||||
result = evaluator.evaluate_heavy_cross_split(development)
|
||||
assert result["pass"] is False
|
||||
assert result["development_pass"] is True
|
||||
assert result["eligible_for_confirmation"] is True
|
||||
assert result["score_failed_gate_count"] == 0
|
||||
|
||||
|
||||
def test_evaluator_rejects_missing_or_inconsistent_evidence():
|
||||
development = create_synthetic_artifact(
|
||||
"development", acc_delta=2.5, nll_delta=-0.05
|
||||
)
|
||||
confirmation = create_synthetic_artifact(
|
||||
"confirmation", acc_delta=2.5, nll_delta=-0.05
|
||||
)
|
||||
|
||||
missing_fingerprint = create_synthetic_artifact(
|
||||
"development", acc_delta=2.5, nll_delta=-0.05
|
||||
)
|
||||
del missing_fingerprint["splits"]["20260905"]["candidates"]["mnist_compact"][
|
||||
"data_fingerprint"
|
||||
]
|
||||
result = evaluator.evaluate_heavy_cross_split(
|
||||
missing_fingerprint, confirmation
|
||||
)
|
||||
assert "split_and_fingerprint_matched" in result["failed_gates"]
|
||||
|
||||
inconsistent_stats = create_synthetic_artifact(
|
||||
"development", acc_delta=2.5, nll_delta=-0.05
|
||||
)
|
||||
inconsistent_stats["splits"]["20260905"]["candidates"]["mnist_compact"][
|
||||
"stats"
|
||||
]["val_acc"]["mean"] += 1.0
|
||||
result = evaluator.evaluate_heavy_cross_split(
|
||||
inconsistent_stats, confirmation
|
||||
)
|
||||
assert "schema_and_phase_seeds" in result["failed_gates"]
|
||||
|
||||
confirmation["candidate_config"]["ratio"] = 0.25
|
||||
result = evaluator.evaluate_heavy_cross_split(development, confirmation)
|
||||
assert "configuration_and_policy_matched" in result["failed_gates"]
|
||||
def test_evaluator_rejects_missing_selected_metrics():
|
||||
conf_art = create_synthetic_artifact("confirmation", acc_delta=2.5, nll_delta=-0.05)
|
||||
|
||||
for metric in ("val_selected_acc", "val_selected_loss"):
|
||||
# Test key deletion
|
||||
dev_art_del = create_synthetic_artifact("development", acc_delta=2.5, nll_delta=-0.05)
|
||||
run_del = dev_art_del["splits"]["20260905"]["candidates"]["mnist_compact"]["per_seed_runs"][0]
|
||||
del run_del[metric]
|
||||
result_del = evaluator.evaluate_heavy_cross_split(dev_art_del, conf_art)
|
||||
assert result_del["pass"] is False
|
||||
assert result_del["failed_hard_gate_count"] > 0
|
||||
assert (
|
||||
"schema_and_phase_seeds" in result_del["failed_gates"]
|
||||
or "all_runs_finite" in result_del["failed_gates"]
|
||||
)
|
||||
|
||||
# Test non-numeric string
|
||||
dev_art_str = create_synthetic_artifact("development", acc_delta=2.5, nll_delta=-0.05)
|
||||
run_str = dev_art_str["splits"]["20260905"]["candidates"]["mnist_compact"]["per_seed_runs"][0]
|
||||
run_str[metric] = "invalid_string"
|
||||
result_str = evaluator.evaluate_heavy_cross_split(dev_art_str, conf_art)
|
||||
assert result_str["pass"] is False
|
||||
assert result_str["failed_hard_gate_count"] > 0
|
||||
assert (
|
||||
"schema_and_phase_seeds" in result_str["failed_gates"]
|
||||
or "all_runs_finite" in result_str["failed_gates"]
|
||||
)
|
||||
|
||||
# Test None
|
||||
dev_art_none = create_synthetic_artifact("development", acc_delta=2.5, nll_delta=-0.05)
|
||||
run_none = dev_art_none["splits"]["20260905"]["candidates"]["mnist_compact"]["per_seed_runs"][0]
|
||||
run_none[metric] = None
|
||||
result_none = evaluator.evaluate_heavy_cross_split(dev_art_none, conf_art)
|
||||
assert result_none["pass"] is False
|
||||
assert result_none["failed_hard_gate_count"] > 0
|
||||
assert (
|
||||
"schema_and_phase_seeds" in result_none["failed_gates"]
|
||||
or "all_runs_finite" in result_none["failed_gates"]
|
||||
)
|
||||
@@ -0,0 +1,368 @@
|
||||
"""
|
||||
Unit tests for Heavy Task Feasibility Study (MNIST & FashionMNIST).
|
||||
|
||||
Covers:
|
||||
1. Exact model parameter counts and forward shapes (CompactCNN vs WideCNN)
|
||||
2. Immutable workload matrix definitions (mnist_compact, mnist_wide, fashion_compact, fashion_wide)
|
||||
3. Train-only loader guard for both datasets (train=True only, test_samples=0, test_evals=0)
|
||||
4. Deterministic normalized-method selection logic (G0, G5, G6)
|
||||
5. Feasibility threshold boundaries (execution_feasible and optimization_feasible)
|
||||
6. Finite and artifact schema properties
|
||||
7. Exact fixed2k/fixed10k query, sample evaluation, and swarm state accounting
|
||||
8. CPU smoke runner execution without network
|
||||
"""
|
||||
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
# Ensure test directory and repo root are in Python path
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent if Path(__file__).resolve().parent.name != "PSO" else Path(__file__).resolve().parent
|
||||
TEST_DIR = REPO_ROOT / "test"
|
||||
if str(TEST_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TEST_DIR))
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
import heavy_task_feasibility as heavy_task
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def synthetic_heavy_data(monkeypatch):
|
||||
"""Keep runner tests deterministic and independent of dataset downloads."""
|
||||
x_search = torch.zeros(100, 1, 28, 28)
|
||||
y_search = torch.arange(100) % 10
|
||||
x_val = torch.zeros(100, 1, 28, 28)
|
||||
y_val = torch.arange(100) % 10
|
||||
nested_subsets = {
|
||||
size: torch.arange(size) % len(y_search)
|
||||
for size in (2000, 10000, 50000)
|
||||
}
|
||||
|
||||
def fake_prepare(dataset_name, split_seed=20260902, cache_dir=None):
|
||||
provenance = {
|
||||
"dataset_name": dataset_name,
|
||||
"official_test_data_loaded": False,
|
||||
"official_test_evaluations": 0,
|
||||
"test_samples": 0,
|
||||
"search_samples": 50000,
|
||||
"val_samples": 10000,
|
||||
"split_fingerprint": "synthetic-split",
|
||||
"data_fingerprint": "synthetic-data",
|
||||
}
|
||||
return (
|
||||
x_search,
|
||||
y_search,
|
||||
x_val,
|
||||
y_val,
|
||||
nested_subsets,
|
||||
"synthetic-data",
|
||||
provenance,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(heavy_task, "prepare_heavy_task_data", fake_prepare)
|
||||
|
||||
|
||||
from heavy_task_feasibility import (
|
||||
PROTOCOL_VERSION,
|
||||
WORKLOADS,
|
||||
HEAVY_METHODS,
|
||||
CompactCNN,
|
||||
WideCNN,
|
||||
make_compact_cnn,
|
||||
make_wide_cnn,
|
||||
create_model,
|
||||
prepare_heavy_task_data,
|
||||
evaluate_untrained_baseline,
|
||||
select_best_normalized_method,
|
||||
evaluate_feasibility,
|
||||
run_heavy_task_screen,
|
||||
run_heavy_task_confirm,
|
||||
run_heavy_task_study,
|
||||
WorkloadConfig,
|
||||
)
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 1. Parameter Counts & Forward Shapes
|
||||
# =====================================================================
|
||||
|
||||
def test_exact_model_parameter_counts_and_forward_shapes():
|
||||
compact_model = make_compact_cnn(seed=41)
|
||||
compact_params = sum(p.numel() for p in compact_model.parameters())
|
||||
assert compact_params == 9098, f"CompactCNN should have 9,098 parameters, got {compact_params}"
|
||||
|
||||
wide_model = make_wide_cnn(seed=41)
|
||||
wide_params = sum(p.numel() for p in wide_model.parameters())
|
||||
assert wide_params == 55338, f"WideCNN should have 55,338 parameters, got {wide_params}"
|
||||
|
||||
# Forward shape test with (B, 1, 28, 28)
|
||||
x_img = torch.randn(2, 1, 28, 28)
|
||||
out_c_img = compact_model(x_img)
|
||||
out_w_img = wide_model(x_img)
|
||||
assert out_c_img.shape == (2, 10), f"CompactCNN image output shape should be (2, 10), got {out_c_img.shape}"
|
||||
assert out_w_img.shape == (2, 10), f"WideCNN image output shape should be (2, 10), got {out_w_img.shape}"
|
||||
|
||||
# Forward shape test with flattened (B, 784)
|
||||
x_flat = torch.randn(2, 784)
|
||||
out_c_flat = compact_model(x_flat)
|
||||
out_w_flat = wide_model(x_flat)
|
||||
assert out_c_flat.shape == (2, 10), f"CompactCNN flat output shape should be (2, 10), got {out_c_flat.shape}"
|
||||
assert out_w_flat.shape == (2, 10), f"WideCNN flat output shape should be (2, 10), got {out_w_flat.shape}"
|
||||
|
||||
# Deterministic factory behavior
|
||||
c1 = make_compact_cnn(seed=41)
|
||||
c2 = make_compact_cnn(seed=41)
|
||||
for p1, p2 in zip(c1.parameters(), c2.parameters()):
|
||||
assert torch.equal(p1, p2)
|
||||
|
||||
w1 = make_wide_cnn(seed=41)
|
||||
w2 = make_wide_cnn(seed=41)
|
||||
for p1, p2 in zip(w1.parameters(), w2.parameters()):
|
||||
assert torch.equal(p1, p2)
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 2. Immutable Workload Matrix
|
||||
# =====================================================================
|
||||
|
||||
def test_immutable_workload_matrix():
|
||||
expected_workloads = {"mnist_compact", "mnist_wide", "fashion_compact", "fashion_wide"}
|
||||
assert set(WORKLOADS.keys()) == expected_workloads
|
||||
|
||||
assert WORKLOADS["mnist_compact"].dataset_name == "mnist"
|
||||
assert WORKLOADS["mnist_compact"].model_name == "compact_cnn"
|
||||
|
||||
assert WORKLOADS["mnist_wide"].dataset_name == "mnist"
|
||||
assert WORKLOADS["mnist_wide"].model_name == "wide_cnn"
|
||||
|
||||
assert WORKLOADS["fashion_compact"].dataset_name == "fashion_mnist"
|
||||
assert WORKLOADS["fashion_compact"].model_name == "compact_cnn"
|
||||
|
||||
assert WORKLOADS["fashion_wide"].dataset_name == "fashion_mnist"
|
||||
assert WORKLOADS["fashion_wide"].model_name == "wide_cnn"
|
||||
|
||||
assert HEAVY_METHODS == ["G0", "G5", "G6", "G8"]
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 3. Train-Only Loader Guard
|
||||
# =====================================================================
|
||||
|
||||
def test_train_only_loader_guard(monkeypatch, tmp_path):
|
||||
import torchvision.datasets
|
||||
|
||||
calls = []
|
||||
|
||||
class DatasetConstructionStopped(Exception):
|
||||
pass
|
||||
|
||||
def reject_after_recording(name):
|
||||
def constructor(*, root, train, download):
|
||||
calls.append((name, train, download))
|
||||
raise DatasetConstructionStopped
|
||||
return constructor
|
||||
|
||||
monkeypatch.setattr(
|
||||
torchvision.datasets,
|
||||
"MNIST",
|
||||
reject_after_recording("mnist"),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
torchvision.datasets,
|
||||
"FashionMNIST",
|
||||
reject_after_recording("fashion_mnist"),
|
||||
)
|
||||
|
||||
for dataset_name in ("mnist", "fashion_mnist"):
|
||||
with pytest.raises(DatasetConstructionStopped):
|
||||
prepare_heavy_task_data(
|
||||
dataset_name=dataset_name,
|
||||
split_seed=20260902,
|
||||
cache_dir=tmp_path,
|
||||
)
|
||||
|
||||
assert calls == [
|
||||
("mnist", True, True),
|
||||
("fashion_mnist", True, True),
|
||||
]
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 4. Deterministic Normalized Method Selection
|
||||
# =====================================================================
|
||||
|
||||
def test_deterministic_normalized_method_selection():
|
||||
mock_screen_results = [
|
||||
{"method_id": "G0", "val_selected_loss": 0.60, "val_selected_acc": 82.0},
|
||||
{"method_id": "G5", "val_selected_loss": 0.50, "val_selected_acc": 84.0},
|
||||
{"method_id": "G6", "val_selected_loss": 0.52, "val_selected_acc": 84.5},
|
||||
{"method_id": "G8", "val_selected_loss": 0.45, "val_selected_acc": 85.0},
|
||||
]
|
||||
# G8 is excluded from normalized custom selection; G5 has lowest val_selected_loss (0.50)
|
||||
best_m = select_best_normalized_method(mock_screen_results)
|
||||
assert best_m == "G5"
|
||||
|
||||
# Test tiebreak logic: same loss, pick higher accuracy
|
||||
mock_tie = [
|
||||
{"method_id": "G0", "val_selected_loss": 0.50, "val_selected_acc": 83.0},
|
||||
{"method_id": "G5", "val_selected_loss": 0.50, "val_selected_acc": 85.0},
|
||||
{"method_id": "G6", "val_selected_loss": 0.50, "val_selected_acc": 84.0},
|
||||
]
|
||||
best_tie = select_best_normalized_method(mock_tie)
|
||||
assert best_tie == "G5"
|
||||
|
||||
# Diverged candidates cannot win selection; fail explicitly if none are finite.
|
||||
with_nonfinite = [
|
||||
{"method_id": "G0", "val_selected_loss": float("nan"), "val_selected_acc": 99.0},
|
||||
{"method_id": "G5", "val_selected_loss": 0.60, "val_selected_acc": 82.0},
|
||||
{"method_id": "G6", "val_selected_loss": float("inf"), "val_selected_acc": 100.0},
|
||||
]
|
||||
assert select_best_normalized_method(with_nonfinite) == "G5"
|
||||
with pytest.raises(ValueError, match="No finite normalized method"):
|
||||
select_best_normalized_method(with_nonfinite[:1])
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 5. Feasibility Threshold Boundaries
|
||||
# =====================================================================
|
||||
|
||||
def test_feasibility_threshold_boundaries():
|
||||
baseline_nll = 2.30
|
||||
baseline_acc = 10.0
|
||||
|
||||
# 1. Non-finite run
|
||||
bad_runs = [{"val_selected_loss": float("nan"), "val_selected_acc": 50.0}]
|
||||
f1 = evaluate_feasibility(bad_runs, baseline_nll, baseline_acc)
|
||||
assert f1["execution_feasible"] is False
|
||||
assert f1["optimization_feasible"] is False
|
||||
|
||||
# 2. Feasible run passing both NLL and accuracy thresholds
|
||||
# Target NLL <= 2.30 * 0.80 = 1.84
|
||||
# Target Acc >= 10.0 + 20.0 = 30.0
|
||||
good_runs = [
|
||||
{"val_selected_loss": 1.50, "val_selected_acc": 40.0},
|
||||
{"val_selected_loss": 1.60, "val_selected_acc": 42.0},
|
||||
]
|
||||
f2 = evaluate_feasibility(good_runs, baseline_nll, baseline_acc)
|
||||
assert f2["execution_feasible"] is True
|
||||
assert f2["optimization_feasible"] is True
|
||||
assert f2["target_val_nll_threshold"] == 1.84
|
||||
assert f2["target_val_acc_threshold"] == 30.0
|
||||
|
||||
# 3. Failing NLL threshold (1.90 > 1.84)
|
||||
fail_nll_runs = [
|
||||
{"val_selected_loss": 1.90, "val_selected_acc": 40.0},
|
||||
]
|
||||
f3 = evaluate_feasibility(fail_nll_runs, baseline_nll, baseline_acc)
|
||||
assert f3["execution_feasible"] is True
|
||||
assert f3["optimization_feasible"] is False
|
||||
|
||||
# 4. Failing Acc threshold (25.0 < 30.0)
|
||||
fail_acc_runs = [
|
||||
{"val_selected_loss": 1.50, "val_selected_acc": 25.0},
|
||||
]
|
||||
f4 = evaluate_feasibility(fail_acc_runs, baseline_nll, baseline_acc)
|
||||
assert f4["execution_feasible"] is True
|
||||
assert f4["optimization_feasible"] is False
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 6. Artifact Schema & Properties
|
||||
# =====================================================================
|
||||
|
||||
def test_finite_and_artifact_schema(tmp_path, synthetic_heavy_data):
|
||||
device = torch.device("cpu")
|
||||
# Quick smoke call to test output schema structure
|
||||
single_wl = {"mnist_compact": WORKLOADS["mnist_compact"]}
|
||||
screen_results, baselines, meta = run_heavy_task_screen(
|
||||
workloads=single_wl,
|
||||
methods=["G0"],
|
||||
particles=2,
|
||||
epochs=2,
|
||||
seed=91,
|
||||
device=device,
|
||||
cache_dir=tmp_path / "cache",
|
||||
)
|
||||
assert len(screen_results) == 1
|
||||
cell = screen_results[0]
|
||||
assert cell["official_test_evaluations"] == 0
|
||||
assert cell["is_finite"] is True
|
||||
assert "core_swarm_state_bytes" in cell
|
||||
assert cell["core_swarm_state_bytes"] == 5 * 2 * 9098 * 4
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 7. Exact Fixed Accounting
|
||||
# =====================================================================
|
||||
|
||||
def test_exact_fixed_accounting():
|
||||
# Fixed 2k screening cell: 12 particles, 40 epochs
|
||||
particles = 12
|
||||
epochs = 40
|
||||
subset_2k = 2000
|
||||
|
||||
expected_queries_2k = particles * epochs
|
||||
expected_sample_evals_2k = expected_queries_2k * subset_2k
|
||||
|
||||
assert expected_queries_2k == 480
|
||||
assert expected_sample_evals_2k == 960000
|
||||
|
||||
# Swarm state bytes:
|
||||
# Custom method (G0, G5, G6): 5 * particles * param_count * 4
|
||||
# G8 (public Optimizer): (5 * particles + 1) * param_count * 4
|
||||
compact_params = 9098
|
||||
custom_bytes = 5 * 12 * compact_params * 4
|
||||
g8_bytes = (5 * 12 + 1) * compact_params * 4
|
||||
|
||||
assert custom_bytes == 2183520
|
||||
assert g8_bytes == 2219912
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# 8. CPU Smoke Runner Execution
|
||||
# =====================================================================
|
||||
|
||||
def test_cpu_smoke_runner(tmp_path, synthetic_heavy_data):
|
||||
device = torch.device("cpu")
|
||||
single_wl = {"mnist_compact": WORKLOADS["mnist_compact"]}
|
||||
|
||||
# Screening phase smoke
|
||||
screen_res, baselines, wl_meta = run_heavy_task_screen(
|
||||
workloads=single_wl,
|
||||
methods=["G0", "G8"],
|
||||
particles=2,
|
||||
epochs=2,
|
||||
seed=91,
|
||||
device=device,
|
||||
cache_dir=tmp_path / "cache",
|
||||
)
|
||||
assert len(screen_res) == 2
|
||||
|
||||
# Confirmation phase smoke
|
||||
selected_methods = {"mnist_compact": ["G0", "G8"]}
|
||||
confirm_res = run_heavy_task_confirm(
|
||||
workloads=single_wl,
|
||||
selected_methods=selected_methods,
|
||||
particles=2,
|
||||
epochs=2,
|
||||
seeds=[101, 102],
|
||||
device=device,
|
||||
cache_dir=tmp_path / "cache",
|
||||
)
|
||||
assert "mnist_compact" in confirm_res
|
||||
assert "G0" in confirm_res["mnist_compact"]
|
||||
assert "G8" in confirm_res["mnist_compact"]
|
||||
|
||||
# Feasibility smoke
|
||||
base = baselines["mnist_compact"]
|
||||
feas = evaluate_feasibility(
|
||||
confirm_res["mnist_compact"]["G0"]["per_seed_runs"],
|
||||
base["val_nll"],
|
||||
base["val_accuracy"],
|
||||
)
|
||||
assert "execution_feasible" in feas
|
||||
assert "optimization_feasible" in feas
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,70 @@
|
||||
def test_binary_1d_target_normalization_no_broadcasting(model_factory):
|
||||
"""Verify binary [N, 1] logits model with 1-D [N] targets normalizes target shape and fits without broadcasting."""
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(6, 2, dtype=torch.float32)
|
||||
y_1d = torch.tensor([0.0, 1.0, 1.0, 0.0, 1.0, 0.0], dtype=torch.float32) # Shape [6]
|
||||
|
||||
model = model_factory(input_dim=2, units=4, output_dim=1) # Output shape [6, 1]
|
||||
loss = nn.BCEWithLogitsLoss()
|
||||
|
||||
opt = Optimizer(model, loss, task="binary", n_particles=3, seed=42)
|
||||
score = opt.fit(x, y_1d, epochs=2)
|
||||
|
||||
assert isinstance(score, tuple)
|
||||
assert len(score) == 3
|
||||
assert all(math.isfinite(s) for s in score)
|
||||
|
||||
|
||||
def test_regression_1d_target_normalization_and_mse(model_factory):
|
||||
"""Verify regression [N, 1] model output with 1-D [N] targets normalizes shape, loss ≈ MSE, and no broadcasting."""
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(8, 2, dtype=torch.float32)
|
||||
y_1d = torch.randn(8, dtype=torch.float32) # Shape [8]
|
||||
|
||||
model = model_factory(input_dim=2, units=4, output_dim=1) # Output shape [8, 1]
|
||||
loss = nn.MSELoss()
|
||||
|
||||
opt = Optimizer(model, loss, task="regression", n_particles=3, seed=42)
|
||||
score = opt.fit(x, y_1d, epochs=2)
|
||||
|
||||
assert isinstance(score, tuple)
|
||||
assert len(score) == 3
|
||||
assert all(math.isfinite(s) for s in score)
|
||||
assert math.isclose(score[0], score[2], rel_tol=1e-5, abs_tol=1e-5)
|
||||
|
||||
|
||||
def test_binary_regression_incompatible_target_counts_fail_fast(model_factory):
|
||||
"""Verify binary and regression fail with contextual ValueError when target element count mismatches output."""
|
||||
x = torch.randn(4, 2, dtype=torch.float32)
|
||||
# Shape [4, 2] has leading dimension 4 (matches x), but 8 elements (mismatches model output [4, 1] 4 elements)
|
||||
y_bad = torch.randn(4, 2, dtype=torch.float32)
|
||||
|
||||
model = model_factory(input_dim=2, units=4, output_dim=1) # Output shape [4, 1] -> 4 elements
|
||||
|
||||
opt_bin = Optimizer(model, nn.BCEWithLogitsLoss(), task="binary", n_particles=2)
|
||||
with pytest.raises(ValueError, match="(?i)target element count"):
|
||||
opt_bin.fit(x, y_bad)
|
||||
|
||||
opt_reg = Optimizer(model, nn.MSELoss(), task="regression", n_particles=2)
|
||||
with pytest.raises(ValueError, match="(?i)target element count"):
|
||||
opt_reg.fit(x, y_bad)
|
||||
|
||||
|
||||
def test_multiclass_target_shapes_and_incompatible_fail_fast(model_factory):
|
||||
"""Verify multiclass fits with [N, 1] integer targets reshaped to [N], and incompatible target shapes fail."""
|
||||
x = torch.randn(6, 4, dtype=torch.float32)
|
||||
# [N, 1] integer class targets
|
||||
y_col = torch.tensor([[0], [1], [2], [0], [1], [2]], dtype=torch.int64)
|
||||
|
||||
model = model_factory(input_dim=4, units=8, output_dim=3) # Output shape [6, 3]
|
||||
loss = nn.CrossEntropyLoss()
|
||||
|
||||
opt = Optimizer(model, loss, task="multiclass", n_particles=3, seed=42)
|
||||
score = opt.fit(x, y_col, epochs=2)
|
||||
assert isinstance(score, tuple)
|
||||
assert all(math.isfinite(s) for s in score)
|
||||
|
||||
# Incompatible target shape (e.g. 5 columns for 3 classes)
|
||||
y_bad = torch.randn(6, 5, dtype=torch.float32)
|
||||
with pytest.raises(ValueError, match="(?i)target shape"):
|
||||
opt.fit(x, y_bad)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,330 @@
|
||||
"""Behavioral tests for the post-training convergence protocol.
|
||||
|
||||
These tests intentionally use tiny local modules and synthetic artifacts. They
|
||||
exercise protocol boundaries (rather than implementation details) while
|
||||
keeping the production data/model paths completely offline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
TEST_DIR = REPO_ROOT / "test"
|
||||
if str(TEST_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TEST_DIR))
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
import post_training_model_convergence as study
|
||||
|
||||
|
||||
class TinyStatefulModel(nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.feature = nn.Linear(3, 2, bias=False)
|
||||
self.bn = nn.BatchNorm1d(2)
|
||||
self.head = nn.Linear(2, 1)
|
||||
|
||||
def forward(self, value: torch.Tensor) -> torch.Tensor:
|
||||
return self.head(self.bn(self.feature(value)))
|
||||
|
||||
|
||||
def _tiny_model() -> TinyStatefulModel:
|
||||
torch.manual_seed(17)
|
||||
model = TinyStatefulModel()
|
||||
model.train()
|
||||
return model
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value"),
|
||||
[
|
||||
("protocol_version", "post-training-model-convergence-drift"),
|
||||
("split_seed", study.SPLIT_SEED + 1),
|
||||
("base_seeds", (501, 502, 504)),
|
||||
("swarm_seeds", (601, 602, 604)),
|
||||
("projection_seed", study.PROJECTION_SEED + 1),
|
||||
("bootstrap_seed", study.BOOTSTRAP_SEED + 1),
|
||||
("particle_count", study.PARTICLE_COUNT - 1),
|
||||
("pso_generations", study.PSO_GENERATIONS - 1),
|
||||
("residual_dimension", study.RESIDUAL_DIMENSION - 1),
|
||||
("residual_bound", study.RESIDUAL_BOUND / 2),
|
||||
("initial_radius", study.INITIAL_RADIUS / 2),
|
||||
("objective_checkpoints", (0, 1)),
|
||||
],
|
||||
)
|
||||
def test_study_config_rejects_protocol_constant_drift(field: str, value: object) -> None:
|
||||
with pytest.raises(study.ProtocolError):
|
||||
study.StudyConfig(**{field: value})
|
||||
|
||||
|
||||
def test_study_config_round_trip_and_matrix_order_boundary(tmp_path: Path) -> None:
|
||||
config = study.StudyConfig()
|
||||
assert study.StudyConfig.from_dict(config.to_dict()) == config
|
||||
assert config.base_seeds == (501, 502, 503)
|
||||
assert config.swarm_seeds == (601, 602, 603)
|
||||
assert config.objective_checkpoints == (0, 10, 20, 30, 40, 50, 60)
|
||||
with pytest.raises(study.ProtocolError, match="fixed order"):
|
||||
study._make_adapters(
|
||||
study.StudyConfig(workload_ids=(study.DEFAULT_WORKLOAD_IDS[0],)),
|
||||
tmp_path / "run",
|
||||
tmp_path / "data",
|
||||
False,
|
||||
)
|
||||
|
||||
def test_selected_codec_is_deterministic_and_preserves_nonselected_state() -> None:
|
||||
model_a = _tiny_model()
|
||||
model_b = copy.deepcopy(model_a)
|
||||
names = ("feature.weight",)
|
||||
codec_a = study.SelectedResidualCodec(model_a, names, projection_seed=12345)
|
||||
codec_b = study.SelectedResidualCodec(model_b, names, projection_seed=12345)
|
||||
residual = torch.linspace(-0.75, 0.75, study.RESIDUAL_DIMENSION)
|
||||
|
||||
assert codec_a.names == names
|
||||
assert torch.equal(codec_a.projection_indices, codec_b.projection_indices)
|
||||
assert torch.equal(codec_a.decode(residual), codec_b.decode(residual))
|
||||
assert torch.equal(codec_a.decode(torch.zeros(study.RESIDUAL_DIMENSION)), model_a.feature.weight.detach().flatten())
|
||||
assert codec_a.scales == codec_b.scales
|
||||
first_indices = codec_a.projection_indices
|
||||
second_indices = codec_a.projection_indices
|
||||
assert first_indices.data_ptr() != second_indices.data_ptr()
|
||||
|
||||
before = {name: value.detach().clone() for name, value in model_a.named_parameters()}
|
||||
before_buffers = {name: value.detach().clone() for name, value in model_a.named_buffers()}
|
||||
original_modes = {name: child.training for name, child in model_a.named_modules()}
|
||||
with codec_a.applied(model_a, residual):
|
||||
assert not torch.equal(model_a.feature.weight.detach(), before["feature.weight"])
|
||||
assert torch.equal(model_a.head.weight.detach(), before["head.weight"])
|
||||
assert torch.equal(model_a.bn.running_mean, before_buffers["bn.running_mean"])
|
||||
assert model_a.training is False
|
||||
assert {name: child.training for name, child in model_a.named_modules()} == original_modes
|
||||
for name, value in model_a.named_parameters():
|
||||
assert torch.equal(value, before[name])
|
||||
for name, value in model_a.named_buffers():
|
||||
assert torch.equal(value, before_buffers[name])
|
||||
|
||||
|
||||
def test_selected_codec_restores_state_after_exception_and_rejects_nonselected_mutation() -> None:
|
||||
model = _tiny_model()
|
||||
codec = study.SelectedResidualCodec(model, ("feature.weight",))
|
||||
before = {name: value.detach().clone() for name, value in model.state_dict().items()}
|
||||
|
||||
with pytest.raises(RuntimeError, match="callback failure"):
|
||||
with codec.applied(model, codec.zero_residual()):
|
||||
model.bn.running_mean.add_(1.0)
|
||||
raise RuntimeError("callback failure")
|
||||
assert all(torch.equal(model.state_dict()[name], value) for name, value in before.items())
|
||||
|
||||
with pytest.raises(study.ProtocolError, match="non-selected"):
|
||||
with codec.applied(model, torch.ones(study.RESIDUAL_DIMENSION)):
|
||||
with torch.no_grad():
|
||||
model.head.bias.add_(1.0)
|
||||
assert all(torch.equal(model.state_dict()[name], value) for name, value in before.items())
|
||||
|
||||
|
||||
def test_state_neutral_audit_restores_model_and_rng() -> None:
|
||||
model = _tiny_model()
|
||||
before_state = {name: value.detach().clone() for name, value in model.state_dict().items()}
|
||||
before_modes = {name: child.training for name, child in model.named_modules()}
|
||||
before_rng = torch.get_rng_state().clone()
|
||||
|
||||
def callback() -> float:
|
||||
model.eval()
|
||||
with torch.no_grad():
|
||||
model.feature.weight.add_(3.0)
|
||||
model.bn.running_var.mul_(2.0)
|
||||
torch.manual_seed(999)
|
||||
return 1.25
|
||||
|
||||
assert study.run_state_neutral_audit(model, callback) == 1.25
|
||||
assert {name: child.training for name, child in model.named_modules()} == before_modes
|
||||
assert torch.equal(torch.get_rng_state(), before_rng)
|
||||
assert all(torch.equal(model.state_dict()[name], value) for name, value in before_state.items())
|
||||
|
||||
|
||||
def _objective(residual: torch.Tensor) -> study.ObjectiveResult:
|
||||
# Deliberately use every coordinate so a candidate is not a mock echo.
|
||||
return study.ObjectiveResult(
|
||||
loss=float(torch.sum(residual.square()).item()),
|
||||
samples=3,
|
||||
forward_passes=1,
|
||||
backward_passes=1,
|
||||
)
|
||||
|
||||
|
||||
def test_pso_and_random_have_exact_equal_query_and_sample_budgets() -> None:
|
||||
validation_calls: list[torch.Tensor] = []
|
||||
|
||||
def validation(residual: torch.Tensor) -> study.AuditResult:
|
||||
validation_calls.append(residual.detach().clone())
|
||||
return study.AuditResult(loss=float(residual.abs().mean()), samples=2)
|
||||
|
||||
pso = study.run_residual_pso(_objective, seed=study.SWARM_SEEDS[0], validation=validation)
|
||||
random = study.run_equal_budget_random(_objective, seed=study.SWARM_SEEDS[0])
|
||||
|
||||
for result in (pso, random):
|
||||
assert result.objective_queries == study.PARTICLE_COUNT * study.PSO_GENERATIONS == 720
|
||||
assert result.counters.objective_samples == 720 * 3
|
||||
assert result.counters.objective_forward_passes == 720
|
||||
assert result.counters.objective_backward_passes == 720
|
||||
assert result.counters.objective_failures == 0
|
||||
assert len(result.endpoints) == study.PSO_GENERATIONS
|
||||
assert len(result.trajectory) == study.PSO_GENERATIONS
|
||||
assert result.best_objective is not None
|
||||
assert result.best_residual is not None
|
||||
assert result.best_residual.shape == (study.RESIDUAL_DIMENSION,)
|
||||
|
||||
assert len(validation_calls) == len(study.OBJECTIVE_CHECKPOINTS)
|
||||
assert pso.counters.validation_evaluations == len(study.OBJECTIVE_CHECKPOINTS)
|
||||
assert pso.counters.validation_samples == len(study.OBJECTIVE_CHECKPOINTS) * 2
|
||||
assert random.method == "feature_random"
|
||||
assert pso.method == "feature_pso"
|
||||
|
||||
|
||||
def test_state_machine_and_confirmation_seal_boundaries(tmp_path: Path) -> None:
|
||||
config = study.StudyConfig()
|
||||
state = study.prepare_run(tmp_path, config)
|
||||
with pytest.raises(study.StateTransitionError):
|
||||
state.transition(study.StudyState.FROZEN)
|
||||
state.transition(study.StudyState.DEVELOPING)
|
||||
artifact = tmp_path / "evidence.json"
|
||||
study.atomic_write_json(artifact, {"metric": 1.0})
|
||||
|
||||
manifest = study.freeze_run(tmp_path, config, ["evidence.json"], state)
|
||||
assert state.state is study.StudyState.FROZEN
|
||||
assert study.load_frozen_manifest(tmp_path).manifest_hash == manifest.manifest_hash
|
||||
with pytest.raises(study.StateTransitionError):
|
||||
state.transition(study.StudyState.DEVELOPING)
|
||||
|
||||
study.begin_confirmation(tmp_path, state)
|
||||
assert state.state is study.StudyState.CONFIRMING
|
||||
study.finish_confirmation(state, success=True)
|
||||
assert state.state is study.StudyState.COMPLETED
|
||||
with pytest.raises(study.StateTransitionError):
|
||||
study.finish_confirmation(state, success=True)
|
||||
|
||||
|
||||
def test_frozen_manifest_rejects_artifact_hash_drift(tmp_path: Path) -> None:
|
||||
config = study.StudyConfig()
|
||||
state = study.prepare_run(tmp_path, config)
|
||||
state.transition(study.StudyState.DEVELOPING)
|
||||
artifact = tmp_path / "checkpoint.bin"
|
||||
artifact.write_bytes(b"original")
|
||||
study.freeze_run(tmp_path, config, [artifact.name], state)
|
||||
assert study.verify_frozen_manifest(tmp_path).artifacts[artifact.name]
|
||||
|
||||
artifact.write_bytes(b"tampered")
|
||||
with pytest.raises(study.SealError, match="hash mismatch"):
|
||||
study.verify_frozen_manifest(tmp_path)
|
||||
|
||||
|
||||
def _completed_record() -> dict[str, float]:
|
||||
return {"loss": 1.0, "queries": 1}
|
||||
|
||||
|
||||
def _matrix_result(workload_id: str, artifact_name: str, artifact_hash: str) -> dict[str, object]:
|
||||
cell_tree = {
|
||||
str(base): {str(swarm): _completed_record() for swarm in study.SWARM_SEEDS}
|
||||
for base in study.BASE_SEEDS
|
||||
}
|
||||
return {
|
||||
"workload_id": workload_id,
|
||||
"family": "classification",
|
||||
"config": {},
|
||||
"manifests": {},
|
||||
"provenance": {},
|
||||
"baselines": {str(seed): _completed_record() for seed in study.BASE_SEEDS},
|
||||
"arms": {
|
||||
"feature_pso": cell_tree,
|
||||
"feature_random": copy.deepcopy(cell_tree),
|
||||
"feature_adam": {str(seed): _completed_record() for seed in study.BASE_SEEDS},
|
||||
"head_adam": {str(seed): _completed_record() for seed in study.BASE_SEEDS},
|
||||
},
|
||||
"ensemble": {
|
||||
"uniform": _completed_record(),
|
||||
"uniform_temperature": _completed_record(),
|
||||
"slsqp_weights": _completed_record(),
|
||||
"ensemble_pso": [_completed_record() for _ in study.SWARM_SEEDS],
|
||||
},
|
||||
"development_selection": {},
|
||||
"confirmation": {},
|
||||
"integrity": {"official_test_opened": False},
|
||||
"leakage_counters": {},
|
||||
"resource_ledger": {},
|
||||
"artifact_hashes": {artifact_name: artifact_hash},
|
||||
}
|
||||
|
||||
|
||||
def test_strict_matrix_validation_accepts_complete_matrix_and_rejects_missing_cell(tmp_path: Path) -> None:
|
||||
for workload_id in study.DEFAULT_WORKLOAD_IDS:
|
||||
workload_root = tmp_path / "workloads" / workload_id
|
||||
workload_root.mkdir(parents=True)
|
||||
evidence = workload_root / "evidence.bin"
|
||||
evidence.write_bytes(workload_id.encode())
|
||||
relative = str(evidence.relative_to(tmp_path))
|
||||
result = _matrix_result(workload_id, relative, study.fingerprint_file(evidence))
|
||||
(workload_root / "result.json").write_text(json.dumps(result), encoding="utf-8")
|
||||
|
||||
validated = study._validate_matrix_results(tmp_path, strict_development=True)
|
||||
assert set(validated) == set(study.DEFAULT_WORKLOAD_IDS)
|
||||
|
||||
path = tmp_path / "workloads" / study.DEFAULT_WORKLOAD_IDS[0] / "result.json"
|
||||
broken = json.loads(path.read_text(encoding="utf-8"))
|
||||
del broken["arms"]["feature_pso"]["501"]["601"]
|
||||
path.write_text(json.dumps(broken), encoding="utf-8")
|
||||
with pytest.raises(study.SealError, match="feature_pso matrix is incomplete"):
|
||||
study._validate_matrix_results(tmp_path, strict_development=True)
|
||||
|
||||
|
||||
def test_development_reuse_requires_complete_hash_verified_artifacts(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
workload_root = tmp_path / "workloads" / "synthetic"
|
||||
workload_root.mkdir(parents=True)
|
||||
artifact = workload_root / "evidence.bin"
|
||||
artifact.write_bytes(b"complete")
|
||||
relative = str(artifact.relative_to(tmp_path))
|
||||
result = _matrix_result(
|
||||
"synthetic",
|
||||
relative,
|
||||
study.fingerprint_file(artifact),
|
||||
)
|
||||
(workload_root / "result.json").write_text(
|
||||
json.dumps(result),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(workload_root / "development_reuse.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"protocol_version": study.PROTOCOL_VERSION,
|
||||
"source_run": "failed-but-preserved",
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
class ReusedAdapter:
|
||||
workload_id = "synthetic"
|
||||
|
||||
def run_phase(self, phase: str) -> object:
|
||||
raise AssertionError(f"unexpected phase: {phase}")
|
||||
|
||||
assert study._run_adapter_development(
|
||||
[ReusedAdapter()],
|
||||
tmp_path,
|
||||
) == [result]
|
||||
artifact.write_bytes(b"drift")
|
||||
with pytest.raises(study.SealError, match="hash drift"):
|
||||
study._run_adapter_development(
|
||||
[ReusedAdapter()],
|
||||
tmp_path,
|
||||
)
|
||||
@@ -0,0 +1,635 @@
|
||||
"""
|
||||
Unit tests for Post-Training PSO Ensemble Study Runner.
|
||||
|
||||
Covers:
|
||||
1. Protocol version and module export verification.
|
||||
2. CachedProbabilityEnsemble softmax parameterization, forward log-prob normalization, and NLLLoss integration.
|
||||
3. Probability cache validation for finite values, non-negativity, and row-sum normalization.
|
||||
4. Mixture probabilities for uniform and one-hot weight configurations across PyTorch and NumPy arrays.
|
||||
5. Probabilistic metrics computation (accuracy, NLL, Brier, ECE, margin).
|
||||
6. Analytical gradient vs central finite-difference gradient verification for simplex NLL.
|
||||
7. SLSQP solver optimization success, simplex constraint adherence, and NLL improvement.
|
||||
8. Deterministic PSO optimization and exact query/sample accounting on synthetic probability caches.
|
||||
9. Development gate boundary checks for safety, accounting, and quality limits.
|
||||
10. Production-runner enforcement of the official-test seal on development failure.
|
||||
11. Atomic file and CSV report writers.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Ensure test directory and repo root are in sys.path
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent if Path(__file__).resolve().parent.name != "PSO" else Path(__file__).resolve().parent
|
||||
TEST_DIR = REPO_ROOT / "test"
|
||||
if str(TEST_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TEST_DIR))
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import post_training_pso_ensemble as study_module
|
||||
|
||||
from post_training_pso_ensemble import (
|
||||
PROTOCOL_VERSION,
|
||||
CachedProbabilityEnsemble,
|
||||
CompactCNN,
|
||||
atomic_write_file,
|
||||
compute_model_fingerprint,
|
||||
evaluate_development_gates,
|
||||
fit_uniform_temperature,
|
||||
mixture_probabilities,
|
||||
optimize_slsqp_weights,
|
||||
probabilistic_metrics,
|
||||
run_pso_weights,
|
||||
save_csv_report,
|
||||
simplex_nll_and_grad,
|
||||
validate_probability_cache,
|
||||
)
|
||||
|
||||
|
||||
def test_protocol_version():
|
||||
"""Verify protocol version identifier adheres to required format."""
|
||||
assert isinstance(PROTOCOL_VERSION, str)
|
||||
assert PROTOCOL_VERSION.startswith("POST-TRAINING-PSO-ENSEMBLE")
|
||||
assert "1.1.0" in PROTOCOL_VERSION or "1.0.0" in PROTOCOL_VERSION
|
||||
|
||||
|
||||
def test_cached_probability_ensemble_weights_and_forward():
|
||||
"""Verify CachedProbabilityEnsemble parameterization, weight normalization, and log-probability output."""
|
||||
ensemble = CachedProbabilityEnsemble(num_members=5)
|
||||
weights = ensemble.weights()
|
||||
|
||||
assert isinstance(weights, torch.Tensor)
|
||||
assert weights.shape == (5,)
|
||||
assert torch.allclose(weights.sum(), torch.tensor(1.0), atol=1e-6)
|
||||
assert (weights >= 0).all()
|
||||
|
||||
# Custom weight initialization
|
||||
init_w = torch.tensor([2.0, 0.0, 0.0, 0.0, 0.0])
|
||||
ensemble_custom = CachedProbabilityEnsemble(num_members=5, init_weights=init_w)
|
||||
assert torch.allclose(ensemble_custom.raw_weights, init_w)
|
||||
|
||||
# Invalid init shape
|
||||
with pytest.raises(ValueError, match="init_weights must have shape"):
|
||||
CachedProbabilityEnsemble(num_members=5, init_weights=torch.tensor([1.0, 2.0]))
|
||||
|
||||
# CachedProbabilityEnsemble has one canonical input shape: (N, M, K).
|
||||
N, K = 100, 10
|
||||
torch.manual_seed(42)
|
||||
raw_probs = torch.rand(5, N, K)
|
||||
member_probs_mnk = raw_probs / raw_probs.sum(dim=-1, keepdim=True)
|
||||
member_probs_nmk = member_probs_mnk.transpose(0, 1)
|
||||
|
||||
log_probs = ensemble(member_probs_nmk)
|
||||
assert log_probs.shape == (N, K)
|
||||
|
||||
# Verify exponentiated log probabilities sum to 1 per sample.
|
||||
probs = torch.exp(log_probs)
|
||||
assert torch.allclose(probs.sum(dim=-1), torch.ones(N), atol=1e-5)
|
||||
|
||||
# Integration with nn.NLLLoss.
|
||||
targets = torch.randint(0, K, (N,))
|
||||
loss = nn.NLLLoss()(log_probs, targets)
|
||||
assert loss.dim() == 0
|
||||
assert torch.isfinite(loss)
|
||||
assert loss.item() > 0.0
|
||||
|
||||
# Reject the alternate (M, N, K) orientation instead of guessing.
|
||||
with pytest.raises(ValueError, match="canonical"):
|
||||
ensemble(member_probs_mnk)
|
||||
|
||||
# Square N == M caches remain unambiguous because the model always weights
|
||||
# axis 1 and mixture_probabilities always weights axis 0.
|
||||
square_raw = torch.arange(1, 51, dtype=torch.float32).reshape(5, 5, 2)
|
||||
square_mnk = square_raw / square_raw.sum(dim=-1, keepdim=True)
|
||||
raw_logits = torch.tensor([1.5, -0.5, 0.2, 0.8, -1.0])
|
||||
square_ensemble = CachedProbabilityEnsemble(5, init_weights=raw_logits)
|
||||
actual_square = torch.exp(square_ensemble(square_mnk.transpose(0, 1)))
|
||||
expected_square = mixture_probabilities(
|
||||
torch.softmax(raw_logits, dim=0),
|
||||
square_mnk,
|
||||
)
|
||||
assert torch.allclose(actual_square, expected_square, atol=1e-6)
|
||||
|
||||
# Malformed dimension or member count mismatch.
|
||||
with pytest.raises(ValueError):
|
||||
ensemble(torch.rand(N, K))
|
||||
with pytest.raises(ValueError, match="canonical"):
|
||||
ensemble(torch.rand(N, 3, K))
|
||||
|
||||
|
||||
def test_validate_probability_cache():
|
||||
"""Verify probability cache validation logic for valid, negative, unnormalized, and non-finite cases."""
|
||||
N, K = 50, 10
|
||||
raw = torch.rand(5, N, K)
|
||||
valid_tensor = raw / raw.sum(dim=-1, keepdim=True)
|
||||
|
||||
assert validate_probability_cache(valid_tensor) is True
|
||||
assert validate_probability_cache(valid_tensor.numpy()) is True
|
||||
|
||||
# Negative values
|
||||
invalid_neg = valid_tensor.clone()
|
||||
invalid_neg[0, 0, 0] = -0.05
|
||||
assert validate_probability_cache(invalid_neg) is False
|
||||
|
||||
# Unnormalized (row sum != 1.0)
|
||||
invalid_unnorm = valid_tensor.clone()
|
||||
invalid_unnorm[0, 0, :] *= 0.5
|
||||
assert validate_probability_cache(invalid_unnorm) is False
|
||||
|
||||
# Non-finite values
|
||||
invalid_nan = valid_tensor.clone()
|
||||
invalid_nan[0, 0, 0] = float("nan")
|
||||
assert validate_probability_cache(invalid_nan) is False
|
||||
|
||||
|
||||
def test_mixture_probabilities_uniform_and_one_hot():
|
||||
"""Verify mixture_probabilities for uniform and one-hot weight configurations."""
|
||||
M, N, K = 5, 40, 10
|
||||
rng = np.random.RandomState(42)
|
||||
raw = rng.rand(M, N, K)
|
||||
member_probs_np = raw / raw.sum(axis=-1, keepdims=True)
|
||||
member_probs_torch = torch.from_numpy(member_probs_np).float()
|
||||
|
||||
# 1. Uniform weights [0.2, 0.2, 0.2, 0.2, 0.2]
|
||||
uniform_w = np.full(M, 0.2)
|
||||
mix_uniform_np = mixture_probabilities(uniform_w, member_probs_np)
|
||||
expected_uniform = member_probs_np.mean(axis=0)
|
||||
assert np.allclose(mix_uniform_np, expected_uniform, atol=1e-6)
|
||||
|
||||
mix_uniform_torch = mixture_probabilities(uniform_w, member_probs_torch)
|
||||
assert torch.allclose(mix_uniform_torch, torch.from_numpy(expected_uniform).float(), atol=1e-5)
|
||||
|
||||
# 2. One-hot weights [1.0, 0.0, 0.0, 0.0, 0.0]
|
||||
onehot_0 = np.array([1.0, 0.0, 0.0, 0.0, 0.0])
|
||||
mix_onehot_0 = mixture_probabilities(onehot_0, member_probs_np)
|
||||
assert np.allclose(mix_onehot_0, member_probs_np[0], atol=1e-6)
|
||||
|
||||
# 3. One-hot weights for model index 2
|
||||
onehot_2 = np.array([0.0, 0.0, 1.0, 0.0, 0.0])
|
||||
mix_onehot_2 = mixture_probabilities(onehot_2, member_probs_np)
|
||||
assert np.allclose(mix_onehot_2, member_probs_np[2], atol=1e-6)
|
||||
|
||||
# Alternate (N, M, K) orientation is rejected rather than guessed.
|
||||
transposed_np = member_probs_np.transpose(1, 0, 2)
|
||||
with pytest.raises(ValueError, match="canonical"):
|
||||
mixture_probabilities(uniform_w, transposed_np)
|
||||
|
||||
# Dimension mismatch
|
||||
with pytest.raises(ValueError):
|
||||
mixture_probabilities(np.array([0.5, 0.5]), member_probs_np)
|
||||
|
||||
|
||||
def test_probabilistic_metrics():
|
||||
"""Verify calculation of accuracy, NLL, Brier, ECE, and margin metrics."""
|
||||
N, K = 100, 10
|
||||
targets = np.random.RandomState(42).randint(0, K, size=N)
|
||||
|
||||
# Perfect prediction: prob=1.0 at true target index
|
||||
perfect_probs = np.zeros((N, K), dtype=np.float64)
|
||||
perfect_probs[np.arange(N), targets] = 1.0
|
||||
|
||||
metrics_perfect = probabilistic_metrics(perfect_probs, targets)
|
||||
assert metrics_perfect["accuracy"] == 100.0
|
||||
assert metrics_perfect["nll"] < 1e-4
|
||||
assert metrics_perfect["brier"] < 1e-4
|
||||
assert metrics_perfect["ece"] < 1e-4
|
||||
assert metrics_perfect["margin"] == 1.0
|
||||
|
||||
# Uniform prediction (1/K per class)
|
||||
uniform_probs = np.full((N, K), 1.0 / K, dtype=np.float64)
|
||||
metrics_uniform = probabilistic_metrics(uniform_probs, targets)
|
||||
expected_nll = -np.log(1.0 / K)
|
||||
assert np.isclose(metrics_uniform["nll"], expected_nll, atol=1e-3)
|
||||
assert metrics_uniform["margin"] == 0.0
|
||||
|
||||
|
||||
def test_simplex_nll_and_grad_vs_finite_difference():
|
||||
"""Verify analytical simplex NLL gradient against central finite differences."""
|
||||
M, N, K = 5, 200, 10
|
||||
rng = np.random.RandomState(101)
|
||||
raw = rng.rand(M, N, K)
|
||||
member_probs = raw / raw.sum(axis=-1, keepdims=True)
|
||||
targets = rng.randint(0, K, size=N)
|
||||
|
||||
weights = np.array([0.3, 0.2, 0.1, 0.25, 0.15], dtype=np.float64)
|
||||
nll_analytical, grad_analytical = simplex_nll_and_grad(weights, member_probs, targets)
|
||||
|
||||
assert np.isfinite(nll_analytical)
|
||||
assert grad_analytical.shape == (M,)
|
||||
assert np.all(np.isfinite(grad_analytical))
|
||||
|
||||
# Numerical gradient computation via central finite differences
|
||||
h = 1e-6
|
||||
grad_numerical = np.zeros(M, dtype=np.float64)
|
||||
for i in range(M):
|
||||
w_plus = weights.copy()
|
||||
w_plus[i] += h
|
||||
nll_plus, _ = simplex_nll_and_grad(w_plus, member_probs, targets)
|
||||
|
||||
w_minus = weights.copy()
|
||||
w_minus[i] -= h
|
||||
nll_minus, _ = simplex_nll_and_grad(w_minus, member_probs, targets)
|
||||
|
||||
grad_numerical[i] = (nll_plus - nll_minus) / (2.0 * h)
|
||||
|
||||
assert np.allclose(grad_analytical, grad_numerical, atol=1e-4)
|
||||
|
||||
|
||||
def test_optimize_slsqp_weights():
|
||||
"""Verify SLSQP solver optimization success, simplex adherence, and NLL non-regression."""
|
||||
M, N, K = 5, 300, 10
|
||||
rng = np.random.RandomState(202)
|
||||
raw = rng.rand(M, N, K)
|
||||
member_probs = raw / raw.sum(axis=-1, keepdims=True)
|
||||
targets = rng.randint(0, K, size=N)
|
||||
|
||||
# Make member 0 slightly better to give SLSQP a clear target
|
||||
member_probs[0, np.arange(N), targets] += 0.5
|
||||
member_probs = member_probs / member_probs.sum(axis=-1, keepdims=True)
|
||||
|
||||
result = optimize_slsqp_weights(member_probs, targets)
|
||||
|
||||
assert result["success"] is True
|
||||
assert len(result["weights"]) == M
|
||||
weights = np.array(result["weights"])
|
||||
assert np.all(weights >= 0.0)
|
||||
assert np.isclose(weights.sum(), 1.0, atol=1e-6)
|
||||
|
||||
# Verify optimized NLL is no worse than uniform ensemble NLL
|
||||
uniform_p = mixture_probabilities(np.full(M, 1.0 / M), member_probs)
|
||||
uniform_nll = probabilistic_metrics(uniform_p, targets)["nll"]
|
||||
assert result["metrics"]["nll"] <= uniform_nll + 1e-6
|
||||
assert result["evaluations"] > 0
|
||||
assert result["wall_time_seconds"] >= 0.0
|
||||
|
||||
|
||||
def test_run_pso_weights_determinism_and_accounting():
|
||||
"""Verify PSO weight optimization determinism, exact accounting, and output structure."""
|
||||
M, N, K = 5, 100, 10
|
||||
rng = np.random.RandomState(303)
|
||||
raw = rng.rand(M, N, K)
|
||||
member_probs = raw / raw.sum(axis=-1, keepdims=True)
|
||||
targets = rng.randint(0, K, size=N)
|
||||
|
||||
swarm_seeds = [301, 302]
|
||||
res_1 = run_pso_weights(member_probs, targets, swarm_seeds=swarm_seeds, device="cpu")
|
||||
|
||||
# Accounting verification
|
||||
assert res_1["queries_per_seed"] == 900
|
||||
assert res_1["sample_evaluations_per_seed"] == 900 * N
|
||||
assert res_1["total_queries"] == 900 * len(swarm_seeds)
|
||||
assert res_1["total_sample_evaluations"] == 900 * N * len(swarm_seeds)
|
||||
|
||||
per_seed = res_1["per_seed_runs"]
|
||||
assert len(per_seed) == len(swarm_seeds)
|
||||
for run_rec in per_seed:
|
||||
assert run_rec["queries"] == 900
|
||||
assert run_rec["sample_evaluations"] == 900 * N
|
||||
assert np.isclose(sum(run_rec["weights"]), 1.0, atol=1e-5)
|
||||
assert run_rec["wall_time_seconds"] >= 0.0
|
||||
|
||||
# Repeatability / Determinism check
|
||||
res_2 = run_pso_weights(member_probs, targets, swarm_seeds=swarm_seeds, device="cpu")
|
||||
assert res_1["selected_seed"] == res_2["selected_seed"]
|
||||
assert np.allclose(res_1["selected_weights"], res_2["selected_weights"], atol=1e-5)
|
||||
assert np.isclose(
|
||||
res_1["per_seed_runs"][0]["metrics"]["nll"],
|
||||
res_2["per_seed_runs"][0]["metrics"]["nll"],
|
||||
atol=1e-5,
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
def test_evaluate_development_gates_pass_and_boundary_failures():
|
||||
"""Verify development gate boundary evaluations across passing and failing synthetic workloads."""
|
||||
def make_valid_workload(seed_nll=1.5, pso_nll=1.0, pso_acc=90.0, slsqp_nll=1.0):
|
||||
def make_mets(nll_val, acc_val):
|
||||
return {"accuracy": acc_val, "nll": nll_val, "brier": 0.15, "ece": 0.02, "margin": 0.5}
|
||||
|
||||
return {
|
||||
"provenance": {"dataset_name": "mnist"},
|
||||
"training": {"adam_pool_wall_time_seconds": 100.0},
|
||||
"validation_cache": {
|
||||
"pool_forward_passes": 5,
|
||||
"base_cnn_forward_passes_during_optimization": 0,
|
||||
},
|
||||
"official_test_data_loaded_before_freeze": False,
|
||||
"official_test_evaluations_before_freeze": 0,
|
||||
"validation": {
|
||||
"methods": {
|
||||
"reference_single_10e": make_mets(seed_nll, 80.0),
|
||||
"best_single_10e": make_mets(1.4, 82.0),
|
||||
"single_50e": make_mets(1.1, 88.0),
|
||||
"uniform_ensemble": make_mets(1.05, 89.9),
|
||||
"uniform_temperature": {
|
||||
"weights": [0.2, 0.2, 0.2, 0.2, 0.2],
|
||||
"metrics": make_mets(1.04, 90.0),
|
||||
},
|
||||
"slsqp_weights": {
|
||||
"weights": [0.2, 0.2, 0.2, 0.2, 0.2],
|
||||
"success": True,
|
||||
"metrics": make_mets(slsqp_nll, 90.0),
|
||||
},
|
||||
"pso_weights": {
|
||||
"selected_seed": 301,
|
||||
"selected_weights": [0.2, 0.2, 0.2, 0.2, 0.2],
|
||||
"metrics": make_mets(pso_nll, pso_acc),
|
||||
"median_one_seed_wall_time_seconds": 2.0,
|
||||
"per_seed_runs": [
|
||||
{
|
||||
"seed": 301,
|
||||
"queries": 900,
|
||||
"sample_evaluations": 9000000,
|
||||
"metrics": make_mets(pso_nll, pso_acc),
|
||||
"weights": [0.2, 0.2, 0.2, 0.2, 0.2],
|
||||
},
|
||||
{
|
||||
"seed": 302,
|
||||
"queries": 900,
|
||||
"sample_evaluations": 9000000,
|
||||
"metrics": make_mets(pso_nll + 0.01, pso_acc),
|
||||
"weights": [0.2, 0.2, 0.2, 0.2, 0.2],
|
||||
},
|
||||
{
|
||||
"seed": 303,
|
||||
"queries": 900,
|
||||
"sample_evaluations": 9000000,
|
||||
"metrics": make_mets(pso_nll + 0.02, pso_acc),
|
||||
"weights": [0.2, 0.2, 0.2, 0.2, 0.2],
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
valid_workloads = {
|
||||
"mnist": make_valid_workload(),
|
||||
"fashion_mnist": make_valid_workload(),
|
||||
}
|
||||
|
||||
eval_pass = evaluate_development_gates(valid_workloads)
|
||||
assert eval_pass["pass"] is True
|
||||
assert eval_pass["failed_hard_gate_count"] == 0
|
||||
assert len(eval_pass["gate_results"]) == 13
|
||||
|
||||
# Assert exact expected gate names
|
||||
expected_gate_names = {
|
||||
"all_values_finite",
|
||||
"validation_pool_forward_passes_exact",
|
||||
"optimization_base_model_forward_passes",
|
||||
"official_test_data_loaded_before_freeze",
|
||||
"slsqp_solver_success",
|
||||
"query_and_sample_accounting_exact",
|
||||
"maximum_pso_nll_regression_vs_uniform",
|
||||
"maximum_pso_accuracy_regression_vs_uniform_pp",
|
||||
"pso_nll_below_reference_single",
|
||||
"maximum_pso_nll_regression_vs_equal_budget_single",
|
||||
"maximum_relative_pso_nll_gap_vs_slsqp",
|
||||
"cross_dataset_mean_relative_pso_nll_reduction_vs_uniform_minimum",
|
||||
"maximum_median_one_seed_pso_to_pool_training_wall_ratio",
|
||||
}
|
||||
assert set(eval_pass["gate_results"].keys()) == expected_gate_names
|
||||
|
||||
# 1. Test data loaded before freeze failure
|
||||
leak_workloads = {
|
||||
"mnist": make_valid_workload(),
|
||||
"fashion_mnist": make_valid_workload(),
|
||||
}
|
||||
leak_workloads["mnist"]["official_test_data_loaded_before_freeze"] = True
|
||||
assert evaluate_development_gates(leak_workloads)["pass"] is False
|
||||
|
||||
# 2. PSO accuracy regression > 0.1 pp below uniform
|
||||
acc_fail_workloads = {
|
||||
"mnist": make_valid_workload(pso_acc=89.0), # Uniform is 89.9
|
||||
"fashion_mnist": make_valid_workload(),
|
||||
}
|
||||
assert evaluate_development_gates(acc_fail_workloads)["pass"] is False
|
||||
|
||||
# 3. Base model called during optimization
|
||||
base_call_fail_workloads = {
|
||||
"mnist": make_valid_workload(),
|
||||
"fashion_mnist": make_valid_workload(),
|
||||
}
|
||||
base_call_fail_workloads["mnist"]["validation_cache"][
|
||||
"base_cnn_forward_passes_during_optimization"
|
||||
] = 1
|
||||
assert evaluate_development_gates(base_call_fail_workloads)["pass"] is False
|
||||
|
||||
|
||||
def test_global_test_seal_monkeypatch(monkeypatch, tmp_path):
|
||||
"""A failed production development run must never construct train=False data."""
|
||||
import torchvision.datasets
|
||||
|
||||
official_constructor_calls = []
|
||||
|
||||
def guarded_dataset(*args, **kwargs):
|
||||
train = kwargs.get("train", True)
|
||||
official_constructor_calls.append(train)
|
||||
if train is False:
|
||||
raise RuntimeError("Leakage blocked: train=False requested before pass")
|
||||
raise AssertionError("Synthetic split setup must bypass train=True constructors")
|
||||
|
||||
monkeypatch.setattr(torchvision.datasets, "MNIST", guarded_dataset)
|
||||
monkeypatch.setattr(torchvision.datasets, "FashionMNIST", guarded_dataset)
|
||||
|
||||
class TinyCNN(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.logits = nn.Parameter(torch.zeros(10))
|
||||
|
||||
def forward(self, x):
|
||||
return self.logits.unsqueeze(0).expand(len(x), -1)
|
||||
|
||||
def fake_prepare(dataset_name, split_seed, cache_dir):
|
||||
x = torch.zeros(1, 1, 28, 28)
|
||||
y = torch.zeros(1, dtype=torch.long)
|
||||
return x, y, x.clone(), y.clone(), {
|
||||
"dataset_name": dataset_name,
|
||||
"split_seed": split_seed,
|
||||
"search_samples": 1,
|
||||
"validation_samples": 1,
|
||||
"normalization": {"mean": 0.0, "std": 1.0},
|
||||
"data_fingerprint": "synthetic",
|
||||
"split_fingerprint": "synthetic",
|
||||
}
|
||||
|
||||
def fake_probabilities(model, x_data, device, batch_size=1000):
|
||||
with torch.no_grad():
|
||||
return torch.softmax(model(x_data.to(device)), dim=1).cpu(), 0.0
|
||||
|
||||
def fake_temperature(uniform_probs, targets):
|
||||
metrics = probabilistic_metrics(uniform_probs, targets)
|
||||
return 1.0, {
|
||||
"fitted_temperature": 1.0,
|
||||
"wall_time_seconds": 0.0,
|
||||
"evaluations": 1,
|
||||
"metrics": metrics,
|
||||
}
|
||||
|
||||
def fake_slsqp(member_probabilities, targets):
|
||||
weights = [0.2] * 5
|
||||
metrics = probabilistic_metrics(
|
||||
mixture_probabilities(weights, member_probabilities),
|
||||
targets,
|
||||
)
|
||||
return {
|
||||
"weights": weights,
|
||||
"evaluations": 1,
|
||||
"wall_time_seconds": 0.0,
|
||||
"success": True,
|
||||
"message": "synthetic",
|
||||
"metrics": metrics,
|
||||
}
|
||||
|
||||
def fake_pso(
|
||||
member_probabilities,
|
||||
targets,
|
||||
swarm_seeds,
|
||||
particles,
|
||||
epochs,
|
||||
device,
|
||||
):
|
||||
weights = [0.2] * 5
|
||||
metrics = probabilistic_metrics(
|
||||
mixture_probabilities(weights, member_probabilities),
|
||||
targets,
|
||||
)
|
||||
queries = particles * epochs
|
||||
samples = queries * len(targets)
|
||||
runs = [
|
||||
{
|
||||
"seed": seed,
|
||||
"queries": queries,
|
||||
"sample_evaluations": samples,
|
||||
"wall_time_seconds": 0.0,
|
||||
"metrics": metrics,
|
||||
"weights": weights,
|
||||
}
|
||||
for seed in swarm_seeds
|
||||
]
|
||||
return {
|
||||
"per_seed_runs": runs,
|
||||
"selected_seed": swarm_seeds[0],
|
||||
"selected_weights": weights,
|
||||
"metrics": metrics,
|
||||
"queries_per_seed": queries,
|
||||
"sample_evaluations_per_seed": samples,
|
||||
"total_queries": queries * len(swarm_seeds),
|
||||
"total_sample_evaluations": samples * len(swarm_seeds),
|
||||
"median_one_seed_wall_time_seconds": 0.0,
|
||||
"total_wall_time_seconds": 0.0,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(study_module, "CompactCNN", TinyCNN)
|
||||
monkeypatch.setattr(study_module, "prepare_dataset_splits", fake_prepare)
|
||||
monkeypatch.setattr(study_module, "get_model_probabilities", fake_probabilities)
|
||||
monkeypatch.setattr(study_module, "fit_uniform_temperature", fake_temperature)
|
||||
monkeypatch.setattr(study_module, "optimize_slsqp_weights", fake_slsqp)
|
||||
monkeypatch.setattr(study_module, "run_pso_weights", fake_pso)
|
||||
monkeypatch.setattr(
|
||||
study_module,
|
||||
"evaluate_development_gates",
|
||||
lambda workloads: {
|
||||
"pass": False,
|
||||
"failed_hard_gate_count": 1,
|
||||
"gate_results": {"synthetic_failure": False},
|
||||
"issues": ["forced development failure"],
|
||||
},
|
||||
)
|
||||
monkeypatch.setattr(study_module, "save_csv_report", lambda *args: None)
|
||||
monkeypatch.setattr(study_module, "save_publication_plot", lambda *args: None)
|
||||
|
||||
artifact = study_module.run_post_training_study(
|
||||
cache_dir=tmp_path / "cache",
|
||||
device="cpu",
|
||||
output_json=tmp_path / "study.json",
|
||||
output_csv=tmp_path / "study.csv",
|
||||
output_png=tmp_path / "study.png",
|
||||
)
|
||||
|
||||
assert artifact["development_pass"] is False
|
||||
assert artifact["official_test_data_loaded"] is False
|
||||
assert all(
|
||||
workload["confirmation"] is None
|
||||
for workload in artifact["workloads"].values()
|
||||
)
|
||||
assert official_constructor_calls == []
|
||||
|
||||
|
||||
def test_atomic_writers(tmp_path):
|
||||
"""Verify atomic writing and CSV output formatting."""
|
||||
target_file = tmp_path / "report.csv"
|
||||
content = "header1,header2\nval1,val2\n"
|
||||
|
||||
atomic_write_file(target_file, content)
|
||||
assert target_file.exists()
|
||||
assert target_file.read_text() == content
|
||||
|
||||
# Test overwrite
|
||||
new_content = "header1,header2\nval3,val4\n"
|
||||
atomic_write_file(target_file, new_content)
|
||||
assert target_file.read_text() == new_content
|
||||
|
||||
# Synthetic artifact CSV generation
|
||||
def make_mets(acc, nll):
|
||||
return {"accuracy": acc, "nll": nll, "brier": 0.15, "ece": 0.02, "margin": 0.5}
|
||||
|
||||
artifact = {
|
||||
"protocol_version": PROTOCOL_VERSION,
|
||||
"policy_frozen": True,
|
||||
"development_pass": True,
|
||||
"official_test_data_loaded": True,
|
||||
"resource_totals": {
|
||||
"total_pso_queries": 5400,
|
||||
"total_pso_sample_evaluations": 54000000,
|
||||
"total_pso_wall_time_seconds": 12.5,
|
||||
},
|
||||
"workloads": {
|
||||
"mnist": {
|
||||
"validation": {
|
||||
"methods": {
|
||||
"reference_single_10e": make_mets(85.0, 0.50),
|
||||
"best_single_10e": make_mets(87.0, 0.45),
|
||||
"single_50e": make_mets(89.0, 0.40),
|
||||
"uniform_ensemble": make_mets(89.9, 0.36),
|
||||
"uniform_temperature": {
|
||||
"weights": [0.2, 0.2, 0.2, 0.2, 0.2],
|
||||
"metrics": make_mets(90.0, 0.355),
|
||||
},
|
||||
"slsqp_weights": {
|
||||
"weights": [0.2, 0.2, 0.2, 0.2, 0.2],
|
||||
"wall_time_seconds": 0.5,
|
||||
"metrics": make_mets(90.0, 0.35),
|
||||
},
|
||||
"pso_weights": {
|
||||
"selected_seed": 301,
|
||||
"selected_weights": [0.2, 0.2, 0.2, 0.2, 0.2],
|
||||
"metrics": make_mets(92.5, 0.25),
|
||||
"median_one_seed_wall_time_seconds": 2.0,
|
||||
"per_seed_runs": [
|
||||
{
|
||||
"seed": 301,
|
||||
"metrics": make_mets(92.5, 0.25),
|
||||
"weights": [0.2, 0.2, 0.2, 0.2, 0.2],
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
csv_path = tmp_path / "summary.csv"
|
||||
save_csv_report(artifact, csv_path)
|
||||
assert csv_path.exists()
|
||||
lines = csv_path.read_text().splitlines()
|
||||
assert len(lines) >= 2
|
||||
assert "Workload,Phase,Method" in lines[0]
|
||||
@@ -0,0 +1,242 @@
|
||||
"""Behavioral tests for the offline CIFAR/ResNet convergence adapter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import builtins
|
||||
import copy
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from test import post_training_resnet_convergence as resnet # noqa: E402
|
||||
from test.post_training_model_convergence import ( # noqa: E402
|
||||
CandidateEndpoint,
|
||||
ObjectiveResult,
|
||||
ProtocolError,
|
||||
SelectedResidualCodec,
|
||||
StudyConfig,
|
||||
prepare_run,
|
||||
select_endpoint,
|
||||
)
|
||||
|
||||
|
||||
class _TinyBlock(nn.Module):
|
||||
def __init__(self, channels: int = 2) -> None:
|
||||
super().__init__()
|
||||
self.conv = nn.Conv2d(channels, channels, kernel_size=1)
|
||||
self.bn = nn.BatchNorm2d(channels)
|
||||
|
||||
def forward(self, value: torch.Tensor) -> torch.Tensor:
|
||||
return F.relu(self.bn(self.conv(value)))
|
||||
|
||||
|
||||
class _TinyResNet(nn.Module):
|
||||
"""Small module with the same prefix/layer4/suffix contract as ResNet."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.conv1 = nn.Conv2d(3, 2, kernel_size=3, padding=1, bias=False)
|
||||
self.bn1 = nn.BatchNorm2d(2)
|
||||
self.relu = nn.ReLU()
|
||||
self.maxpool = nn.Identity()
|
||||
self.layer1 = nn.Sequential(_TinyBlock())
|
||||
self.layer2 = nn.Sequential(_TinyBlock())
|
||||
self.layer3 = nn.Sequential(_TinyBlock())
|
||||
self.layer4 = nn.Sequential(_TinyBlock(), _TinyBlock())
|
||||
self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
|
||||
self.fc = nn.Linear(2, 3)
|
||||
def forward(self, value: torch.Tensor) -> torch.Tensor:
|
||||
value = self.maxpool(self.relu(self.bn1(self.conv1(value))))
|
||||
value = self.layer1(value)
|
||||
value = self.layer2(value)
|
||||
value = self.layer3(value)
|
||||
value = self.layer4(value)
|
||||
return self.fc(torch.flatten(self.avgpool(value), 1))
|
||||
|
||||
|
||||
def _synthetic_cifar() -> tuple[np.ndarray, np.ndarray, tuple[int, int]]:
|
||||
"""Make valid-shaped, deterministic pixels without constructing a dataset."""
|
||||
count = resnet.TRAIN_SAMPLES
|
||||
labels = np.repeat(np.arange(10, dtype=np.int64), count // 10)
|
||||
images = np.zeros((count, 32, 32, 3), dtype=np.uint8)
|
||||
encoded = np.arange(count, dtype=np.uint32).view(np.uint8).reshape(count, 4)
|
||||
images[:, 0, 0, :] = encoded[:, :3]
|
||||
rng = np.random.default_rng(20260908)
|
||||
initial_assignment = np.full(count, "", dtype=object)
|
||||
for cls in range(10):
|
||||
members = np.flatnonzero(labels == cls)
|
||||
members = members[rng.permutation(len(members))]
|
||||
initial_assignment[members[:3500]] = "bp_train"
|
||||
initial_assignment[members[3500:4000]] = "refine_search"
|
||||
initial_assignment[members[4000:5000]] = "selection_val"
|
||||
first = 0
|
||||
second = next(
|
||||
index
|
||||
for index in range(1, count // 10)
|
||||
if initial_assignment[index] != initial_assignment[first]
|
||||
)
|
||||
images[second] = images[first]
|
||||
return images, labels, (first, second)
|
||||
|
||||
|
||||
def test_cifar_manifest_is_deterministic_disjoint_and_group_safe() -> None:
|
||||
images, labels, duplicate_pair = _synthetic_cifar()
|
||||
first = resnet.build_cifar_manifests(images, labels, split_seed=20260908)
|
||||
second = resnet.build_cifar_manifests(images, labels, split_seed=20260908)
|
||||
|
||||
assert first == second
|
||||
roles = first["roles"]
|
||||
role_sets = {role: set(indices) for role, indices in roles.items()}
|
||||
assert sum(len(indices) for indices in role_sets.values()) == len(labels)
|
||||
for role, values in role_sets.items():
|
||||
for other, other_values in role_sets.items():
|
||||
if role != other:
|
||||
assert values.isdisjoint(other_values)
|
||||
assert set().union(*role_sets.values()) == set(range(len(labels)))
|
||||
|
||||
owner_roles = [role for role, values in role_sets.items() if duplicate_pair[0] in values]
|
||||
assert len(owner_roles) == 1
|
||||
assert duplicate_pair[1] in role_sets[owner_roles[0]]
|
||||
assert set(first["objective"]).issubset(role_sets["refine_search"])
|
||||
assert len(first["objective"]) == resnet.OBJECTIVE_SAMPLES
|
||||
assert len(set(first["objective"])) == resnet.OBJECTIVE_SAMPLES
|
||||
objective_labels = labels[np.asarray(first["objective"])]
|
||||
assert np.bincount(objective_labels, minlength=10).tolist() == [103, 103, 103, 103, 102, 102, 102, 102, 102, 102]
|
||||
assert first["normalization_scope"] == "bp_train_only"
|
||||
|
||||
|
||||
def test_real_resnet_selected_suffix_topology_without_downloads() -> None:
|
||||
try:
|
||||
import torchvision # noqa: F401
|
||||
except Exception as exc: # torchvision is optional on lightweight CI workers.
|
||||
pytest.skip(f"torchvision unavailable: {exc}")
|
||||
|
||||
for architecture, block in (("resnet18", "layer4.1"), ("resnet50", "layer4.2")):
|
||||
model = resnet.make_cifar_resnet(architecture, seed=501)
|
||||
assert model.conv1.in_channels == 3
|
||||
assert model.conv1.out_channels == 64
|
||||
assert model.conv1.kernel_size == (3, 3)
|
||||
assert model.conv1.stride == (1, 1)
|
||||
assert isinstance(model.maxpool, nn.Identity)
|
||||
|
||||
names = resnet.selected_parameter_names(model, architecture)
|
||||
expected = tuple(
|
||||
name
|
||||
for name, parameter in model.named_parameters()
|
||||
if name.startswith(block + ".") and parameter.is_floating_point()
|
||||
)
|
||||
assert names == expected
|
||||
assert names and all(name.startswith(block + ".") for name in names)
|
||||
assert resnet.head_parameter_names(model) == ("fc.weight", "fc.bias")
|
||||
|
||||
|
||||
def test_cached_suffix_parity_and_residual_zero_nonzero_restoration() -> None:
|
||||
torch.manual_seed(7)
|
||||
model = _TinyResNet()
|
||||
images = torch.randn(5, 3, 8, 8)
|
||||
labels = torch.tensor([0, 1, 2, 1, 0])
|
||||
model.eval()
|
||||
cache = resnet.ResNetCache.build(model, images, labels, block_index=1, batch_size=2)
|
||||
names = tuple(name for name, _ in model.named_parameters() if name.startswith("layer4.1."))
|
||||
codec = SelectedResidualCodec(model, names, projection_seed=resnet.PROJECTION_SEED)
|
||||
zero = codec.zero_residual()
|
||||
nonzero = torch.full((codec.dimension,), 0.4)
|
||||
|
||||
base_state = {name: value.detach().clone() for name, value in model.state_dict().items()}
|
||||
base_logits = resnet.CachedSuffixEvaluator(model, cache, "cpu").logits()
|
||||
assert torch.equal(codec.decode(zero), torch.cat([value.reshape(-1) for value in codec.base_values]))
|
||||
assert torch.count_nonzero(codec.decode_delta(zero)) == 0
|
||||
assert torch.count_nonzero(codec.decode_delta(nonzero)) > 0
|
||||
|
||||
with codec.applied(model, zero):
|
||||
assert torch.equal(resnet.CachedSuffixEvaluator(model, cache, "cpu").logits(), base_logits)
|
||||
assert all(torch.equal(value, base_state[name]) for name, value in model.state_dict().items())
|
||||
|
||||
model.train()
|
||||
with pytest.raises(RuntimeError, match="candidate failure"):
|
||||
with codec.applied(model, nonzero):
|
||||
selected = dict(model.named_parameters())
|
||||
assert any(not torch.equal(selected[name], base_state[name]) for name in names)
|
||||
assert all(torch.equal(selected[name], base_state[name]) for name in selected if name not in names)
|
||||
assert all(torch.equal(value, base_state[name]) for name, value in model.named_buffers())
|
||||
raise RuntimeError("candidate failure")
|
||||
assert model.training
|
||||
assert all(torch.equal(value, base_state[name]) for name, value in model.state_dict().items())
|
||||
|
||||
parity = resnet.cached_residual_parity(model, images, cache, codec, nonzero)
|
||||
assert parity["passed"] is True
|
||||
assert parity["samples"] == len(images)
|
||||
assert parity["max_abs_difference"] <= 1e-6
|
||||
assert resnet.cached_full_parity(model, images, cache)["passed"] is True
|
||||
|
||||
|
||||
|
||||
def test_endpoint_selection_ties_are_stable() -> None:
|
||||
objective = ObjectiveResult(loss=0.25, samples=4)
|
||||
endpoints = (
|
||||
CandidateEndpoint(20, 1, torch.ones(64), objective),
|
||||
CandidateEndpoint(10, 0, torch.zeros(64), objective),
|
||||
)
|
||||
assert select_endpoint(endpoints, {10: 0.5, 20: 0.5}).generation == 10
|
||||
assert select_endpoint(endpoints, {10: 0.8, 20: 0.8}, maximize=True).generation == 10
|
||||
with pytest.raises(ProtocolError):
|
||||
select_endpoint(endpoints, {10: float("nan"), 20: 0.5})
|
||||
|
||||
|
||||
def test_ensemble_fit_uses_objective_pool_and_apply_does_not_refit() -> None:
|
||||
pytest.importorskip("scipy")
|
||||
rng = np.random.default_rng(19)
|
||||
objective_probs = rng.uniform(0.01, 1.0, size=(3, 9, 3))
|
||||
objective_probs /= objective_probs.sum(axis=-1, keepdims=True)
|
||||
selection_probs = np.roll(objective_probs, shift=1, axis=1).copy()
|
||||
objective_labels = np.arange(9, dtype=np.int64) % 3
|
||||
selection_labels = np.roll(objective_labels, 2)
|
||||
|
||||
fitted = resnet.run_ensemble_methods(objective_probs, objective_labels, swarm_seeds=(601,))
|
||||
fitted_snapshot = copy.deepcopy(fitted)
|
||||
applied = resnet.evaluate_fitted_ensemble(fitted, selection_probs, selection_labels)
|
||||
assert fitted == fitted_snapshot
|
||||
|
||||
uniform = np.full(3, 1 / 3)
|
||||
expected_uniform = np.einsum("m,mnk->nk", uniform, selection_probs)
|
||||
expected_nll = float(-np.log(np.clip(expected_uniform[np.arange(9), selection_labels], 1e-300, 1.0)).mean())
|
||||
assert applied["uniform"]["metrics"]["nll"] == pytest.approx(expected_nll, abs=1e-12)
|
||||
assert fitted["uniform"]["metrics"]["nll"] == pytest.approx(
|
||||
float(-np.log(np.clip(np.einsum("m,mnk->nk", uniform, objective_probs)[np.arange(9), objective_labels], 1e-300, 1.0)).mean()),
|
||||
abs=1e-12,
|
||||
)
|
||||
|
||||
for candidate in applied["ensemble_pso"]:
|
||||
weights = np.asarray(candidate["weights"], dtype=np.float64)
|
||||
mixed = np.einsum("m,mnk->nk", weights, selection_probs)
|
||||
expected = float(-np.log(np.clip(mixed[np.arange(9), selection_labels], 1e-300, 1.0)).mean())
|
||||
assert candidate["selection_metrics"]["nll"] == pytest.approx(expected, abs=1e-12)
|
||||
|
||||
|
||||
def test_official_test_loader_refuses_pre_freeze_without_importing_dataset(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
run_root = tmp_path / "run"
|
||||
prepare_run(run_root, StudyConfig())
|
||||
imported = False
|
||||
original_import = builtins.__import__
|
||||
|
||||
def reject_torchvision(name: str, *args: object, **kwargs: object):
|
||||
nonlocal imported
|
||||
if name.startswith("torchvision"):
|
||||
imported = True
|
||||
raise AssertionError("official dataset import must be behind the frozen seal")
|
||||
return original_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", reject_torchvision)
|
||||
with pytest.raises(resnet.TestSealError, match="forbidden before frozen"):
|
||||
resnet.load_official_test_data(tmp_path / "data", run_root, allow_download=False)
|
||||
assert imported is False
|
||||
@@ -0,0 +1,331 @@
|
||||
"""Offline behavioral tests for the pinned VOC/YOLO convergence adapter."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import builtins
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from test import post_training_yolo_convergence as study
|
||||
from test.post_training_model_convergence import SealError, StudyConfig, prepare_run
|
||||
|
||||
|
||||
def _record(index: int, *, fingerprint: str | None = None) -> study.VOCRecord:
|
||||
"""Build a cheap, label-complete synthetic record for manifest tests."""
|
||||
labels = tuple((class_id, 0.5, 0.5, 0.25, 0.25) for class_id in range(20))
|
||||
return study.VOCRecord(
|
||||
year="2007" if index % 2 == 0 else "2012",
|
||||
image_id=f"item-{index:05d}",
|
||||
image_path=f"/synthetic/{index}.jpg",
|
||||
annotation_path=f"/synthetic/{index}.xml",
|
||||
width=640,
|
||||
height=480,
|
||||
labels=labels,
|
||||
difficult_excluded=0,
|
||||
fingerprint=fingerprint or f"{index:064x}",
|
||||
)
|
||||
|
||||
|
||||
def test_optional_detection_imports_are_lazy(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Importing the adapter stays safe when optional detection packages are absent."""
|
||||
real_import = builtins.__import__
|
||||
|
||||
def block_ultralytics(name, *args, **kwargs):
|
||||
if name == "ultralytics":
|
||||
raise ImportError("blocked optional dependency")
|
||||
return real_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", block_ultralytics)
|
||||
with pytest.raises(study.YoloProtocolError, match="Ultralytics is required"):
|
||||
study._ultralytics()
|
||||
|
||||
def block_ensemble_boxes(name, *args, **kwargs):
|
||||
if name == "ensemble_boxes":
|
||||
raise ImportError("blocked optional dependency")
|
||||
return real_import(name, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(builtins, "__import__", block_ensemble_boxes)
|
||||
with pytest.raises(study.YoloProtocolError, match="ensemble-boxes is required"):
|
||||
study._wbf()
|
||||
|
||||
|
||||
def test_parse_voc_xml_excludes_difficult_and_uses_pinned_coordinates(tmp_path: Path) -> None:
|
||||
Image = pytest.importorskip("PIL.Image")
|
||||
image_path = tmp_path / "sample.jpg"
|
||||
Image.new("RGB", (20, 20), (10, 20, 30)).save(image_path)
|
||||
xml_path = tmp_path / "sample.xml"
|
||||
xml_path.write_text(
|
||||
"""<annotation>
|
||||
<size><width>20</width><height>20</height><depth>3</depth></size>
|
||||
<object><name>cat</name><difficult>0</difficult>
|
||||
<bndbox><xmin>1</xmin><ymin>2</ymin><xmax>9</xmax><ymax>10</ymax></bndbox>
|
||||
</object>
|
||||
<object><name>dog</name><difficult>1</difficult>
|
||||
<bndbox><xmin>0</xmin><ymin>0</ymin><xmax>19</xmax><ymax>19</ymax></bndbox>
|
||||
</object>
|
||||
</annotation>""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
record = study.parse_voc_xml(xml_path, image_path, year="2007", image_id="sample")
|
||||
|
||||
assert record.width == 20 and record.height == 20
|
||||
assert record.difficult_excluded == 1
|
||||
assert record.labels == ((7, 0.2, 0.25, 0.4, 0.4),)
|
||||
assert record.fingerprint == study.image_fingerprint(image_path)
|
||||
|
||||
|
||||
def test_duplicate_grouping_keeps_group_members_together() -> None:
|
||||
duplicate_a = _record(0, fingerprint="same")
|
||||
duplicate_b = _record(1, fingerprint="same")
|
||||
unique = _record(2, fingerprint="unique")
|
||||
|
||||
ordered, groups = study._assign_duplicate_groups(
|
||||
[duplicate_a, duplicate_b, unique], seed=20260908
|
||||
)
|
||||
|
||||
assert groups["same"] == tuple(f"{item.year}:{item.image_id}" for item in ordered if item.fingerprint == "same")
|
||||
same_positions = [index for index, item in enumerate(ordered) if item.fingerprint == "same"]
|
||||
assert same_positions == list(range(min(same_positions), max(same_positions) + 1))
|
||||
assert {item.image_id for item in ordered} == {"item-00000", "item-00001", "item-00002"}
|
||||
|
||||
|
||||
def test_manifest_has_exact_disjoint_partitions_and_objective_prefix() -> None:
|
||||
records = [_record(index) for index in range(16551)]
|
||||
manifest = study.make_voc_manifests(records, seed=20260908)
|
||||
|
||||
assert manifest.counts == {
|
||||
"bp_train": study.BP_COUNT,
|
||||
"refine_search": study.REFINE_COUNT,
|
||||
"selection_val": study.SELECTION_COUNT,
|
||||
}
|
||||
partitions = (manifest.bp_train, manifest.refine_search, manifest.selection_val)
|
||||
keys = [
|
||||
{f"{item.year}:{item.image_id}" for item in partition}
|
||||
for partition in partitions
|
||||
]
|
||||
assert [len(partition) for partition in partitions] == [11551, 2500, 2500]
|
||||
assert not (keys[0] & keys[1] or keys[0] & keys[2] or keys[1] & keys[2])
|
||||
assert manifest.objective_keys == tuple(
|
||||
(item.year, item.image_id) for item in manifest.refine_search[: study.OBJECTIVE_COUNT]
|
||||
)
|
||||
for partition in partitions:
|
||||
assert {label[0] for item in partition for label in item.labels} == set(range(20))
|
||||
|
||||
|
||||
def test_letterbox_box_round_trip_preserves_original_coordinates() -> None:
|
||||
original = np.array([[10.0, 5.0, 190.0, 95.0, 0.87]], dtype=np.float64)
|
||||
ratio_pad = (3.2, (0.0, 160.0)) # 200x100 image letterboxed to 640x640
|
||||
|
||||
letterboxed = study.transform_boxes_to_letterbox(original, ratio_pad=ratio_pad)
|
||||
restored = study.transform_boxes_to_original(
|
||||
letterboxed, ratio_pad=ratio_pad, shape=(100, 200)
|
||||
)
|
||||
|
||||
assert np.allclose(restored, original, atol=1e-12)
|
||||
assert np.allclose(letterboxed[0, :4], [32.0, 176.0, 608.0, 464.0])
|
||||
clipped = study.transform_boxes_to_original(
|
||||
np.array([[-10.0, 150.0, 650.0, 500.0]]),
|
||||
ratio_pad=ratio_pad,
|
||||
shape=(100, 200),
|
||||
)
|
||||
assert np.array_equal(clipped, np.array([[0.0, 0.0, 200.0, 100.0]]))
|
||||
|
||||
def test_native_target_uses_non_square_letterbox_geometry() -> None:
|
||||
record = study.VOCRecord(
|
||||
year="2007",
|
||||
image_id="wide",
|
||||
image_path="/synthetic/wide.jpg",
|
||||
annotation_path="/synthetic/wide.xml",
|
||||
width=200,
|
||||
height=100,
|
||||
labels=((0, 0.5, 0.5, 0.5, 0.5),),
|
||||
difficult_excluded=0,
|
||||
fingerprint="a" * 64,
|
||||
)
|
||||
target = study._native_target(
|
||||
record,
|
||||
index=3,
|
||||
ratio_pad=(3.2, (0.0, 160.0)),
|
||||
)
|
||||
assert target["batch_idx"].tolist() == [3]
|
||||
assert target["cls"].tolist() == [[0.0]]
|
||||
assert np.allclose(
|
||||
target["bboxes"].numpy(),
|
||||
np.array([[0.5, 0.5, 0.5, 0.25]]),
|
||||
)
|
||||
|
||||
|
||||
def test_wbf_uses_normalized_weights_and_stable_score_order(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_fusion(boxes, scores, labels, **kwargs):
|
||||
captured["weights"] = kwargs["weights"]
|
||||
captured["kwargs"] = kwargs
|
||||
return (
|
||||
[[0.1, 0.1, 0.2, 0.2], [0.3, 0.3, 0.4, 0.4], [0.5, 0.5, 0.6, 0.6]],
|
||||
[0.20, 0.90, 0.50],
|
||||
[2, 1, 0],
|
||||
)
|
||||
|
||||
monkeypatch.setattr(study, "_wbf", lambda: fake_fusion)
|
||||
result = study.weighted_box_fusion(
|
||||
[
|
||||
{"boxes": np.array([[0.1, 0.1, 0.2, 0.2]]), "scores": [0.8], "labels": [2]},
|
||||
{"boxes": np.array([[0.3, 0.3, 0.4, 0.4]]), "scores": [0.7], "labels": [1]},
|
||||
],
|
||||
[2.0, 6.0],
|
||||
)
|
||||
|
||||
assert captured["weights"] == pytest.approx([0.25, 0.75])
|
||||
assert sum(captured["weights"]) == pytest.approx(1.0)
|
||||
assert captured["kwargs"]["iou_thr"] == 0.55
|
||||
assert result["scores"].tolist() == [0.90, 0.50, 0.20]
|
||||
assert result["labels"].tolist() == [1, 0, 2]
|
||||
|
||||
|
||||
def test_wbf_pso_and_random_have_exact_12x20_query_accounting(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
calls: list[tuple[float, ...]] = []
|
||||
|
||||
def tiny_metric(member_predictions, targets, weights):
|
||||
values = tuple(float(value) for value in weights)
|
||||
calls.append(values)
|
||||
assert sum(values) == pytest.approx(1.0)
|
||||
return {"map50_95": values[0]}
|
||||
|
||||
monkeypatch.setattr(study, "_wbf_dataset_metrics", tiny_metric)
|
||||
targets = [{"image_id": "a"}, {"image_id": "b"}]
|
||||
members = [[{} for _ in targets] for _ in range(3)]
|
||||
|
||||
pso = study.run_wbf_weight_search(members, targets, seed=601, random_mode=False)
|
||||
random_result = study.run_wbf_weight_search(members, targets, seed=601, random_mode=True)
|
||||
|
||||
assert pso["method"] == "ensemble_pso"
|
||||
assert random_result["method"] == "ensemble_random"
|
||||
for result in (pso, random_result):
|
||||
assert result["queries"] == 12 * 20
|
||||
assert result["sample_evaluations"] == 12 * 20 * len(targets)
|
||||
assert len(result["trajectory"]) == 20
|
||||
assert sum(result["weights"]) == pytest.approx(1.0)
|
||||
assert all(0.0 <= weight <= 1.0 for weight in result["weights"])
|
||||
assert len(calls) == 2 * 12 * 20
|
||||
|
||||
|
||||
class C3k2(nn.Module):
|
||||
pass
|
||||
|
||||
|
||||
class Detect(nn.Module):
|
||||
def __init__(self, nc: int = 20) -> None:
|
||||
super().__init__()
|
||||
self.nc = nc
|
||||
self.cv2 = nn.ModuleList([nn.Sequential(nn.Linear(1, 42)) for _ in range(3)])
|
||||
self.cv3 = nn.ModuleList([nn.Sequential(nn.Linear(1, 42)) for _ in range(3)])
|
||||
|
||||
|
||||
class WrongDetect(Detect):
|
||||
pass
|
||||
|
||||
|
||||
class WrongBlock(nn.Module):
|
||||
pass
|
||||
|
||||
|
||||
def _tiny_graph(*, block: nn.Module | None = None, detect: nn.Module | None = None) -> nn.Module:
|
||||
graph = nn.Module()
|
||||
graph.model = nn.ModuleList([nn.Identity() for _ in range(22)] + [block or C3k2(), detect or Detect()])
|
||||
return graph
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("graph", "message"),
|
||||
[
|
||||
(nn.Module(), "shorter than"),
|
||||
(_tiny_graph(block=WrongBlock()), "expected model.22 C3k2"),
|
||||
(_tiny_graph(detect=WrongDetect()), "expected model.23 Detect"),
|
||||
(_tiny_graph(detect=Detect(nc=19)), "expected Detect.nc=20"),
|
||||
],
|
||||
)
|
||||
def test_topology_guard_rejects_tiny_mismatched_graphs(graph: nn.Module, message: str) -> None:
|
||||
if not hasattr(graph, "model"):
|
||||
graph.model = nn.ModuleList([nn.Identity(), nn.Identity()])
|
||||
with pytest.raises(study.YoloProtocolError, match=message):
|
||||
study.assert_yolo_topology(graph)
|
||||
|
||||
|
||||
def test_official_test_loader_refuses_before_frozen_confirmation(tmp_path: Path) -> None:
|
||||
run_root = tmp_path / "run"
|
||||
data_root = tmp_path / "data"
|
||||
state = prepare_run(run_root, StudyConfig(device="cpu"))
|
||||
|
||||
with pytest.raises(SealError, match="sealed until confirm phase"):
|
||||
study.guarded_voc_test_loader(data_root, run_root, confirmation=False)
|
||||
assert state.state.value == "prepared"
|
||||
assert not (data_root / "VOCdevkit").exists()
|
||||
|
||||
with pytest.raises(SealError, match="sealed until frozen confirmation"):
|
||||
study.VOCTestGuard(str(run_root), "", False).require_open()
|
||||
|
||||
|
||||
def test_native_baseline_reuse_verifies_complete_epoch_artifacts(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
run_root = tmp_path / "run"
|
||||
baseline_root = (
|
||||
run_root
|
||||
/ "workloads"
|
||||
/ study.WORKLOAD_ID
|
||||
/ "baselines"
|
||||
/ "501"
|
||||
)
|
||||
baseline_root.mkdir(parents=True)
|
||||
checkpoint = baseline_root / "ema_fp32.pt"
|
||||
torch.save({"weight": torch.ones(2)}, checkpoint)
|
||||
results = (
|
||||
run_root
|
||||
/ "ultralytics"
|
||||
/ "base-501-100e"
|
||||
/ "results.csv"
|
||||
)
|
||||
results.parent.mkdir(parents=True)
|
||||
results.write_text(
|
||||
"epoch,train/loss\n"
|
||||
+ "".join(f"{epoch},{1 / epoch}\n" for epoch in range(1, 101)),
|
||||
encoding="utf-8",
|
||||
)
|
||||
marker = {
|
||||
"protocol_version": study.PROTOCOL_VERSION,
|
||||
"source_run": "source",
|
||||
"checkpoint_hash": study.fingerprint_file(checkpoint),
|
||||
"results_hash": study.fingerprint_file(results),
|
||||
}
|
||||
(baseline_root / "baseline_reuse.json").write_text(
|
||||
json.dumps(marker),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
reused = study._reused_native_baseline(
|
||||
run_root,
|
||||
study.StrictScratchTrainer(device="cpu"),
|
||||
501,
|
||||
)
|
||||
assert reused is not None
|
||||
assert reused["reused"] is True
|
||||
assert len(reused["telemetry"]) == 11
|
||||
results.write_text("epoch,train/loss\n1,1\n", encoding="utf-8")
|
||||
with pytest.raises(study.YoloProtocolError, match="reused baseline marker"):
|
||||
study._reused_native_baseline(
|
||||
run_root,
|
||||
study.StrictScratchTrainer(device="cpu"),
|
||||
501,
|
||||
)
|
||||
@@ -0,0 +1,180 @@
|
||||
import inspect
|
||||
import subprocess
|
||||
import sys
|
||||
import pytest
|
||||
import torch
|
||||
import pso
|
||||
from pso import Optimizer, Particle, __version__
|
||||
|
||||
|
||||
def test_canonical_exports_and_all():
|
||||
"""Verify pso exports Optimizer, Particle, __version__, stage plugins and defines __all__ correctly."""
|
||||
expected_all = [
|
||||
"Optimizer",
|
||||
"Particle",
|
||||
"__version__",
|
||||
"BasePlugin",
|
||||
"InitializationPlugin",
|
||||
"EvaluationPlugin",
|
||||
"MovementPlugin",
|
||||
"ConvergencePlugin",
|
||||
"RefinementPlugin",
|
||||
"PluginMetadata",
|
||||
"SwarmState",
|
||||
"available_plugins",
|
||||
]
|
||||
assert pso.__all__ == expected_all
|
||||
assert pso.Optimizer is Optimizer
|
||||
assert pso.Particle is Particle
|
||||
assert pso.__version__ == "4.0.0"
|
||||
assert __version__ == "4.0.0"
|
||||
|
||||
from pso.plugins import (
|
||||
BasePlugin,
|
||||
InitializationPlugin,
|
||||
EvaluationPlugin,
|
||||
MovementPlugin,
|
||||
ConvergencePlugin,
|
||||
RefinementPlugin,
|
||||
PluginMetadata,
|
||||
SwarmState,
|
||||
available_plugins,
|
||||
)
|
||||
assert pso.BasePlugin is BasePlugin
|
||||
assert pso.InitializationPlugin is InitializationPlugin
|
||||
assert pso.EvaluationPlugin is EvaluationPlugin
|
||||
assert pso.MovementPlugin is MovementPlugin
|
||||
assert pso.ConvergencePlugin is ConvergencePlugin
|
||||
assert pso.RefinementPlugin is RefinementPlugin
|
||||
assert pso.PluginMetadata is PluginMetadata
|
||||
assert pso.SwarmState is SwarmState
|
||||
assert pso.available_plugins is available_plugins
|
||||
|
||||
|
||||
def test_lowercase_aliases_and_legacy_api_absent():
|
||||
"""Verify lowercase names and legacy get_best_weights are excluded/absent."""
|
||||
assert "optimizer" not in pso.__all__
|
||||
assert "particle" not in pso.__all__
|
||||
assert not hasattr(Optimizer, "get_best_weights")
|
||||
assert hasattr(Optimizer, "get_best_state_dict")
|
||||
|
||||
if hasattr(pso, "optimizer"):
|
||||
obj = getattr(pso, "optimizer")
|
||||
assert not isinstance(obj, type)
|
||||
|
||||
if hasattr(pso, "particle"):
|
||||
obj = getattr(pso, "particle")
|
||||
assert not isinstance(obj, type)
|
||||
|
||||
|
||||
def test_optimizer_init_signature_and_kwonly():
|
||||
"""Verify Optimizer.__init__ parameter names and keyword-only positions."""
|
||||
sig = inspect.signature(Optimizer.__init__)
|
||||
params = sig.parameters
|
||||
|
||||
assert "model" in params
|
||||
assert "loss" in params
|
||||
|
||||
# Positional parameters (excluding self)
|
||||
assert params["model"].kind in (
|
||||
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
||||
inspect.Parameter.POSITIONAL_ONLY,
|
||||
)
|
||||
assert params["loss"].kind in (
|
||||
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
||||
inspect.Parameter.POSITIONAL_ONLY,
|
||||
)
|
||||
|
||||
kwonly_expected = [
|
||||
"method",
|
||||
"initialization",
|
||||
"evaluation",
|
||||
"convergence",
|
||||
"refinement",
|
||||
"method_options",
|
||||
"n_particles",
|
||||
"c0",
|
||||
"c1",
|
||||
"w_min",
|
||||
"w_max",
|
||||
"negative_swarm",
|
||||
"mutation_swarm",
|
||||
"particle_min",
|
||||
"particle_max",
|
||||
"velocity_limit_ratio",
|
||||
"boundary_strategy",
|
||||
"initial_position_noise",
|
||||
"seed",
|
||||
"device",
|
||||
"fitness_size",
|
||||
"convergence_patience",
|
||||
"convergence_min_delta",
|
||||
"convergence_monitor",
|
||||
"refinement_epochs",
|
||||
"refinement_lr",
|
||||
"moment_blend",
|
||||
"moment_beta1",
|
||||
"moment_beta2",
|
||||
"moment_step_size",
|
||||
"moment_epsilon",
|
||||
]
|
||||
|
||||
for name in kwonly_expected:
|
||||
assert name in params, f"Missing parameter {name} in Optimizer.__init__"
|
||||
assert params[name].kind == inspect.Parameter.KEYWORD_ONLY, (
|
||||
f"Parameter {name} must be KEYWORD_ONLY"
|
||||
)
|
||||
|
||||
|
||||
def test_optimizer_fit_signature_and_kwonly():
|
||||
"""Verify Optimizer.fit parameter names and keyword-only positions."""
|
||||
sig = inspect.signature(Optimizer.fit)
|
||||
params = sig.parameters
|
||||
|
||||
assert "x" in params
|
||||
assert "y" in params
|
||||
|
||||
kwonly_expected = [
|
||||
"epochs",
|
||||
"batch_size",
|
||||
"fitness_size",
|
||||
"renewal",
|
||||
"validation_data",
|
||||
"validation_split",
|
||||
"output_dir",
|
||||
"log_format",
|
||||
"checkpoint_interval",
|
||||
"save_info",
|
||||
]
|
||||
|
||||
for name in kwonly_expected:
|
||||
assert name in params, f"Missing parameter {name} in Optimizer.fit"
|
||||
assert params[name].kind == inspect.Parameter.KEYWORD_ONLY, (
|
||||
f"Parameter {name} must be KEYWORD_ONLY"
|
||||
)
|
||||
|
||||
|
||||
def test_kwonly_positional_and_unknown_kwargs(model_factory, xor_data):
|
||||
"""Verify passing keyword-only arguments positionally or unknown kwargs raises TypeError."""
|
||||
x, y = xor_data
|
||||
model = model_factory()
|
||||
loss = torch.nn.BCEWithLogitsLoss()
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
Optimizer(model, loss, "binary") # type: ignore[call-arg]
|
||||
|
||||
opt = Optimizer(model, loss, task="binary")
|
||||
with pytest.raises(TypeError):
|
||||
opt.fit(x, y, invalid_unknown_arg=123) # type: ignore[call-arg]
|
||||
|
||||
|
||||
def test_subprocess_import_quiet_stdout():
|
||||
"""Verify importing pso in a fresh subprocess produces exit code 0 and empty stdout."""
|
||||
res = subprocess.run(
|
||||
[sys.executable, "-c", "import pso"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert res.returncode == 0, f"Import failed with stderr: {res.stderr}"
|
||||
assert res.stdout == "", f"Expected empty stdout from import pso, got: {res.stdout!r}"
|
||||
@@ -0,0 +1,400 @@
|
||||
"""
|
||||
Unit tests for Heavy PSO Cross-Split Results Publisher.
|
||||
|
||||
Covers:
|
||||
1. Valid publication pipeline execution on synthetic 9-variant cross-split source data.
|
||||
2. Verification of compact JSON schema, cumulative resources (432 runs, 414720 queries, 4147200000 samples),
|
||||
official test seals (0 evaluations), confirmation_executed=false, retained_policy=null.
|
||||
3. Verification of exact CSV output shape (72 data rows) and deterministic byte-for-byte reproducibility.
|
||||
4. Validation error enforcement for cell count mismatch, official test unsealing, unexpected development pass,
|
||||
and variant count mismatch.
|
||||
5. CLI entrypoint invocation.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# Ensure test directory and repo root are in sys.path
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent if Path(__file__).resolve().parent.name != "PSO" else Path(__file__).resolve().parent
|
||||
TEST_DIR = REPO_ROOT / "test"
|
||||
if str(TEST_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TEST_DIR))
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
import publish_heavy_cross_split as publisher
|
||||
from pso import __version__ as pso_version
|
||||
|
||||
|
||||
def make_synthetic_source(
|
||||
source_path: Path,
|
||||
num_variants: int = 9,
|
||||
cell_count: int = 8,
|
||||
test_evals: int = 0,
|
||||
test_loaded: bool = False,
|
||||
dev_pass: bool = False,
|
||||
queries_per_run: int = 960,
|
||||
samples_per_run: int = 9600000,
|
||||
) -> Path:
|
||||
"""
|
||||
Creates a synthetic cross-split experiment run directory with candidates/ and evaluations/
|
||||
matching expected structure for testing publisher integrity checks.
|
||||
"""
|
||||
cand_dir = source_path / "candidates"
|
||||
eval_dir = source_path / "evaluations"
|
||||
cand_dir.mkdir(parents=True, exist_ok=True)
|
||||
eval_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
splits = [20260905, 20260906]
|
||||
workloads = ["mnist_compact", "mnist_wide", "fashion_compact", "fashion_wide"]
|
||||
swarm_seeds = [101, 102, 103]
|
||||
|
||||
for variant_id in publisher.EXPECTED_VARIANT_IDS[:num_variants]:
|
||||
cf_path = cand_dir / f"{variant_id}.json"
|
||||
ef_path = eval_dir / f"{variant_id}.json"
|
||||
|
||||
# Construct Candidate Artifact
|
||||
splits_dict = {}
|
||||
for split_seed in splits:
|
||||
split_key = str(split_seed)
|
||||
baselines_dict = {}
|
||||
candidates_dict = {}
|
||||
|
||||
for wl in workloads:
|
||||
baseline_method = "G8" if "compact" in wl else "G5"
|
||||
|
||||
def make_runs():
|
||||
runs_list = []
|
||||
for seed in swarm_seeds:
|
||||
runs_list.append({
|
||||
"seed": seed,
|
||||
"val_selected_loss": 0.50,
|
||||
"val_selected_acc": 80.0,
|
||||
"gbest_loss": 0.48,
|
||||
"gbest_acc": 81.0,
|
||||
"wall_time_sec": 0.01,
|
||||
"optimization_wall_time_sec": 0.01,
|
||||
"validation_wall_time_sec": 0.001,
|
||||
"throughput_samples_per_sec": 1000000.0,
|
||||
"total_queries": queries_per_run,
|
||||
"total_sample_evaluations": samples_per_run,
|
||||
"official_test_evaluations": test_evals,
|
||||
"val_metrics": {"brier": 0.1, "ece": 0.02},
|
||||
"is_finite": True,
|
||||
"core_swarm_state_bytes": 1000,
|
||||
})
|
||||
return runs_list
|
||||
|
||||
baselines_dict[wl] = {
|
||||
"workload_id": wl,
|
||||
"method_id": baseline_method,
|
||||
"subset_size": 10000,
|
||||
"particles": 12,
|
||||
"epochs": 80,
|
||||
"seeds": swarm_seeds,
|
||||
"split_seed": split_seed,
|
||||
"data_fingerprint": f"data-fp-{wl}-{split_seed}",
|
||||
"split_fingerprint": f"split-fp-{wl}-{split_seed}",
|
||||
"per_seed_runs": make_runs(),
|
||||
}
|
||||
candidates_dict[wl] = {
|
||||
"workload_id": wl,
|
||||
"ratio": 0.5,
|
||||
"subset_size": 10000,
|
||||
"particles": 12,
|
||||
"epochs": 80,
|
||||
"seeds": swarm_seeds,
|
||||
"split_seed": split_seed,
|
||||
"data_fingerprint": f"data-fp-{wl}-{split_seed}",
|
||||
"split_fingerprint": f"split-fp-{wl}-{split_seed}",
|
||||
"per_seed_runs": make_runs(),
|
||||
}
|
||||
|
||||
splits_dict[split_key] = {
|
||||
"split_seed": split_seed,
|
||||
"baselines": baselines_dict,
|
||||
"candidates": candidates_dict,
|
||||
}
|
||||
|
||||
candidate_payload = {
|
||||
"version": "HEAVY-PSO-CROSS-SPLIT 1.0.0",
|
||||
"protocol_version": "HEAVY-PSO-CROSS-SPLIT 1.0.0",
|
||||
"phase": "development",
|
||||
"split_seeds": splits,
|
||||
"swarm_seeds": swarm_seeds,
|
||||
"official_test_data_loaded": test_loaded,
|
||||
"official_test_evaluations": test_evals * 48,
|
||||
"candidate_config": {
|
||||
"ratio": 0.5,
|
||||
"geometry_policy": "baseline_aligned",
|
||||
"particles": 12,
|
||||
"epochs": 80,
|
||||
"subset_size": 10000,
|
||||
},
|
||||
"workloads": {wl: {"workload_id": wl, "baseline_method": "G8" if "compact" in wl else "G5"} for wl in workloads},
|
||||
"splits": splits_dict,
|
||||
"resource_totals": {
|
||||
"total_runs": 48,
|
||||
"total_queries": queries_per_run * 48,
|
||||
"total_samples_evaluated": samples_per_run * 48,
|
||||
"official_test_evaluations": test_evals * 48,
|
||||
"wall_time_sec": 1.0,
|
||||
},
|
||||
}
|
||||
|
||||
# Construct Evaluation Artifact
|
||||
cell_metrics = []
|
||||
for s_idx, split_seed in enumerate(splits):
|
||||
for wl in workloads:
|
||||
cell_metrics.append({
|
||||
"phase": "development",
|
||||
"split_seed": split_seed,
|
||||
"workload_id": wl,
|
||||
"baseline_acc": 80.0,
|
||||
"candidate_acc": 80.5,
|
||||
"baseline_nll": 0.50,
|
||||
"candidate_nll": 0.49,
|
||||
"acc_gain_pp": 0.5,
|
||||
"nll_reduction_fraction": 0.02,
|
||||
})
|
||||
|
||||
cell_metrics = cell_metrics[:cell_count]
|
||||
|
||||
evaluation_payload = {
|
||||
"pass": False,
|
||||
"development_pass": dev_pass,
|
||||
"eligible_for_confirmation": False,
|
||||
"score": -300.0 + publisher.EXPECTED_VARIANT_IDS.index(variant_id) * 10.0,
|
||||
"evaluator_version": "HEAVY-PSO-CROSS-SPLIT-EVALUATOR 1.0.0",
|
||||
"failed_hard_gate_count": 3,
|
||||
"failed_gates": ["maximum_accuracy_regression_percentage_points_each_split_workload"],
|
||||
"gates": {
|
||||
"official_test_sealed": {
|
||||
"pass": not test_loaded and test_evals == 0,
|
||||
}
|
||||
},
|
||||
"summary_metrics": {
|
||||
"development_cells": len(cell_metrics),
|
||||
"development_grand_mean_accuracy_gain_pp": 0.5,
|
||||
"development_grand_mean_nll_reduction_fraction": 0.02,
|
||||
"development_mnist_wide_accuracy_gain_pp": -0.5,
|
||||
"development_mnist_wide_nll_reduction_fraction": -0.01,
|
||||
},
|
||||
"state_ratios": {
|
||||
"development": {
|
||||
"mnist_compact": 0.4918032786885246,
|
||||
"mnist_wide": 0.5,
|
||||
"fashion_compact": 0.4918032786885246,
|
||||
"fashion_wide": 0.5,
|
||||
}
|
||||
},
|
||||
"cell_metrics": cell_metrics,
|
||||
}
|
||||
|
||||
with cf_path.open("w", encoding="utf-8") as f:
|
||||
json.dump(candidate_payload, f, indent=2)
|
||||
with ef_path.open("w", encoding="utf-8") as f:
|
||||
json.dump(evaluation_payload, f, indent=2)
|
||||
|
||||
return source_path
|
||||
|
||||
|
||||
def test_publish_synthetic_success(tmp_path):
|
||||
"""Verifies successful end-to-end publication on valid synthetic 9-variant cross-split source data."""
|
||||
source_dir = make_synthetic_source(tmp_path / "source")
|
||||
out_json = tmp_path / "pso_v7_heavy_cross_split.json"
|
||||
out_csv = tmp_path / "pso_v7_heavy_cross_split.csv"
|
||||
out_plot = tmp_path / "pso_v7_heavy_cross_split.png"
|
||||
|
||||
payload = publisher.publish_heavy_cross_split(
|
||||
source_dir=source_dir,
|
||||
output_json=out_json,
|
||||
output_csv=out_csv,
|
||||
output_plot=out_plot,
|
||||
)
|
||||
|
||||
# 1. JSON Verification
|
||||
assert out_json.is_file()
|
||||
assert payload["protocol_version"] == publisher.PUBLISH_PROTOCOL_VERSION
|
||||
assert payload["pso_version"] == pso_version
|
||||
assert payload["official_test_data_loaded"] is False
|
||||
assert payload["official_test_evaluations"] == 0
|
||||
assert payload["confirmation_executed"] is False
|
||||
assert payload["retained_policy"] is None
|
||||
|
||||
assert payload["total_runs"] == 432
|
||||
assert payload["total_queries"] == 414720
|
||||
assert payload["total_sample_evaluations"] == 4147200000
|
||||
assert payload["total_wall_time_sec"] == 9.0
|
||||
assert len(payload["variants"]) == 9
|
||||
assert payload["verdict"]["status"] == "NO_RETAINED_POLICY_NO_CONFIRMATION"
|
||||
|
||||
# Best-observed variant should be the final expected variant.
|
||||
best_v = [v for v in payload["variants"] if v["is_best_observed"]]
|
||||
assert len(best_v) == 1
|
||||
assert best_v[0]["variant_id"] == publisher.EXPECTED_VARIANT_IDS[-1]
|
||||
|
||||
# 2. CSV Verification
|
||||
assert out_csv.is_file()
|
||||
with out_csv.open("r", encoding="utf-8") as f:
|
||||
reader = list(csv.reader(f))
|
||||
# 1 header line + 72 data rows = 73 lines
|
||||
assert len(reader) == 73
|
||||
header = reader[0]
|
||||
assert "variant_id" in header
|
||||
assert "baseline_acc" in header
|
||||
assert "candidate_acc" in header
|
||||
assert "acc_gain_pp" in header
|
||||
|
||||
# 3. Plot Verification
|
||||
assert out_plot.is_file()
|
||||
assert out_plot.stat().st_size > 0
|
||||
|
||||
|
||||
def test_publish_mismatch_cell_count(tmp_path):
|
||||
"""Verifies ValueError when an evaluation artifact has a cell count other than 8."""
|
||||
source_dir = make_synthetic_source(tmp_path / "source", cell_count=7)
|
||||
out_json = tmp_path / "out.json"
|
||||
out_csv = tmp_path / "out.csv"
|
||||
out_plot = tmp_path / "out.png"
|
||||
|
||||
with pytest.raises(ValueError, match="Expected 8 development cells"):
|
||||
publisher.publish_heavy_cross_split(source_dir, out_json, out_csv, out_plot)
|
||||
|
||||
|
||||
def test_publish_official_test_unsealed(tmp_path):
|
||||
"""Verifies ValueError when official test evaluations > 0 or official_test_data_loaded is True."""
|
||||
source_dir = make_synthetic_source(tmp_path / "source", test_evals=10)
|
||||
out_json = tmp_path / "out.json"
|
||||
out_csv = tmp_path / "out.csv"
|
||||
out_plot = tmp_path / "out.png"
|
||||
|
||||
with pytest.raises(ValueError, match="official_test_evaluations must be 0"):
|
||||
publisher.publish_heavy_cross_split(source_dir, out_json, out_csv, out_plot)
|
||||
|
||||
|
||||
def test_publish_unexpected_pass(tmp_path):
|
||||
"""Verifies ValueError when development_pass is True."""
|
||||
source_dir = make_synthetic_source(tmp_path / "source", dev_pass=True)
|
||||
out_json = tmp_path / "out.json"
|
||||
out_csv = tmp_path / "out.csv"
|
||||
out_plot = tmp_path / "out.png"
|
||||
|
||||
with pytest.raises(ValueError, match="development_pass must be False"):
|
||||
publisher.publish_heavy_cross_split(source_dir, out_json, out_csv, out_plot)
|
||||
|
||||
|
||||
def test_publish_variant_count_mismatch(tmp_path):
|
||||
"""Verifies that omitting an expected variant is rejected."""
|
||||
source_dir = make_synthetic_source(tmp_path / "source", num_variants=8)
|
||||
out_json = tmp_path / "out.json"
|
||||
out_csv = tmp_path / "out.csv"
|
||||
out_plot = tmp_path / "out.png"
|
||||
|
||||
with pytest.raises(ValueError, match="Expected exact development variants"):
|
||||
publisher.publish_heavy_cross_split(source_dir, out_json, out_csv, out_plot)
|
||||
|
||||
|
||||
def test_deterministic_csv_shape(tmp_path):
|
||||
"""Verifies that running publication twice yields identical CSV byte content."""
|
||||
source_dir = make_synthetic_source(tmp_path / "source")
|
||||
out_json = tmp_path / "out.json"
|
||||
out_csv1 = tmp_path / "out1.csv"
|
||||
out_csv2 = tmp_path / "out2.csv"
|
||||
out_plot = tmp_path / "out.png"
|
||||
|
||||
publisher.publish_heavy_cross_split(source_dir, out_json, out_csv1, out_plot)
|
||||
publisher.publish_heavy_cross_split(source_dir, out_json, out_csv2, out_plot)
|
||||
|
||||
assert out_csv1.read_bytes() == out_csv2.read_bytes()
|
||||
|
||||
def test_publish_rejects_wrong_variant_identity(tmp_path):
|
||||
source_dir = make_synthetic_source(tmp_path / "source")
|
||||
candidate = source_dir / "candidates" / f"{publisher.EXPECTED_VARIANT_IDS[-1]}.json"
|
||||
evaluation = source_dir / "evaluations" / f"{publisher.EXPECTED_VARIANT_IDS[-1]}.json"
|
||||
candidate.rename(candidate.with_name("iteration-9999-development.json"))
|
||||
evaluation.rename(evaluation.with_name("iteration-9999-development.json"))
|
||||
|
||||
with pytest.raises(ValueError, match="Expected exact development variants"):
|
||||
publisher.publish_heavy_cross_split(
|
||||
source_dir,
|
||||
tmp_path / "out.json",
|
||||
tmp_path / "out.csv",
|
||||
tmp_path / "out.png",
|
||||
)
|
||||
|
||||
|
||||
def test_publish_rejects_resource_total_mismatch(tmp_path):
|
||||
source_dir = make_synthetic_source(tmp_path / "source")
|
||||
candidate = source_dir / "candidates" / f"{publisher.EXPECTED_VARIANT_IDS[0]}.json"
|
||||
payload = json.loads(candidate.read_text(encoding="utf-8"))
|
||||
payload["resource_totals"]["total_queries"] -= 1
|
||||
candidate.write_text(json.dumps(payload), encoding="utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match=r"resource_totals\.total_queries"):
|
||||
publisher.publish_heavy_cross_split(
|
||||
source_dir,
|
||||
tmp_path / "out.json",
|
||||
tmp_path / "out.csv",
|
||||
tmp_path / "out.png",
|
||||
)
|
||||
|
||||
|
||||
def test_publish_rejects_invalid_wall_time(tmp_path):
|
||||
source_dir = make_synthetic_source(tmp_path / "source")
|
||||
candidate = source_dir / "candidates" / f"{publisher.EXPECTED_VARIANT_IDS[0]}.json"
|
||||
candidate_payload = json.loads(candidate.read_text(encoding="utf-8"))
|
||||
candidate_payload["resource_totals"]["wall_time_sec"] = float("nan")
|
||||
candidate.write_text(json.dumps(candidate_payload), encoding="utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match=r"resource_totals\.wall_time_sec"):
|
||||
publisher.publish_heavy_cross_split(
|
||||
source_dir,
|
||||
tmp_path / "out.json",
|
||||
tmp_path / "out.csv",
|
||||
tmp_path / "out.png",
|
||||
)
|
||||
|
||||
|
||||
def test_publish_uses_repository_relative_source_path(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(publisher, "REPO_ROOT", tmp_path)
|
||||
source_dir = make_synthetic_source(tmp_path / "source")
|
||||
csv_path = tmp_path / "out.csv"
|
||||
payload = publisher.publish_heavy_cross_split(
|
||||
source_dir,
|
||||
tmp_path / "out.json",
|
||||
csv_path,
|
||||
tmp_path / "out.png",
|
||||
)
|
||||
assert payload["source_provenance"]["source_dir"] == "source"
|
||||
with csv_path.open(encoding="utf-8") as handle:
|
||||
rows = list(csv.DictReader(handle))
|
||||
assert rows[0]["candidate_path"].startswith("source/candidates/")
|
||||
assert rows[0]["evaluation_path"].startswith("source/evaluations/")
|
||||
|
||||
|
||||
def test_cli_invocation(tmp_path, monkeypatch):
|
||||
"""Verifies CLI main entrypoint executes cleanly."""
|
||||
source_dir = make_synthetic_source(tmp_path / "source")
|
||||
out_json = tmp_path / "cli.json"
|
||||
out_csv = tmp_path / "cli.csv"
|
||||
out_plot = tmp_path / "cli.png"
|
||||
|
||||
cli_args = [
|
||||
"publish_heavy_cross_split.py",
|
||||
"--source-dir", str(source_dir),
|
||||
"--output-json", str(out_json),
|
||||
"--output-csv", str(out_csv),
|
||||
"--output-plot", str(out_plot),
|
||||
]
|
||||
monkeypatch.setattr(sys, "argv", cli_args)
|
||||
|
||||
publisher.main()
|
||||
|
||||
assert out_json.is_file()
|
||||
assert out_csv.is_file()
|
||||
assert out_plot.is_file()
|
||||
Reference in New Issue
Block a user