mirror of
https://github.com/jung-geun/PSO.git
synced 2026-09-20 14:11:48 +09:00
feat: modernize PSO and add convergence research
Migrate the package and examples to the tensor-native PyTorch implementation, add benchmark evidence, and add the guarded post-training convergence protocol with TensorBoard progress monitoring and hash-verified recovery. Constraint: Preserve one-shot official-test sealing and auditable research artifacts Rejected: Commit local .omc runs and downloaded datasets | multi-gigabyte runtime state is machine-local Confidence: high Scope-risk: broad Not-tested: Production CUDA run on pieroot-server
This commit is contained in:
@@ -0,0 +1,400 @@
|
||||
"""
|
||||
Unit tests for Heavy PSO Cross-Split Results Publisher.
|
||||
|
||||
Covers:
|
||||
1. Valid publication pipeline execution on synthetic 9-variant cross-split source data.
|
||||
2. Verification of compact JSON schema, cumulative resources (432 runs, 414720 queries, 4147200000 samples),
|
||||
official test seals (0 evaluations), confirmation_executed=false, retained_policy=null.
|
||||
3. Verification of exact CSV output shape (72 data rows) and deterministic byte-for-byte reproducibility.
|
||||
4. Validation error enforcement for cell count mismatch, official test unsealing, unexpected development pass,
|
||||
and variant count mismatch.
|
||||
5. CLI entrypoint invocation.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# Ensure test directory and repo root are in sys.path
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent if Path(__file__).resolve().parent.name != "PSO" else Path(__file__).resolve().parent
|
||||
TEST_DIR = REPO_ROOT / "test"
|
||||
if str(TEST_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TEST_DIR))
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
import publish_heavy_cross_split as publisher
|
||||
from pso import __version__ as pso_version
|
||||
|
||||
|
||||
def make_synthetic_source(
|
||||
source_path: Path,
|
||||
num_variants: int = 9,
|
||||
cell_count: int = 8,
|
||||
test_evals: int = 0,
|
||||
test_loaded: bool = False,
|
||||
dev_pass: bool = False,
|
||||
queries_per_run: int = 960,
|
||||
samples_per_run: int = 9600000,
|
||||
) -> Path:
|
||||
"""
|
||||
Creates a synthetic cross-split experiment run directory with candidates/ and evaluations/
|
||||
matching expected structure for testing publisher integrity checks.
|
||||
"""
|
||||
cand_dir = source_path / "candidates"
|
||||
eval_dir = source_path / "evaluations"
|
||||
cand_dir.mkdir(parents=True, exist_ok=True)
|
||||
eval_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
splits = [20260905, 20260906]
|
||||
workloads = ["mnist_compact", "mnist_wide", "fashion_compact", "fashion_wide"]
|
||||
swarm_seeds = [101, 102, 103]
|
||||
|
||||
for variant_id in publisher.EXPECTED_VARIANT_IDS[:num_variants]:
|
||||
cf_path = cand_dir / f"{variant_id}.json"
|
||||
ef_path = eval_dir / f"{variant_id}.json"
|
||||
|
||||
# Construct Candidate Artifact
|
||||
splits_dict = {}
|
||||
for split_seed in splits:
|
||||
split_key = str(split_seed)
|
||||
baselines_dict = {}
|
||||
candidates_dict = {}
|
||||
|
||||
for wl in workloads:
|
||||
baseline_method = "G8" if "compact" in wl else "G5"
|
||||
|
||||
def make_runs():
|
||||
runs_list = []
|
||||
for seed in swarm_seeds:
|
||||
runs_list.append({
|
||||
"seed": seed,
|
||||
"val_selected_loss": 0.50,
|
||||
"val_selected_acc": 80.0,
|
||||
"gbest_loss": 0.48,
|
||||
"gbest_acc": 81.0,
|
||||
"wall_time_sec": 0.01,
|
||||
"optimization_wall_time_sec": 0.01,
|
||||
"validation_wall_time_sec": 0.001,
|
||||
"throughput_samples_per_sec": 1000000.0,
|
||||
"total_queries": queries_per_run,
|
||||
"total_sample_evaluations": samples_per_run,
|
||||
"official_test_evaluations": test_evals,
|
||||
"val_metrics": {"brier": 0.1, "ece": 0.02},
|
||||
"is_finite": True,
|
||||
"core_swarm_state_bytes": 1000,
|
||||
})
|
||||
return runs_list
|
||||
|
||||
baselines_dict[wl] = {
|
||||
"workload_id": wl,
|
||||
"method_id": baseline_method,
|
||||
"subset_size": 10000,
|
||||
"particles": 12,
|
||||
"epochs": 80,
|
||||
"seeds": swarm_seeds,
|
||||
"split_seed": split_seed,
|
||||
"data_fingerprint": f"data-fp-{wl}-{split_seed}",
|
||||
"split_fingerprint": f"split-fp-{wl}-{split_seed}",
|
||||
"per_seed_runs": make_runs(),
|
||||
}
|
||||
candidates_dict[wl] = {
|
||||
"workload_id": wl,
|
||||
"ratio": 0.5,
|
||||
"subset_size": 10000,
|
||||
"particles": 12,
|
||||
"epochs": 80,
|
||||
"seeds": swarm_seeds,
|
||||
"split_seed": split_seed,
|
||||
"data_fingerprint": f"data-fp-{wl}-{split_seed}",
|
||||
"split_fingerprint": f"split-fp-{wl}-{split_seed}",
|
||||
"per_seed_runs": make_runs(),
|
||||
}
|
||||
|
||||
splits_dict[split_key] = {
|
||||
"split_seed": split_seed,
|
||||
"baselines": baselines_dict,
|
||||
"candidates": candidates_dict,
|
||||
}
|
||||
|
||||
candidate_payload = {
|
||||
"version": "HEAVY-PSO-CROSS-SPLIT 1.0.0",
|
||||
"protocol_version": "HEAVY-PSO-CROSS-SPLIT 1.0.0",
|
||||
"phase": "development",
|
||||
"split_seeds": splits,
|
||||
"swarm_seeds": swarm_seeds,
|
||||
"official_test_data_loaded": test_loaded,
|
||||
"official_test_evaluations": test_evals * 48,
|
||||
"candidate_config": {
|
||||
"ratio": 0.5,
|
||||
"geometry_policy": "baseline_aligned",
|
||||
"particles": 12,
|
||||
"epochs": 80,
|
||||
"subset_size": 10000,
|
||||
},
|
||||
"workloads": {wl: {"workload_id": wl, "baseline_method": "G8" if "compact" in wl else "G5"} for wl in workloads},
|
||||
"splits": splits_dict,
|
||||
"resource_totals": {
|
||||
"total_runs": 48,
|
||||
"total_queries": queries_per_run * 48,
|
||||
"total_samples_evaluated": samples_per_run * 48,
|
||||
"official_test_evaluations": test_evals * 48,
|
||||
"wall_time_sec": 1.0,
|
||||
},
|
||||
}
|
||||
|
||||
# Construct Evaluation Artifact
|
||||
cell_metrics = []
|
||||
for s_idx, split_seed in enumerate(splits):
|
||||
for wl in workloads:
|
||||
cell_metrics.append({
|
||||
"phase": "development",
|
||||
"split_seed": split_seed,
|
||||
"workload_id": wl,
|
||||
"baseline_acc": 80.0,
|
||||
"candidate_acc": 80.5,
|
||||
"baseline_nll": 0.50,
|
||||
"candidate_nll": 0.49,
|
||||
"acc_gain_pp": 0.5,
|
||||
"nll_reduction_fraction": 0.02,
|
||||
})
|
||||
|
||||
cell_metrics = cell_metrics[:cell_count]
|
||||
|
||||
evaluation_payload = {
|
||||
"pass": False,
|
||||
"development_pass": dev_pass,
|
||||
"eligible_for_confirmation": False,
|
||||
"score": -300.0 + publisher.EXPECTED_VARIANT_IDS.index(variant_id) * 10.0,
|
||||
"evaluator_version": "HEAVY-PSO-CROSS-SPLIT-EVALUATOR 1.0.0",
|
||||
"failed_hard_gate_count": 3,
|
||||
"failed_gates": ["maximum_accuracy_regression_percentage_points_each_split_workload"],
|
||||
"gates": {
|
||||
"official_test_sealed": {
|
||||
"pass": not test_loaded and test_evals == 0,
|
||||
}
|
||||
},
|
||||
"summary_metrics": {
|
||||
"development_cells": len(cell_metrics),
|
||||
"development_grand_mean_accuracy_gain_pp": 0.5,
|
||||
"development_grand_mean_nll_reduction_fraction": 0.02,
|
||||
"development_mnist_wide_accuracy_gain_pp": -0.5,
|
||||
"development_mnist_wide_nll_reduction_fraction": -0.01,
|
||||
},
|
||||
"state_ratios": {
|
||||
"development": {
|
||||
"mnist_compact": 0.4918032786885246,
|
||||
"mnist_wide": 0.5,
|
||||
"fashion_compact": 0.4918032786885246,
|
||||
"fashion_wide": 0.5,
|
||||
}
|
||||
},
|
||||
"cell_metrics": cell_metrics,
|
||||
}
|
||||
|
||||
with cf_path.open("w", encoding="utf-8") as f:
|
||||
json.dump(candidate_payload, f, indent=2)
|
||||
with ef_path.open("w", encoding="utf-8") as f:
|
||||
json.dump(evaluation_payload, f, indent=2)
|
||||
|
||||
return source_path
|
||||
|
||||
|
||||
def test_publish_synthetic_success(tmp_path):
|
||||
"""Verifies successful end-to-end publication on valid synthetic 9-variant cross-split source data."""
|
||||
source_dir = make_synthetic_source(tmp_path / "source")
|
||||
out_json = tmp_path / "pso_v7_heavy_cross_split.json"
|
||||
out_csv = tmp_path / "pso_v7_heavy_cross_split.csv"
|
||||
out_plot = tmp_path / "pso_v7_heavy_cross_split.png"
|
||||
|
||||
payload = publisher.publish_heavy_cross_split(
|
||||
source_dir=source_dir,
|
||||
output_json=out_json,
|
||||
output_csv=out_csv,
|
||||
output_plot=out_plot,
|
||||
)
|
||||
|
||||
# 1. JSON Verification
|
||||
assert out_json.is_file()
|
||||
assert payload["protocol_version"] == publisher.PUBLISH_PROTOCOL_VERSION
|
||||
assert payload["pso_version"] == pso_version
|
||||
assert payload["official_test_data_loaded"] is False
|
||||
assert payload["official_test_evaluations"] == 0
|
||||
assert payload["confirmation_executed"] is False
|
||||
assert payload["retained_policy"] is None
|
||||
|
||||
assert payload["total_runs"] == 432
|
||||
assert payload["total_queries"] == 414720
|
||||
assert payload["total_sample_evaluations"] == 4147200000
|
||||
assert payload["total_wall_time_sec"] == 9.0
|
||||
assert len(payload["variants"]) == 9
|
||||
assert payload["verdict"]["status"] == "NO_RETAINED_POLICY_NO_CONFIRMATION"
|
||||
|
||||
# Best-observed variant should be the final expected variant.
|
||||
best_v = [v for v in payload["variants"] if v["is_best_observed"]]
|
||||
assert len(best_v) == 1
|
||||
assert best_v[0]["variant_id"] == publisher.EXPECTED_VARIANT_IDS[-1]
|
||||
|
||||
# 2. CSV Verification
|
||||
assert out_csv.is_file()
|
||||
with out_csv.open("r", encoding="utf-8") as f:
|
||||
reader = list(csv.reader(f))
|
||||
# 1 header line + 72 data rows = 73 lines
|
||||
assert len(reader) == 73
|
||||
header = reader[0]
|
||||
assert "variant_id" in header
|
||||
assert "baseline_acc" in header
|
||||
assert "candidate_acc" in header
|
||||
assert "acc_gain_pp" in header
|
||||
|
||||
# 3. Plot Verification
|
||||
assert out_plot.is_file()
|
||||
assert out_plot.stat().st_size > 0
|
||||
|
||||
|
||||
def test_publish_mismatch_cell_count(tmp_path):
|
||||
"""Verifies ValueError when an evaluation artifact has a cell count other than 8."""
|
||||
source_dir = make_synthetic_source(tmp_path / "source", cell_count=7)
|
||||
out_json = tmp_path / "out.json"
|
||||
out_csv = tmp_path / "out.csv"
|
||||
out_plot = tmp_path / "out.png"
|
||||
|
||||
with pytest.raises(ValueError, match="Expected 8 development cells"):
|
||||
publisher.publish_heavy_cross_split(source_dir, out_json, out_csv, out_plot)
|
||||
|
||||
|
||||
def test_publish_official_test_unsealed(tmp_path):
|
||||
"""Verifies ValueError when official test evaluations > 0 or official_test_data_loaded is True."""
|
||||
source_dir = make_synthetic_source(tmp_path / "source", test_evals=10)
|
||||
out_json = tmp_path / "out.json"
|
||||
out_csv = tmp_path / "out.csv"
|
||||
out_plot = tmp_path / "out.png"
|
||||
|
||||
with pytest.raises(ValueError, match="official_test_evaluations must be 0"):
|
||||
publisher.publish_heavy_cross_split(source_dir, out_json, out_csv, out_plot)
|
||||
|
||||
|
||||
def test_publish_unexpected_pass(tmp_path):
|
||||
"""Verifies ValueError when development_pass is True."""
|
||||
source_dir = make_synthetic_source(tmp_path / "source", dev_pass=True)
|
||||
out_json = tmp_path / "out.json"
|
||||
out_csv = tmp_path / "out.csv"
|
||||
out_plot = tmp_path / "out.png"
|
||||
|
||||
with pytest.raises(ValueError, match="development_pass must be False"):
|
||||
publisher.publish_heavy_cross_split(source_dir, out_json, out_csv, out_plot)
|
||||
|
||||
|
||||
def test_publish_variant_count_mismatch(tmp_path):
|
||||
"""Verifies that omitting an expected variant is rejected."""
|
||||
source_dir = make_synthetic_source(tmp_path / "source", num_variants=8)
|
||||
out_json = tmp_path / "out.json"
|
||||
out_csv = tmp_path / "out.csv"
|
||||
out_plot = tmp_path / "out.png"
|
||||
|
||||
with pytest.raises(ValueError, match="Expected exact development variants"):
|
||||
publisher.publish_heavy_cross_split(source_dir, out_json, out_csv, out_plot)
|
||||
|
||||
|
||||
def test_deterministic_csv_shape(tmp_path):
|
||||
"""Verifies that running publication twice yields identical CSV byte content."""
|
||||
source_dir = make_synthetic_source(tmp_path / "source")
|
||||
out_json = tmp_path / "out.json"
|
||||
out_csv1 = tmp_path / "out1.csv"
|
||||
out_csv2 = tmp_path / "out2.csv"
|
||||
out_plot = tmp_path / "out.png"
|
||||
|
||||
publisher.publish_heavy_cross_split(source_dir, out_json, out_csv1, out_plot)
|
||||
publisher.publish_heavy_cross_split(source_dir, out_json, out_csv2, out_plot)
|
||||
|
||||
assert out_csv1.read_bytes() == out_csv2.read_bytes()
|
||||
|
||||
def test_publish_rejects_wrong_variant_identity(tmp_path):
|
||||
source_dir = make_synthetic_source(tmp_path / "source")
|
||||
candidate = source_dir / "candidates" / f"{publisher.EXPECTED_VARIANT_IDS[-1]}.json"
|
||||
evaluation = source_dir / "evaluations" / f"{publisher.EXPECTED_VARIANT_IDS[-1]}.json"
|
||||
candidate.rename(candidate.with_name("iteration-9999-development.json"))
|
||||
evaluation.rename(evaluation.with_name("iteration-9999-development.json"))
|
||||
|
||||
with pytest.raises(ValueError, match="Expected exact development variants"):
|
||||
publisher.publish_heavy_cross_split(
|
||||
source_dir,
|
||||
tmp_path / "out.json",
|
||||
tmp_path / "out.csv",
|
||||
tmp_path / "out.png",
|
||||
)
|
||||
|
||||
|
||||
def test_publish_rejects_resource_total_mismatch(tmp_path):
|
||||
source_dir = make_synthetic_source(tmp_path / "source")
|
||||
candidate = source_dir / "candidates" / f"{publisher.EXPECTED_VARIANT_IDS[0]}.json"
|
||||
payload = json.loads(candidate.read_text(encoding="utf-8"))
|
||||
payload["resource_totals"]["total_queries"] -= 1
|
||||
candidate.write_text(json.dumps(payload), encoding="utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match=r"resource_totals\.total_queries"):
|
||||
publisher.publish_heavy_cross_split(
|
||||
source_dir,
|
||||
tmp_path / "out.json",
|
||||
tmp_path / "out.csv",
|
||||
tmp_path / "out.png",
|
||||
)
|
||||
|
||||
|
||||
def test_publish_rejects_invalid_wall_time(tmp_path):
|
||||
source_dir = make_synthetic_source(tmp_path / "source")
|
||||
candidate = source_dir / "candidates" / f"{publisher.EXPECTED_VARIANT_IDS[0]}.json"
|
||||
candidate_payload = json.loads(candidate.read_text(encoding="utf-8"))
|
||||
candidate_payload["resource_totals"]["wall_time_sec"] = float("nan")
|
||||
candidate.write_text(json.dumps(candidate_payload), encoding="utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match=r"resource_totals\.wall_time_sec"):
|
||||
publisher.publish_heavy_cross_split(
|
||||
source_dir,
|
||||
tmp_path / "out.json",
|
||||
tmp_path / "out.csv",
|
||||
tmp_path / "out.png",
|
||||
)
|
||||
|
||||
|
||||
def test_publish_uses_repository_relative_source_path(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(publisher, "REPO_ROOT", tmp_path)
|
||||
source_dir = make_synthetic_source(tmp_path / "source")
|
||||
csv_path = tmp_path / "out.csv"
|
||||
payload = publisher.publish_heavy_cross_split(
|
||||
source_dir,
|
||||
tmp_path / "out.json",
|
||||
csv_path,
|
||||
tmp_path / "out.png",
|
||||
)
|
||||
assert payload["source_provenance"]["source_dir"] == "source"
|
||||
with csv_path.open(encoding="utf-8") as handle:
|
||||
rows = list(csv.DictReader(handle))
|
||||
assert rows[0]["candidate_path"].startswith("source/candidates/")
|
||||
assert rows[0]["evaluation_path"].startswith("source/evaluations/")
|
||||
|
||||
|
||||
def test_cli_invocation(tmp_path, monkeypatch):
|
||||
"""Verifies CLI main entrypoint executes cleanly."""
|
||||
source_dir = make_synthetic_source(tmp_path / "source")
|
||||
out_json = tmp_path / "cli.json"
|
||||
out_csv = tmp_path / "cli.csv"
|
||||
out_plot = tmp_path / "cli.png"
|
||||
|
||||
cli_args = [
|
||||
"publish_heavy_cross_split.py",
|
||||
"--source-dir", str(source_dir),
|
||||
"--output-json", str(out_json),
|
||||
"--output-csv", str(out_csv),
|
||||
"--output-plot", str(out_plot),
|
||||
]
|
||||
monkeypatch.setattr(sys, "argv", cli_args)
|
||||
|
||||
publisher.main()
|
||||
|
||||
assert out_json.is_file()
|
||||
assert out_csv.is_file()
|
||||
assert out_plot.is_file()
|
||||
Reference in New Issue
Block a user