jung-geun/PSO
1
0
Fork 0
mirror of https://github.com/jung-geun/PSO.git synced 2026-09-20 14:11:48 +09:00
Code Issues Packages Projects Releases Wiki Activity

fix(research): isolate resumable YOLO seed stages

Browse Source 
Release native trainer state before cached refinement, rebuild each detector from the persisted EMA checkpoint, save gradient controls incrementally, and reuse hash-verified completed controls after interruption.

Constraint: Preserve baseline checkpoints and fixed optimization budgets

Rejected: Retry candidates in retained trainer process | keeps opaque CUDA state alive

Confidence: high

Scope-risk: moderate

Not-tested: Full three-seed remote development matrix
This commit is contained in:
jung-geun
2026-09-08 16:56:47 +09:00
parent 6fe31394b9
commit 4b18c0ed5e
1 changed files with 62 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/post_training_yolo_convergence.py
+62 -45
View File
@@ -11,6 +11,7 @@ import copy
import csv import csv
import dataclasses import dataclasses
import hashlib import hashlib
import gc
import json import json
import math import math
import os import os
@@ -1729,12 +1730,12 @@ class YoloConvergenceAdapter:
self.result.get("baselines", {}) self.result.get("baselines", {})
) )
for base_seed in BASE_SEEDS: for base_seed in BASE_SEEDS:
model = make_yolo11n(device=self.device)
trainer = StrictScratchTrainer(device=self.device, batch=16) trainer = StrictScratchTrainer(device=self.device, batch=16)
baseline = baselines.get(str(base_seed)) baseline = baselines.get(str(base_seed))
if not _checkpoint_record_valid(self.run_root, baseline): if not _checkpoint_record_valid(self.run_root, baseline):
training_model = make_yolo11n(device=self.device)
baseline = train_baseline( baseline = train_baseline(
model=model, model=training_model,
yaml_path=str(yaml_path), yaml_path=str(yaml_path),
trainer=trainer, trainer=trainer,
run_root=self.run_root, run_root=self.run_root,
@@ -1743,7 +1744,11 @@ class YoloConvergenceAdapter:
baselines[str(base_seed)] = baseline baselines[str(base_seed)] = baseline
self.result["baselines"] = baselines self.result["baselines"] = baselines
self._save() self._save()
detector = model.model del training_model
gc.collect()
if self.device == "cuda":
torch.cuda.empty_cache()
detector = make_yolo11n(device=self.device).model
state = torch.load( state = torch.load(
_resolve_run_path( _resolve_run_path(
self.run_root, self.run_root,
@@ -1916,49 +1921,52 @@ class YoloConvergenceAdapter:
] = record ] = record
arm_record[f"{method}:{swarm_seed}"] = record arm_record[f"{method}:{swarm_seed}"] = record
self._save() self._save()
feature_detector = make_yolo11n(device=self.device).model for method in ("feature_adam", "head_adam"):
feature_detector.load_state_dict(pristine_state, strict=True) existing = self.result["arms"].get(method, {}).get(
head_detector = make_yolo11n(device=self.device).model str(base_seed)
head_detector.load_state_dict(pristine_state, strict=True)
feature_names = selected_block_names(feature_detector)
feature_codec = SelectedResidualCodec(
feature_detector,
feature_names,
projection_seed=PROJECTION_SEED,
)
feature_bounds = {
name: RESIDUAL_BOUND * scale
for name, scale in zip(
feature_codec.names,
feature_codec.scales,
) )
} if _checkpoint_record_valid(self.run_root, existing):
feature_adam = run_bounded_adam( arm_record[method] = existing
feature_detector, continue
feature_names, control_model = make_yolo11n(device=self.device).model
lambda backward, detector=feature_detector, cache=cache: control_model.load_state_dict(pristine_state, strict=True)
cached_detection_loss_tensor( if method == "feature_adam":
detector, parameter_names = selected_block_names(control_model)
cache, feature_codec = SelectedResidualCodec(
model_device=self.device, control_model,
backward=backward, parameter_names,
), projection_seed=PROJECTION_SEED,
bounds=feature_bounds, )
) feature_bounds = {
head_adam = run_head_adam( name: RESIDUAL_BOUND * scale
head_detector, for name, scale in zip(
lambda backward, detector=head_detector, cache=cache: feature_codec.names,
cached_detection_loss_tensor( feature_codec.scales,
detector, )
cache, }
model_device=self.device, record = run_bounded_adam(
backward=backward, control_model,
), parameter_names,
) lambda backward, detector=control_model, cache=cache:
for method, control_model, record in ( cached_detection_loss_tensor(
("feature_adam", feature_detector, feature_adam), detector,
("head_adam", head_detector, head_adam), cache,
): model_device=self.device,
backward=backward,
),
bounds=feature_bounds,
)
else:
record = run_head_adam(
control_model,
lambda backward, detector=control_model, cache=cache:
cached_detection_loss_tensor(
detector,
cache,
model_device=self.device,
backward=backward,
),
)
selection_metrics, _ = evaluate_detection_records( selection_metrics, _ = evaluate_detection_records(
control_model, control_model,
selection_records, selection_records,
@@ -1983,6 +1991,11 @@ class YoloConvergenceAdapter:
str(base_seed) str(base_seed)
] = record ] = record
arm_record[method] = record arm_record[method] = record
self._save()
del control_model
gc.collect()
if self.device == "cuda":
torch.cuda.empty_cache()
selected: dict[str, Any] = {} selected: dict[str, Any] = {}
for method in ("feature_pso", "feature_random"): for method in ("feature_pso", "feature_random"):
records = self.result["arms"][method][str(base_seed)] records = self.result["arms"][method][str(base_seed)]
@@ -2006,6 +2019,10 @@ class YoloConvergenceAdapter:
self.result["development_selection"][str(base_seed)] = selected self.result["development_selection"][str(base_seed)] = selected
arm_path = self.root / "arms" / str(base_seed) / "record.json" arm_path = self.root / "arms" / str(base_seed) / "record.json"
atomic_write_json(arm_path, arm_record) atomic_write_json(arm_path, arm_record)
del detector, cache, objective
gc.collect()
if self.device == "cuda":
torch.cuda.empty_cache()
pool_records = objective_records + selection_records pool_records = objective_records + selection_records
member_rows = [[] for _ in pool_records] member_rows = [[] for _ in pool_records]
for baseline in baselines.values(): for baseline in baselines.values():