mirror of
https://github.com/jung-geun/PSO.git
synced 2026-09-20 14:11:48 +09:00
feat: modernize PSO and add convergence research
Migrate the package and examples to the tensor-native PyTorch implementation, add benchmark evidence, and add the guarded post-training convergence protocol with TensorBoard progress monitoring and hash-verified recovery. Constraint: Preserve one-shot official-test sealing and auditable research artifacts Rejected: Commit local .omc runs and downloaded datasets | multi-gigabyte runtime state is machine-local Confidence: high Scope-risk: broad Not-tested: Production CUDA run on pieroot-server
This commit is contained in:
@@ -1,23 +0,0 @@
|
||||
name: Release Tag
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- master
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Get Version
|
||||
run: echo "##[set-output name=version;]$(echo '${{ github.event.head_commit.message }}' | egrep -o '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}')"
|
||||
id: extract_version_name
|
||||
- name: Create Release
|
||||
uses: actions/create-release@v1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
tag_name: ${{ steps.extract_version_name.outputs.version }}
|
||||
release_name: Release ${{ steps.extract_version_name.outputs.version }}
|
||||
body: |
|
||||
Release ${{ steps.extract_version_name.outputs.version }}
|
||||
+40
-23
@@ -1,38 +1,55 @@
|
||||
name: PyPI package
|
||||
name: Publish to PyPI
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "setup.py"
|
||||
- "pso/__init__.py"
|
||||
tags:
|
||||
- "v*"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
jobs:
|
||||
build-linux:
|
||||
test:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
max-parallel: 5
|
||||
matrix:
|
||||
python-version: ["3.9"]
|
||||
python-version: ["3.10", "3.11"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v3
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install uv and Python ${{ matrix.python-version }}
|
||||
uses: astral-sh/setup-uv@v9
|
||||
with:
|
||||
version: "0.12.7"
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||
pip install setuptools wheel twine
|
||||
- name: Build and publish
|
||||
env:
|
||||
TWINE_USERNAME: __token__
|
||||
TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
|
||||
run: |
|
||||
python setup.py bdist_wheel sdist
|
||||
twine upload dist/*.whl dist/*.tar.gz
|
||||
enable-cache: true
|
||||
- name: Sync dependencies
|
||||
run: uv sync --locked --group dev
|
||||
- name: Run tests
|
||||
run: uv run --no-sync pytest -q
|
||||
- name: Build package
|
||||
run: uv build
|
||||
- name: Check package distribution
|
||||
run: uv run --no-sync twine check dist/*
|
||||
|
||||
publish:
|
||||
needs: test
|
||||
runs-on: ubuntu-22.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install uv and Python 3.11
|
||||
uses: astral-sh/setup-uv@v9
|
||||
with:
|
||||
version: "0.12.7"
|
||||
python-version: "3.11"
|
||||
enable-cache: true
|
||||
- name: Sync dependencies
|
||||
run: uv sync --locked --group dev
|
||||
- name: Build package
|
||||
run: uv build
|
||||
- name: Check package distribution
|
||||
run: uv run --no-sync twine check dist/*
|
||||
- name: Publish package to PyPI
|
||||
uses: pypa/gh-action-pypa-publish@release/v1
|
||||
with:
|
||||
password: ${{ secrets.PYPI_TOKEN }}
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
name: Python Package using Conda
|
||||
|
||||
on: [push]
|
||||
|
||||
jobs:
|
||||
build-linux:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
max-parallel: 5
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Python 3.9
|
||||
uses: actions/setup-python@v3
|
||||
with:
|
||||
python-version: "3.9"
|
||||
- name: Add conda to system path
|
||||
run: |
|
||||
# $CONDA is an environment variable pointing to the root of the miniconda directory
|
||||
echo $CONDA/bin >> $GITHUB_PATH
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
conda env create --file conda_env/environment.yaml --name pso
|
||||
conda activate pso
|
||||
python mnist.py
|
||||
@@ -0,0 +1,41 @@
|
||||
name: Python package test
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "pso/**"
|
||||
- "pyproject.toml"
|
||||
- "uv.lock"
|
||||
- ".python-version"
|
||||
- "README.md"
|
||||
- "tests/**"
|
||||
- ".github/workflows/python-package.yml"
|
||||
pull_request:
|
||||
paths:
|
||||
- "pso/**"
|
||||
- "pyproject.toml"
|
||||
- "uv.lock"
|
||||
- ".python-version"
|
||||
- "README.md"
|
||||
- "tests/**"
|
||||
- ".github/workflows/python-package.yml"
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-22.04
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Install uv and Python ${{ matrix.python-version }}
|
||||
uses: astral-sh/setup-uv@v9
|
||||
with:
|
||||
version: "0.12.7"
|
||||
python-version: ${{ matrix.python-version }}
|
||||
enable-cache: true
|
||||
- name: Sync dependencies
|
||||
run: uv sync --locked --group dev
|
||||
- name: Run tests
|
||||
run: uv run --no-sync pytest -q
|
||||
@@ -27,3 +27,11 @@ logs/
|
||||
|
||||
.vscode/
|
||||
metacode/
|
||||
|
||||
# uv / venv
|
||||
.venv/
|
||||
.uv/
|
||||
|
||||
# Local research execution state and large downloaded datasets
|
||||
.omc/
|
||||
runs/
|
||||
@@ -0,0 +1 @@
|
||||
3.11
|
||||
@@ -0,0 +1,796 @@
|
||||
# PSO v4.0.0 실증 벤치마크 평가 보고서 (Benchmark 2.0.0 + Tuning 1.0.0 + Replication 1.0.0 + Epoch Convergence 1.0.0 + Full MNIST 1.0.0 + Deep Accuracy 1.0.0 + MNIST-PSO-RAW-V5 1.0.0 + MNIST-PSO-RAW-V6 1.0.0 + HEAVY-TASK-PSO-V6 1.0.0 + HEAVY-PSO-CROSS-SPLIT 1.0.0)
|
||||
|
||||
## 1. 요약 및 핵심 발견사항 (Executive Findings)
|
||||
|
||||
본 보고서는 `pso2keras` 라이브러리의 v4.0.0 5단계 플러그인 아키텍처 기반 미분 무관(Derivative-Free) Particle Swarm Optimization (PSO) 알고리즘 수렴 성능 및 특성에 대한 종합 실증 평가 결과입니다. 벤치마크 프로토콜 v2.0.0에 따라 총 225회의 독립 측정 실행(메인 벤치마크 7기법 × 5워크로드 × 5시드 = 175회, MNIST Ablation 10프로필 × 5시드 = 50회)을 수행하였으며, 모든 실행은 에러 없이 100% 성공적으로 완료되었습니다.
|
||||
|
||||
추가로 Tuning Protocol 1.0.0에서 32개 하이퍼파라미터 후보의 검증 선택, 5개 기법의 held-out 확인, `adaptive_moment` 파티클 스케일링을 수행했습니다. 이 확장 연구는 161개 결과 레코드(96 search + 25 confirmation + 40 scaling)를 포함하며, 공유 30×80 scaling 셀을 재사용했으므로 실제 scaling 적합 실행은 35회입니다. 이후 120×80 exact replay/fresh-seed 10회, 같은 시드 5개의 240-epoch subset 궤적, 공식 MNIST 60,000/10,000 전체 split의 240-epoch 궤적 5회를 각각 별도 프로토콜로 실행했습니다. Deep Accuracy Protocol 1.0.0은 아키텍처 레인 9개와 최적화기 레인 9개의 논리 레코드를 포함하며, 최적화기 레인의 `adam_only` 3개는 아키텍처 레인의 Compact CNN 결과를 명시적으로 재사용하므로 실제 고유 학습 워크플로는 15개입니다. 이 후속 실행들은 기존 161개 Tuning Protocol 레코드나 225회 Protocol 2.0.0 결과에 합산하지 않습니다.
|
||||
|
||||
HEAVY-TASK-PSO-V6 1.0.0은 공식 test split을 사용하지 않고 MNIST/FashionMNIST의 train 60,000개를 각각 search 50,000/validation 10,000으로 분할해, Compact CNN(9,098 params)과 WideCNN(55,338 params)의 전가중치 PSO 실행 가능성을 추가로 평가했습니다. 단일시드 16-cell screen 뒤 G8과 workload별 validation-selected normalized 방법을 3개 시드로 확인했으며, 이 결과는 기존 프로토콜 실행 수에 합산하지 않습니다.
|
||||
|
||||
### 핵심 평가 요약
|
||||
1. **고전 무브먼트 기법의 강세**: 5개 워크로드 종합 평균 순위에서 `constriction` (수축 계수 PSO)과 `inertia` (관성 가중치 감쇄 PSO)가 평가 정확도 평균 순위 **1.60**으로 공동 1위를 기록했습니다. 손실(Loss) 측면에서는 `constriction`이 평균 순위 **1.60**으로 `inertia` (**1.80**) 대비 더 낮은 손실 수렴 성향을 나타냈습니다.
|
||||
2. **미분 기반 후처리(Adam)의 효과**: MNIST 소형 분류 모델 실험에서 하이브리드 경사하강 미세조정(`tuned_adam_100_lr.01`)을 적용할 경우 검증 정확도 **85.58% ± 0.24%**를 기록하여 최고 성과를 달성했습니다. 다만, 이는 역전파 경사도(`loss.backward()`)를 활용하는 하이브리드 방식입니다.
|
||||
3. **미분 무관 Ablation 탐색 최고 성과**: pure derivative-free 기법 중 MNIST Ablation 최고 검증 정확도는 `adaptive_moment_.10` (적응형 경로 모멘트 혼합비 0.10)로 **63.00% ± 1.83%**를 기록했습니다 (단, 검증 손실은 **1.243339**로 `inertia_tuned`의 **1.236600** 대비 약간 높음).
|
||||
4. **평가 방식 및 샘플 수 교란 요인**: 미분 무관 ablation 중 최저 검증 손실은 `tuned_full_evaluation`의 **1.179009 ± 0.050059**였으나, 이는 고정 서브셋(2,000개)이 아닌 전체 학습 데이터(3,000개) 평가 및 배치 처리 차이에 따른 교란 요인(Confound)이 반영된 수치입니다.
|
||||
5. **파티클 재초기화(Particle Reset)**: `tuned_particle_reset` 프로필은 측정된 시드 세트(46~50)에서 `inertia_tuned`와 동일한 최종 성능 수치(**61.62% / 1.236600**)를 기록했습니다. 재초기화 이벤트 수를 별도로 계측하지 않았으므로, 이벤트가 없었는지 또는 최종 전역 최적해에 영향을 주지 않았는지는 이 결과만으로 구분할 수 없습니다.
|
||||
6. **확장 튜닝의 동일 예산 확인**: 파티클 30개 × 80세대 held-out 확인에서 `local_best`가 **62.64% ± 2.94%**, `inertia`가 **62.30% ± 3.01%**, `adaptive_moment`가 **61.06% ± 0.91%**를 기록했습니다. 따라서 Adaptive Moment는 이 동일 예산 비교의 최상위 기법이 아닙니다.
|
||||
7. **Adaptive Moment 파티클 증가**: 80세대를 유지하면서 파티클을 30개에서 120개로 늘리면 정확도가 **62.12%에서 72.34%**로 상승했지만 particle-evaluations와 fit 시간도 각각 4배와 **4.16배**로 증가했습니다. 약 2,400 particle-epochs를 고정하면 파티클 증가와 세대 감소 조합의 정확도는 오히려 낮아졌습니다.
|
||||
8. **120×80 재현성 확인**: 동일 시드 71~75 exact replay는 baseline과 같은 **72.34% ± 1.82%**를 재현했고 시드별 최대 차이는 **0.00%p**였습니다. 독립 시드 81~85는 **73.60% ± 1.66%**로 baseline 대비 **+1.26%p**였으며, 사전 선언한 ±3%p 및 95% t-신뢰구간 중첩 조건을 모두 충족했습니다.
|
||||
9. **Epoch 80은 조기 수렴 지점이 아님**: 120개 파티클의 연속 실행을 240세대까지 늘리자 테스트 정확도가 **72.34%에서 83.52%**로 **+11.18%p** 상승하고 training best loss가 **47.30%** 감소했습니다. 200→240에서도 loss가 **6.68%** 감소하고 정확도가 **+0.94%p** 상승해 epoch 240에서도 완전한 plateau는 확인되지 않았습니다.
|
||||
10. **전체 MNIST 학습 결과**: 공식 train 60,000개를 모든 파티클의 매 epoch fitness에 사용하고 test 10,000개 전체를 평가했습니다. 120 particles × 240 epochs에서 정확도는 **87.70% ± 0.36%**, 테스트 loss는 **0.412078 ± 0.006848**이었습니다. Epoch 200→240에서도 정확도가 **+0.85%p**, training loss가 **6.13%** 개선되어 plateau는 확인되지 않았습니다.
|
||||
11. **원본 MNIST 이미지 및 딥 신경망 아키텍처/최적화 기법 평가 (Deep Accuracy 1.0.0)**: PCA 없이 공식 train 60,000개와 test 10,000개 원본 $1 \times 28 \times 28$ 입력을 평가했습니다($n=3$, seeds 101~103). Adam 10 epochs에서 Raw Linear(7,850 params)는 **92.45% ± 0.10%**, Raw MLP(109,386 params)는 **97.70% ± 0.08%**, Compact CNN(9,098 params)은 **98.53% ± 0.16%**였고 CNN은 모든 시드가 5 epochs 이내 98%에 도달했습니다. 같은 CNN에서 30 particles × 40 generations·fixed-2k all-weight PSO는 **36.76% ± 3.76%**, PSO→Adam은 **97.30% ± 0.75%**였습니다. 측정한 예산에서는 전가중치 PSO와 PSO 초기화가 pure Adam을 대체하거나 개선하지 못했습니다.
|
||||
12. **V5 탐색 구조 회귀 원인 분리 (MNIST-PSO-RAW-V6 1.0.0)**: 공식 test split을 로드하지 않고 train 60,000개를 search 50,000/validation 10,000으로 나눈 뒤, Compact CNN 전가중치 PSO의 탐색 구조 9개를 비교했습니다. 3-시드 확인에서 V5 구조 G0은 **79.53% ± 0.58% / NLL 0.689687**, 기존 `Optimizer` 제어군 G8은 **84.92% ± 0.99% / 0.481440**이었습니다. mutation·초기 속도·normalized ±6 경계를 묶은 G5/G6은 각각 **84.24% / 0.510297**, **84.25% / 0.503809**로 사전 회복 기준을 충족했습니다. 단일시드 screen은 mutation과 경계 확장을 유력 요인으로 지목하지만 개별 3-시드 인과 확인은 아직 수행하지 않았습니다.
|
||||
13. **더 무거운 영상 태스크 실행 가능성 (HEAVY-TASK-PSO-V6 1.0.0)**: 파라미터 수를 **9,098→55,338(6.08배)**로 늘리고 FashionMNIST를 추가한 네 workload에서 G8과 screen-selected G5/G6의 24개 확인 실행이 모두 finite하게 완료됐습니다. 사전 기준(초기 모델 대비 validation NLL 20% 이상 감소와 accuracy 20%p 이상 상승)은 8개 workload-method 집계가 모두 통과했습니다. 그러나 12 particles × 80 epochs·fixed-10k에서 최종 정확도는 **41.03~49.15%**에 그쳐, 이는 계산상 최적화 가능성이지 실용적 학습 성능이나 Adam 대체 가능성을 의미하지 않습니다.
|
||||
14. **Heavy PSO 교차 분할 강건성 검증 실패 (HEAVY-PSO-CROSS-SPLIT 1.0.0)**: 개발 분할 2개(20260905/20260906) 및 시드 101~103, 매칭 baseline 재실행 조건(12p×80e×fixed10k)에서 동결 정책 `fixed_global_hybrid_v3`를 교차 평가한 결과, 전체 정확도 개선은 +0.1533%p, NLL 감소는 0.5546%에 그쳤고 최악 accuracy 회귀 -7.1767%p, 최악 NLL 회귀 +14.5682%, MNIST Wide acc -0.3617%p(NLL 2.4600% 악화)로 게이트를 통과하지 못해 실패(FAIL)로 판정되었습니다. 총 8회 결정 탐색(9개 개발 변형) 중 최상위 스코어 후보(Iteration 5, 스코어 -185.610686)도 MNIST Wide acc -2.6633%p(NLL 2.1874% 악화)로 탈락했습니다. 이에 따라 보존 정책은 `null`로 확정되었고 확인 분할(20260907) 실행은 과학적 규칙에 따라 보류되었으며 공식 test split은 0회 로드/평가되었습니다 (총 432 runs, 414,720 queries, 41.472억 샘플 평가, 3162.9717s).
|
||||
|
||||
---
|
||||
|
||||
## 2. 실험 환경 및 워크로드 스펙 (Hardware, Software & Workload Spec)
|
||||
|
||||
### 2.1 하드웨어 및 소프트웨어 프로비넌스
|
||||
|
||||
| 환경 항목 | 세부 사양 / 버전 |
|
||||
| --- | --- |
|
||||
| **플랫폼 (OS)** | macOS 26.5.2 (Darwin 25.5.0 arm64) |
|
||||
| **프로세서 (CPU/GPU)** | Apple M5 Max (System Apple Silicon, MPS 가속) |
|
||||
| **Python 버전** | 3.11.15 |
|
||||
| **PyTorch 버전** | 2.13.0 (`device="mps"`) |
|
||||
| **패키지 버전** | `pso2keras` v4.0.0 |
|
||||
| **벤치마크 프로토콜** | Protocol v2.0.0 |
|
||||
|
||||
### 2.2 워크로드 및 신경망 모델 매트릭스
|
||||
|
||||
본 벤치마크는 비선형 논리 회로(XOR), 소형 다변량 표형 데이터(Iris, Seeds), 중형 이미지 수치 데이터(Digits), PCA 32차원 축소 MNIST 데이터셋을 대상으로 수행되었습니다.
|
||||
|
||||
| 워크로드 | 데이터 크기 (학습/검증) | 모델 구조 | 파라미터 수 | 손실 함수 및 작업 | 파티클 수 ($N$) | 세대 수 ($T$) | 평가 대상 집합 | 시드 범위 |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| **XOR** | 4 / 4 | `Linear(2,4) - Tanh - Linear(4,1)` | 17 | `BCEWithLogitsLoss` (Binary) | 24 | 80 | Train Eval | 41 ~ 45 |
|
||||
| **Iris** | 120 / 30 | `Linear(4,10) - ReLU - Linear(10,10) - ReLU - Linear(10,3)` | 193 | `CrossEntropyLoss` (Multiclass) | 24 | 60 | Held-out Eval | 41 ~ 45 |
|
||||
| **Seeds** | 168 / 42 | `Linear(7,16) - ReLU - Linear(16,32) - ReLU - Linear(32,3)` | 771 | `CrossEntropyLoss` (Multiclass) | 24 | 60 | Held-out Eval | 41 ~ 45 |
|
||||
| **Digits** | 1,437 / 360 | `Linear(64,12) - ReLU - Linear(12,10) - ReLU - Linear(10,10)` | 1,020 | `CrossEntropyLoss` (Multiclass) | 24 | 50 | Held-out Eval | 41 ~ 45 |
|
||||
| **MNIST** | 3,000 / 1,000 (PCA32) | `Linear(32,10)` | 330 | `CrossEntropyLoss` (Multiclass) | 30 | 80 | Held-out Eval | 41 ~ 45 |
|
||||
|
||||
> **참고**: MNIST 실험은 784차원 원본 이미지 텐서가 아닌 PCA 32차원 로짓 분류기 `Linear(32,10)` 환경(학습 3,000개, 검증 1,000개)에서 수행되었습니다.
|
||||
> **타이밍 예산 및 웜업**: 각 측정은 별도의 모델과 Optimizer로 기법별 비측정 웜업(PSO 2세대)을 수행한 뒤, 동일 시드로 측정 대상을 다시 생성하여 `fit()` 호출만 측정했습니다. Adam 프로필의 웜업은 refinement 1세대를 포함하며, 메인 미분 무관 실험의 웜업은 refinement를 사용하지 않습니다.
|
||||
|
||||
---
|
||||
|
||||
## 3. 메인 벤치마크 평가 결과 (Main Benchmark Matrix)
|
||||
|
||||
7가지 주요 PSO 무브먼트 기법(`original`, `inertia`, `constriction`, `fips`, `clpso`, `bare_bones`, `adaptive_moment`)에 대한 5개 워크로드별 5-시드 평균 및 표준편차(Mean ± SD, $n=5$) 결과입니다.
|
||||
|
||||
### 3.1 정확도 Matrix (Eval Accuracy %, Mean ± SD)
|
||||
|
||||
| 워크로드 (평가 구분) | original | inertia | constriction | fips | clpso | bare_bones | adaptive_moment |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| **XOR** (Train) | 100.00% ± 0.00% | 100.00% ± 0.00% | **100.00% ± 0.00%** | 75.00% ± 25.00% | 60.00% ± 13.69% | 100.00% ± 0.00% | 100.00% ± 0.00% |
|
||||
| **Iris** (Held-out) | 92.00% ± 8.69% | **94.00% ± 2.79%** | 94.00% ± 3.65% | 74.00% ± 5.48% | 79.33% ± 7.60% | 84.67% ± 5.58% | 84.67% ± 7.67% |
|
||||
| **Seeds** (Held-out) | 87.62% ± 4.26% | **91.43% ± 5.22%** | 89.05% ± 9.16% | 88.57% ± 3.53% | 85.71% ± 2.92% | 87.14% ± 6.86% | 89.05% ± 4.64% |
|
||||
| **Digits** (Held-out) | 16.44% ± 2.23% | 30.33% ± 6.17% | **37.28% ± 1.79%** | 24.94% ± 6.15% | 20.78% ± 5.06% | 19.00% ± 5.63% | 26.78% ± 1.83% |
|
||||
| **MNIST** (Held-out) | 14.40% ± 1.66% | 46.84% ± 7.53% | **52.00% ± 4.62%** | 21.06% ± 4.81% | 20.68% ± 2.28% | 41.26% ± 3.80% | 26.20% ± 1.04% |
|
||||
|
||||
### 3.2 손실 Matrix (Eval Raw Loss, Mean ± SD)
|
||||
|
||||
| 워크로드 (평가 구분) | original | inertia | constriction | fips | clpso | bare_bones | adaptive_moment |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| **XOR** (Train) | 0.004558 ± 0.002672 | 0.004431 ± 0.003308 | **0.004086 ± 0.002357** | 0.546355 ± 0.059307 | 0.548166 ± 0.031946 | 0.082100 ± 0.056963 | 0.190431 ± 0.178122 |
|
||||
| **Iris** (Held-out) | 0.222324 ± 0.155767 | **0.106901 ± 0.053063** | 0.171796 ± 0.076965 | 0.605010 ± 0.060601 | 0.448270 ± 0.053877 | 0.405791 ± 0.172336 | 0.349701 ± 0.178108 |
|
||||
| **Seeds** (Held-out) | 0.492240 ± 0.433653 | 0.331656 ± 0.342636 | 0.351420 ± 0.388200 | 0.383732 ± 0.103951 | 0.469817 ± 0.073254 | 0.421920 ± 0.120052 | **0.315947 ± 0.120895** |
|
||||
| **Digits** (Held-out) | 2.269628 ± 0.025331 | 1.980667 ± 0.143070 | **1.765113 ± 0.057309** | 2.064165 ± 0.091569 | 2.204543 ± 0.023215 | 2.250203 ± 0.084448 | 2.108837 ± 0.023357 |
|
||||
| **MNIST** (Held-out) | 2.428194 ± 0.091877 | 1.716089 ± 0.260764 | **1.507968 ± 0.149152** | 2.223230 ± 0.073048 | 2.285833 ± 0.068177 | 1.965562 ± 0.175909 | 2.219058 ± 0.104724 |
|
||||
|
||||
### 3.3 워크로드별 최고 성과 기법 요약 (Per-workload Winners)
|
||||
|
||||
| 워크로드 | 최고 정확도 기법 (Eval Acc) | 최저 손실 기법 (Eval Loss) | 비고 |
|
||||
| --- | --- | --- | --- |
|
||||
| **XOR** | `constriction` (100.00% ± 0.00%) | `constriction` (0.004086 ± 0.002357) | 5개 기법 100% 동률 (손실로 순위 구분) |
|
||||
| **Iris** | `inertia` (94.00% ± 2.79%) | `inertia` (0.106901 ± 0.053063) | `constriction`과 정확도 동률, 손실로 순위 구분 |
|
||||
| **Seeds** | `inertia` (91.43% ± 5.22%) | `adaptive_moment` (0.315947 ± 0.120895) | 정확도 inertia 우수, 손실 AM 우수 |
|
||||
| **Digits** | `constriction` (37.28% ± 1.79%) | `constriction` (1.765113 ± 0.057309) | 고차원 다중 분류에서 수축 계수 우수 |
|
||||
| **MNIST** | `constriction` (52.00% ± 4.62%) | `constriction` (1.507968 ± 0.149152) | 이 고정 예산에서 정확도와 손실 모두 1위 |
|
||||
|
||||
### 3.4 평균 순위 및 수렴 실행 시간 (Ranks & Warmed Runtime)
|
||||
|
||||
각 워크로드 내에서 1위(최고)부터 7위(최저)까지 순위를 부여한 후 5개 워크로드에 대해 평균한 결과 및 MPS 디바이스 웜업 완료 후 측정된 실행 시간입니다.
|
||||
|
||||
| 기법 (`method`) | 정확도 순위 목록 [XOR, Iris, Seeds, Digits, MNIST] | 평균 정확도 순위 | 손실 순위 목록 [XOR, Iris, Seeds, Digits, MNIST] | 평균 손실 순위 | 평균 실행 시간 ($n=25$) |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| **`constriction`** | [1, 2, 3, 1, 1] | **1.60** | [1, 2, 3, 1, 1] | **1.60** | 2.0821s ± 0.766s |
|
||||
| **`inertia`** | [2, 1, 1, 2, 2] | **1.60** | [2, 1, 2, 2, 2] | **1.80** | 1.9824s ± 0.614s |
|
||||
| **`adaptive_moment`** | [5, 4, 2, 3, 4] | **3.60** | [5, 4, 1, 4, 4] | **3.60** | 2.1264s ± 0.874s |
|
||||
| **`bare_bones`** | [4, 5, 6, 6, 3] | **4.80** | [4, 5, 5, 6, 3] | **4.60** | 2.2476s ± 0.724s |
|
||||
| **`original`** | [3, 3, 5, 7, 7] | **5.00** | [3, 3, 7, 7, 7] | **5.40** | 2.1112s ± 0.857s |
|
||||
| **`fips`** | [6, 7, 4, 4, 5] | **5.20** | [6, 7, 4, 3, 5] | **5.00** | 2.5636s ± 0.708s |
|
||||
| **`clpso`** | [7, 6, 7, 5, 6] | **6.20** | [7, 6, 6, 5, 6] | **6.00** | 2.3507s ± 0.726s |
|
||||
|
||||
> **핵심 요약**: `inertia`와 `constriction`은 평균 정확도 순위 **1.60**으로 공동 1위를 기록했습니다. 평균 손실 순위는 `constriction` **1.60**, `inertia` **1.80**이었으며, 이는 이 다섯 워크로드의 표본 평균 서열입니다.
|
||||
|
||||
---
|
||||
|
||||
## 4. 메인 시각화 차트 (Main Figures)
|
||||
|
||||
벤치마크 결과 시각화 차트는 `history_plt/` 디렉터리에 저장되어 있습니다.
|
||||
|
||||
| 정확도 비교 (Accuracy) | 손실 비교 (Loss) |
|
||||
| --- | --- |
|
||||
|  |  |
|
||||
|
||||
| 순위 히트맵 (Rank Heatmap) | 실행 시간 비교 (Runtime) |
|
||||
| --- | --- |
|
||||
|  |  |
|
||||
|
||||
---
|
||||
|
||||
## 5. MNIST Ablation 연구 (MNIST 10-Profile Ablation Study)
|
||||
|
||||
MNIST PCA32 `Linear(32,10)` 워크로드(파티클 30, 세대 80, 시드 46~50)에서 초기화, 적합도 평가, 수렴 제어, 미세조정, 적응형 경로 모멘트 하이퍼파라미터 변형 10개 프로필을 비교 분석하였습니다.
|
||||
|
||||
### 5.1 Ablation 종합 성과 매트릭스
|
||||
|
||||
| 프로필 식별자 (`profile`) | 검증 정확도 (Eval Acc) | 검증 손실 (Eval Loss) | 학습 정확도 (Train Acc) | 학습 손실 (Train Loss) | 실행 시간 (Runtime) | Acc 순위 | Loss 순위 |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| `tuned_adam_100_lr.01` | **85.58% ± 0.24%** | **0.470271 ± 0.011931** | 91.58% ± 0.39% | 0.298524 ± 0.012987 | 4.991s ± 0.434s | 1 | 1 |
|
||||
| `adaptive_moment_.10` | **63.00% ± 1.83%** | 1.243339 ± 0.102561 | 70.99% ± 1.28% | 0.978244 ± 0.021198 | 4.287s ± 0.289s | 2 | 5 |
|
||||
| `tuned_full_evaluation` | **62.54% ± 2.80%** | **1.179009 ± 0.050059** | 68.79% ± 1.07% | 1.005140 ± 0.028281 | 3.377s ± 0.469s | 3 | 2 |
|
||||
| `inertia_tuned` (기준) | 61.62% ± 5.13% | 1.236600 ± 0.124281 | 69.31% ± 2.93% | 0.999514 ± 0.075496 | 3.132s ± 0.409s | 4 | 3 |
|
||||
| `tuned_particle_reset` | 61.62% ± 5.13% | 1.236600 ± 0.124281 | 69.31% ± 2.93% | 0.999514 ± 0.075496 | 4.176s ± 0.449s | 5 | 4 |
|
||||
| `tuned_no_mutation` | 60.42% ± 3.04% | 1.249836 ± 0.084985 | 66.86% ± 1.28% | 1.074396 ± 0.043372 | 3.348s ± 0.343s | 6 | 6 |
|
||||
| `tuned_uniform_initialization` | 56.98% ± 2.40% | 1.619196 ± 0.157329 | 61.60% ± 0.90% | 1.345857 ± 0.048557 | 4.307s ± 0.443s | 7 | 8 |
|
||||
| `adaptive_moment_.25` | 56.32% ± 2.94% | 1.499979 ± 0.119449 | 63.79% ± 2.42% | 1.228023 ± 0.088967 | 3.291s ± 0.426s | 8 | 7 |
|
||||
| `adaptive_moment_.50` | 51.86% ± 3.91% | 1.683781 ± 0.121571 | 57.05% ± 2.75% | 1.463700 ± 0.097485 | 3.203s ± 0.270s | 9 | 9 |
|
||||
| `inertia_canonical` | 46.76% ± 2.44% | 1.720983 ± 0.118092 | 52.77% ± 2.89% | 1.528197 ± 0.079997 | 3.120s ± 0.306s | 10 | 10 |
|
||||
|
||||
### 5.2 MNIST Ablation 시각화 차트
|
||||
|
||||
|
||||
|
||||
### 5.3 `inertia_tuned` 기준 상대 델타 (Paired Deltas vs `inertia_tuned`)
|
||||
|
||||
`inertia_tuned` 프로필(Eval Acc: **61.62%**, Eval Loss: **1.236600**, Runtime: **3.132s**) 대비 각 변형 프로필의 절대 편차량입니다:
|
||||
|
||||
1. **`inertia_canonical` vs `inertia_tuned`**:
|
||||
- $\Delta$ Eval Acc: **-14.86%p** (46.76% vs 61.62%)
|
||||
- $\Delta$ Eval Loss: **+0.484383** (1.720983 vs 1.236600)
|
||||
- $\Delta$ Runtime: **-0.012s** (3.120s vs 3.132s)
|
||||
- *해석*: 인지·사회 계수, 관성, 속도 제한 및 변이를 함께 조정한 튜닝 프로필이 이 PCA32 워크로드에서 canonical inertia 프로필보다 14.86%p 높은 평균 정확도를 기록했습니다. 개별 요소의 기여는 아래 단일요인 비교로만 제한적으로 해석해야 합니다.
|
||||
2. **`tuned_no_mutation` vs `inertia_tuned`**:
|
||||
- $\Delta$ Eval Acc: **-1.20%p** (60.42% vs 61.62%)
|
||||
- $\Delta$ Eval Loss: **+0.013236** (1.249836 vs 1.236600)
|
||||
- $\Delta$ Runtime: **+0.216s** (3.348s vs 3.132s)
|
||||
- *해석*: `mutation_swarm=0.02`를 제거한 프로필의 평균 정확도가 1.20%p 낮았습니다. $n=5$ 변동 범위가 겹치므로, 변이가 국소 최적점 탈출을 입증했다고 단정할 수는 없습니다.
|
||||
3. **`tuned_full_evaluation` vs `inertia_tuned`**:
|
||||
- $\Delta$ Eval Acc: **+0.92%p** (62.54% vs 61.62%)
|
||||
- $\Delta$ Eval Loss: **-0.057591** (1.179009 vs 1.236600)
|
||||
- $\Delta$ Runtime: **+0.245s** (3.377s vs 3.132s)
|
||||
- *해석*: 미분 무관 프로필 중 최저 손실(**1.179009**)을 기록했습니다. 단, 2,000개 고정 서브셋 평가인 `inertia_tuned`와 달리 3,000개 전체 데이터셋 평가 방식이 적용되어 샘플 수 차이 및 배치 연산 차이에 따른 교란 요인(Confound)이 존재합니다.
|
||||
4. **`tuned_uniform_initialization` vs `inertia_tuned`**:
|
||||
- $\Delta$ Eval Acc: **-4.64%p** (56.98% ± 2.40% vs 61.62%)
|
||||
- $\Delta$ Eval Loss: **+0.382596** (1.619196 vs 1.236600)
|
||||
- $\Delta$ Runtime: **+1.175s** (4.307s vs 3.132s)
|
||||
- *해석*: PyTorch 가중치 기준 노이즈 부가 대신 유니폼 무작위 위치 초기화를 사용할 경우 신경망 적합도 탐색에 불리함을 나타냅니다.
|
||||
5. **`tuned_particle_reset` vs `inertia_tuned`**:
|
||||
- $\Delta$ Eval Acc: **+0.00%p** (61.62% vs 61.62%)
|
||||
- $\Delta$ Eval Loss: **0.000000** (1.236600 vs 1.236600)
|
||||
- $\Delta$ Runtime: **+1.045s** (4.176s vs 3.132s)
|
||||
- *해석*: 최종 평가지표는 기준과 완전히 동일했지만 재초기화 이벤트 텔레메트리가 없으므로, 정체 조건 미충족과 전역 최적해 비영향을 구분할 수 없습니다. 추가 이벤트 계측 없이 수렴 개선 효과를 주장하지 않습니다.
|
||||
6. **`tuned_adam_100_lr.01` vs `inertia_tuned`**:
|
||||
- $\Delta$ Eval Acc: **+23.96%p** (85.58% vs 61.62%)
|
||||
- $\Delta$ Eval Loss: **-0.766329** (0.470271 vs 1.236600)
|
||||
- $\Delta$ Runtime: **+1.859s** (4.991s vs 3.132s)
|
||||
- *해석*: 전체 10개 프로필 중 가장 높은 정확도와 가장 낮은 손실을 기록했습니다. 다만, 본 기법은 순수 미분 무관 기법이 아닌 PSO 수렴 위치 후 100 세대 역전파 Adam 미세조정(Refinement)을 수행하는 경사도 활용(Gradient-assisted) 하이브리드 기법입니다.
|
||||
7. **`adaptive_moment_.10` vs `inertia_tuned`**:
|
||||
- $\Delta$ Eval Acc: **+1.38%p** (63.00% vs 61.62%)
|
||||
- $\Delta$ Eval Loss: **+0.006739** (1.243339 vs 1.236600)
|
||||
- $\Delta$ Runtime: **+1.155s** (4.287s vs 3.132s)
|
||||
- *해석*: 미분 무관(Derivative-Free) 프로필 중 **최고 검증 정확도(63.00% ± 1.83%)**를 기록했습니다. 정확도는 `inertia_tuned` 대비 1.38%p 우수하나, 검증 손실은 1.243339로 `inertia_tuned` (1.236600) 대비 약간 높게 유지되었습니다.
|
||||
8. **`adaptive_moment_.25` 및 `.50` vs `inertia_tuned`**:
|
||||
- `.25`: $\Delta$ Eval Acc **-5.30%p** (56.32%), $\Delta$ Eval Loss **+0.263379** (1.499979)
|
||||
- `.50`: $\Delta$ Eval Acc **-9.76%p** (51.86%), $\Delta$ Eval Loss **+0.447181** (1.683781)
|
||||
- *해석*: 이 3개 혼합비 설정에서는 $\lambda$가 0.10에서 0.25, 0.50으로 증가할수록 평균 정확도가 낮아지고 손실이 높아지는 패턴이 관찰되었습니다. 다른 모델·예산으로 일반화되는 단조 관계로 해석하지 않습니다.
|
||||
|
||||
## 6. 확장 튜닝 및 파티클 스케일링 (Tuning Protocol 1.0.0)
|
||||
|
||||
### 6.1 방법론과 데이터 분리
|
||||
|
||||
- **모델/데이터**: MNIST 첫 학습 3,000개와 테스트 1,000개, PCA32 whitening, `Linear(32,10)`, `CrossEntropyLoss`.
|
||||
- **Search**: 첫 3,000개를 stratified 2,400 inner-train / 600 validation으로 분리하고 PCA를 inner-train에만 적합했습니다. 5개 기법의 32개 후보를 파티클 30개, 80세대, 시드 51~53에서 비교하고 평균 validation accuracy 내림차순, 동률 시 loss 오름차순으로 기법별 후보를 선택했습니다.
|
||||
- **Confirmation**: 선택된 기법별 후보를 전체 학습 3,000개로 다시 적합했습니다. PCA도 전체 학습 데이터에만 다시 적합하고 search에 사용하지 않은 테스트 1,000개를 시드 61~65에서 평가했습니다.
|
||||
- **Scaling**: 선택된 Adaptive Moment 후보를 시드 71~75에서 파티클 30/60/90/120개로 평가했습니다. 80세대 고정과 약 2,400 particle-epochs 고정을 분리했습니다.
|
||||
- **공통 조건**: `fixed_subset=2000`, batch 1,000, 반사 경계 ±3, 미분 무관, Adam refinement 없음. 시간은 기법별 비계측 웜업 후 `fit()`만 측정했습니다.
|
||||
|
||||
Search가 테스트셋을 사용하지 않도록 데이터와 PCA 적합 범위를 분리했습니다. 모든 161개 레코드는 MPS에서 완료되었고 오류는 0건이었습니다. 총 계측 fit 시간은 search **259.479s**, confirmation **64.745s**, 중복을 제외한 scaling **184.978s**였습니다.
|
||||
|
||||
### 6.2 검증 선택 결과
|
||||
|
||||
| 기법 | 선택 후보 | Validation Accuracy | Validation Loss |
|
||||
| --- | --- | ---: | ---: |
|
||||
| `local_best` | `local_best_r4_constant` | **72.44%** | **0.944913** |
|
||||
| `constriction` | `constriction_c205_canonical` | 70.89% | 0.984401 |
|
||||
| `adaptive_moment` | `am_b0.06_s0.5` | 70.17% | 0.950972 |
|
||||
| `inertia` | `inertia_asymmetric` | 69.72% | 1.029132 |
|
||||
| `quantum` | `quantum_beta_0.4_0.9` | 56.50% | 1.337709 |
|
||||
|
||||
Adaptive Moment 선택값은 $c_0=c_1=1.49618$, $w=0.7298$, velocity limit ratio 0.025, mutation 0.02, `moment_blend=0.06`, `moment_step_size=0.5`, $\beta_1=0.9$입니다. 이는 저장소 기본값이 아니라 이 search 범위에서 선택된 후보입니다.
|
||||
|
||||
### 6.3 Held-out 테스트 확인
|
||||
|
||||
| 기법 | 선택 후보 | Test Accuracy (Mean ± SD) | Test Loss (Mean ± SD) | Fit Time (Mean ± SD) |
|
||||
| --- | --- | ---: | ---: | ---: |
|
||||
| `local_best` | `local_best_r4_constant` | **62.64% ± 2.94%** | **1.211368 ± 0.059575** | 2.512s ± 0.048s |
|
||||
| `inertia` | `inertia_asymmetric` | 62.30% ± 3.01% | 1.212971 ± 0.078548 | 2.570s ± 0.035s |
|
||||
| `adaptive_moment` | `am_b0.06_s0.5` | 61.06% ± 0.91% | 1.229627 ± 0.022179 | 2.943s ± 0.256s |
|
||||
| `constriction` | `constriction_c205_canonical` | 60.66% ± 2.30% | 1.246275 ± 0.044657 | 2.521s ± 0.045s |
|
||||
| `quantum` | `quantum_beta_0.4_0.9` | 48.58% ± 2.71% | 1.519246 ± 0.052511 | 2.403s ± 0.022s |
|
||||
|
||||
|
||||
|
||||
검증 1위였던 `local_best`가 held-out 확인에서도 가장 높은 평균 정확도와 가장 낮은 평균 손실을 기록했습니다. `local_best`와 `inertia`는 동일 30×80 예산에서 Adaptive Moment보다 높은 평균 정확도를 기록했습니다. $n=5$이므로 이 순서의 통계적 유의성이나 다른 워크로드로의 일반화를 주장하지 않습니다.
|
||||
|
||||
### 6.4 Adaptive Moment 파티클 스케일링
|
||||
|
||||
| 비교 방식 | 파티클 | 세대 | Particle-epochs | Test Accuracy (Mean ± SD) | Test Loss (Mean ± SD) | Fit Time (Mean ± SD) | 30×80 대비 정확도 |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| Fixed epochs | 30 | 80 | 2,400 | 62.12% ± 3.07% | 1.193469 ± 0.086618 | 2.685s ± 0.090s | 기준 |
|
||||
| Fixed epochs | 60 | 80 | 4,800 | 65.94% ± 0.87% | 1.074629 ± 0.046106 | 5.586s ± 0.517s | +3.82%p |
|
||||
| Fixed epochs | 90 | 80 | 7,200 | 70.32% ± 2.16% | 0.931880 ± 0.038073 | 8.627s ± 0.448s | +8.20%p |
|
||||
| Fixed epochs | 120 | 80 | 9,600 | **72.34% ± 1.82%** | **0.902448 ± 0.058380** | 11.174s ± 0.718s | **+10.22%p** |
|
||||
| Fixed particle-epochs | 30 | 80 | 2,400 | 62.12% ± 3.07% | 1.193469 ± 0.086618 | 2.685s ± 0.090s | 기준 |
|
||||
| Fixed particle-epochs | 60 | 40 | 2,400 | 50.74% ± 2.66% | 1.534634 ± 0.078628 | 2.786s ± 0.095s | -11.38%p |
|
||||
| Fixed particle-epochs | 90 | 27 | 2,430 | 47.82% ± 6.31% | 1.610771 ± 0.098843 | 3.109s ± 0.203s | -14.30%p |
|
||||
| Fixed particle-epochs | 120 | 20 | 2,400 | 41.06% ± 2.36% | 1.800035 ± 0.072261 | 3.029s ± 0.179s | -21.06%p |
|
||||
|
||||
|
||||
|
||||
80세대를 유지한 비교에서는 60, 90, 120개 파티클이 각각 **+3.82%p**, **+8.20%p**, **+10.22%p**였고 각 파티클 수에서 5개 paired seed가 모두 30개 기준보다 높았습니다. 그러나 120개 설정의 평균 fit 시간은 30개 대비 **4.16배**였습니다. 약 2,400 particle-epochs를 고정하면 더 많은 파티클 때문에 세대가 40/27/20으로 줄어 모든 비교에서 정확도가 낮았습니다. 이 결과는 파티클 수 자체의 무비용 효과가 아니라 추가 평가 예산과 충분한 세대 수의 결합 효과를 보여줍니다.
|
||||
|
||||
### 6.5 확장 연구 해석 한계
|
||||
|
||||
1. Search는 시드 3개, confirmation/scaling은 시드 5개로 제한됩니다. 통계적 유의성 검정이나 보편적 우위를 주장하지 않습니다.
|
||||
2. PCA32 선형 MNIST 한 워크로드의 결과이며 전체 MNIST, CNN, 대형 모델로 직접 일반화할 수 없습니다.
|
||||
3. validation 평균이 가까운 후보는 시드 수가 늘면 선택 순서가 바뀔 수 있습니다.
|
||||
4. 테스트셋은 search에 사용하지 않았지만, 공개된 scaling 비교를 추가적인 테스트셋 기반 하이퍼파라미터 선택으로 사용해서는 안 됩니다.
|
||||
5. fixed-epochs 비교는 총 particle-evaluations가 2,400에서 9,600으로 증가하므로 compute-equal 비교가 아닙니다. fixed particle-epochs도 디바이스 벡터화와 세대별 오버헤드까지 동일하게 만들지는 않습니다.
|
||||
6. 120×80 replication은 같은 PCA32 데이터와 테스트셋을 다시 평가한 수치 재현성 검증입니다. fresh seed는 스웜 난수에 대해서만 독립적이며, 새로운 표본이나 외부 데이터셋에 대한 독립 검증은 아닙니다.
|
||||
|
||||
### 6.6 120p×80e 파티클 스케일링 재현성 검증 (Replication Protocol 1.0.0)
|
||||
|
||||
발표된 Adaptive Moment 120-particle × 80-epoch fixed-epoch 결과를 별도 수트([`test/reproduce_scaling.py`](test/reproduce_scaling.py))로 다시 실행했습니다. 데이터 fingerprint `dfe645918ece54c0`, 선택 후보 `am_b0.06_s0.5`, MPS, PyTorch 2.13.0, pso2keras 4.0.0을 baseline과 일치시켰습니다.
|
||||
|
||||
사전 선언 합격 조건은 (1) 시드 71~75 exact replay의 시드별 테스트 정확도 최대 절대 차이 $\le 0.005$와 초기 모델 fingerprint 일치, (2) 시드 81~85 fresh-seed 평균 정확도의 baseline 대비 절대 차이 $\le 0.03$, (3) 두 집단 95% t-신뢰구간 중첩입니다.
|
||||
|
||||
| 집단 | 시드 | Test Accuracy (Mean ± SD) | 95% t-CI | Baseline 대비 | 판정 |
|
||||
| --- | --- | ---: | ---: | ---: | --- |
|
||||
| Baseline | 71~75 | 72.34% ± 1.82% | [70.08%, 74.60%] | 기준 | 기준 |
|
||||
| Exact replay | 71~75 | 72.34% ± 1.82% | [70.08%, 74.60%] | 시드별 최대 차이 **0.00%p** | **PASS** |
|
||||
| Fresh seed | 81~85 | **73.60% ± 1.66%** | [71.54%, 75.66%] | **+1.26%p** | **PASS** |
|
||||
|
||||
Exact replay의 초기 모델 fingerprint는 5개 시드 모두 baseline과 일치했습니다. Fresh-seed 평균 차이는 1.26%p로 3%p 허용 범위 안이고 두 95% t-신뢰구간도 중첩되어 전체 판정은 **PASS**입니다. 추가 적합 실행은 10회이며 모두 MPS에서 오류 없이 완료되었습니다.
|
||||
|
||||
- **검증 실행 명령**: `uv run --locked --extra examples python test/reproduce_scaling.py --device mps`
|
||||
- **출력 아티팩트**: [`benchmark_results/pso_v4_120p80_replication.json`](benchmark_results/pso_v4_120p80_replication.json), [`benchmark_results/pso_v4_120p80_replication.csv`](benchmark_results/pso_v4_120p80_replication.csv)
|
||||
|
||||
### 6.7 120p epoch 확장 수렴 진단 (Epoch Convergence Protocol 1.0.0)
|
||||
|
||||
`am_b0.06_s0.5`, 파티클 120개, 시드 71~75를 각각 240세대까지 한 번에 연속 실행하고 20세대 간격의 global-best 체크포인트를 같은 테스트셋에서 진단했습니다. Epoch 80의 시드별 정확도는 기존 120×80 baseline과 최대 차이 0.00%p로 일치하여 궤적 prefix가 재현됐습니다.
|
||||
|
||||
| Epoch | Particle-epochs | Training Best Loss (Mean ± SD) | Test Accuracy (Mean ± SD) | Test Loss (Mean ± SD) |
|
||||
| ---: | ---: | ---: | ---: | ---: |
|
||||
| 80 | 9,600 | 0.684245 ± 0.046414 | 72.34% ± 1.82% | 0.902448 ± 0.058380 |
|
||||
| 120 | 14,400 | 0.508942 ± 0.026551 | 78.56% ± 0.88% | 0.686080 ± 0.033467 |
|
||||
| 160 | 19,200 | 0.426009 ± 0.011667 | 81.58% ± 1.17% | 0.599888 ± 0.027074 |
|
||||
| 200 | 24,000 | 0.385320 ± 0.008996 | 82.58% ± 0.64% | 0.555652 ± 0.020276 |
|
||||
| 240 | 28,800 | **0.359582 ± 0.009652** | **83.52% ± 1.05%** | **0.515820 ± 0.018827** |
|
||||
|
||||
사전 기준은 (1) 80→240 mean training loss 감소율 1% 이상이면 post-80 optimization 지속, (2) mean test accuracy +1%p 이상이면 유의미한 held-out 개선, (3) 200→240 loss 감소율 1% 미만과 정확도 절대 변화 0.5%p 미만을 동시에 만족하면 late plateau로 분류하는 방식입니다.
|
||||
|
||||
실측 80→240 training loss 감소율은 **47.30%**, 정확도 증가는 **+11.18%p**였고 5개 시드 모두 정확도가 상승했습니다. 200→240에서도 loss가 **6.68%** 감소하고 정확도가 평균 **+0.94%p** 변했으며 5개 중 4개 시드가 상승했습니다. 각 시드의 마지막 training-best 갱신 epoch는 240/240/240/239/240이었습니다. 따라서 early stagnation, overfitting, late plateau는 모두 false입니다. 다만 구간별 정확도 이득은 80→120 **+6.22%p**, 120→160 **+3.02%p**, 160→200 **+1.00%p**, 200→240 **+0.94%p**로 감소해 한계효용은 줄고 있습니다.
|
||||
|
||||
|
||||
|
||||
- **실행 명령**: `uv run --locked --extra examples python test/epoch_convergence.py --device mps`
|
||||
- **출력 아티팩트**: [`benchmark_results/pso_v4_epoch_convergence.json`](benchmark_results/pso_v4_epoch_convergence.json), [`benchmark_results/pso_v4_epoch_convergence.csv`](benchmark_results/pso_v4_epoch_convergence.csv)
|
||||
- **해석 제한**: 체크포인트 테스트 정확도는 같은 테스트셋을 반복 관찰한 진단값이며, epoch 선택이나 자동 중단에 사용해서는 안 됩니다. 실제 stopping rule은 별도 validation split에 정의해야 합니다.
|
||||
|
||||
### 6.8 전체 MNIST 60,000/10,000 학습 (Full MNIST Protocol 1.0.0)
|
||||
|
||||
공식 MNIST train 60,000개와 test 10,000개 전체를 사용했습니다. 픽셀 정규화와 flatten 후 PCA32 whitening을 train 60,000개에만 적합했으며 설명 분산 비율 합은 0.743600입니다. 고정된 `am_b0.06_s0.5`를 `evaluation="full"`, `fitness_size=None`, batch 60,000, 파티클 120개, 240 epochs, 시드 71~75로 실행했습니다. 따라서 각 파티클은 매 epoch마다 학습 60,000개 전체에서 평가됐습니다.
|
||||
|
||||
| Epoch | Training Best Loss (Mean ± SD) | Full Test Accuracy (Mean ± SD) | Full Test Loss (Mean ± SD) | 2k-fitness study 대비 |
|
||||
| ---: | ---: | ---: | ---: | ---: |
|
||||
| 80 | 0.768448 ± 0.026961 | 77.67% ± 1.50% | 0.725023 ± 0.033552 | +5.33%p |
|
||||
| 120 | 0.589822 ± 0.014141 | 83.28% ± 0.83% | 0.551133 ± 0.018616 | +4.72%p |
|
||||
| 160 | 0.511683 ± 0.013526 | 85.56% ± 0.56% | 0.482441 ± 0.012016 | +3.98%p |
|
||||
| 200 | 0.465194 ± 0.010498 | 86.85% ± 0.32% | 0.438875 ± 0.008499 | +4.27%p |
|
||||
| 240 | **0.436664 ± 0.008101** | **87.70% ± 0.36%** | **0.412078 ± 0.006848** | **+4.18%p** |
|
||||
|
||||
80→240에서 training best loss는 **43.18%** 감소하고 테스트 정확도는 **+10.03%p** 상승했습니다. 200→240에서도 loss가 **6.13%** 감소하고 정확도가 **+0.85%p** 상승했으며 모든 시드가 개선됐습니다. 5개 시드 모두 마지막 training-best가 epoch 240에서 갱신되어 full-data 조건에서도 late plateau는 false입니다.
|
||||
|
||||
이 프로토콜은 5개 시드 합계 144,000 particle-epochs와 8,640,000,000 particle-sample evaluations를 수행했습니다. 계측된 `fit()` 합계는 165.856초입니다. Full-data 정확도는 공통 checkpoint마다 2k-fitness 연구보다 3.98~5.33%p 높았지만, PCA 적합 범위, fitness objective, evaluation plugin의 RNG 소비가 함께 바뀌므로 정확도 차이는 기술적 비교이며 인과 추정이 아닙니다. 서로 다른 학습 objective의 training loss 절댓값도 직접 비교하지 않습니다.
|
||||
|
||||
|
||||
|
||||
- **실행 명령**: `uv run --locked --extra examples python test/full_mnist_study.py --device mps`
|
||||
- **출력 아티팩트**: [`benchmark_results/pso_v4_full_mnist.json`](benchmark_results/pso_v4_full_mnist.json), [`benchmark_results/pso_v4_full_mnist.csv`](benchmark_results/pso_v4_full_mnist.csv)
|
||||
- **해석 제한**: 공식 split 전체를 사용했지만 입력은 PCA32이고 모델은 `Linear(32,10)`입니다. 원본 784차원 PSO나 CNN 결과가 아닙니다. 반복 test checkpoint는 사후 진단일 뿐 stopping rule 선택에 사용하지 않습니다.
|
||||
|
||||
### 6.9 원본 MNIST 딥 신경망 아키텍처 및 최적화기 비교 (Deep Accuracy Protocol 1.0.0)
|
||||
|
||||
PCA 차원 축소 없이 공식 MNIST 데이터셋 전체(학습 60,000개, 테스트 10,000개) 원본 $1 \times 28 \times 28$ 입력을 대상으로 딥 신경망 아키텍처 및 최적화기 특성을 다중 트랙으로 평가했습니다 (`Deep Accuracy Protocol 1.0.0`, $n=3$, seeds 101~103). 픽셀 정규화 mean(`0.13066`)과 std(`0.308108`)는 학습 60,000개에서만 산출하여 평가 데이터 누수를 차단했습니다. 테스트 10,000개는 히스토리 모니터링 전용으로 사용되었으며, 테스트 성능에 기반한 하이퍼파라미터 선택이나 조기 종료(stopping rule)는 수행하지 않았습니다.
|
||||
|
||||
실험 조건: Apple Silicon MPS, PyTorch 2.13.0, batch size 256, lr 0.001. Adam 실행은 10 epochs, PSO 실행은 파티클 30개 × 40세대에 2,000개 고정 서브셋(`fitness_size=2000`) 평가 및 선택된 Adaptive Moment 후보 설정(`am_b0.06_s0.5`, `c0=c1=1.49618`, `w=0.7298`, `velocity_limit_ratio=0.025`, `mutation_swarm=0.02`, `moment_step_size=0.5`)을 적용했습니다. 하이브리드(PSO→Adam) 방식은 PSO 40세대 탐색 후 Adam 10세대를 수행하므로 pure Adam 대비 추가 PSO 계산량을 포함하는 비동등 계산량 설정입니다.
|
||||
|
||||
#### 트랙 1: 아키텍처 레인 성과 (Adam 10 Epochs, Mean ± Sample SD, $n=3$)
|
||||
|
||||
| 아키텍처 (`arch`) | 모델 구조 및 파라미터 수 | 테스트 정확도 (Mean ± SD) | 테스트 손실 (Mean ± SD) | 98% 정확도 도달 속도 |
|
||||
| --- | --- | ---: | ---: | --- |
|
||||
| `raw_linear` | `Linear(784, 10)` (7,850 params) | 92.45% ± 0.10% | 0.268945 ± 0.002050 | 미도달 |
|
||||
| `raw_mlp` | `Linear(784,128) - ReLU - Linear(128,64) - ReLU - Linear(64,10)` (109,386 params) | 97.70% ± 0.08% | 0.078368 ± 0.002818 | 미도달 |
|
||||
| `compact_cnn` | `Conv2d(1,8,3) - ReLU - MaxPool2d - Conv2d(8,16,3) - ReLU - MaxPool2d - Linear(784,10)` (9,098 params) | **98.53% ± 0.16%** | **0.043809 ± 0.004907** | **5 Epoch 이내 전 시드 달성** |
|
||||
|
||||
`compact_cnn` 아키텍처의 경우 시드 101(4 epoch), 시드 102(3 epoch), 시드 103(5 epoch)에서 모두 5 epoch 이내에 테스트 정확도 98% 이상을 달성했습니다.
|
||||
|
||||
#### 트랙 2: 최적화기 레인 성과 (Compact CNN 9,098 Params, Mean ± Sample SD, $n=3$)
|
||||
|
||||
| 최적화 기법 (`optimizer`) | 실행 구성 | 테스트 정확도 (Mean ± SD) | 테스트 손실 (Mean ± SD) | 판정 및 특성 |
|
||||
| --- | --- | ---: | ---: | --- |
|
||||
| `adam_only` | Adam 10 epochs | **98.53% ± 0.16%** | **0.043809 ± 0.004907** | 역전파 경사하강법 기반 최적화 |
|
||||
| `pso_only` | PSO 40 epochs (30p × 40e) | 36.76% ± 3.76% | 14.104417 ± 8.075411 | 고차원 전가중치 수렴 실패 |
|
||||
| `hybrid` (PSO→Adam) | PSO 40e + Adam 10e | 97.30% ± 0.75% | 0.086542 ± 0.023969 | 추가 탐색 연산에도 pure Adam 대비 저조 |
|
||||
|
||||
|
||||
|
||||
- **실행 명령**: `uv run --locked --extra examples python test/deep_accuracy_study.py --device mps`
|
||||
- **출력 아티팩트**: [`benchmark_results/pso_v4_deep_accuracy.json`](benchmark_results/pso_v4_deep_accuracy.json), [`benchmark_results/pso_v4_deep_accuracy.csv`](benchmark_results/pso_v4_deep_accuracy.csv)
|
||||
- **결과 해석 및 한계점**:
|
||||
1. **아키텍처 인덕티브 바이어스**: Adam 조건을 고정한 아키텍처 레인에서 Compact CNN은 MLP보다 약 1/12의 파라미터로 정확도가 0.83%p 높았습니다. 원본 이미지의 공간적 구조를 사용하는 것이 측정된 차이에 중요했습니다.
|
||||
2. **고차원 전가중치 PSO의 한계**: 이 프로토콜의 30p × 40e·fixed-2k 예산에서 9,098개 CNN 파라미터를 직접 탐색한 PSO는 36.76% ± 3.76%였습니다. 이 결과는 해당 설정에서 역전파를 대체하지 못했음을 보이지만, 더 큰 예산이나 다른 간접 인코딩을 포함한 PSO의 이론적 한계를 증명하지는 않습니다.
|
||||
3. **하이브리드 초기화 결과**: PSO→Adam은 97.30% ± 0.75%로 동일한 10-epoch pure Adam의 98.53% ± 0.16%보다 낮았습니다. 추가 PSO 계산량도 포함되므로 측정한 초기화 방식의 실용적 이점은 관측되지 않았습니다.
|
||||
4. **표본 및 적용 범위 한계**: 본 실험은 $n=3$ 기술적(descriptive) 표본 측정 결과이며 통계적 유의성이나 일반적 보편 우위로 확언하지 않습니다.
|
||||
|
||||
|
||||
### 6.10 희소 부호 해시 부분공간·단계적 평가·스웜 앙상블 탐색 (MNIST-PSO-RAW-V5 1.0.0)
|
||||
|
||||
공식 MNIST 데이터셋(학습 60,000개, 테스트 10,000개) 원본 $1 \times 28 \times 28$ 입력을 대상으로 Compact CNN 아키텍처(9,098개 파라미터)의 고정 희소 부호 해시 부분공간, 단계적 표본 확장 평가, 검증 다양성 기반 스웜 앙상블 탐색을 수행했습니다 (`MNIST-PSO-RAW-V5 1.0.0`).
|
||||
|
||||
학습 데이터 60,000개는 탐색 전용 학습 세트 50,000개와 검증 세트 10,000개로 분할되었으며 (`split_seed=20260902`, `split_fingerprint=51b289d9f503a9f3`), 픽셀 정규화 평균(`0.130682`) 및 표준편차(`0.308127`)는 50,000개 탐색 학습 세트에서만 산출하여 데이터 누수를 차단했습니다. 부분공간 차원과 최종 단일 모델·앙상블 구성은 검증 세트에서 선택했으며, 테스트 세트 10,000개는 선택 완료 후 단일 모델과 앙상블의 최종 엔드포인트에만 사용했습니다.
|
||||
|
||||
#### 트랙 1: 부분공간 차원 탐색 파일럿 (Pilot Subspace Search, 30p × 160e, fixed-2k subset)
|
||||
|
||||
파티클 30개 × 160세대 고정 2,000개 서브셋 조건에서 결정론적 희소 부호 해시 부분공간(290, 1024, 4096차원)과 전가중치(9,098차원) 탐색 성능을 비교했습니다 (`pilot_seed=91`). 손실 함수는 Cross-Entropy Loss를 주 목적(primary)으로 하고 Accuracy를 동률 처리(tiebreak) 지표로 사용했습니다.
|
||||
|
||||
| 부분공간 차원 (`dimension`) | 검증 정확도 (%) | 검증 손실 (CE Loss) | 쿼리 수 (Queries) | 샘플 평가 수 (Sample Evals) | 계산 시간 (Wall Time) |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| `290` | 22.33% | 2.216933 | 4,800 | 9,600,000 | 6.0698s |
|
||||
| `1024` | 41.34% | 1.960143 | 4,800 | 9,600,000 | 6.0666s |
|
||||
| `4096` | 55.97% | 1.550738 | 4,800 | 9,600,000 | 6.2638s |
|
||||
| `full` (9,098-D) | **67.18%** | **1.205564** | 4,800 | 9,600,000 | 6.1629s |
|
||||
|
||||
파일럿 예산에서는 전가중치 9,098차원 탐색의 검증 정확도(67.18%)가 가장 높고 손실(1.205564)이 가장 낮아, 확인 프로토콜의 차원으로 선택했습니다.
|
||||
|
||||
#### 트랙 2: 단계적 표본 확장 확인 프로토콜 (Confirmation Protocol, 60p × 600e, $n=3$)
|
||||
|
||||
파티클 60개 × 600세대 설정으로 3개 무작위 시드(101, 102, 103)에 대해 단계적 표본 확장 스케줄(`2000:420,10000:135,50000:45`)을 적용했습니다. Epoch 1~420은 2,000개 서브셋, 421~555는 10,000개 서브셋, 556~600은 50,000개 전체 탐색 세트로 계층적 평가를 진행했습니다. Epoch 421 및 556의 새 목적함수 평가 전에 모든 파티클의 pbest 적합도를 새 서브셋으로 전수 재평가한 후 gbest를 재구성하여 서로 다른 목적함수의 과거 점수를 직접 비교하지 않았습니다.
|
||||
|
||||
- **확인 실행 검증 요약** ($n=3$, seeds 101~103):
|
||||
- Validation Best-pbest Accuracy (Mean ± SD): **82.34% ± 0.35%** (std: 0.346987%, 95% t-CI: ±0.861973%)
|
||||
- Validation Best-pbest NLL (Mean ± SD): **0.592376 ± 0.018705** (95% t-CI: ±0.046467)
|
||||
|
||||
#### 트랙 3: 최종 엔드포인트 및 스웜 앙상블 성과 (Final Endpoint & Swarm Ensemble)
|
||||
|
||||
검증 세트 성능에 따라 선택된 단일 최적 모델(Seed 103, Particle 54)과 검증 다양성 기준 상위 Top-5 파티클로 구성된 스웜 앙상블(Top-5 distinct pbest models across seeds: 103/P54, 102/P50, 101/P48, 102/P29, 101/P22)의 최종 공식 Held-out 테스트(10,000개) 성과 비교입니다.
|
||||
|
||||
| 엔드포인트 구분 (`endpoint`) | 모델 구성 및 선택 기준 | 검증 정확도 | 검증 NLL | 테스트 정확도 | 테스트 NLL | 테스트 Brier | 테스트 ECE | 상호 불일치도 (Disagreement) |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| **단일 최적 모델** (`single_model`) | Validation NLL 최저 (Seed 103, P54) | 82.64% | 0.571431 | 83.23% | 0.556332 | 0.253921 | 0.085242 | - |
|
||||
| **스웜 앙상블 (Top-5)** (`ensemble`) | Validation 다양성 Top-5 pbest | 86.60% | 0.548785 | **87.00%** | **0.535928** | **0.236233** | 0.164284 | 0.1585 |
|
||||
| **앙상블 대비 단일 차이** ($\Delta$) | Ensemble - Single | +3.96%p | -0.022646 | **+3.77%p** | **-0.020404** | **-0.017688** | **+0.079042** | - |
|
||||
|
||||
#### 자원 사용량 계측 (Resource Accounting)
|
||||
|
||||
- **총 후보 목적함수 평가 쿼리 수**: 127,560 회 (파일럿 19,200 + 확인 108,360)
|
||||
- **총 샘플 수준 평가 수**: 848,400,000 회 (848.4M)
|
||||
- **합산 최적화 벽시계 시간**: 473.4657 초 (파일럿 24.5631초 + 확인 448.9026초)
|
||||
|
||||
|
||||
|
||||
- **실행 명령**: `uv run --locked --extra examples python test/deep_pso_methods.py --device mps`
|
||||
- **출력 아티팩트**: [`benchmark_results/pso_v5_deep_methods.json`](benchmark_results/pso_v5_deep_methods.json), [`benchmark_results/pso_v5_deep_methods.csv`](benchmark_results/pso_v5_deep_methods.csv)
|
||||
- **결과 해석 및 특성**:
|
||||
1. **파일럿 차원 비교**: 290~4096차원 희소 부호 해시 부분공간은 동일한 30p × 160e 파일럿 예산에서 전가중치(9,098차원)보다 검증 정확도가 낮고 NLL이 높았습니다. 다른 투영, 차원, 예산의 결과까지 일반화하지 않습니다.
|
||||
2. **단계적 표본 전환과 회복**: 2k→10k→50k 전환 시 새 목적함수에서 pbest를 재평가하자 손실이 일시 상승하고 정확도가 하락했으며, 이후 각 단계에서 다시 개선됐습니다. 이 점프는 서로 다른 표본 목적함수의 난이도 차이를 반영하며 동일 궤적 손실의 악화로 해석하지 않습니다.
|
||||
3. **최종 단일 모델 수렴 한계**: 60p × 600e 단계적 탐색의 단일 모델 테스트 정확도는 83.23%였습니다. 보존된 v4 30p × 40e accuracy-primary PSO의 36.76%보다 높지만 목적함수·데이터 분리·예산이 모두 달라 인과적 개선량으로 해석할 수 없습니다. v4 Adam 10-epoch 평균 98.53%와도 계산 방식·예산이 비동등하며, 측정 정확도는 여전히 15.30%p 낮았습니다.
|
||||
4. **앙상블 다양성 및 확률 보정(ECE) 트레이드오프**: 검증 다양성으로 선택한 Top-5 앙상블은 단일 모델보다 테스트 정확도가 3.77%p 높고 NLL/Brier가 낮았지만 ECE는 0.079042 높았습니다(0.085242 → 0.164284). 이 실행은 예측 불일치가 활용 가능한 앙상블 이득과 함께 나타날 수 있음을 보였지만, 높은 손실이나 다양성만으로 일반화·보정 향상을 판정할 수는 없습니다.
|
||||
5. **종료 시점의 수렴 상태**: 50k 최종 단계에서 세 시드 모두 기록된 최저 학습 손실이 마지막 Epoch 600에 갱신됐습니다. 따라서 600세대 결과를 수렴 한계로 해석할 수 없으며, 더 긴 50k 단계의 효용은 별도 validation 기반 중단 규칙으로 확인해야 합니다.
|
||||
|
||||
### 6.11 V5 탐색 구조 회귀 원인 분리 (MNIST-PSO-RAW-V6 1.0.0)
|
||||
|
||||
V5의 83.23% 단일 모델 결과가 고차원 자체의 한계인지, 탐색 구조 변경에 따른 회귀인지 분리하기 위해 Phase A/B 진단을 수행했습니다. 공식 MNIST test split은 로드하지 않았고, train 60,000개만 `split_seed=20260902`로 search 50,000개와 validation 10,000개로 분할했습니다. 모든 비교는 동일한 결정론적 2,000개 search subset, Compact CNN 초기 모델 seed 41, CE loss-primary/accuracy-tiebreak 선택을 사용했습니다.
|
||||
|
||||
#### 단일시드 구조 screen (seed 91, 30p × 160e)
|
||||
|
||||
| ID | 핵심 변경 | Validation Accuracy | Validation NLL |
|
||||
| --- | --- | ---: | ---: |
|
||||
| G8 | 기존 `Optimizer` 의미론 제어군 | **70.99%** | **0.883557** |
|
||||
| G5 | per-tensor SD + velocity + mutation + bound ±6 | 70.93% | 0.901510 |
|
||||
| G6 | G5 + initial radius 1.5 | 69.56% | 0.961290 |
|
||||
| G4 | per-tensor SD + velocity + mutation + bound ±3 | 65.76% | 1.116280 |
|
||||
| G3 | G0 + mutation 0.02 | 68.07% | 1.130839 |
|
||||
| G0 | V5 구조 제어군 | 67.73% | 1.183117 |
|
||||
| G7 | G2 + independent positions | 58.79% | 1.291266 |
|
||||
| G2 | G0 + initial velocity | 60.72% | 1.296105 |
|
||||
| G1 | global RMS coordinate scale | 62.32% | 1.437604 |
|
||||
|
||||
한 시드에서 G3 mutation은 G0 대비 NLL을 0.052278 낮췄고, G5의 ±6 경계는 G4 대비 정확도를 5.17%p 높이고 NLL을 0.214770 낮췄습니다. 반면 초기 속도만 추가한 G2와 global RMS scale G1은 G0보다 낮았습니다. 이 screen은 확인 대상 선택용이며 단일시드 차이를 일반적 인과 효과로 확정하지 않습니다.
|
||||
|
||||
#### 3-시드 확인 (seeds 101–103, 60p × 420e)
|
||||
|
||||
| ID | Validation Accuracy (Mean ± Sample SD) | Validation NLL (Mean ± Sample SD) | G8 대비 Accuracy | G8 대비 NLL |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| G8 | **84.92% ± 0.99%** | **0.481440 ± 0.029447** | 기준 | 기준 |
|
||||
| G6 | 84.25% ± 0.22% | 0.503809 ± 0.015789 | -0.67%p | +0.022369 |
|
||||
| G5 | 84.24% ± 0.90% | 0.510297 ± 0.024899 | -0.68%p | +0.028857 |
|
||||
| G0 | 79.53% ± 0.58% | 0.689687 ± 0.008079 | -5.39%p | +0.208247 |
|
||||
| G1 | 76.95% ± 2.49% | 0.833759 ± 0.123667 | -7.96%p | +0.352319 |
|
||||
|
||||
G5와 G6은 사전 선언한 회복 기준(G8 대비 accuracy 1.0%p 이내, NLL 0.05 이내)을 모두 통과했습니다. 따라서 V5 G0의 격차는 9,098차원 자체만으로 설명되지 않으며, 제거했던 탐색 다양성과 좁은 normalized 경계의 묶음이 주요 원인입니다. G6은 사전 순위 규칙인 validation NLL 오름차순에서 G5보다 0.006488 낮아 다음 단계 기준으로 선택됐지만, 차이는 회복 기준보다 작습니다.
|
||||
|
||||
개별 요인 판정은 제한적입니다. G1은 G0보다 정확도가 2.57%p 낮고 NLL이 0.144072 높아 per-tensor SD scaling을 global RMS로 교체하는 가설은 기각됐습니다. G5/G6의 회복은 확인됐지만 G2–G4를 3개 시드로 확인하지 않았으므로 초기 속도, mutation, mutation 상호작용, 경계 확장의 독립 효과는 아직 unresolved입니다. G6과 G5 차이는 accuracy +0.01%p/NLL -0.006488로 넓은 초기 반경의 material improvement 기준을 충족하지 못했습니다.
|
||||
|
||||
#### 자원·누수 계측 및 다음 단계
|
||||
|
||||
- 후보 목적함수 쿼리: **421,200**
|
||||
- 후보 샘플 평가: **842,400,000**
|
||||
- 합산 최적화 시간: **596.9980초**
|
||||
- 합산 validation 평가 시간: **9.1062초**
|
||||
- 공식 test 데이터 로드 및 평가: **0회**
|
||||
|
||||
|
||||
|
||||
- **실행 명령**: `uv run --locked --extra examples python test/deep_pso_v6.py --phase all --device mps`
|
||||
- **출력 아티팩트**: [`benchmark_results/pso_v6_phase_b.json`](benchmark_results/pso_v6_phase_b.json), [`benchmark_results/pso_v6_phase_b.csv`](benchmark_results/pso_v6_phase_b.csv)
|
||||
- **다음 단계**: G6 구조를 기준으로 `sqrt(9098/d)` 반경·속도·경계 보정을 적용한 공정한 subspace 비교(Phase C)를 수행합니다. 이후에만 objective schedule과 transition moment reset을 분리합니다. CCPSO/CSO는 whole-vector validation 성능이 90% 미만에서 plateau일 때까지 보류합니다.
|
||||
|
||||
### 6.12 더 무거운 태스크의 전가중치 PSO 실행 가능성 (HEAVY-TASK-PSO-V6 1.0.0)
|
||||
|
||||
동일한 보존 기법 G0/G5/G6/G8을 더 큰 모델 및 다른 영상 분류 데이터에 적용했습니다. MNIST와 FashionMNIST 모두 공식 train split만 로드한 뒤 `split_seed=20260902`로 search 50,000/validation 10,000을 층화 분할하고, search 50,000개에서만 정규화 통계를 적합했습니다. 모델 축은 Compact CNN 9,098개와 Conv16→32·FC32 구조의 WideCNN 55,338개 파라미터입니다. 공식 test split 로드와 평가는 모두 0회입니다.
|
||||
|
||||
단일시드 screen은 네 workload와 G0/G5/G6/G8의 16개 셀을 12 particles × 40 epochs·fixed-2k로 평가했습니다. Validation NLL 우선, accuracy 동률 판정으로 선택된 normalized 방법은 MNIST Compact/Fashion Compact에서 G6, MNIST Wide/Fashion Wide에서 G5였습니다. 이 선택은 같은 validation split을 사용했으므로 독립적인 일반화 추정치가 아니라 확인 대상 축소 절차입니다.
|
||||
|
||||
3-시드 확인은 workload마다 G8과 선택된 normalized 방법을 12 particles × 80 epochs·fixed-10k로 실행했습니다.
|
||||
|
||||
| Workload | Method | Validation Accuracy (Mean ± Sample SD) | Validation NLL (Mean ± Sample SD) | 초기 모델 대비 Accuracy | NLL 감소율 | 사전 최적화 기준 |
|
||||
| --- | --- | ---: | ---: | ---: | ---: | --- |
|
||||
| MNIST Compact (9,098) | G8 | **49.15% ± 1.32%** | **1.518089 ± 0.061523** | +42.50%p | 34.71% | 통과 |
|
||||
| MNIST Compact (9,098) | G6 | 45.97% ± 3.55% | 1.642081 ± 0.091398 | +39.32%p | 29.38% | 통과 |
|
||||
| MNIST Wide (55,338) | G8 | 41.03% ± 4.06% | 1.733351 ± 0.084331 | +31.72%p | 25.43% | 통과 |
|
||||
| MNIST Wide (55,338) | G5 | **43.86% ± 3.51%** | **1.721259 ± 0.098656** | +34.55%p | 25.95% | 통과 |
|
||||
| Fashion Compact (9,098) | G8 | **47.00% ± 5.23%** | **1.511217 ± 0.168628** | +38.92%p | 34.87% | 통과 |
|
||||
| Fashion Compact (9,098) | G6 | 45.62% ± 3.59% | 1.584443 ± 0.053077 | +37.54%p | 31.71% | 통과 |
|
||||
| Fashion Wide (55,338) | G8 | 41.67% ± 2.27% | 1.615937 ± 0.036692 | +32.90%p | 30.08% | 통과 |
|
||||
| Fashion Wide (55,338) | G5 | **46.31% ± 7.57%** | **1.525747 ± 0.149706** | +37.54%p | 33.99% | 통과 |
|
||||
|
||||
모든 실행은 수치적으로 finite했고 사전 feasibility 기준을 통과했으므로, 약 55k 파라미터 및 FashionMNIST 범위에서도 이 PSO 기법들이 목적함수를 낮추고 초기 정확도를 높이는 것은 확인됐습니다. 다만 최고 평균 정확도가 49.15%에 불과해 유용한 분류기 학습이나 수렴 완료는 확인되지 않았습니다. 80 epochs의 endpoint만 비교했고 plateau 판정용 궤적을 보존하지 않았으므로, 이 프로토콜은 수렴 한계나 추가 세대의 효용을 결정하지 않습니다.
|
||||
|
||||
모델 파라미터가 6.08배 증가하면서 12-particle core swarm state는 custom 방법 기준 약 2.08 MiB에서 12.67 MiB, G8 기준 약 2.12 MiB에서 12.88 MiB로 선형 증가했습니다. 확인 실행 처리량은 Compact CNN의 약 1.90~2.19M samples/s에서 WideCNN의 약 0.98~1.04M samples/s로 감소했습니다. G8의 Wide 모델은 Compact 모델보다 MNIST에서 정확도가 8.12%p, FashionMNIST에서 5.34%p 낮았습니다. 반면 screen-selected G5는 두 Wide workload에서 G8 평균을 앞섰지만, 방법 선택에 사용한 validation 재사용과 $n=3$의 큰 변동성(Fashion Wide G5 SD 7.57%p) 때문에 고차원 우월성으로 일반화하지 않습니다.
|
||||
|
||||
- 후보 목적함수 쿼리: **30,720**
|
||||
- 후보 샘플 평가: **245,760,000**
|
||||
- 합산 최적화 시간: **188.7283초**
|
||||
- 합산 validation 평가 시간: **6.3826초**
|
||||
- 공식 test 데이터 로드 및 평가: **0회**
|
||||
|
||||
|
||||
|
||||
- **실행 명령**: `uv run --no-sync python test/heavy_task_feasibility.py --stage screen --device mps --out-dir /tmp/pso-v6-heavy-screen` 이후 `uv run --no-sync python test/heavy_task_feasibility.py --stage confirm --device mps --screen-artifact /tmp/pso-v6-heavy-screen/pso_v6_heavy_tasks_screen.json`
|
||||
- **출력 아티팩트**: [`benchmark_results/pso_v6_heavy_tasks.json`](benchmark_results/pso_v6_heavy_tasks.json), [`benchmark_results/pso_v6_heavy_tasks.csv`](benchmark_results/pso_v6_heavy_tasks.csv)
|
||||
- **판정**: 더 무거운 measured workload에서도 PSO 실행과 제한적 최적화는 가능하지만, 현재 예산의 전가중치 탐색은 실용적인 학습기로 판정하지 않습니다.
|
||||
|
||||
### 6.13 고정 부분공간 PSO의 품질·상태 Pareto 반복 연구 (HEAVY-PSO-AUTORESEARCH 1.0.0)
|
||||
|
||||
6.12의 네 workload를 대상으로 방법 제안 → 고정 평가 → 분석 → 유지/기각을 반복했습니다. 공식 test split은 끝까지 로드하지 않았고, primary evaluator는 12 particles × 80 epochs × fixed-10k, seeds 101~103, workload별 validation accuracy/NLL, 그리고 persistent core swarm state를 고정했습니다. 후보는 모든 workload에서 baseline state의 50% 이하, accuracy 하락 1%p 이하, NLL 악화 5% 이하를 만족하고, 가장 약한 baseline인 MNIST Wide에서 accuracy 2%p 또는 NLL 5% 이상을 개선해야 통과합니다. 점수는 평균 상대 NLL 개선율 + 평균 accuracy 개선(%p) + 상태 절감 보너스에서 실패 gate당 100점을 차감합니다.
|
||||
|
||||
#### 반복 결과와 유지 정책
|
||||
|
||||
초기 전역 signed-hash 부분공간은 swarm seed마다 projection도 함께 바꾸어 optimizer 변동과 표현 변동을 혼합했습니다. 별도 projection replica에서 이 문제가 확인됐고, tensor-local 비례 hash는 모든 품질 gate를 크게 악화시켜 기각했습니다. 이후 하나의 전역 projection을 swarm seed 전체에서 고정했습니다. 세 workload에는 ratio 0.5 고정 projection을 사용하고, MNIST Wide에는 기존 단일 실행에서 발견된 projection 592157828을 세 matched seed에서 다시 확인한 뒤 사용했습니다. 반경/속도/경계 multiplier 0.75와 0.5는 MNIST Wide 성능을 각각 41.71%와 36.78%로 낮춰 기각했으며 multiplier 1.0을 유지했습니다.
|
||||
|
||||
| Workload | Latent 구성 | State ratio | Baseline Acc / NLL | Development 101~103 Acc / NLL | Confirmation 111~113 Acc / NLL |
|
||||
| --- | --- | ---: | ---: | ---: | ---: |
|
||||
| MNIST Compact | fixed global, ratio 0.5, projection 1800044939 | 0.4918 | 49.1533% / 1.518089 | 49.0533% / 1.525704 | 48.4667% / 1.550212 |
|
||||
| MNIST Wide | fixed global, ratio 0.5, projection 592157828 | 0.5000 | 43.8600% / 1.721259 | 48.3467% / 1.677712 | 45.7233% / 1.734905 |
|
||||
| Fashion Compact | fixed global, ratio 0.5, projection 1363313651 | 0.4918 | 47.0033% / 1.511217 | 50.7967% / 1.385324 | 53.1967% / 1.328168 |
|
||||
| Fashion Wide | fixed global, ratio 0.5, projection 189641451 | 0.5000 | 46.3100% / 1.525747 | 48.2633% / 1.531421 | 49.4767% / 1.513876 |
|
||||
|
||||
Development 정책 `fixed_global_hybrid_v3`는 모든 고정 gate를 통과했습니다. Score는 **15.030089**, 평균 accuracy 개선은 **+2.5333%p**, 평균 상대 NLL 개선은 **2.4968%**, 최대 state ratio는 **0.5**였습니다. 그러나 정책을 재선택하지 않고 swarm seeds 111~113에서 실행한 confirmation은 **한 gate만 실패**했습니다. 모든 workload의 accuracy/NLL 비열화 gate와 state/safety/accounting gate는 통과했지만 MNIST Wide 개선이 **+1.8633%p**로 사전 기준 +2%p에 0.1367%p 미달했습니다. 따라서 독립 확인된 Pareto 승리로 판정하지 않습니다.
|
||||
|
||||
두 seed 집합을 합친 6-seed 수치는 사후 기술 통계일 뿐 gate 판정값이 아닙니다. MNIST Compact/MNIST Wide/Fashion Compact/Fashion Wide accuracy 변화는 각각 **-0.3933/+3.1750/+4.9933/+2.5600%p**, 상대 NLL 변화는 **-1.3088/+0.8686/+10.2216/+0.2031%**였습니다. 이 결과는 고정 projection이 projection-coupled 정책보다 해석 가능하고 평균 품질·상태 Pareto를 개선할 가능성을 보이지만, 새 validation split이나 공식 test 일반화 증거는 아닙니다.
|
||||
|
||||
- 총 실제 실행: **312 runs**
|
||||
- 총 목적함수 쿼리: **299,520**
|
||||
- 총 샘플 평가: **2,995,200,000**
|
||||
- 실행 wall time 합: **2,264.4978초**
|
||||
- 공식 test 데이터 로드 및 평가: **0회**
|
||||
- 실행기/평가기: [`test/heavy_pso_autoresearch.py`](test/heavy_pso_autoresearch.py), [`test/evaluate_heavy_autoresearch.py`](test/evaluate_heavy_autoresearch.py)
|
||||
- 요약 아티팩트: [`benchmark_results/pso_v6_heavy_autoresearch.json`](benchmark_results/pso_v6_heavy_autoresearch.json), [`benchmark_results/pso_v6_heavy_autoresearch.csv`](benchmark_results/pso_v6_heavy_autoresearch.csv)
|
||||
- 전체 반복 결정 로그: [`.omc/autoresearch/heavy-pso-progressive-improvement/runs/20260902T153426Z/decision-log.md`](.omc/autoresearch/heavy-pso-progressive-improvement/runs/20260902T153426Z/decision-log.md)
|
||||
|
||||
- **※ 주의 (후속 검증 결과)**: 후속 교차 분할 평가([`REPORT.md` §6.14](#614-heavy-pso-교차-분할-강건성-검증-heavy-pso-cross-split-100))에서 동일 정책이 개발 분할 평가 게이트를 통과하지 못했습니다. 따라서 본 절의 단일 개발 분할 수치는 새 validation split에 대한 강건성 증거가 아닙니다.
|
||||
|
||||
|
||||
### 6.14 Heavy PSO 교차 분할 강건성 검증 (HEAVY-PSO-CROSS-SPLIT 1.0.0)
|
||||
|
||||
§6.13의 고정 부분공간 수축 정책(`fixed_global_hybrid_v3`)이 단일 개발 분할(split 20260902) 이외의 validation 분할에서도 결과를 재현하는지 확인하기 위해, 개발 분할 20260905/20260906과 시드 101~103에서 매칭 baseline을 다시 실행했습니다. 예산은 12 particles × 80 epochs × fixed-10k로 고정했습니다 (`HEAVY-PSO-CROSS-SPLIT 1.0.0`).
|
||||
|
||||
#### 6.14.1 실험 설계 및 개념적 구분
|
||||
|
||||
1. **결정 탐색(Decision Iterations) vs 개발 변형(Development Variants)**:
|
||||
- 총 8회 결정 탐색(Decision Iterations 1~8)을 통해 다양한 부분공간 하이퍼파라미터 및 해시 프로젝션을 탐색했습니다.
|
||||
- Iteration 3에서 projection seed replica 1과 replica 2를 별도 평가했으므로, 8회 결정에서 총 9개 개발 변형(Development Variants)을 평가했습니다.
|
||||
2. **개발 단계(Development Phase) vs 확인 단계(Confirmation Phase)**:
|
||||
- **개발 단계**: 2개 무작위 개발 분할(20260905, 20260906) 및 시드 101~103 (변형당 8개 셀, 48개 실행)을 기반으로 엄격한 품질/회귀 게이트를 적용했습니다.
|
||||
- **확인 단계**: 개발 단계의 모든 게이트를 통과한 보존 후보에 한해 세 번째 확인 분할(20260907) 및 시드 111~113에서 독립 확인 평가를 수행하도록 정의했습니다.
|
||||
3. **검증 분할(Validation Split) vs 공식 테스트(Official Test Split)**:
|
||||
- 학습 50,000개 / 검증 10,000개 층화 분할(Search/Validation)을 사용했으며, 데이터 정규화 통계는 Search 50,000개에서만 산출했습니다.
|
||||
- 공식 테스트 데이터셋(10,000개)은 0회 로드 및 0회 평가로 미사용 완전히 봉인 유지되었습니다 (`official_test_data_loaded = false`, `official_test_evaluations = 0`).
|
||||
4. **관측 최상위 후보(Best Observed Candidate) vs 최종 보존 정책(Retained Policy)**:
|
||||
- 평가된 9개 개발 변형 중 탐색 스코어가 가장 높은 후보(Best Observed Candidate)와 모든 게이트를 통과하여 채택되는 최종 보존 정책(Retained Policy)을 명확히 구분했습니다.
|
||||
|
||||
#### 6.14.2 주요 결과 및 게이트 평가
|
||||
|
||||
1. **동결 정책 (`fixed_global_hybrid_v3`, Iteration 1)**:
|
||||
- §6.13에서 수립된 고정 global projection 및 latent ratio 0.5 정책을 동결하여 교차 평가한 결과, 전체 평균 accuracy 개선은 **+0.1533%p**, NLL 감소는 **0.5546%**에 그쳤습니다.
|
||||
- Accuracy·NLL·MNIST Wide 개선의 3개 development 품질 gate를 위반하여 **FAIL**로 판정했습니다. 개발 gate가 닫혔으므로 confirmation gate는 의도대로 실행하지 않았습니다.
|
||||
2. **관측 최상위 후보 (Iteration 5, Largest-Tensor Hash)**:
|
||||
- 8회 결정 탐색 중 최고 스코어(**-185.610686**)를 기록한 Iteration 5(텐서 크기순 비례 해시)는 overall accuracy 개선 **+0.4400%p**, NLL 감소 **3.9493%**, 최악 accuracy 회귀 **-3.0667%p**를 기록했습니다.
|
||||
- 그러나 가장 민감한 게이트인 MNIST Wide accuracy에서 **-2.6633%p** (NLL **2.1874%** 악화)로 여전히 회귀가 관측되어 게이트 미달로 **FAIL** 판정되었습니다.
|
||||
3. **확인 단계 과학적 보류 및 보존 실패**:
|
||||
- 9개 개발 변형 전체가 개발 단계 하드 게이트를 통과하지 못함에 따라, 사전 선언된 과학적 엄격성 규칙에 의거하여 확인 분할(20260907) 평가 실행은 **완전히 보류(withheld, confirmation_executed = false)**되었습니다.
|
||||
- 최종 보존 정책은 `retained_policy = null`로 확정되었으며, 공식 test split 역시 0회 평가로 미사용 봉인 상태를 유지했습니다.
|
||||
|
||||
#### 6.14.3 실험 및 성과 요약 표
|
||||
|
||||
| 항목 (Category) | 세부 사양 및 평가 수치 (Details & Metrics) |
|
||||
| --- | --- |
|
||||
| **프로토콜 명칭** | `HEAVY-PSO-CROSS-SPLIT 1.0.0` (Publish Protocol: `1.0.0`) |
|
||||
| **평가 대상 모델/데이터** | MNIST Compact (9,098), MNIST Wide (55,338), Fashion Compact (9,098), Fashion Wide (55,338) |
|
||||
| **스웜/평가 하이퍼파라미터** | 12 particles × 80 epochs × fixed-10k subset, seeds 101~103, matched baseline 재실행 |
|
||||
| **데이터 분할 구성** | 개발 분할 2개 (20260905, 20260906) / 확인 분할 1개 (20260907, 실행 보류) |
|
||||
| **공식 테스트 데이터** | 0회 로드, 0회 평가 (완전 봉인 유지) |
|
||||
| **결정 탐색 및 개발 변형** | 8회 결정 탐색 (Decision Iterations 1~8) / 9개 개발 변형 (Development Variants, Iter 3 Replica 1/2) |
|
||||
| **동결 정책 (Iter 1) 결과** | Overall Acc **+0.1533%p**, NLL 감소 **0.5546%**, Worst Acc 회귀 **-7.1767%p**, Worst NLL 회귀 **+14.5682%**, MNIST Wide Acc **-0.3617%p** (FAIL) |
|
||||
| **관측 최상위 (Iter 5) 결과** | Score **-185.610686**, Overall Acc **+0.4400%p**, NLL 감소 **3.9493%**, Worst Acc 회귀 **-3.0667%p**, MNIST Wide Acc **-2.6633%p** (FAIL) |
|
||||
| **최종 판정 및 보존 정책** | **FAIL** (`retained_policy = null`), 확인 단계 보류 (`confirmation_executed = false`) |
|
||||
| **누적 자원 사용 계측** | 432 runs, 414,720 queries, 4,147,200,000 sample evaluations, 3,162.9717s wall time (~52.7 min) |
|
||||
| **공개 아티팩트 및 시각화** | [`benchmark_results/pso_v7_heavy_cross_split.json`](benchmark_results/pso_v7_heavy_cross_split.json), [`benchmark_results/pso_v7_heavy_cross_split.csv`](benchmark_results/pso_v7_heavy_cross_split.csv), [`history_plt/pso_v7_heavy_cross_split.png`](history_plt/pso_v7_heavy_cross_split.png) |
|
||||
| **반복 결정 로그 경로** | [`.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/decision-log.md`](.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/decision-log.md) |
|
||||
|
||||
|
||||
|
||||
#### 6.14.4 재현 실행 및 아티팩트 발행 명령
|
||||
|
||||
```shell
|
||||
# 1. 교차 분할 탐색 미션 실행 (개발 단계)
|
||||
uv run --no-sync python test/heavy_pso_cross_split.py --phase development --device mps
|
||||
|
||||
# 2. 개별 변형 평가 실행 예시
|
||||
uv run --no-sync python test/evaluate_heavy_cross_split.py --development .omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0001-development.json --output .omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0001-development.json
|
||||
|
||||
# 3. 공개 아티팩트 및 시각화 생성 명령
|
||||
uv run --no-sync python test/publish_heavy_cross_split.py --source-dir .omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z --output-json benchmark_results/pso_v7_heavy_cross_split.json --output-csv benchmark_results/pso_v7_heavy_cross_split.csv --output-plot history_plt/pso_v7_heavy_cross_split.png
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
### 6.15 학습 후 예측공간 PSO 앙상블 연구 (POST-TRAINING-PSO-ENSEMBLE 1.1.0)
|
||||
|
||||
#### 6.15.1 연구 질문과 설계
|
||||
|
||||
본 연구는 이미 학습된 CompactCNN의 예측을 재사용하여, 다섯 모델의 출력 확률을 하나의 가중 평균으로 결합할 때 저차원 PSO가 유용한지를 평가했습니다. MNIST와 FashionMNIST 각각에 대해 split seed **20260904**, 50,000개 search 샘플과 10,000개 validation 샘플을 사용했고, 정규화 통계도 search 샘플에서만 계산했습니다. 모델 pool은 seeds 201~205의 독립 CompactCNN(각 **9,098 parameters**) 다섯 개이며, 각 모델은 Adam 10 epochs, learning rate 0.001, batch size 256으로 학습했습니다. 다섯 모델의 validation 확률 캐시 shape은 `[5, 10000, 10]`이고, 최적화 중 base CNN forward pass는 0회였습니다.
|
||||
|
||||
최적화 변수는 다섯 출력의 비음수 합이 1인 simplex 가중치 $w$뿐입니다. 따라서 이는 **prediction-space ensemble**이며, 파라미터를 평균내는 model soup가 아닙니다. Model soup의 weight averaging과 permutation 정렬을 다루는 문헌은 이 연구의 조작이나 결론을 뒷받침하는 직접적인 비교가 아닙니다 [arXiv:2203.05482](https://arxiv.org/abs/2203.05482), [arXiv:2209.04836](https://arxiv.org/abs/2209.04836). 비교 방법은 uniform, validation NLL에 대한 uniform temperature scaling, simplex 제약 SLSQP, 그리고 PSO입니다. PSO는 constriction movement, loss renewal, 30 particles, particle bounds $[-4,4]$, reflective boundary, velocity limit ratio 0.1, initial position noise 0, swarm seeds 301~303을 고정했습니다.
|
||||
|
||||
#### 6.15.2 누수 방지와 사전 선언된 반복
|
||||
|
||||
- 모델 학습, temperature fitting, SLSQP/PSO 가중치 선택은 모두 search/validation 단계에서만 수행했습니다. 공식 test split은 정책 동결 전 로드 0회·평가 0회였습니다.
|
||||
- Iteration 0의 독립 evaluator가 per-workload test-seal field lookup을 교정한 뒤 wall-efficiency gate만 실패한 것을 확인했습니다. 품질·안정성 gate와 세 swarm seed의 수렴 결과는 유지하고, 결정 로그에 따라 PSO epochs만 50에서 30으로 줄였습니다. pool, objective, seed, split, baseline, threshold, selection rule은 바꾸지 않았습니다.
|
||||
- 동결된 정책이 모든 development gate를 통과한 뒤에만 공식 test를 dataset별 1회 로드하여 pool 5회 forward와 50-epoch single 1회 forward로 확인했습니다. 그 뒤의 tuning/rerun은 0회입니다. 이 순서로 validation 선택과 one-shot test 확인을 분리했지만, 하나의 split과 하나의 final test endpoint만 사용했다는 한계는 남습니다.
|
||||
|
||||
Iteration 0(30 particles × 50 epochs)은 seed당 1,500 objective evaluations를 사용했습니다. PSO는 SLSQP와 같은 six-decimal validation NLL에 도달했지만, median one-seed wall ratio가 MNIST **13.14%**, FashionMNIST **12.05%**로 frozen 10% ceiling을 넘었습니다. MNIST validation NLL은 PSO/SLSQP **0.045902**, uniform **0.046385**, 10-epoch reference **0.060105**, equal-epoch-budget 50-epoch single **0.073988**였고, FashionMNIST는 각각 **0.285338**, **0.286751**, **0.303799**, **0.289660**이었습니다. SLSQP는 workload당 23 evaluations와 약 0.011초, PSO는 workload당 1,500 evaluations와 약 2.98~3.29초를 사용했으며, uniform temperature는 두 workload에서 PSO보다 낮은 validation NLL을 보였습니다. 이 결정과 실패 사유는 [post-training decision log](.omc/autoresearch/post-training-pso-ensemble/runs/20260904T093144Z/decision-log.md)에 기록되어 있습니다.
|
||||
|
||||
Iteration 1은 다른 조건을 바꾸지 않고 30 particles × 30 epochs로 축소했습니다. seed당 **900 queries**와 **9,000,000 candidate-sample evaluations**를 사용했고, 세 swarm seed가 모두 같은 six-decimal PSO NLL을 재현했습니다. Runner의 13개 집계 gate와 독립 evaluator가 재계산한 14개 development hard gate가 모두 통과했습니다.
|
||||
|
||||
#### 6.15.3 Validation 및 one-shot test 결과
|
||||
|
||||
아래 값은 accuracy(%) / NLL이며, PSO 행은 validation에서 frozen selected seed 301을 사용한 결과입니다. SLSQP와 PSO의 validation NLL은 두 dataset 모두 six-decimal 수준에서 같지만, test에서는 마지막 decimal 차이가 남습니다.
|
||||
|
||||
| 방법 | MNIST validation | MNIST test (one-shot) | FashionMNIST validation | FashionMNIST test (one-shot) |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| 10-epoch single | 98.23 / 0.060105 | 98.47 / 0.044991 | 89.47 / 0.303799 | 88.90 / 0.314516 |
|
||||
| 50-epoch single (equal epoch budget) | 98.52 / 0.073988 | 98.60 / 0.062102 | 90.32 / 0.289660 | 89.93 / 0.302348 |
|
||||
| Uniform ensemble | 98.63 / 0.046385 | 98.86 / 0.036184 | 90.28 / 0.286751 | 89.65 / 0.293522 |
|
||||
| Uniform + temperature | 98.63 / 0.045355 | 98.86 / 0.034129 | 90.28 / 0.285048 | 89.65 / 0.291996 |
|
||||
| SLSQP simplex weights | 98.60 / 0.045902 | 98.83 / 0.036179 | 90.42 / 0.285338 | 89.54 / 0.291696 |
|
||||
| PSO simplex weights | 98.61 / 0.045902 | 98.83 / 0.036178 | 90.42 / 0.285338 | 89.54 / 0.291700 |
|
||||
|
||||
Evaluator의 cross-dataset mean은 equal-budget single 대비 validation NLL **19.726% 감소**와 accuracy **+0.095%p**, test NLL **22.633% 감소**와 accuracy **-0.080%p**였습니다. 이는 PSO가 단일 모델보다 NLL을 낮추면서도 uniform 대비 사전 선언된 regression gate를 넘지 않았다는 기술통계이지, PSO의 일반적 정확도 우월성은 아닙니다. Test에서 uniform+temperature는 MNIST에서 PSO보다 NLL이 낮고 accuracy가 **0.03%p 높았습니다**. FashionMNIST에서는 uniform+temperature NLL이 **0.291996**으로 PSO의 **0.291700**보다 높았지만, accuracy는 **0.11%p 높았습니다**.
|
||||
|
||||
#### 6.15.4 계산비용과 full-weight G8과의 기술적 비교
|
||||
|
||||
두 iteration을 합친 post-training study의 PSO 연구 비용은 **14,400 queries**, **144,000,000 candidate-sample evaluations**, research wall time **30.2907초**였습니다. 이 중 최종 Iteration 1은 **5,400 queries**, **54,000,000 candidate-sample evaluations**, research wall time **11.1512초**였고, validation에서 선택된 endpoint의 production wall time은 **3.7888초**였습니다. 다음 비율은 모두 **Iteration 1 내부 비교**이며 두 iteration 합산 비용을 한 iteration의 Adam/pool 학습 비용으로 나눈 값이 아닙니다. 선택된 production PSO 총시간을 Iteration 1의 다섯 모델 pool 학습 총시간 **44.4624초**로 나눈 비율은 정확히 **8.521376%**(소수 셋째 자리 반올림 **8.521%**)입니다. Development gate가 직접 확인한 workload별 median one-seed 비율은 MNIST **8.563655%**, FashionMNIST **8.479599%**이며, Iteration 1 artifact의 `pso_to_pool_wall_ratio`는 이 두 workload 비율의 중앙값 **8.521627%**(소수 셋째 자리 반올림 **8.522%**)입니다. 두 workload 모두 10% ceiling 아래였습니다. Iteration 1의 SLSQP는 workload당 23회, 총 **46 evaluations**, 총 **0.0201초**였습니다. Validation cache는 dataset당 pool 5회와 long single 1회, 총 12 forward passes를 사용했습니다. 앙상블은 모델 다섯 개를 보존하고 prediction inference도 다섯 번 수행하므로 single model 대비 storage/inference 경로가 **5배**입니다. 이 비용을 포함해도 …
|
||||
|
||||
다음 표는 같은 CompactCNN parameter count를 사용한 [HEAVY-TASK-PSO-V6 artifact](benchmark_results/pso_v6_heavy_tasks.json)의 full-weight G8 semantic control과의 **기술적(descriptive) 비교**입니다. G8은 9,098개 전가중치를 직접 탐색한 12 particles × 80 epochs, fixed-10k validation, seeds 101~103 실행입니다. Post-training 행은 validation에서 선택된 PSO seed 301의 endpoint이고, G8 행은 세 seed의 평균입니다. 또한 post-training은 5개 모델의 5차원 prediction-space weight를 탐색하므로 두 행은 요약 통계, 목적함수, 최적화 공간이 모두 다릅니다.
|
||||
|
||||
| 연구/방법 | 요약 통계 | 최적화 공간 | MNIST validation accuracy / NLL | FashionMNIST validation accuracy / NLL | 공식 test |
|
||||
| --- | --- | --- | ---: | ---: | --- |
|
||||
| Post-training PSO (Iteration 1) | validation-selected seed 301 | 5-way prediction simplex | 98.61% / 0.045902 | 90.42% / 0.285338 | 98.83% / 0.036178; 89.54% / 0.291700 |
|
||||
| Full-weight CompactCNN G8 (V6) | seeds 101~103 mean | 9,098 weights | 49.1533% / 1.518089 | 47.0033% / 1.511217 | 로드/평가 0회 |
|
||||
|
||||
선택 endpoint와 3-seed mean 사이의 기술통계 차이는 MNIST에서 **+49.4567%p, NLL -1.472187**, FashionMNIST에서 **+43.4167%p, NLL -1.225879**(post-training PSO minus G8)입니다. 그러나 이 차이는 짝지어진 추정량이 아니며, model-soup 대 full-weight PSO의 인과 비교나 일반화 증거도 아닙니다. V6 G8의 split seed는 **20260902**, 공식 test는 봉인되었고, post-training study는 split seed **20260904**에서 이미 Adam으로 학습된 pool을 사용했습니다. V6 전체 study의 자원은 30,720 queries, 245,760,000 sample evaluations, summed optimization wall time 188.7283초, validation wall time 6.3826초였으며, 이 수치 역시 네 workload와 G0/G5/G6/G8 실행을 합친 값입니다. 따라서 full-weight G8 수치는 “전가중치 PSO를 실용적 학습기로 판정하지 않는다”는 기존 feasibility 결과를 보강하지만, post-training 앙상블의 test accuracy를 설명하는 대조군으로 사용하지 않습니다.
|
||||
|
||||
#### 6.15.5 문헌과 해석의 경계
|
||||
|
||||
Deep ensembles는 여러 독립 predictor의 predictive uncertainty를 실용적으로 추정할 수 있음을 보였고([arXiv:1612.01474](https://arxiv.org/abs/1612.01474)), temperature scaling은 단일 scalar로 calibration을 개선하는 간단한 post-processing으로 제시되었습니다([arXiv:1706.04599](https://arxiv.org/abs/1706.04599)). 본 연구에서 uniform+temperature가 validation/test NLL과 ECE를 개선한 관측은 이 두 문헌과 방향이 일치하지만, 두 dataset·한 split의 결과를 넘어선 보장은 아닙니다. PSO로 diversity와 accuracy를 함께 고려한 weighted ensemble을 구성한 선행 연구([DOI:10.3390/a13100255](https://doi.org/10.3390/a13100255))도 있으나, 그 연구는 mixed-binary learner selection과 UCI dataset들을 포함하는 다른 설계입니다. 또한 초기 신경망 ensemble 연구([DOI:10.1016/0893-6080(92)90023-1](https://doi.org/10.1016/0893-6080(92)90023-1))는 ensemble의 일반적 동기를 제공할 수 있을 뿐, 본 PSO 비용 비교의 실험적 근거는 아닙니다.
|
||||
|
||||
#### 6.15.6 한계와 결정
|
||||
|
||||
1. **선택 편향과 표본 범위**: validation split 하나, dataset 두 개, CompactCNN 하나, PSO swarm seed 세 개뿐입니다. validation에서 방법과 PSO seed를 선택한 뒤 test를 한 번 확인했으므로 test leakage는 피했지만, 반복 split·외부 dataset·독립 replication은 없습니다.
|
||||
2. **비동등한 대조군**: 50-epoch single은 pool의 합산 epoch와 맞춘 equal-epoch-budget 비교이며, 정확한 hardware FLOP 또는 병렬화 비용 동등성을 증명하지 않습니다. V6 G8은 전가중치 직접 탐색이고 split과 objective가 달라 post-training과의 accuracy/NLL 차이를 causal effect로 읽을 수 없습니다.
|
||||
3. **저차원 smooth objective의 특수성**: 다섯 확률 출력의 simplex NLL은 비교적 매끄러운 저차원 함수라서, PSO 900 evaluations가 SLSQP 23 evaluations와 같은 NLL을 얻은 사실은 이 설정에서의 redundancy를 보여줍니다. 모든 비선형·불연속 목적함수에서 PSO가 중복된다는 뜻은 아닙니다.
|
||||
4. **저장·추론 비용**: 다섯 모델을 보존하고 다섯 출력 forward를 수행해야 하므로 single model보다 5배 경로가 필요합니다. 본 연구는 이 ensemble overhead를 제거하거나 model soup으로 대체하지 않았습니다.
|
||||
|
||||
**결정**: smooth prediction-space NLL에는 먼저 **uniform + temperature**를 사용하고, 명시적인 simplex weight가 필요하면 **SLSQP**를 우선합니다. PSO는 현재 연구에서 SLSQP보다 품질을 추가로 개선하지 못하면서 더 많은 query와 wall time을 사용했으므로 이 경로의 기본 optimizer로 채택하지 않습니다. 향후 PSO는 gradient가 없거나 불연속·이산인 architecture/subset/diversity 선택 문제처럼 SLSQP가 직접 다루기 어려운 목적함수에서만 별도 protocol과 leakage 방지 확인을 거쳐 연구합니다. 이 결론은 PSO 전체의 실패 선언이나 full-weight G8과의 일반적 우열 주장이 아니라, 본 post-training prediction-space study의 bounded decision입니다.
|
||||
|
||||
**출력 아티팩트**: [`benchmark_results/pso_v8_post_training_ensemble.json`](benchmark_results/pso_v8_post_training_ensemble.json), [`benchmark_results/pso_v8_post_training_ensemble.csv`](benchmark_results/pso_v8_post_training_ensemble.csv), [`benchmark_results/pso_v8_post_training_ensemble_evaluation.json`](benchmark_results/pso_v8_post_training_ensemble_evaluation.json), [`history_plt/pso_v8_post_training_ensemble.png`](history_plt/pso_v8_post_training_ensemble.png), [decision log](.omc/autoresearch/post-training-pso-ensemble/runs/20260904T093144Z/decision-log.md).
|
||||
|
||||
|
||||
|
||||
|
||||
## 7. 결과 해석 및 실무 권고사항 (Interpretation & Recommendations)
|
||||
|
||||
1. **고전 무브먼트 기법 선택**:
|
||||
- Protocol 2.0.0의 5개 워크로드 고정 예산에서는 `constriction`과 `inertia`가 가장 낮은 평균 순위를 기록했습니다. 새로운 워크로드에서는 둘을 우선 비교하되 보편적 우위로 간주하지 않아야 합니다.
|
||||
- 확장 MNIST 연구에서는 새 `local_best`의 반경 4 후보가 동일 30×80 held-out 확인에서 가장 높은 평균 정확도(62.64%)와 가장 낮은 손실(1.211368)을 기록했습니다. 다른 데이터셋에서는 별도 검증이 필요합니다.
|
||||
2. **`adaptive_moment` 독자 기법**:
|
||||
- 기존 ablation의 $\lambda=0.10$은 해당 10개 프로필 중 미분 무관 최고 정확도였지만, 확장 search에서 선택된 $\lambda=0.06$/step 0.5는 동일 예산 held-out 확인에서 `local_best`와 `inertia`보다 낮았습니다.
|
||||
- 더 많은 파티클은 80세대를 유지해 총 평가량을 늘릴 때 성능을 높였고, particle-epochs를 고정하면 낮아졌습니다. 파티클 수와 계산 예산을 함께 보고 선택해야 합니다.
|
||||
- 120개 파티클에서는 80세대가 충분한 수렴 horizon이 아니었습니다. 이 워크로드에서 계산 예산이 허용되면 더 긴 horizon을 사용하되, 테스트 체크포인트가 아니라 별도 validation metric으로 중단 시점을 결정해야 합니다.
|
||||
- 공식 MNIST 전체 학습에서는 240세대 정확도가 87.70%로 2k-fitness 연구보다 높았습니다. 다만 full objective와 PCA 적합 범위가 함께 달라졌으므로 full-data 사용 하나의 인과 효과로 분리하지 않습니다.
|
||||
3. **`quantum` 기법**:
|
||||
- 이 후보 범위의 QPSO는 held-out 정확도 48.58%로 가장 낮았습니다. 하나의 워크로드 결과이며 QPSO 일반 성능으로 해석하지 않습니다.
|
||||
4. **Adam refinement**:
|
||||
- 경사도 사용이 허용될 때 `tuned_adam_100_lr.01`은 기존 ablation에서 가장 높은 정확도와 가장 낮은 손실을 기록했습니다. 순수 미분 무관 방법과 별도 범주로 비교해야 합니다.
|
||||
5. **초기화 선택**:
|
||||
- 기존 PCA32 ablation에서 `uniform` 초기화는 `model_noise`보다 평균 정확도가 4.64%p 낮았습니다. 이는 해당 경계·모델·예산에 한정된 관찰입니다.
|
||||
|
||||
6. **딥러닝 아키텍처 및 PSO 역할 권고사항 (Deep Accuracy Insights)**:
|
||||
- 측정된 설정에서 실무적 권장 경로는 **Compact CNN + Adam**입니다.
|
||||
- 9,098개 CNN 전가중치를 직접 탐색한 PSO는 이 고정 예산에서 역전파를 대체하지 못했습니다. PSO를 딥러닝 파이프라인에 유지하려면 전가중치 대체보다 저차원 아키텍처·하이퍼파라미터 탐색 후 Adam으로 가중치를 학습하는 역할 분담이 합리적입니다. 이 역할 분담 자체는 본 프로토콜에서 비교하지 않은 공학적 권고입니다.
|
||||
7. **희소 부호 해시 부분공간·단계적 평가·스웜 앙상블 탐색 권고사항 (V5 Insights)**:
|
||||
- V5의 동일 30p × 160e 파일럿에서는 희소 부호 해시 부분공간(290~4096차원)보다 전가중치(9,098차원) 검증 성능이 높았습니다. 그러나 V6 진단에서 동일 latent radius가 차원이 작을수록 decoded per-parameter RMS를 축소하는 교란이 확인됐으므로, 이 결과는 부분공간 자체의 열위 근거가 아닙니다.
|
||||
- 단계적 표본 확장(2k→10k→50k) 적용 시 목적 함수 변경에 맞춰 pbest 재평가를 수행하여 수렴 편향을 방지해야 합니다.
|
||||
- 스웜 내 검증 상위 다양성 파티클 기반 Top-5 앙상블은 단일 모델 대비 정확도(+3.77%p)와 NLL/Brier를 개선하였으나, ECE가 악화(+0.079042)되었습니다. 단순히 파티클 다양성이나 손실 잔여물이 높다고 해서 일반화나 신뢰성이 비례하여 향상된다고 가정하지 않아야 합니다.
|
||||
8. **V6 탐색 구조 권고사항**:
|
||||
- V5의 zero-velocity/no-mutation/normalized ±3 묶음은 동일 2k·60p×420e 조건에서 기존 `Optimizer`보다 5.39%p 낮았습니다. mutation·초기 속도·±6 경계를 복원한 G5/G6은 사전 회복 기준을 통과했으므로 다음 탐색은 G6을 기준으로 진행합니다.
|
||||
- global RMS coordinate scale은 per-tensor SD보다 낮았으므로 좌표 정규화 자체를 제거하지 않습니다. mutation과 경계 확장의 독립 효과는 아직 3-시드 확인 전이므로 각각을 확정 원인으로 표현하지 않습니다.
|
||||
9. **더 무거운 태스크에 대한 PSO 역할 권고사항**:
|
||||
- 55,338개 파라미터와 FashionMNIST에서도 모든 실행이 finite했고 초기 모델 대비 개선됐으므로 기술적 실행 가능성은 확인됐습니다. 그러나 41.03~49.15% validation accuracy는 전가중치 PSO를 실용적 학습 경로로 권고할 근거가 아닙니다.
|
||||
- 파라미터 수 증가로 swarm state가 6.08배 증가하고 처리량이 대략 절반으로 감소했습니다. 다음 확장 실험은 더 큰 전가중치 모델보다 저차원 subspace/협력적 block 탐색 또는 PSO 기반 하이퍼파라미터 탐색을 우선해야 합니다. 이 대안들의 성능은 본 프로토콜에서 측정하지 않았습니다.
|
||||
---
|
||||
|
||||
## 8. 연구 한계점 (Limitations)
|
||||
|
||||
1. **소규모 샘플 사이즈 ($n=5$) 및 신뢰구간 측정 한계**:
|
||||
- 본 벤치마크는 무작위 시드 5개(41~45 및 46~50)에 대한 측정을 바탕으로 하였습니다. 샘플 크기가 $n=5$로 제한되어 일부 지표에서 표준편차가 크며, 95% 신뢰구간(CI)의 범위를 좁히는 데 한계가 있습니다.
|
||||
2. **순위(Rank) 지표의 서열적(Ordinal) 특성**:
|
||||
- 평균 순위(Mean Rank) 지표는 절대적인 성능 격차 수치를 반영하지 않고 상대적인 순위 수치만을 반영합니다. 따라서 순위 차이가 절대적인 손실/정확도 차이와 정비례하지 않습니다.
|
||||
3. **`tuned_full_evaluation` 비교 시 교란 요인 (Confounders)**:
|
||||
- `tuned_full_evaluation` (3,000개 전체 데이터)과 고정 서브셋 기법(2,000개 서브셋) 간의 손실 수치 비교는 평가에 사용된 데이터 샘플 수 및 배치 연산 차이가 개입된 교란 요인을 포함하고 있습니다.
|
||||
4. **소형/얕은 신경망 모델 범위 한계**:
|
||||
- 메인 및 튜닝 평가 대부분은 1~2개 레이어의 소형 MLP와 PCA32 선형 모델이며, 딥 프로토콜도 9,098개 파라미터 Compact CNN 하나로 제한됩니다. 수만~수억 개 파라미터 모델로의 직접 일반화에는 근거가 부족합니다.
|
||||
|
||||
5. **확장 튜닝의 선택 불확실성**:
|
||||
- 32개 후보 search는 시드 3개만 사용했습니다. validation 차이가 작은 후보의 순위는 추가 시드에서 바뀔 수 있습니다.
|
||||
6. **파티클 스케일링의 계산량 차이**:
|
||||
- 80세대 고정 비교는 파티클 수와 함께 총 평가량이 증가합니다. 약 2,400 particle-epochs 고정 비교는 평가 횟수만 근사적으로 맞추며 실제 MPS 실행 비용은 동일하지 않습니다.
|
||||
|
||||
7. **Epoch checkpoint 테스트 반복 관찰**:
|
||||
- 80~240세대 궤적은 같은 테스트 1,000개를 반복 평가한 진단 결과입니다. 이를 근거로 epoch를 선택하면 테스트셋 누수가 되므로 실제 stopping rule에는 별도 validation split이 필요합니다.
|
||||
|
||||
8. **Full MNIST 모델 범위와 반복 테스트 관찰**:
|
||||
- train/test 전체 split을 사용했지만 PCA32 선형 모델 실험입니다. 원본 영상 공간 및 CNN으로 일반화하지 않으며, test 10,000개 checkpoint를 반복 관찰해 epoch를 선택하지 않습니다.
|
||||
|
||||
9. **Deep Accuracy 프로토콜의 표본 수($n=3$) 및 탐색 공간 제약**:
|
||||
- Deep Accuracy 실험은 $n=3$ 시드로 수행된 기술적(descriptive) 비교이며 통계적 유의성 검정을 제공하지 않습니다. 또한 PSO 30 파티클 × 40 세대의 고정 예산과 고정 서브셋(2,000개) 평가 조건을 사용하였으므로, 파티클 수나 세대를 무한히 늘린 경우의 가설적 한계 성능을 수식적으로 증명한 것은 아닙니다. 그럼에도 9,098개 파라미터 전가중치 탐색에서 경사도 대비 극심한 열위(36.76% vs 98.53%)는 고차원 전가중치 PSO 적용의 실질적 한계를 명확히 나타냅니다.
|
||||
|
||||
10. **MNIST-PSO-RAW-V5 프로토콜의 범위와 불확실성**:
|
||||
- 부분공간 선택은 단일 파일럿 시드(91), 확인은 3개 시드와 Compact CNN 한 구조, 원본 MNIST 한 데이터셋에 한정됩니다. 60p × 600e 단계적 탐색의 단일 모델 정확도(83.23%)와 앙상블 정확도(87.00%)는 비동등 계산량의 v4 Adam 10-epoch 평균(98.53% ± 0.16%)보다 각각 15.30%p와 11.53%p 낮았습니다.
|
||||
- 앙상블은 단일 모델보다 정확도와 NLL/Brier가 개선되고 예측 불일치도 0.1585를 보였지만, ECE도 0.085242에서 0.164284로 높아졌습니다. 이 동시 관찰만으로 다양성이나 손실이 개선 또는 보정 악화의 원인이라고 결론 내릴 수 없습니다.
|
||||
11. **MNIST-PSO-RAW-V6 screen/confirmation 범위**:
|
||||
- G0–G8 screen은 단일 seed 91이므로 요인별 material 판정은 후보 선별 근거입니다. 3-시드 confirmation은 G0/G1/G5/G6/G8만 포함하여 전체 구조 회복과 scale/radius 판단만 확인했습니다.
|
||||
- G5와 G6이 G8에 근접했다는 결과는 동일 2k 목적함수와 60p×420e 예산에 한정됩니다. 50k objective, 더 긴 horizon, 다른 초기 모델, 외부 데이터에 대한 성능을 보장하지 않습니다.
|
||||
- 공식 test split을 사용하지 않았으므로 V6 Phase B 수치는 validation 결과이며 V5의 공식 test 정확도와 직접적인 paired test 비교가 아닙니다.
|
||||
12. **HEAVY-TASK-PSO-V6의 validation 재사용 및 예산 한계**:
|
||||
- 단일 seed screen에서 normalized 방법을 선택한 뒤 같은 validation 10,000개로 3-시드 확인 결과를 보고했으므로 선택 편향이 남습니다. 공식 test split을 로드하지 않았고 외부 일반화 성능은 측정하지 않았습니다.
|
||||
- 12 particles × 80 epochs·fixed-10k endpoint는 계산상 feasibility만 판정합니다. 세대 궤적, validation 기반 조기 중단, 50k objective, Adam 동예산 비교가 없으므로 수렴 완료·최대 도달 성능·계산 효율 우월성을 주장하지 않습니다.
|
||||
- FashionMNIST는 설계상 데이터 난도 축으로 사용했지만 measured endpoint가 MNIST보다 일관되게 낮지 않았습니다. 따라서 이 실행만으로 데이터 난도 증가의 인과 효과를 분리하지 않습니다.
|
||||
13. **Heavy PSO 교차 분할 강건성 한계 (Cross-Split Robustness Failure)**: §6.14의 두 개발 분할(20260905/20260906)에서 동결 정책과 8회 결정의 9개 변형 모두 gate를 통과하지 못했습니다. 따라서 `fixed_global_hybrid_v3`의 이전 단일 분할 개선은 측정한 새 분할에서 재현되지 않았고, 이 정책은 보존하지 않습니다(`retained_policy = null`). 이 결과는 두 validation 분할과 고정 예산에 한정되며, 공식 test나 외부 데이터 일반화 성능을 측정하지 않았습니다.
|
||||
---
|
||||
## 9. 재현 명령 및 결과 데이터 아티팩트 (Reproducibility & Artifacts)
|
||||
|
||||
### 9.1 재현 실행 명령 (Reproducibility Command)
|
||||
|
||||
```shell
|
||||
# uv 환경에서 동일한 벤치마크 수트 전체 실행 (MPS 디바이스 사용)
|
||||
uv run --locked --extra examples python test/benchmark_suite.py --device mps
|
||||
|
||||
# 검증 선택, held-out 확인, Adaptive Moment 파티클 스케일링
|
||||
uv run --locked --extra examples python test/tuning_suite.py --device mps
|
||||
|
||||
# 120p×80e Adaptive Moment 파티클 스케일링 재현성 검증 (exact-replay + independent seeds)
|
||||
uv run --locked --extra examples python test/reproduce_scaling.py --device mps
|
||||
# 120p Adaptive Moment 연속 240-epoch 수렴 진단
|
||||
uv run --locked --extra examples python test/epoch_convergence.py --device mps
|
||||
# 공식 MNIST 60k/10k 전체 학습, 120p×240e
|
||||
uv run --locked --extra examples python test/full_mnist_study.py --device mps
|
||||
# 공식 MNIST 딥 신경망 아키텍처 및 최적화기 비교 (Deep Accuracy 1.0.0)
|
||||
uv run --locked --extra examples python test/deep_accuracy_study.py --device mps
|
||||
# 희소 부호 해시 부분공간·단계적 평가·스웜 앙상블 탐색 (MNIST-PSO-RAW-V5 1.0.0)
|
||||
uv run --locked --extra examples python test/deep_pso_methods.py --device mps
|
||||
# V5 탐색 구조 회귀 원인 분리 (MNIST-PSO-RAW-V6 1.0.0 Phase A/B)
|
||||
uv run --locked --extra examples python test/deep_pso_v6.py --phase all --device mps
|
||||
# 더 큰 CNN 및 FashionMNIST 전가중치 PSO feasibility screen/confirmation
|
||||
uv run --no-sync python test/heavy_task_feasibility.py --stage all --device mps
|
||||
# Heavy PSO 교차 분할 강건성 탐색/평가/발행
|
||||
uv run --no-sync python test/heavy_pso_cross_split.py --phase development --device mps
|
||||
uv run --no-sync python test/evaluate_heavy_cross_split.py --development .omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0001-development.json --output .omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0001-development.json
|
||||
uv run --no-sync python test/publish_heavy_cross_split.py --source-dir .omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z --output-json benchmark_results/pso_v7_heavy_cross_split.json --output-csv benchmark_results/pso_v7_heavy_cross_split.csv --output-plot history_plt/pso_v7_heavy_cross_split.png
|
||||
```
|
||||
|
||||
### 9.2 원천 결과 데이터 및 아티팩트 경로
|
||||
|
||||
- **종합 JSON 벤치마크 데이터**: [`benchmark_results/pso_v4_benchmark.json`](benchmark_results/pso_v4_benchmark.json)
|
||||
- **메인 벤치마크 CSV**: [`benchmark_results/pso_v4_main_benchmark.csv`](benchmark_results/pso_v4_main_benchmark.csv)
|
||||
- **Ablation 벤치마크 CSV**: [`benchmark_results/pso_v4_ablation_benchmark.csv`](benchmark_results/pso_v4_ablation_benchmark.csv)
|
||||
- **확장 튜닝 JSON**: [`benchmark_results/pso_v4_tuning.json`](benchmark_results/pso_v4_tuning.json)
|
||||
- **후보 Search CSV**: [`benchmark_results/pso_v4_tuning_search.csv`](benchmark_results/pso_v4_tuning_search.csv)
|
||||
- **Held-out Confirmation CSV**: [`benchmark_results/pso_v4_tuning_confirmation.csv`](benchmark_results/pso_v4_tuning_confirmation.csv)
|
||||
- **Particle Scaling CSV**: [`benchmark_results/pso_v4_particle_scaling.csv`](benchmark_results/pso_v4_particle_scaling.csv)
|
||||
- **120p×80e 재현성 검증 JSON**: [`benchmark_results/pso_v4_120p80_replication.json`](benchmark_results/pso_v4_120p80_replication.json)
|
||||
- **120p×80e 재현성 검증 CSV**: [`benchmark_results/pso_v4_120p80_replication.csv`](benchmark_results/pso_v4_120p80_replication.csv)
|
||||
- **120p epoch 수렴 진단 JSON**: [`benchmark_results/pso_v4_epoch_convergence.json`](benchmark_results/pso_v4_epoch_convergence.json)
|
||||
- **120p epoch 수렴 진단 CSV**: [`benchmark_results/pso_v4_epoch_convergence.csv`](benchmark_results/pso_v4_epoch_convergence.csv)
|
||||
- **Full MNIST JSON**: [`benchmark_results/pso_v4_full_mnist.json`](benchmark_results/pso_v4_full_mnist.json)
|
||||
- **Full MNIST CSV**: [`benchmark_results/pso_v4_full_mnist.csv`](benchmark_results/pso_v4_full_mnist.csv)
|
||||
- **Deep Accuracy JSON**: [`benchmark_results/pso_v4_deep_accuracy.json`](benchmark_results/pso_v4_deep_accuracy.json)
|
||||
- **Deep Accuracy CSV**: [`benchmark_results/pso_v4_deep_accuracy.csv`](benchmark_results/pso_v4_deep_accuracy.csv)
|
||||
- **V5 Deep Methods JSON**: [`benchmark_results/pso_v5_deep_methods.json`](benchmark_results/pso_v5_deep_methods.json)
|
||||
- **V5 Deep Methods CSV**: [`benchmark_results/pso_v5_deep_methods.csv`](benchmark_results/pso_v5_deep_methods.csv)
|
||||
- **V6 Geometry Ablation JSON**: [`benchmark_results/pso_v6_phase_b.json`](benchmark_results/pso_v6_phase_b.json)
|
||||
- **V6 Geometry Ablation CSV**: [`benchmark_results/pso_v6_phase_b.csv`](benchmark_results/pso_v6_phase_b.csv)
|
||||
- **Heavy Task Feasibility JSON**: [`benchmark_results/pso_v6_heavy_tasks.json`](benchmark_results/pso_v6_heavy_tasks.json)
|
||||
- **Heavy Task Feasibility CSV**: [`benchmark_results/pso_v6_heavy_tasks.csv`](benchmark_results/pso_v6_heavy_tasks.csv)
|
||||
- **Heavy PSO Cross-Split JSON**: [`benchmark_results/pso_v7_heavy_cross_split.json`](benchmark_results/pso_v7_heavy_cross_split.json)
|
||||
- **Heavy PSO Cross-Split CSV**: [`benchmark_results/pso_v7_heavy_cross_split.csv`](benchmark_results/pso_v7_heavy_cross_split.csv)
|
||||
- **결과 시각화 이미지**:
|
||||
- [`history_plt/pso_v4_accuracy.png`](history_plt/pso_v4_accuracy.png)
|
||||
- [`history_plt/pso_v4_loss.png`](history_plt/pso_v4_loss.png)
|
||||
- [`history_plt/pso_v4_rank_heatmap.png`](history_plt/pso_v4_rank_heatmap.png)
|
||||
- [`history_plt/pso_v4_runtime.png`](history_plt/pso_v4_runtime.png)
|
||||
- [`history_plt/pso_v4_mnist_ablation.png`](history_plt/pso_v4_mnist_ablation.png)
|
||||
- [`history_plt/pso_v4_extended_tuning.png`](history_plt/pso_v4_extended_tuning.png)
|
||||
- [`history_plt/pso_v4_particle_scaling.png`](history_plt/pso_v4_particle_scaling.png)
|
||||
- [`history_plt/pso_v4_epoch_convergence.png`](history_plt/pso_v4_epoch_convergence.png)
|
||||
- [`history_plt/pso_v4_full_mnist.png`](history_plt/pso_v4_full_mnist.png)
|
||||
- [`history_plt/pso_v4_deep_accuracy.png`](history_plt/pso_v4_deep_accuracy.png)
|
||||
- [`history_plt/pso_v5_deep_methods.png`](history_plt/pso_v5_deep_methods.png)
|
||||
- [`history_plt/pso_v6_phase_b.png`](history_plt/pso_v6_phase_b.png)
|
||||
- [`history_plt/pso_v6_heavy_tasks.png`](history_plt/pso_v6_heavy_tasks.png)
|
||||
- [`history_plt/pso_v7_heavy_cross_split.png`](history_plt/pso_v7_heavy_cross_split.png)
|
||||
|
||||
### 9.3 소스 코드 및 레퍼런스 문헌 참조
|
||||
- **5단계 플러그인 구현 소스**: [`pso/plugins.py`](pso/plugins.py) 및 [`pso/optimizer.py`](pso/optimizer.py)
|
||||
- **V5 탐색 및 앙상블 스크립트**: [`test/deep_pso_methods.py`](test/deep_pso_methods.py)
|
||||
- **V6 탐색 구조 진단 스크립트**: [`test/deep_pso_v6.py`](test/deep_pso_v6.py)
|
||||
- **Heavy Task Feasibility 스크립트**: [`test/heavy_task_feasibility.py`](test/heavy_task_feasibility.py)
|
||||
- **Heavy PSO Cross-Split 탐색/평가/발행 스크립트**: [`test/heavy_pso_cross_split.py`](test/heavy_pso_cross_split.py), [`test/evaluate_heavy_cross_split.py`](test/evaluate_heavy_cross_split.py), [`test/publish_heavy_cross_split.py`](test/publish_heavy_cross_split.py)
|
||||
- **고전 PSO 논문 DOI 및 알고리즘 구현 레퍼런스**: [`README.md` 참고 문헌 섹션](README.md#참고-문헌-primary-references--dois) 참조
|
||||
@@ -0,0 +1,16 @@
|
||||
cohort,method,candidate_label,regimen,seed,n_particles,epochs,particle_epochs,train_loss,train_acc,test_loss,test_acc,test_mse,fit_time_sec,data_fingerprint,model_fingerprint,device,completed,error
|
||||
baseline,adaptive_moment,am_b0.06_s0.5,fixed_epoch,71,120,80,9600,0.6516683101654053,0.7950000166893005,0.8584634065628052,0.7360000014305115,0.037673790007829666,11.962705624988303,dfe645918ece54c0,0777bd52fd76272d,mps,True,
|
||||
baseline,adaptive_moment,am_b0.06_s0.5,fixed_epoch,72,120,80,9600,0.6982801556587219,0.7914999723434448,0.9025362133979797,0.7239999771118164,0.038956169039011,11.906837583053857,dfe645918ece54c0,6fcb6e473bdacbd2,mps,True,
|
||||
baseline,adaptive_moment,am_b0.06_s0.5,fixed_epoch,73,120,80,9600,0.7527623772621155,0.7749999761581421,1.001193642616272,0.6919999718666077,0.04309915751218796,10.664651792030782,dfe645918ece54c0,2e6c351372592f10,mps,True,
|
||||
baseline,adaptive_moment,am_b0.06_s0.5,fixed_epoch,74,120,80,9600,0.6858600974082947,0.784500002861023,0.8601324558258057,0.7350000143051147,0.03845023736357689,10.921957665821537,dfe645918ece54c0,4000fe3fb26ef207,mps,True,
|
||||
baseline,adaptive_moment,am_b0.06_s0.5,fixed_epoch,75,120,80,9600,0.6326538324356079,0.8125,0.889915406703949,0.7300000190734863,0.03931796923279762,10.411899874918163,dfe645918ece54c0,0966039f5ef7af88,mps,True,
|
||||
replay,adaptive_moment,am_b0.06_s0.5,fixed_epoch,71,120,80,9600,0.6516683101654053,0.7950000166893005,0.8584634065628052,0.7360000014305115,0.037673790007829666,11.090910458937287,dfe645918ece54c0,0777bd52fd76272d,mps,True,
|
||||
replay,adaptive_moment,am_b0.06_s0.5,fixed_epoch,72,120,80,9600,0.6982801556587219,0.7914999723434448,0.9025362133979797,0.7239999771118164,0.038956169039011,10.668682500021532,dfe645918ece54c0,6fcb6e473bdacbd2,mps,True,
|
||||
replay,adaptive_moment,am_b0.06_s0.5,fixed_epoch,73,120,80,9600,0.7527623772621155,0.7749999761581421,1.001193642616272,0.6919999718666077,0.04309915751218796,11.617381499847397,dfe645918ece54c0,2e6c351372592f10,mps,True,
|
||||
replay,adaptive_moment,am_b0.06_s0.5,fixed_epoch,74,120,80,9600,0.6858600974082947,0.784500002861023,0.8601324558258057,0.7350000143051147,0.03845023736357689,12.46007908298634,dfe645918ece54c0,4000fe3fb26ef207,mps,True,
|
||||
replay,adaptive_moment,am_b0.06_s0.5,fixed_epoch,75,120,80,9600,0.6326538324356079,0.8125,0.889915406703949,0.7300000190734863,0.03931796923279762,11.499297459144145,dfe645918ece54c0,0966039f5ef7af88,mps,True,
|
||||
independent,adaptive_moment,am_b0.06_s0.5,fixed_epoch,81,120,80,9600,0.7000858783721924,0.781000018119812,0.9081020355224609,0.7250000238418579,0.039418451488018036,10.6923490408808,dfe645918ece54c0,b9e1b8cbb9345add,mps,True,
|
||||
independent,adaptive_moment,am_b0.06_s0.5,fixed_epoch,82,120,80,9600,0.6899945139884949,0.796999990940094,0.9245963096618652,0.718999981880188,0.03965267166495323,11.110405791085213,dfe645918ece54c0,f1a0025f5b3b5b7c,mps,True,
|
||||
independent,adaptive_moment,am_b0.06_s0.5,fixed_epoch,83,120,80,9600,0.6943607330322266,0.7929999828338623,0.8056192398071289,0.7609999775886536,0.034950967878103256,10.71645870897919,dfe645918ece54c0,9cb7fe904bf992ac,mps,True,
|
||||
independent,adaptive_moment,am_b0.06_s0.5,fixed_epoch,84,120,80,9600,0.7076351046562195,0.7885000109672546,0.8160682320594788,0.7429999709129333,0.0366184301674366,10.859140583081171,dfe645918ece54c0,b833ebd886fce382,mps,True,
|
||||
independent,adaptive_moment,am_b0.06_s0.5,fixed_epoch,85,120,80,9600,0.6802449822425842,0.7875000238418579,0.8681835532188416,0.7319999933242798,0.038917701691389084,10.97266870806925,dfe645918ece54c0,e6bdf9e5d849521b,mps,True,
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
||||
profile,dataset,method,n_particles,epochs,n_seeds,eval_acc_mean,eval_acc_std,eval_acc_median,eval_acc_iqr,eval_acc_ci95,eval_loss_mean,eval_loss_std,eval_loss_median,eval_loss_iqr,eval_loss_ci95,eval_mse_mean,eval_mse_std,train_acc_mean,train_loss_mean,runtime_seconds_mean,runtime_seconds_std,rank_acc,rank_loss
|
||||
adaptive_moment_.10,MNIST,adaptive_moment,30,80,5,0.63,0.018276,0.641,0.027,0.022692,1.243339,0.102561,1.253713,0.068197,0.127344,0.051633,0.002578,0.7099,0.978244,4.286662,0.289397,2,5
|
||||
adaptive_moment_.25,MNIST,adaptive_moment,30,80,5,0.5632,0.02938,0.569,0.034,0.03648,1.499979,0.119449,1.483628,0.043413,0.148313,0.06119,0.003677,0.6379,1.228023,3.290951,0.425997,8,7
|
||||
adaptive_moment_.50,MNIST,adaptive_moment,30,80,5,0.5186,0.039087,0.51,0.019,0.048532,1.683781,0.121571,1.683062,0.08627,0.150948,0.067331,0.003822,0.5705,1.4637,3.202502,0.269844,9,9
|
||||
inertia_canonical,MNIST,inertia,30,80,5,0.4676,0.024358,0.472,0.033,0.030244,1.720983,0.118092,1.699598,0.020267,0.146628,0.070356,0.002531,0.5277,1.528197,3.119898,0.30594,10,10
|
||||
inertia_tuned,MNIST,inertia,30,80,5,0.6162,0.05131,0.635,0.007,0.063709,1.2366,0.124281,1.194831,0.029657,0.154312,0.05278,0.005923,0.6931,0.999514,3.131882,0.408534,4,3
|
||||
tuned_adam_100_lr.01,MNIST,inertia,30,80,5,0.8558,0.002387,0.856,0.003,0.002964,0.470271,0.011931,0.472233,0.019931,0.014814,0.02164,0.000368,0.9158,0.298524,4.990688,0.434352,1,1
|
||||
tuned_full_evaluation,MNIST,inertia,30,80,5,0.6254,0.02805,0.624,0.029,0.034828,1.179009,0.050059,1.194071,0.052015,0.062156,0.051537,0.00241,0.687867,1.00514,3.377083,0.469302,3,2
|
||||
tuned_no_mutation,MNIST,inertia,30,80,5,0.6042,0.030376,0.607,0.003,0.037716,1.249836,0.084985,1.215945,0.022012,0.105521,0.053483,0.003055,0.6686,1.074396,3.347668,0.3431,6,6
|
||||
tuned_particle_reset,MNIST,inertia,30,80,5,0.6162,0.05131,0.635,0.007,0.063709,1.2366,0.124281,1.194831,0.029657,0.154312,0.05278,0.005923,0.6931,0.999514,4.176388,0.449217,5,4
|
||||
tuned_uniform_initialization,MNIST,inertia,30,80,5,0.5698,0.023952,0.575,0.017,0.02974,1.619196,0.157329,1.641514,0.149905,0.195347,0.061757,0.0042,0.616,1.345857,4.306866,0.442651,7,8
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,19 @@
|
||||
lane,profile_or_arch,seed,model_name,param_count,initial_test_acc,final_test_acc,final_test_loss,subset_fitness_acc,subset_fitness_loss,pso_epochs,adam_epochs,elapsed_sec,model_fingerprint,data_fingerprint
|
||||
architecture,raw_linear,101,Raw Linear (784->10),7850,0.1449,0.9239,0.269691,,,0,10,2.6001,12cef21a80b85b0b,8dd702555745641a
|
||||
architecture,raw_linear,102,Raw Linear (784->10),7850,0.1308,0.924,0.270518,,,0,10,2.6265,0f099d5ae79e7863,8dd702555745641a
|
||||
architecture,raw_linear,103,Raw Linear (784->10),7850,0.0559,0.9257,0.266626,,,0,10,2.8535,23a4a6362abdad3d,8dd702555745641a
|
||||
architecture,raw_mlp,101,Raw MLP (784->128->64->10),109386,0.0812,0.9773,0.076661,,,0,10,8.8041,9ac96d5f45d71cb3,8dd702555745641a
|
||||
architecture,raw_mlp,102,Raw MLP (784->128->64->10),109386,0.0522,0.976,0.081621,,,0,10,5.7141,55966246db4dfff1,8dd702555745641a
|
||||
architecture,raw_mlp,103,Raw MLP (784->128->64->10),109386,0.1111,0.9776,0.076823,,,0,10,4.5491,723217e2418edcfd,8dd702555745641a
|
||||
architecture,compact_cnn,101,"Compact CNN (9,098 params)",9098,0.0851,0.9866,0.039444,,,0,10,6.228,db41fb515dcb49fa,8dd702555745641a
|
||||
architecture,compact_cnn,102,"Compact CNN (9,098 params)",9098,0.0963,0.9858,0.042864,,,0,10,5.396,efd743cad60c7530,8dd702555745641a
|
||||
architecture,compact_cnn,103,"Compact CNN (9,098 params)",9098,0.1072,0.9836,0.04912,,,0,10,5.4935,17891ec08e79ee74,8dd702555745641a
|
||||
optimizer,adam_only,101,Compact CNN (Adam-Only),9098,0.0851,0.9866,0.039444,,,0,10,6.228,db41fb515dcb49fa,8dd702555745641a
|
||||
optimizer,adam_only,102,Compact CNN (Adam-Only),9098,0.0963,0.9858,0.042864,,,0,10,5.396,efd743cad60c7530,8dd702555745641a
|
||||
optimizer,adam_only,103,Compact CNN (Adam-Only),9098,0.1072,0.9836,0.04912,,,0,10,5.4935,17891ec08e79ee74,8dd702555745641a
|
||||
optimizer,pso_only,101,Compact CNN (PSO-Only),9098,0.0851,0.3901,22.182518,0.399,21.674469,40,0,2.5761,db41fb515dcb49fa,8dd702555745641a
|
||||
optimizer,pso_only,102,Compact CNN (PSO-Only),9098,0.0963,0.3886,6.031698,0.3925,5.977414,40,0,2.331,efd743cad60c7530,8dd702555745641a
|
||||
optimizer,pso_only,103,Compact CNN (PSO-Only),9098,0.1072,0.3242,14.099036,0.3405,14.065619,40,0,2.7469,17891ec08e79ee74,8dd702555745641a
|
||||
optimizer,hybrid,101,Compact CNN (Hybrid),9098,0.0851,0.9658,0.109308,0.399,21.674469,40,10,7.4414,db41fb515dcb49fa,8dd702555745641a
|
||||
optimizer,hybrid,102,Compact CNN (Hybrid),9098,0.0963,0.9807,0.061528,0.3925,5.977414,40,10,7.226,efd743cad60c7530,8dd702555745641a
|
||||
optimizer,hybrid,103,Compact CNN (Hybrid),9098,0.1072,0.9725,0.088789,0.3405,14.065619,40,10,7.7426,17891ec08e79ee74,8dd702555745641a
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,61 @@
|
||||
seed,epoch,train_loss,train_acc,train_mse,test_loss,test_acc,test_mse,fit_time_sec
|
||||
71,20,1.5967158079147339,0.4830000102519989,0.0660470575094223,1.7100492715835571,0.43799999356269836,0.07075551897287369,32.5467
|
||||
71,40,1.1239886283874512,0.6434999704360962,0.04822305589914322,1.3276519775390625,0.578000009059906,0.05614830181002617,32.5467
|
||||
71,60,0.8301568627357483,0.7429999709129333,0.03629826754331589,1.0045816898345947,0.6800000071525574,0.04412851482629776,32.5467
|
||||
71,80,0.6516683101654053,0.7950000166893005,0.028850017115473747,0.8584634065628052,0.7360000014305115,0.037673790007829666,32.5467
|
||||
71,100,0.5542685985565186,0.824999988079071,0.024905934929847717,0.7590173482894897,0.7620000243186951,0.03418450802564621,32.5467
|
||||
71,120,0.49202004075050354,0.8399999737739563,0.021997792646288872,0.7061744332313538,0.7889999747276306,0.03144212067127228,32.5467
|
||||
71,140,0.45162707567214966,0.8665000200271606,0.01984463632106781,0.6375347971916199,0.796999990940094,0.029180902987718582,32.5467
|
||||
71,160,0.4246227741241455,0.8740000128746033,0.018856780603528023,0.6248053312301636,0.8009999990463257,0.028511321172118187,32.5467
|
||||
71,180,0.39958131313323975,0.8799999952316284,0.01768091320991516,0.6031026244163513,0.8159999847412109,0.027248937636613846,32.5467
|
||||
71,200,0.3831179141998291,0.8924999833106995,0.016940467059612274,0.5841547846794128,0.8180000185966492,0.026774849742650986,32.5467
|
||||
71,220,0.36869707703590393,0.8970000147819519,0.016194477677345276,0.5548750162124634,0.8259999752044678,0.025661807507276535,32.5467
|
||||
71,240,0.3557758927345276,0.8989999890327454,0.015656106173992157,0.5201151371002197,0.8420000076293945,0.023789195343852043,32.5467
|
||||
72,20,1.673862338066101,0.46149998903274536,0.0692000463604927,1.8141505718231201,0.4099999964237213,0.07267315685749054,32.606
|
||||
72,40,1.1490310430526733,0.6324999928474426,0.04914539307355881,1.372908592224121,0.5709999799728394,0.05662866309285164,32.606
|
||||
72,60,0.8807169198989868,0.7369999885559082,0.03719912841916084,1.1267516613006592,0.6420000195503235,0.04854830726981163,32.606
|
||||
72,80,0.6982801556587219,0.7914999723434448,0.030153820291161537,0.9025362133979797,0.7239999771118164,0.038956169039011,32.606
|
||||
72,100,0.5910305380821228,0.8165000081062317,0.026041019707918167,0.7481632232666016,0.7570000290870667,0.03314316272735596,32.606
|
||||
72,120,0.5200771689414978,0.8464999794960022,0.022959064692258835,0.6597934365272522,0.7889999747276306,0.029753563925623894,32.606
|
||||
72,140,0.4719507694244385,0.8554999828338623,0.020908480510115623,0.58547043800354,0.8100000023841858,0.02705816738307476,32.606
|
||||
72,160,0.4356405735015869,0.871999979019165,0.019257094711065292,0.583936333656311,0.8220000267028809,0.026691943407058716,32.606
|
||||
72,180,0.4127918779850006,0.8725000023841858,0.018300892785191536,0.5632762312889099,0.8199999928474426,0.025847580283880234,32.606
|
||||
72,200,0.3940938115119934,0.8805000185966492,0.017671920359134674,0.5363063216209412,0.8339999914169312,0.024680841714143753,32.606
|
||||
72,220,0.37782323360443115,0.8845000267028809,0.017062701284885406,0.508123517036438,0.8399999737739563,0.023583590984344482,32.606
|
||||
72,240,0.36558520793914795,0.8899999856948853,0.01645873300731182,0.49120959639549255,0.8429999947547913,0.022604094818234444,32.606
|
||||
73,20,1.7135908603668213,0.4449999928474426,0.07240503281354904,1.8838708400726318,0.375,0.0778227150440216,32.7255
|
||||
73,40,1.2155654430389404,0.6340000033378601,0.051504913717508316,1.4863072633743286,0.5569999814033508,0.06111995875835419,32.7255
|
||||
73,60,0.9407793283462524,0.722000002861023,0.03982725366950035,1.1702646017074585,0.6439999938011169,0.049435749650001526,32.7255
|
||||
73,80,0.7527623772621155,0.7749999761581421,0.032556530088186264,1.001193642616272,0.6919999718666077,0.04309915751218796,32.7255
|
||||
73,100,0.6268301606178284,0.8140000104904175,0.027759678661823273,0.8296219706535339,0.7319999933242798,0.03728931397199631,32.7255
|
||||
73,120,0.5454505681991577,0.8320000171661377,0.02451479621231556,0.7347204089164734,0.7730000019073486,0.03278779238462448,32.7255
|
||||
73,140,0.46880221366882324,0.8575000166893005,0.021052677184343338,0.6670262813568115,0.7889999747276306,0.030432477593421936,32.7255
|
||||
73,160,0.43280255794525146,0.8659999966621399,0.01961500570178032,0.6248233914375305,0.8080000281333923,0.028731103986501694,32.7255
|
||||
73,180,0.4033927321434021,0.8790000081062317,0.017949724569916725,0.5680584907531738,0.8230000138282776,0.02612500637769699,32.7255
|
||||
73,200,0.3850976228713989,0.8880000114440918,0.017122117802500725,0.5508874654769897,0.8209999799728394,0.025604458525776863,32.7255
|
||||
73,220,0.3726767599582672,0.8899999856948853,0.016703158617019653,0.5412379503250122,0.8209999799728394,0.025124624371528625,32.7255
|
||||
73,240,0.3600545823574066,0.8989999890327454,0.016138451173901558,0.5263404250144958,0.8190000057220459,0.024716168642044067,32.7255
|
||||
74,20,1.5981744527816772,0.49950000643730164,0.06673180311918259,1.7434691190719604,0.42399999499320984,0.07240629941225052,32.9017
|
||||
74,40,1.1163724660873413,0.6600000262260437,0.047663308680057526,1.2812504768371582,0.6079999804496765,0.05496295168995857,32.9017
|
||||
74,60,0.8247233629226685,0.7429999709129333,0.03677733987569809,0.996366560459137,0.6800000071525574,0.044562000781297684,32.9017
|
||||
74,80,0.6858600974082947,0.784500002861023,0.030640259385108948,0.8601324558258057,0.7350000143051147,0.03845023736357689,32.9017
|
||||
74,100,0.5762760043144226,0.8274999856948853,0.025603292509913445,0.7484227418899536,0.7580000162124634,0.03352699801325798,32.9017
|
||||
74,120,0.5109706521034241,0.8500000238418579,0.02288639359176159,0.6725562810897827,0.7960000038146973,0.030220312997698784,32.9017
|
||||
74,140,0.4668312966823578,0.8600000143051147,0.021057307720184326,0.6381762623786926,0.8069999814033508,0.028759241104125977,32.9017
|
||||
74,160,0.430540531873703,0.8709999918937683,0.01937052235007286,0.6036783456802368,0.8169999718666077,0.026846695691347122,32.9017
|
||||
74,180,0.4092761278152466,0.8755000233650208,0.018336936831474304,0.5836188793182373,0.8199999928474426,0.0263815987855196,32.9017
|
||||
74,200,0.3926721215248108,0.8865000009536743,0.017603637650609016,0.5680423378944397,0.8270000219345093,0.02558077871799469,32.9017
|
||||
74,220,0.38078930974006653,0.8865000009536743,0.017137007787823677,0.5442376136779785,0.8379999995231628,0.024594470858573914,32.9017
|
||||
74,240,0.37086427211761475,0.8880000114440918,0.016686489805579185,0.5385385155677795,0.8299999833106995,0.024481549859046936,32.9017
|
||||
75,20,1.635738730430603,0.46950000524520874,0.06960180401802063,1.8486356735229492,0.4059999883174896,0.07580890506505966,32.7233
|
||||
75,40,1.0605015754699707,0.6604999899864197,0.04849516972899437,1.2981681823730469,0.5630000233650208,0.05867462605237961,32.7233
|
||||
75,60,0.7738548517227173,0.7615000009536743,0.03458299860358238,1.0294034481048584,0.671999990940094,0.04577324911952019,32.7233
|
||||
75,80,0.6326538324356079,0.8125,0.02839995175600052,0.889915406703949,0.7300000190734863,0.03931796923279762,32.7233
|
||||
75,100,0.529951810836792,0.8370000123977661,0.02384800836443901,0.762531042098999,0.7620000243186951,0.034515053033828735,32.7233
|
||||
75,120,0.47619307041168213,0.8575000166893005,0.021528739482164383,0.6571563482284546,0.781000018119812,0.030564110726118088,32.7233
|
||||
75,140,0.43610602617263794,0.8709999918937683,0.019426772370934486,0.6097874641418457,0.8140000104904175,0.028030628338456154,32.7233
|
||||
75,160,0.4064372479915619,0.8794999718666077,0.01836566999554634,0.5621976256370544,0.8309999704360962,0.025522591546177864,32.7233
|
||||
75,180,0.3866714835166931,0.8865000009536743,0.01732715405523777,0.557114839553833,0.8270000219345093,0.025415310636162758,32.7233
|
||||
75,200,0.37161633372306824,0.8855000138282776,0.016865627840161324,0.5388693809509277,0.8289999961853027,0.025061823427677155,32.7233
|
||||
75,220,0.3574405014514923,0.8939999938011169,0.016058821231126785,0.528910756111145,0.8289999961853027,0.024624072015285492,32.7233
|
||||
75,240,0.3456318974494934,0.8945000171661377,0.0156160369515419,0.5028988122940063,0.8420000076293945,0.023111792281270027,32.7233
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,61 @@
|
||||
seed,epoch,train_loss,train_acc,train_mse,test_loss,test_acc,test_mse,fit_time_sec
|
||||
71,20,1.6637481451034546,0.4778333306312561,0.06879015266895294,1.6244964599609375,0.4961000084877014,0.0671548843383789,33.649
|
||||
71,40,1.165610432624817,0.6545166373252869,0.04875154793262482,1.0854827165603638,0.6718000173568726,0.046549420803785324,33.649
|
||||
71,60,0.905483067035675,0.7269166707992554,0.0389271154999733,0.8291375041007996,0.7445999979972839,0.036550913006067276,33.649
|
||||
71,80,0.7415720224380493,0.7783499956130981,0.03182263299822807,0.6753337979316711,0.7944999933242798,0.029618915170431137,33.649
|
||||
71,100,0.6563442945480347,0.8064000010490417,0.02799912355840206,0.5993074178695679,0.8216999769210815,0.025761328637599945,33.649
|
||||
71,120,0.588559091091156,0.8279500007629395,0.025105126202106476,0.5335480570793152,0.8414999842643738,0.022965701296925545,33.649
|
||||
71,140,0.5534469485282898,0.8387500047683716,0.023768901824951172,0.5105180144309998,0.8489999771118164,0.022098202258348465,33.649
|
||||
71,160,0.5169084668159485,0.8464000225067139,0.022431854158639908,0.4750884771347046,0.8555999994277954,0.02082662656903267,33.649
|
||||
71,180,0.48863890767097473,0.8547000288963318,0.021347153931856155,0.4477296769618988,0.8686000108718872,0.01953563280403614,33.649
|
||||
71,200,0.4686008393764496,0.8610333204269409,0.020526422187685966,0.43183833360671997,0.8719000220298767,0.018971838057041168,33.649
|
||||
71,220,0.45222264528274536,0.8653833270072937,0.0198610108345747,0.4211212396621704,0.8756999969482422,0.01851881667971611,33.649
|
||||
71,240,0.438436359167099,0.8698999881744385,0.019294776022434235,0.41157108545303345,0.878000020980835,0.018099937587976456,33.649
|
||||
72,20,1.7392971515655518,0.4580833315849304,0.07111531496047974,1.7067090272903442,0.4602999985218048,0.07063549011945724,33.7062
|
||||
72,40,1.2618558406829834,0.6182666420936584,0.05362313985824585,1.2296088933944702,0.6273000240325928,0.05239206552505493,33.7062
|
||||
72,60,0.9506728053092957,0.7125833630561829,0.04073568433523178,0.90444415807724,0.725600004196167,0.03892349451780319,33.7062
|
||||
72,80,0.7690414786338806,0.7677000164985657,0.0332583524286747,0.7225316762924194,0.7825000286102295,0.03151436522603035,33.7062
|
||||
72,100,0.6607407331466675,0.7977833151817322,0.029053399339318275,0.6213417053222656,0.8091999888420105,0.02738099917769432,33.7062
|
||||
72,120,0.5884125232696533,0.8225333094596863,0.02557925134897232,0.5514547824859619,0.8324999809265137,0.024056605994701385,33.7062
|
||||
72,140,0.5509796142578125,0.8328999876976013,0.02395990863442421,0.5114421844482422,0.8463000059127808,0.022018717601895332,33.7062
|
||||
72,160,0.5122449398040771,0.8464499711990356,0.022468935698270798,0.4830068349838257,0.8574000000953674,0.021029997617006302,33.7062
|
||||
72,180,0.4828203022480011,0.8543833494186401,0.021225325763225555,0.45465460419654846,0.8646000027656555,0.01990104280412197,33.7062
|
||||
72,200,0.46293872594833374,0.8621000051498413,0.020341966301202774,0.4369881749153137,0.8700000047683716,0.019186409190297127,33.7062
|
||||
72,220,0.4459497630596161,0.8664166927337646,0.0197211392223835,0.41942450404167175,0.8730000257492065,0.0184775423258543,33.7062
|
||||
72,240,0.4350419342517853,0.871566653251648,0.01916772872209549,0.41060513257980347,0.875,0.018150048330426216,33.7062
|
||||
73,20,1.703110694885254,0.4322333335876465,0.07236693799495697,1.662993311882019,0.4422999918460846,0.07139508426189423,33.0256
|
||||
73,40,1.234723448753357,0.605400025844574,0.05369972065091133,1.2019942998886108,0.6128000020980835,0.05271701514720917,33.0256
|
||||
73,60,0.9583781361579895,0.7071499824523926,0.04126281663775444,0.9148314595222473,0.7215999960899353,0.03978179767727852,33.0256
|
||||
73,80,0.781466543674469,0.7578666806221008,0.034513432532548904,0.7499686479568481,0.7670999765396118,0.03327890485525131,33.0256
|
||||
73,100,0.6772935390472412,0.7944999933242798,0.029873577877879143,0.6402292251586914,0.8046000003814697,0.02839311771094799,33.0256
|
||||
73,120,0.6110365986824036,0.8133666515350342,0.027105839923024178,0.5728253722190857,0.821399986743927,0.02574964240193367,33.0256
|
||||
73,140,0.5631463527679443,0.8302500247955322,0.024875810369849205,0.5334057807922363,0.8353999853134155,0.023713810369372368,33.0256
|
||||
73,160,0.5311679840087891,0.8410833477973938,0.023450637236237526,0.501768171787262,0.8464000225067139,0.022286171093583107,33.0256
|
||||
73,180,0.5000290870666504,0.8493833541870117,0.022157883271574974,0.4740225672721863,0.8550000190734863,0.021153515204787254,33.0256
|
||||
73,200,0.48113930225372314,0.8564833402633667,0.02121545933187008,0.4526425898075104,0.8634999990463257,0.020089736208319664,33.0256
|
||||
73,220,0.46380415558815,0.8618666529655457,0.020463503897190094,0.4359147846698761,0.8684999942779541,0.019339669495821,33.0256
|
||||
73,240,0.449687123298645,0.8664166927337646,0.019846484065055847,0.4234127998352051,0.8718000054359436,0.018784264102578163,33.0256
|
||||
74,20,1.696816325187683,0.4334833323955536,0.07303200662136078,1.685478925704956,0.4397999942302704,0.07274489104747772,32.3565
|
||||
74,40,1.2137997150421143,0.6062666773796082,0.053883783519268036,1.193790316581726,0.6158999800682068,0.05286615714430809,32.3565
|
||||
74,60,0.9459328651428223,0.7093166708946228,0.04073885455727577,0.926419734954834,0.7172999978065491,0.03994119539856911,32.3565
|
||||
74,80,0.7440351843833923,0.7731500267982483,0.03230465576052666,0.7158955335617065,0.7832000255584717,0.0311629269272089,32.3565
|
||||
74,100,0.6222507357597351,0.8106666803359985,0.02721019648015499,0.5782299041748047,0.826200008392334,0.025308359414339066,32.3565
|
||||
74,120,0.5712149143218994,0.8261666893959045,0.025098947808146477,0.5316479206085205,0.8402000069618225,0.023361243307590485,32.3565
|
||||
74,140,0.5278127193450928,0.8391500115394592,0.02332845889031887,0.5003852248191833,0.8485000133514404,0.022033091634511948,32.3565
|
||||
74,160,0.4974406957626343,0.8500166535377502,0.02210366539657116,0.4702143669128418,0.8614000082015991,0.02081647887825966,32.3565
|
||||
74,180,0.4760556221008301,0.8567833304405212,0.0210970938205719,0.4514496922492981,0.8648999929428101,0.01998041942715645,32.3565
|
||||
74,200,0.4600929617881775,0.8631166815757751,0.020222675055265427,0.43229734897613525,0.8694999814033508,0.01901264674961567,32.3565
|
||||
74,220,0.44254931807518005,0.8678500056266785,0.019469955936074257,0.417081743478775,0.8754000067710876,0.018321329727768898,32.3565
|
||||
74,240,0.43052077293395996,0.8725666403770447,0.018974633887410164,0.40485620498657227,0.8798999786376953,0.01778729446232319,32.3565
|
||||
75,20,1.7082723379135132,0.42640000581741333,0.07211916148662567,1.6778934001922607,0.43220001459121704,0.07167963683605194,33.1184
|
||||
75,40,1.2474853992462158,0.628333330154419,0.05103134736418724,1.1880637407302856,0.6402000188827515,0.04922466725111008,33.1184
|
||||
75,60,0.9827106595039368,0.7029333114624023,0.041922468692064285,0.9340476393699646,0.7074000239372253,0.040672920644283295,33.1184
|
||||
75,80,0.8061239719390869,0.7508833408355713,0.035569027066230774,0.7613834738731384,0.7562999725341797,0.03404615819454193,33.1184
|
||||
75,100,0.6677705645561218,0.7959833145141602,0.029427148401737213,0.6365524530410767,0.8015999794006348,0.027956314384937286,33.1184
|
||||
75,120,0.5898874998092651,0.8204500079154968,0.025896690785884857,0.5661898255348206,0.8285999894142151,0.024743687361478806,33.1184
|
||||
75,140,0.5386065244674683,0.8367666602134705,0.023783763870596886,0.5188254714012146,0.8396999835968018,0.022942783311009407,33.1184
|
||||
75,160,0.5006535053253174,0.8497833609580994,0.022141018882393837,0.48212501406669617,0.8569999933242798,0.021169248968362808,33.1184
|
||||
75,180,0.47602227330207825,0.8565999865531921,0.021099673584103584,0.45692178606987,0.861299991607666,0.020099563524127007,33.1184
|
||||
75,200,0.4532005488872528,0.8648999929428101,0.020130231976509094,0.4406105577945709,0.8676999807357788,0.019379951059818268,33.1184
|
||||
75,220,0.4393870532512665,0.8705000281333923,0.019430004060268402,0.42279207706451416,0.8766999840736389,0.018535815179347992,33.1184
|
||||
75,240,0.4296344220638275,0.8724499940872192,0.019024165347218513,0.40994536876678467,0.8802000284194946,0.01800552010536194,33.1184
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,36 @@
|
||||
dataset,method,n_particles,epochs,n_seeds,eval_acc_mean,eval_acc_std,eval_acc_median,eval_acc_iqr,eval_acc_ci95,eval_loss_mean,eval_loss_std,eval_loss_median,eval_loss_iqr,eval_loss_ci95,eval_mse_mean,eval_mse_std,train_acc_mean,train_loss_mean,runtime_seconds_mean,runtime_seconds_std,rank_acc,rank_loss
|
||||
Digits,adaptive_moment,24,50,5,0.267778,0.018279,0.272222,0.002778,0.022695,2.108837,0.023357,2.10196,0.039265,0.029001,0.084021,0.001996,0.2718,2.086962,2.499448,0.161274,3,4
|
||||
Digits,bare_bones,24,50,5,0.19,0.056266,0.155556,0.1,0.069863,2.250203,0.084448,2.251535,0.020136,0.104855,0.088622,0.003141,0.2,2.230929,2.564876,0.124195,6,6
|
||||
Digits,clpso,24,50,5,0.207778,0.050629,0.213889,0.05,0.062863,2.204543,0.023215,2.210108,0.008743,0.028825,0.087396,0.001398,0.223,2.176502,2.519316,0.326734,5,5
|
||||
Digits,constriction,24,50,5,0.372778,0.017938,0.380556,0.025,0.022272,1.765113,0.057309,1.760801,0.0677,0.071157,0.074815,0.001608,0.382,1.720719,2.330013,0.148841,1,1
|
||||
Digits,fips,24,50,5,0.249444,0.06147,0.263889,0.069444,0.076324,2.064165,0.091569,2.064055,0.108705,0.113696,0.084041,0.002578,0.2662,2.056891,2.621102,0.126036,4,3
|
||||
Digits,inertia,24,50,5,0.303333,0.061658,0.305556,0.088889,0.076557,1.980667,0.14307,1.956439,0.144242,0.177642,0.080715,0.004026,0.297,2.014632,2.006851,0.09822,2,2
|
||||
Digits,original,24,50,5,0.164444,0.022343,0.158333,0.036111,0.027743,2.269628,0.025331,2.255083,0.030712,0.031453,0.089494,0.000451,0.1798,2.266102,2.272324,0.113557,7,7
|
||||
Iris,adaptive_moment,24,60,5,0.846667,0.076739,0.866667,0.133333,0.095283,0.349701,0.178108,0.334009,0.15044,0.221147,0.071416,0.035403,0.89,0.294127,1.392557,0.004708,4,4
|
||||
Iris,bare_bones,24,60,5,0.846667,0.055777,0.833333,0.066667,0.069256,0.405791,0.172336,0.367909,0.160981,0.21398,0.072802,0.025814,0.885,0.30248,1.66449,0.144454,5,5
|
||||
Iris,clpso,24,60,5,0.793333,0.076012,0.766667,0.066667,0.094379,0.44827,0.053877,0.474597,0.049216,0.066896,0.091464,0.016877,0.78,0.462534,1.86465,0.145025,6,6
|
||||
Iris,constriction,24,60,5,0.94,0.036515,0.966667,0.066667,0.045338,0.171796,0.076965,0.148386,0.105914,0.095563,0.033793,0.017284,0.991667,0.032715,1.4309,0.079265,2,2
|
||||
Iris,fips,24,60,5,0.74,0.054772,0.733333,0.033333,0.068008,0.60501,0.060601,0.625894,0.064748,0.075245,0.117407,0.008091,0.766667,0.595101,2.072251,0.154361,7,7
|
||||
Iris,inertia,24,60,5,0.94,0.027889,0.933333,0.033333,0.034628,0.106901,0.053063,0.108104,0.053939,0.065885,0.024266,0.011903,0.983333,0.040899,1.569569,0.121148,1,1
|
||||
Iris,original,24,60,5,0.92,0.086923,0.966667,0.033333,0.107927,0.222324,0.155767,0.192657,0.065706,0.193407,0.043721,0.035081,0.966667,0.08742,1.502316,0.042929,3,3
|
||||
MNIST,adaptive_moment,30,80,5,0.262,0.01044,0.269,0.014,0.012963,2.219058,0.104724,2.175837,0.105964,0.13003,0.087089,0.001349,0.2678,2.184856,3.547601,0.614982,4,4
|
||||
MNIST,bare_bones,30,80,5,0.4126,0.037958,0.408,0.057,0.04713,1.965562,0.175909,1.989087,0.298842,0.218416,0.077204,0.004802,0.4408,1.8435,3.435835,0.303983,3,3
|
||||
MNIST,clpso,30,80,5,0.2068,0.022775,0.2,0.019,0.028278,2.285833,0.068177,2.307686,0.037065,0.084651,0.089514,0.00163,0.2342,2.199232,3.536445,0.226486,6,6
|
||||
MNIST,constriction,30,80,5,0.52,0.046244,0.528,0.047,0.057418,1.507968,0.149152,1.509533,0.108216,0.185194,0.063327,0.005427,0.5968,1.284408,3.298112,0.589527,1,1
|
||||
MNIST,fips,30,80,5,0.2106,0.048066,0.229,0.04,0.05968,2.22323,0.073048,2.176908,0.130584,0.0907,0.088006,0.001556,0.2302,2.177675,3.806571,0.141017,5,5
|
||||
MNIST,inertia,30,80,5,0.4684,0.075252,0.497,0.108,0.093436,1.716089,0.260764,1.620183,0.331954,0.323776,0.070705,0.008553,0.552,1.44457,3.048485,0.155303,2,2
|
||||
MNIST,original,30,80,5,0.144,0.016628,0.143,0.026,0.020646,2.428194,0.091877,2.415645,0.154207,0.114079,0.093035,0.001872,0.1563,2.385448,3.490117,0.862784,7,7
|
||||
Seeds,adaptive_moment,24,60,5,0.890476,0.046413,0.904762,0.047619,0.057629,0.315947,0.120895,0.278367,0.077865,0.150108,0.053174,0.012296,0.917857,0.225176,1.407326,0.041435,2,1
|
||||
Seeds,bare_bones,24,60,5,0.871429,0.068595,0.857143,0.095238,0.08517,0.42192,0.120052,0.439151,0.13714,0.149062,0.069299,0.016028,0.882143,0.348336,1.54165,0.018779,6,5
|
||||
Seeds,clpso,24,60,5,0.857143,0.029161,0.857143,0.023809,0.036207,0.469817,0.073254,0.510473,0.12722,0.090955,0.0841,0.011752,0.892857,0.369897,1.537528,0.018265,7,6
|
||||
Seeds,constriction,24,60,5,0.890476,0.091597,0.880952,0.095238,0.113731,0.35142,0.3882,0.252366,0.243302,0.482006,0.056642,0.052203,0.960714,0.100819,1.36371,0.053261,3,3
|
||||
Seeds,fips,24,60,5,0.885714,0.035315,0.880952,0.02381,0.043849,0.383732,0.103951,0.392303,0.094168,0.12907,0.067541,0.016554,0.9,0.336734,1.815592,0.045394,4,4
|
||||
Seeds,inertia,24,60,5,0.914286,0.052164,0.928571,0.023809,0.064769,0.331656,0.342636,0.202104,0.068956,0.425432,0.048999,0.036322,0.963095,0.103132,1.318196,0.015319,1,2
|
||||
Seeds,original,24,60,5,0.87619,0.042592,0.880952,0.071429,0.052884,0.49224,0.433653,0.324954,0.104917,0.538443,0.065762,0.028096,0.916667,0.246239,1.385803,0.082957,5,7
|
||||
XOR,adaptive_moment,24,80,5,1.0,0.0,1.0,0.0,0.0,0.190431,0.178122,0.197448,0.188515,0.221164,0.04504,0.056878,1.0,0.190431,1.785274,0.092073,5,5
|
||||
XOR,bare_bones,24,80,5,1.0,0.0,1.0,0.0,0.0,0.0821,0.056963,0.081613,0.055056,0.070728,0.010907,0.010338,1.0,0.0821,2.031221,0.13707,4,4
|
||||
XOR,clpso,24,80,5,0.6,0.136931,0.5,0.25,0.170019,0.548166,0.031946,0.55628,0.024237,0.039666,0.183052,0.012602,0.6,0.548166,2.295572,0.260868,7,7
|
||||
XOR,constriction,24,80,5,1.0,0.0,1.0,0.0,0.0,0.004086,0.002357,0.005281,0.001585,0.002927,3.2e-05,1.9e-05,1.0,0.004086,1.987776,0.182034,1,1
|
||||
XOR,fips,24,80,5,0.75,0.25,0.75,0.5,0.310411,0.546355,0.059307,0.568609,0.092147,0.073638,0.178887,0.029466,0.75,0.546355,2.502509,0.055447,6,6
|
||||
XOR,inertia,24,80,5,1.0,0.0,1.0,0.0,0.0,0.004431,0.003308,0.003801,0.001559,0.004108,3.6e-05,3.1e-05,1.0,0.004431,1.969097,0.170892,2,2
|
||||
XOR,original,24,80,5,1.0,0.0,1.0,0.0,0.0,0.004558,0.002672,0.004584,0.001454,0.003318,4.2e-05,2.8e-05,1.0,0.004558,1.905231,0.246635,3,3
|
||||
|
@@ -0,0 +1,41 @@
|
||||
method,candidate_label,regimen,n_particles,epochs,particle_epochs,seed,train_loss,train_acc,test_loss,test_acc,test_mse,fit_time_sec,data_fingerprint,model_fingerprint,device
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,30,80,2400,71,1.0238605737686157,0.6834999918937683,1.2778329849243164,0.5789999961853027,0.05719960853457451,2.591740084113553,dfe645918ece54c0,0777bd52fd76272d,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,30,80,2400,72,1.0670214891433716,0.6604999899864197,1.199388861656189,0.6060000061988831,0.052410222589969635,2.7087553329765797,dfe645918ece54c0,6fcb6e473bdacbd2,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,30,80,2400,73,0.9569582939147949,0.7014999985694885,1.1562552452087402,0.6389999985694885,0.051246583461761475,2.6669440830592066,dfe645918ece54c0,2e6c351372592f10,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,30,80,2400,74,0.8988860845565796,0.718999981880188,1.0667707920074463,0.6589999794960022,0.047151338309049606,2.6316912909969687,dfe645918ece54c0,4000fe3fb26ef207,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,30,80,2400,75,0.9674410820007324,0.7139999866485596,1.2670954465866089,0.6230000257492065,0.052300941199064255,2.8266918330918998,dfe645918ece54c0,0966039f5ef7af88,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,60,80,4800,71,0.8341852426528931,0.7419999837875366,1.021918535232544,0.6729999780654907,0.0449918694794178,6.297774499980733,dfe645918ece54c0,0777bd52fd76272d,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,60,80,4800,72,0.8334679007530212,0.7580000162124634,1.1486669778823853,0.6499999761581421,0.048473432660102844,5.944013542030007,dfe645918ece54c0,6fcb6e473bdacbd2,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,60,80,4800,73,0.7845665216445923,0.7599999904632568,1.059795618057251,0.6589999794960022,0.04623967781662941,5.409191000042483,dfe645918ece54c0,2e6c351372592f10,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,60,80,4800,74,0.8425021767616272,0.7400000095367432,1.0703872442245483,0.6610000133514404,0.046978432685136795,5.092122667003423,dfe645918ece54c0,4000fe3fb26ef207,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,60,80,4800,75,0.7727756500244141,0.753000020980835,1.0723776817321777,0.6539999842643738,0.04744390770792961,5.18646250013262,dfe645918ece54c0,0966039f5ef7af88,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,90,80,7200,71,0.6898615956306458,0.7910000085830688,0.8688502311706543,0.7310000061988831,0.03894031420350075,8.216789624886587,dfe645918ece54c0,0777bd52fd76272d,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,90,80,7200,72,0.7159585952758789,0.781499981880188,0.9239839911460876,0.718999981880188,0.040356870740652084,8.96910895803012,dfe645918ece54c0,6fcb6e473bdacbd2,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,90,80,7200,73,0.738656759262085,0.7760000228881836,0.9498289227485657,0.6959999799728394,0.04225980117917061,9.117825584020466,dfe645918ece54c0,2e6c351372592f10,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,90,80,7200,74,0.7267816066741943,0.7770000100135803,0.954045295715332,0.6769999861717224,0.04218713194131851,8.722332582809031,dfe645918ece54c0,4000fe3fb26ef207,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,90,80,7200,75,0.7104133367538452,0.7889999747276306,0.962693989276886,0.6930000185966492,0.04196783900260925,8.110321166925132,dfe645918ece54c0,0966039f5ef7af88,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,120,80,9600,71,0.6516683101654053,0.7950000166893005,0.8584634065628052,0.7360000014305115,0.037673790007829666,11.962705624988303,dfe645918ece54c0,0777bd52fd76272d,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,120,80,9600,72,0.6982801556587219,0.7914999723434448,0.9025362133979797,0.7239999771118164,0.038956169039011,11.906837583053857,dfe645918ece54c0,6fcb6e473bdacbd2,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,120,80,9600,73,0.7527623772621155,0.7749999761581421,1.001193642616272,0.6919999718666077,0.04309915751218796,10.664651792030782,dfe645918ece54c0,2e6c351372592f10,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,120,80,9600,74,0.6858600974082947,0.784500002861023,0.8601324558258057,0.7350000143051147,0.03845023736357689,10.921957665821537,dfe645918ece54c0,4000fe3fb26ef207,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_epoch,120,80,9600,75,0.6326538324356079,0.8125,0.889915406703949,0.7300000190734863,0.03931796923279762,10.411899874918163,dfe645918ece54c0,0966039f5ef7af88,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,30,80,2400,71,1.0238605737686157,0.6834999918937683,1.2778329849243164,0.5789999961853027,0.05719960853457451,2.591740084113553,dfe645918ece54c0,0777bd52fd76272d,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,30,80,2400,72,1.0670214891433716,0.6604999899864197,1.199388861656189,0.6060000061988831,0.052410222589969635,2.7087553329765797,dfe645918ece54c0,6fcb6e473bdacbd2,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,30,80,2400,73,0.9569582939147949,0.7014999985694885,1.1562552452087402,0.6389999985694885,0.051246583461761475,2.6669440830592066,dfe645918ece54c0,2e6c351372592f10,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,30,80,2400,74,0.8988860845565796,0.718999981880188,1.0667707920074463,0.6589999794960022,0.047151338309049606,2.6316912909969687,dfe645918ece54c0,4000fe3fb26ef207,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,30,80,2400,75,0.9674410820007324,0.7139999866485596,1.2670954465866089,0.6230000257492065,0.052300941199064255,2.8266918330918998,dfe645918ece54c0,0966039f5ef7af88,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,60,40,2400,71,1.3702143430709839,0.5690000057220459,1.5396113395690918,0.49000000953674316,0.06583584100008011,2.6427467500325292,dfe645918ece54c0,0777bd52fd76272d,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,60,40,2400,72,1.3834563493728638,0.5490000247955322,1.564630150794983,0.5270000100135803,0.06432151794433594,2.897082625189796,dfe645918ece54c0,6fcb6e473bdacbd2,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,60,40,2400,73,1.239696979522705,0.6129999756813049,1.4340611696243286,0.5109999775886536,0.06299576163291931,2.8362932079471648,dfe645918ece54c0,2e6c351372592f10,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,60,40,2400,74,1.3051151037216187,0.593500018119812,1.4914799928665161,0.5370000004768372,0.06303515285253525,2.7601085831411183,dfe645918ece54c0,4000fe3fb26ef207,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,60,40,2400,75,1.23734712600708,0.6150000095367432,1.6433888673782349,0.47200000286102295,0.07071221619844437,2.791566374944523,dfe645918ece54c0,0966039f5ef7af88,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,90,27,2430,71,1.454797387123108,0.5099999904632568,1.7139235734939575,0.40400001406669617,0.07408219575881958,3.001450875075534,dfe645918ece54c0,0777bd52fd76272d,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,90,27,2430,72,1.436651349067688,0.5364999771118164,1.525922417640686,0.5180000066757202,0.06329239904880524,2.9281624578870833,dfe645918ece54c0,6fcb6e473bdacbd2,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,90,27,2430,73,1.5700972080230713,0.4805000126361847,1.7064924240112305,0.4180000126361847,0.07409033179283142,3.446030291961506,dfe645918ece54c0,2e6c351372592f10,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,90,27,2430,74,1.4872608184814453,0.5425000190734863,1.6065630912780762,0.5059999823570251,0.06642712652683258,3.13685941696167,dfe645918ece54c0,4000fe3fb26ef207,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,90,27,2430,75,1.4402897357940674,0.5699999928474426,1.5009552240371704,0.5450000166893005,0.06177634745836258,3.0350140419322997,dfe645918ece54c0,0966039f5ef7af88,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,120,20,2400,71,1.5967158079147339,0.4830000102519989,1.7100492715835571,0.43799999356269836,0.07075551897287369,3.14462749985978,dfe645918ece54c0,0777bd52fd76272d,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,120,20,2400,72,1.673862338066101,0.46149998903274536,1.8141505718231201,0.4099999964237213,0.07267315685749054,3.102293625008315,dfe645918ece54c0,6fcb6e473bdacbd2,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,120,20,2400,73,1.7135908603668213,0.4449999928474426,1.8838708400726318,0.375,0.0778227150440216,2.7550693340599537,dfe645918ece54c0,2e6c351372592f10,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,120,20,2400,74,1.5981744527816772,0.49950000643730164,1.7434691190719604,0.42399999499320984,0.07240629941225052,2.9445771670434624,dfe645918ece54c0,4000fe3fb26ef207,mps
|
||||
adaptive_moment,am_b0.06_s0.5,fixed_budget,120,20,2400,75,1.635738730430603,0.46950000524520874,1.8486356735229492,0.4059999883174896,0.07580890506505966,3.1960245410446078,dfe645918ece54c0,0966039f5ef7af88,mps
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,26 @@
|
||||
method,candidate_label,seed,n_particles,epochs,particle_epochs,train_loss,train_acc,test_loss,test_acc,test_mse,fit_time_sec,data_fingerprint,model_fingerprint,device
|
||||
adaptive_moment,am_b0.06_s0.5,61,30,80,2400,1.0297304391860962,0.6644999980926514,1.2476433515548706,0.5960000157356262,0.054337989538908005,3.3018275420181453,dfe645918ece54c0,2a1760dc9ec95c0a,mps
|
||||
adaptive_moment,am_b0.06_s0.5,62,30,80,2400,1.0248669385910034,0.6800000071525574,1.2172735929489136,0.6150000095367432,0.05301284044981003,3.082969541894272,dfe645918ece54c0,8722b0d78b78d382,mps
|
||||
adaptive_moment,am_b0.06_s0.5,63,30,80,2400,1.0421160459518433,0.6690000295639038,1.2344101667404175,0.6190000176429749,0.052518151700496674,2.7472599998582155,dfe645918ece54c0,f9f62f1942f9151e,mps
|
||||
adaptive_moment,am_b0.06_s0.5,64,30,80,2400,0.9773346185684204,0.7059999704360962,1.1978241205215454,0.6150000095367432,0.052042748779058456,2.670909541891888,dfe645918ece54c0,4cd40d38555ee5ce,mps
|
||||
adaptive_moment,am_b0.06_s0.5,65,30,80,2400,1.0047204494476318,0.6959999799728394,1.2509828805923462,0.6079999804496765,0.05341806635260582,2.9130489169619977,dfe645918ece54c0,70f99d26e1303b76,mps
|
||||
inertia,inertia_asymmetric,61,30,80,2400,1.0274338722229004,0.6735000014305115,1.2075469493865967,0.6320000290870667,0.051997773349285126,2.564156916923821,dfe645918ece54c0,2a1760dc9ec95c0a,mps
|
||||
inertia,inertia_asymmetric,62,30,80,2400,0.9986715912818909,0.6855000257492065,1.2740414142608643,0.578000009059906,0.056466199457645416,2.5913295838981867,dfe645918ece54c0,8722b0d78b78d382,mps
|
||||
inertia,inertia_asymmetric,63,30,80,2400,0.8828525543212891,0.7315000295639038,1.0797629356384277,0.6620000004768372,0.04682194069027901,2.6188287080731243,dfe645918ece54c0,f9f62f1942f9151e,mps
|
||||
inertia,inertia_asymmetric,64,30,80,2400,0.9942940473556519,0.6865000128746033,1.2426005601882935,0.621999979019165,0.05173056200146675,2.534188667079434,dfe645918ece54c0,4cd40d38555ee5ce,mps
|
||||
inertia,inertia_asymmetric,65,30,80,2400,1.0921698808670044,0.6729999780654907,1.2609055042266846,0.6209999918937683,0.052379049360752106,2.542070833966136,dfe645918ece54c0,70f99d26e1303b76,mps
|
||||
constriction,constriction_c205_canonical,61,30,80,2400,1.0248574018478394,0.6919999718666077,1.2035431861877441,0.6299999952316284,0.05152203515172005,2.5165979999583215,dfe645918ece54c0,2a1760dc9ec95c0a,mps
|
||||
constriction,constriction_c205_canonical,62,30,80,2400,1.1310851573944092,0.6700000166893005,1.3196481466293335,0.5849999785423279,0.055950723588466644,2.481917541939765,dfe645918ece54c0,8722b0d78b78d382,mps
|
||||
constriction,constriction_c205_canonical,63,30,80,2400,1.0529502630233765,0.6644999980926514,1.2498247623443604,0.5789999961853027,0.05548400804400444,2.593008500058204,dfe645918ece54c0,f9f62f1942f9151e,mps
|
||||
constriction,constriction_c205_canonical,64,30,80,2400,0.9444444179534912,0.7145000100135803,1.2384077310562134,0.6179999709129333,0.0526747927069664,2.4862103750929236,dfe645918ece54c0,4cd40d38555ee5ce,mps
|
||||
constriction,constriction_c205_canonical,65,30,80,2400,0.9420643448829651,0.7105000019073486,1.219950795173645,0.6209999918937683,0.051569391041994095,2.5256920421961695,dfe645918ece54c0,70f99d26e1303b76,mps
|
||||
local_best,local_best_r4_constant,61,30,80,2400,1.0233083963394165,0.703499972820282,1.1751943826675415,0.6549999713897705,0.050412945449352264,2.535716458922252,dfe645918ece54c0,2a1760dc9ec95c0a,mps
|
||||
local_best,local_best_r4_constant,62,30,80,2400,1.0379226207733154,0.6970000267028809,1.2656233310699463,0.593999981880188,0.056342579424381256,2.5843862500041723,dfe645918ece54c0,8722b0d78b78d382,mps
|
||||
local_best,local_best_r4_constant,63,30,80,2400,0.9664440751075745,0.7275000214576721,1.167758584022522,0.6570000052452087,0.049540925770998,2.485073500080034,dfe645918ece54c0,f9f62f1942f9151e,mps
|
||||
local_best,local_best_r4_constant,64,30,80,2400,1.0115971565246582,0.703000009059906,1.1620168685913086,0.625,0.052118122577667236,2.467889874940738,dfe645918ece54c0,4cd40d38555ee5ce,mps
|
||||
local_best,local_best_r4_constant,65,30,80,2400,1.0574767589569092,0.6959999799728394,1.286249041557312,0.6010000109672546,0.05573433265089989,2.488020292017609,dfe645918ece54c0,70f99d26e1303b76,mps
|
||||
quantum,quantum_beta_0.4_0.9,61,30,80,2400,1.4583405256271362,0.5460000038146973,1.464557409286499,0.5320000052452087,0.06282258033752441,2.3820620418991894,dfe645918ece54c0,2a1760dc9ec95c0a,mps
|
||||
quantum,quantum_beta_0.4_0.9,62,30,80,2400,1.5153175592422485,0.5084999799728394,1.5834708213806152,0.4650000035762787,0.0668596550822258,2.4394417498260736,dfe645918ece54c0,8722b0d78b78d382,mps
|
||||
quantum,quantum_beta_0.4_0.9,63,30,80,2400,1.4202407598495483,0.5490000247955322,1.5065674781799316,0.4860000014305115,0.06405868381261826,2.3948492500931025,dfe645918ece54c0,f9f62f1942f9151e,mps
|
||||
quantum,quantum_beta_0.4_0.9,64,30,80,2400,1.3938108682632446,0.5199999809265137,1.4775264263153076,0.46799999475479126,0.06637361645698547,2.3948124169837683,dfe645918ece54c0,4cd40d38555ee5ce,mps
|
||||
quantum,quantum_beta_0.4_0.9,65,30,80,2400,1.3520328998565674,0.5724999904632568,1.5641074180603027,0.4779999852180481,0.06454122811555862,2.402772541856393,dfe645918ece54c0,70f99d26e1303b76,mps
|
||||
|
@@ -0,0 +1,97 @@
|
||||
method,candidate_label,seed,n_particles,epochs,particle_epochs,train_loss,train_acc,val_loss,val_acc,val_mse,fit_time_sec,data_fingerprint,model_fingerprint,device
|
||||
adaptive_moment,am_b0.03_s0.5,51,30,80,2400,1.112807035446167,0.6570000052452087,1.0964455604553223,0.6633333563804626,0.046677425503730774,2.5230953749269247,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
adaptive_moment,am_b0.03_s0.5,52,30,80,2400,0.995387077331543,0.6869999766349792,1.0115970373153687,0.6816666722297668,0.04280940815806389,2.5447300830855966,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
adaptive_moment,am_b0.03_s0.5,53,30,80,2400,0.9480092525482178,0.7080000042915344,0.9940392971038818,0.6883333325386047,0.0421285405755043,2.493663167115301,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
adaptive_moment,am_b0.03_s1.0,51,30,80,2400,0.9984938502311707,0.7024999856948853,1.0240328311920166,0.6850000023841858,0.04404553398489952,2.54264641716145,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
adaptive_moment,am_b0.03_s1.0,52,30,80,2400,1.0163198709487915,0.6890000104904175,0.9720749855041504,0.6933333277702332,0.042471032589673996,2.491340707987547,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
adaptive_moment,am_b0.03_s1.0,53,30,80,2400,1.0211265087127686,0.6884999871253967,1.081996202468872,0.6850000023841858,0.04483034089207649,2.633713499875739,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
adaptive_moment,am_b0.03_s1.5,51,30,80,2400,1.0612047910690308,0.6735000014305115,1.030881404876709,0.6850000023841858,0.04405777156352997,2.8367540831677616,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
adaptive_moment,am_b0.03_s1.5,52,30,80,2400,0.9462777972221375,0.7149999737739563,0.8990310430526733,0.7266666889190674,0.03923330828547478,2.5477614579722285,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
adaptive_moment,am_b0.03_s1.5,53,30,80,2400,0.9608960747718811,0.7120000123977661,1.0034033060073853,0.6833333373069763,0.042279649525880814,2.5643632498104125,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
adaptive_moment,am_b0.06_s0.5,51,30,80,2400,1.068866491317749,0.6855000257492065,1.0220483541488647,0.675000011920929,0.04446922242641449,2.6285997920203954,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
adaptive_moment,am_b0.06_s0.5,52,30,80,2400,0.9650915265083313,0.7080000042915344,0.9152445197105408,0.7083333134651184,0.04066821560263634,2.7918378338217735,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
adaptive_moment,am_b0.06_s0.5,53,30,80,2400,0.9414446949958801,0.7014999985694885,0.9156243801116943,0.721666693687439,0.03982962667942047,2.729898874880746,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
adaptive_moment,am_b0.06_s1.0,51,30,80,2400,1.0662047863006592,0.6754999756813049,0.9721739888191223,0.6899999976158142,0.042344048619270325,2.5812953328713775,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
adaptive_moment,am_b0.06_s1.0,52,30,80,2400,0.9808334112167358,0.6924999952316284,0.9488086104393005,0.6933333277702332,0.04174558073282242,2.8018474159762263,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
adaptive_moment,am_b0.06_s1.0,53,30,80,2400,0.9464473724365234,0.7085000276565552,0.9320653080940247,0.7083333134651184,0.040938157588243484,2.7171109160408378,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
adaptive_moment,am_b0.06_s1.5,51,30,80,2400,1.1136488914489746,0.6549999713897705,1.1235008239746094,0.6650000214576721,0.04781530797481537,2.7500752080231905,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
adaptive_moment,am_b0.06_s1.5,52,30,80,2400,1.0324392318725586,0.6825000047683716,1.0213521718978882,0.6833333373069763,0.04513593390583992,3.103288209065795,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
adaptive_moment,am_b0.06_s1.5,53,30,80,2400,1.0860146284103394,0.6589999794960022,1.0920277833938599,0.6499999761581421,0.04753243178129196,2.862214791122824,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
adaptive_moment,am_b0.1_s0.5,51,30,80,2400,1.0369712114334106,0.6644999980926514,1.0877516269683838,0.6766666769981384,0.04590357467532158,2.8528131251223385,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
adaptive_moment,am_b0.1_s0.5,52,30,80,2400,0.9936539530754089,0.6965000033378601,0.9451570510864258,0.6983333230018616,0.040929101407527924,2.7851123749278486,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
adaptive_moment,am_b0.1_s0.5,53,30,80,2400,0.9365023374557495,0.7014999985694885,0.8658249974250793,0.7283333539962769,0.038747914135456085,2.7191387079656124,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
adaptive_moment,am_b0.1_s1.0,51,30,80,2400,1.0722532272338867,0.6995000243186951,0.9926102757453918,0.7016666531562805,0.04247405380010605,2.8380547501146793,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
adaptive_moment,am_b0.1_s1.0,52,30,80,2400,0.9977437853813171,0.6930000185966492,1.009635329246521,0.6966666579246521,0.04288069158792496,2.905045999912545,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
adaptive_moment,am_b0.1_s1.0,53,30,80,2400,1.0425642728805542,0.6664999723434448,1.0561128854751587,0.6783333420753479,0.0449373684823513,2.7804793331306428,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
adaptive_moment,am_b0.1_s1.5,51,30,80,2400,1.0099478960037231,0.6775000095367432,1.00997793674469,0.6899999976158142,0.042956266552209854,2.8892802079208195,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
adaptive_moment,am_b0.1_s1.5,52,30,80,2400,1.0597009658813477,0.6744999885559082,1.063821792602539,0.6966666579246521,0.044883932918310165,2.669369083130732,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
adaptive_moment,am_b0.1_s1.5,53,30,80,2400,0.986152708530426,0.7080000042915344,1.0491483211517334,0.6850000023841858,0.04293783754110336,2.626303083030507,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
adaptive_moment,am_b0.15_s0.5,51,30,80,2400,1.1084171533584595,0.652999997138977,1.1322306394577026,0.6600000262260437,0.04928039386868477,2.5766194579191506,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
adaptive_moment,am_b0.15_s0.5,52,30,80,2400,0.970059335231781,0.7129999995231628,0.9312149882316589,0.7300000190734863,0.03986109420657158,2.731992583023384,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
adaptive_moment,am_b0.15_s0.5,53,30,80,2400,0.9970248341560364,0.6995000243186951,0.9656715989112854,0.70333331823349,0.04226723685860634,3.0274115409702063,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
adaptive_moment,am_b0.15_s1.0,51,30,80,2400,1.0099536180496216,0.7009999752044678,0.9564719796180725,0.7166666388511658,0.04047820717096329,2.9793542500119656,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
adaptive_moment,am_b0.15_s1.0,52,30,80,2400,1.0434995889663696,0.6819999814033508,0.9963273406028748,0.6949999928474426,0.04349374398589134,2.8151185419410467,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
adaptive_moment,am_b0.15_s1.0,53,30,80,2400,1.0348174571990967,0.6840000152587891,1.0127019882202148,0.6866666674613953,0.04363732784986496,3.0412363330833614,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
adaptive_moment,am_b0.15_s1.5,51,30,80,2400,1.1573493480682373,0.6299999952316284,1.2029211521148682,0.6133333444595337,0.05152761936187744,3.1714885828550905,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
adaptive_moment,am_b0.15_s1.5,52,30,80,2400,1.060623288154602,0.6949999928474426,1.0831577777862549,0.6966666579246521,0.04344262182712555,2.993674041936174,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
adaptive_moment,am_b0.15_s1.5,53,30,80,2400,1.0753278732299805,0.6769999861717224,0.9602290987968445,0.6983333230018616,0.0414869599044323,2.7658886671997607,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
adaptive_moment,am_b0.06_s1.0_beta0.8,51,30,80,2400,1.1594665050506592,0.6480000019073486,1.1239941120147705,0.6700000166893005,0.047122661024332047,2.764871333958581,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
adaptive_moment,am_b0.06_s1.0_beta0.8,52,30,80,2400,0.9945915341377258,0.7064999938011169,1.0324488878250122,0.6683333516120911,0.04436164349317551,2.830462665995583,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
adaptive_moment,am_b0.06_s1.0_beta0.8,53,30,80,2400,0.9297089576721191,0.7110000252723694,0.938503623008728,0.70333331823349,0.04101715609431267,2.8122877080459148,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
adaptive_moment,am_b0.06_s1.0_beta0.95,51,30,80,2400,1.1143780946731567,0.6514999866485596,1.0093973875045776,0.6800000071525574,0.04380199685692787,2.9217382080387324,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
adaptive_moment,am_b0.06_s1.0_beta0.95,52,30,80,2400,1.0144661664962769,0.6884999871253967,0.9828318357467651,0.6833333373069763,0.04375388100743294,2.777603624854237,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
adaptive_moment,am_b0.06_s1.0_beta0.95,53,30,80,2400,1.1004270315170288,0.6600000262260437,1.0834088325500488,0.6683333516120911,0.04595582187175751,3.0502532080281526,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
inertia,inertia_canonical,51,30,80,2400,1.4416260719299316,0.5649999976158142,1.4125466346740723,0.6000000238418579,0.05729174241423607,3.098805333022028,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
inertia,inertia_canonical,52,30,80,2400,1.665074348449707,0.4894999861717224,1.6294100284576416,0.5066666603088379,0.06497763097286224,2.652766958111897,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
inertia,inertia_canonical,53,30,80,2400,1.4698249101638794,0.5565000176429749,1.4316622018814087,0.5683333277702332,0.059196680784225464,2.529014667030424,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
inertia,inertia_tuned,51,30,80,2400,1.0468239784240723,0.699999988079071,1.0345858335494995,0.6816666722297668,0.04465937986969948,2.626064541982487,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
inertia,inertia_tuned,52,30,80,2400,0.9420707821846008,0.703000009059906,0.9224241375923157,0.7250000238418579,0.040230974555015564,2.5009557919111103,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
inertia,inertia_tuned,53,30,80,2400,1.0711344480514526,0.6754999756813049,1.0678696632385254,0.6766666769981384,0.04436859115958214,2.4420957090333104,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
inertia,inertia_low_w,51,30,80,2400,1.113174319267273,0.6365000009536743,1.0575727224349976,0.6850000023841858,0.0461583249270916,2.484879707917571,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
inertia,inertia_low_w,52,30,80,2400,1.0245047807693481,0.6790000200271606,0.9806669354438782,0.70333331823349,0.042586930096149445,2.643615792039782,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
inertia,inertia_low_w,53,30,80,2400,1.0131726264953613,0.6859999895095825,0.9656168818473816,0.6983333230018616,0.04216707497835159,2.4834530411753803,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
inertia,inertia_w_decay,51,30,80,2400,1.0393675565719604,0.6915000081062317,1.025821566581726,0.6983333230018616,0.04372088611125946,2.426888459129259,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
inertia,inertia_w_decay,52,30,80,2400,1.1047112941741943,0.6600000262260437,1.1661359071731567,0.6700000166893005,0.04753515496850014,2.480164624983445,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
inertia,inertia_w_decay,53,30,80,2400,0.9673198461532593,0.6909999847412109,0.9870904684066772,0.6816666722297668,0.04321449622511864,2.422041916055605,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
inertia,inertia_asymmetric,51,30,80,2400,1.0810083150863647,0.6775000095367432,1.0954482555389404,0.6949999928474426,0.04539918154478073,2.485565959010273,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
inertia,inertia_asymmetric,52,30,80,2400,0.9861184358596802,0.6909999847412109,1.0121054649353027,0.6933333277702332,0.043754804879426956,2.5209125420078635,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
inertia,inertia_asymmetric,53,30,80,2400,0.9526417851448059,0.722000002861023,0.9798429608345032,0.70333331823349,0.04217696189880371,2.5959840829018503,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
constriction,constriction_c201,51,30,80,2400,1.0345860719680786,0.6869999766349792,0.9738073945045471,0.6916666626930237,0.042279358953237534,2.698073250008747,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
constriction,constriction_c201,52,30,80,2400,1.0233638286590576,0.6865000128746033,1.0523117780685425,0.6883333325386047,0.043930795043706894,3.4117308750282973,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
constriction,constriction_c201,53,30,80,2400,1.0302053689956665,0.6880000233650208,0.9275049567222595,0.7166666388511658,0.040264155715703964,3.1805959579069167,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
constriction,constriction_c205_canonical,51,30,80,2400,1.0515944957733154,0.6759999990463257,1.0619410276412964,0.6933333277702332,0.044216156005859375,3.237764042103663,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
constriction,constriction_c205_canonical,52,30,80,2400,1.061699390411377,0.6744999885559082,1.016150712966919,0.6866666674613953,0.0442710816860199,2.7161605830769986,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
constriction,constriction_c205_canonical,53,30,80,2400,0.9327093362808228,0.7289999723434448,0.8751127123832703,0.746666669845581,0.03667077049612999,2.968678707955405,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
constriction,constriction_c205_tuned,51,30,80,2400,1.053567886352539,0.6825000047683716,0.991266667842865,0.699999988079071,0.04196731001138687,3.042221749899909,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
constriction,constriction_c205_tuned,52,30,80,2400,0.9388713836669922,0.7139999866485596,0.9303705096244812,0.7116666436195374,0.0403158962726593,2.8448961251415312,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
constriction,constriction_c205_tuned,53,30,80,2400,0.9965179562568665,0.6995000243186951,0.9846652150154114,0.7133333086967468,0.04187808558344841,3.1848887908272445,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
constriction,constriction_c250,51,30,80,2400,2.2795634269714355,0.18150000274181366,2.250452756881714,0.18666666746139526,0.08861013501882553,2.9961123750545084,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
constriction,constriction_c250,52,30,80,2400,2.2547407150268555,0.20350000262260437,2.2187206745147705,0.2150000035762787,0.08802558481693268,3.024313250090927,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
constriction,constriction_c250,53,30,80,2400,2.19023060798645,0.19300000369548798,2.178219795227051,0.20499999821186066,0.08775272965431213,2.961929291021079,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
constriction,constriction_asymmetric,51,30,80,2400,1.2645200490951538,0.6150000095367432,1.2170491218566895,0.6316666603088379,0.05265685170888901,3.023697500117123,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
constriction,constriction_asymmetric,52,30,80,2400,1.2607256174087524,0.6014999747276306,1.2684078216552734,0.5883333086967468,0.05453150346875191,2.8057189998216927,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
constriction,constriction_asymmetric,53,30,80,2400,1.2320771217346191,0.6140000224113464,1.2376829385757446,0.6200000047683716,0.05186690762639046,2.709051915910095,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
local_best,local_best_r1_constant,51,30,80,2400,1.4435429573059082,0.5354999899864197,1.4319947957992554,0.528333306312561,0.062414027750492096,2.6544057081919163,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
local_best,local_best_r1_constant,52,30,80,2400,1.4914970397949219,0.527999997138977,1.4473850727081299,0.5333333611488342,0.06336859613656998,2.7407990000210702,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
local_best,local_best_r1_constant,53,30,80,2400,1.4734760522842407,0.5680000185966492,1.5045198202133179,0.5366666913032532,0.06426934152841568,2.693694499786943,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
local_best,local_best_r2_constant,51,30,80,2400,1.2284750938415527,0.6345000267028809,1.191932201385498,0.628333330154419,0.0514327734708786,2.4563205409795046,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
local_best,local_best_r2_constant,52,30,80,2400,1.1704559326171875,0.6504999995231628,1.1325551271438599,0.6700000166893005,0.049076274037361145,2.5833707919809967,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
local_best,local_best_r2_constant,53,30,80,2400,1.243313193321228,0.6320000290870667,1.2125868797302246,0.6516666412353516,0.05214923247694969,2.475700625218451,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
local_best,local_best_r4_constant,51,30,80,2400,1.0172255039215088,0.6884999871253967,1.0000126361846924,0.699999988079071,0.0425647497177124,2.4682277080137283,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
local_best,local_best_r4_constant,52,30,80,2400,0.9992414712905884,0.7089999914169312,0.9295963644981384,0.7283333539962769,0.04094172641634941,2.4259307920001447,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
local_best,local_best_r4_constant,53,30,80,2400,0.9578440189361572,0.7139999866485596,0.9051287770271301,0.7450000047683716,0.03751235082745552,2.4171505419071764,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
local_best,local_best_r1_decay,51,30,80,2400,1.4906078577041626,0.5640000104904175,1.4867504835128784,0.5400000214576721,0.06405453383922577,2.455931582953781,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
local_best,local_best_r1_decay,52,30,80,2400,1.4803287982940674,0.5559999942779541,1.4699759483337402,0.5733333230018616,0.0630197748541832,2.5800107079558074,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
local_best,local_best_r1_decay,53,30,80,2400,1.4953768253326416,0.5370000004768372,1.4344968795776367,0.5416666865348816,0.06267009675502777,2.5192452499177307,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
quantum,quantum_beta_0.5_1.0,51,30,80,2400,1.4666624069213867,0.5055000185966492,1.4257338047027588,0.5183333158493042,0.06076965853571892,2.411743083037436,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
quantum,quantum_beta_0.5_1.0,52,30,80,2400,1.4086458683013916,0.5370000004768372,1.3660792112350464,0.5383333563804626,0.05962810665369034,2.398069208022207,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
quantum,quantum_beta_0.5_1.0,53,30,80,2400,1.3147938251495361,0.5580000281333923,1.295978307723999,0.574999988079071,0.05797567218542099,2.4090406668838114,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
quantum,quantum_beta_0.6_1.0,51,30,80,2400,1.7944316864013672,0.3955000042915344,1.7316218614578247,0.4050000011920929,0.07277336716651917,2.4751413341145962,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
quantum,quantum_beta_0.6_1.0,52,30,80,2400,1.770385980606079,0.3785000145435333,1.7482523918151855,0.3866666555404663,0.07508829981088638,2.41624170797877,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
quantum,quantum_beta_0.6_1.0,53,30,80,2400,1.711459994316101,0.4320000112056732,1.684290885925293,0.4399999976158142,0.07037489116191864,2.4244089999701828,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
quantum,quantum_beta_0.5_1.2,51,30,80,2400,1.6640392541885376,0.4580000042915344,1.6313401460647583,0.43833333253860474,0.0688972994685173,2.379183917073533,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
quantum,quantum_beta_0.5_1.2,52,30,80,2400,1.5431870222091675,0.49149999022483826,1.515285849571228,0.5133333206176758,0.06496633589267731,2.3730562501586974,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
quantum,quantum_beta_0.5_1.2,53,30,80,2400,1.7410157918930054,0.4059999883174896,1.7098183631896973,0.3916666805744171,0.07386184483766556,2.438973000040278,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
quantum,quantum_beta_0.4_0.9,51,30,80,2400,1.4330989122390747,0.512499988079071,1.369968295097351,0.5433333516120911,0.06111948564648628,2.5881015001796186,bfc8de485755d7f3,82b3214d02e2e634,mps
|
||||
quantum,quantum_beta_0.4_0.9,52,30,80,2400,1.4685417413711548,0.5249999761581421,1.3982360363006592,0.5716666579246521,0.06072157621383667,2.5672016669996083,bfc8de485755d7f3,ddffbe335bef15ad,mps
|
||||
quantum,quantum_beta_0.4_0.9,53,30,80,2400,1.3139362335205078,0.5789999961853027,1.244922161102295,0.5799999833106995,0.056043192744255066,2.4583064999897033,bfc8de485755d7f3,00868f788440b8a0,mps
|
||||
|
@@ -0,0 +1,10 @@
|
||||
section,metric,value
|
||||
protocol,version,MNIST-PSO-RAW-V5 1.0.0
|
||||
pilot,selected_dimension,full
|
||||
single_model,test_accuracy,83.23
|
||||
single_model,test_nll,0.556332
|
||||
single_model,test_brier,0.253921
|
||||
single_model,test_ece,0.085242
|
||||
ensemble,test_accuracy,87.0
|
||||
ensemble,test_nll,0.535928
|
||||
ensemble,pairwise_disagreement,0.1585
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,9 @@
|
||||
workload,split,seeds,accuracy_mean,accuracy_sd,nll_mean,nll_sd,accuracy_gain_vs_baseline_pp,relative_nll_reduction_vs_baseline_pct,state_ratio,ratio,projection_mode,projection_seed
|
||||
mnist_compact,development_seeds_101_103,101-103,49.053333,0.417652,1.525704,0.01494,-0.1,-0.501617,0.4918032786885246,0.5,fixed,1800044939
|
||||
mnist_compact,confirmation_seeds_111_113,111-113,48.466667,2.515002,1.550212,0.046101,-0.686666,-2.116016,0.4918032786885246,0.5,fixed,1800044939
|
||||
mnist_wide,development_seeds_101_103,101-103,48.346667,3.214052,1.677712,0.072138,4.486667,2.52995,0.5,0.5,explicit,592157828
|
||||
mnist_wide,confirmation_seeds_111_113,111-113,45.723333,1.05633,1.734905,0.034461,1.863333,-0.792792,0.5,0.5,explicit,592157828
|
||||
fashion_compact,development_seeds_101_103,101-103,50.796667,3.095228,1.385324,0.035646,3.793334,8.330571,0.4918032786885246,0.5,fixed,1363313651
|
||||
fashion_compact,confirmation_seeds_111_113,111-113,53.196667,2.025669,1.328168,0.078488,6.193334,12.112688,0.4918032786885246,0.5,fixed,1363313651
|
||||
fashion_wide,development_seeds_101_103,101-103,48.263333,3.71862,1.531421,0.069986,1.953333,-0.371883,0.5,0.5,fixed,189641451
|
||||
fashion_wide,confirmation_seeds_111_113,111-113,49.476667,4.250251,1.513876,0.067096,3.166667,0.778045,0.5,0.5,fixed,189641451
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,164 @@
|
||||
section,workload,method,metric,value
|
||||
protocol,global,all,version,HEAVY-TASK-PSO-V6 1.0.0
|
||||
protocol,global,all,official_test_data_loaded,False
|
||||
protocol,global,all,official_test_evaluations,0
|
||||
baseline,mnist_compact,untrained,val_nll,2.325095
|
||||
baseline,mnist_compact,untrained,val_accuracy,6.65
|
||||
baseline,mnist_wide,untrained,val_nll,2.32458
|
||||
baseline,mnist_wide,untrained,val_accuracy,9.31
|
||||
baseline,fashion_compact,untrained,val_nll,2.320302
|
||||
baseline,fashion_compact,untrained,val_accuracy,8.08
|
||||
baseline,fashion_wide,untrained,val_nll,2.311222
|
||||
baseline,fashion_wide,untrained,val_accuracy,8.77
|
||||
screen,mnist_compact,G0,val_nll,2.019988
|
||||
screen,mnist_compact,G0,val_acc,33.32
|
||||
screen,mnist_compact,G0,gbest_loss,2.014459
|
||||
screen,mnist_compact,G0,gbest_acc,34.4
|
||||
screen,mnist_compact,G0,throughput_sps,1106883.43
|
||||
screen,mnist_compact,G5,val_nll,1.959743
|
||||
screen,mnist_compact,G5,val_acc,33.95
|
||||
screen,mnist_compact,G5,gbest_loss,1.971574
|
||||
screen,mnist_compact,G5,gbest_acc,32.35
|
||||
screen,mnist_compact,G5,throughput_sps,1139465.88
|
||||
screen,mnist_compact,G6,val_nll,1.957307
|
||||
screen,mnist_compact,G6,val_acc,35.11
|
||||
screen,mnist_compact,G6,gbest_loss,1.941859
|
||||
screen,mnist_compact,G6,gbest_acc,35.35
|
||||
screen,mnist_compact,G6,throughput_sps,1340969.41
|
||||
screen,mnist_compact,G8,val_nll,1.690442
|
||||
screen,mnist_compact,G8,val_acc,43.88
|
||||
screen,mnist_compact,G8,gbest_loss,1.675063
|
||||
screen,mnist_compact,G8,gbest_acc,43.6
|
||||
screen,mnist_compact,G8,throughput_sps,984615.38
|
||||
screen,mnist_wide,G0,val_nll,2.088156
|
||||
screen,mnist_wide,G0,val_acc,33.19
|
||||
screen,mnist_wide,G0,gbest_loss,2.087951
|
||||
screen,mnist_wide,G0,gbest_acc,32.45
|
||||
screen,mnist_wide,G0,throughput_sps,595459.62
|
||||
screen,mnist_wide,G5,val_nll,1.913566
|
||||
screen,mnist_wide,G5,val_acc,39.87
|
||||
screen,mnist_wide,G5,gbest_loss,1.924357
|
||||
screen,mnist_wide,G5,gbest_acc,39.5
|
||||
screen,mnist_wide,G5,throughput_sps,772511.47
|
||||
screen,mnist_wide,G6,val_nll,2.017511
|
||||
screen,mnist_wide,G6,val_acc,29.46
|
||||
screen,mnist_wide,G6,gbest_loss,2.014518
|
||||
screen,mnist_wide,G6,gbest_acc,27.8
|
||||
screen,mnist_wide,G6,throughput_sps,777013.35
|
||||
screen,mnist_wide,G8,val_nll,2.056341
|
||||
screen,mnist_wide,G8,val_acc,20.43
|
||||
screen,mnist_wide,G8,gbest_loss,2.062368
|
||||
screen,mnist_wide,G8,gbest_acc,20.3
|
||||
screen,mnist_wide,G8,throughput_sps,855005.34
|
||||
screen,fashion_compact,G0,val_nll,1.969548
|
||||
screen,fashion_compact,G0,val_acc,28.25
|
||||
screen,fashion_compact,G0,gbest_loss,1.96974
|
||||
screen,fashion_compact,G0,gbest_acc,28.15
|
||||
screen,fashion_compact,G0,throughput_sps,1219977.13
|
||||
screen,fashion_compact,G5,val_nll,1.892402
|
||||
screen,fashion_compact,G5,val_acc,34.31
|
||||
screen,fashion_compact,G5,gbest_loss,1.874629
|
||||
screen,fashion_compact,G5,gbest_acc,36.6
|
||||
screen,fashion_compact,G5,throughput_sps,1122281.97
|
||||
screen,fashion_compact,G6,val_nll,1.824273
|
||||
screen,fashion_compact,G6,val_acc,35.2
|
||||
screen,fashion_compact,G6,gbest_loss,1.844914
|
||||
screen,fashion_compact,G6,gbest_acc,33.5
|
||||
screen,fashion_compact,G6,throughput_sps,1152322.65
|
||||
screen,fashion_compact,G8,val_nll,1.88018
|
||||
screen,fashion_compact,G8,val_acc,30.85
|
||||
screen,fashion_compact,G8,gbest_loss,1.895269
|
||||
screen,fashion_compact,G8,gbest_acc,28.85
|
||||
screen,fashion_compact,G8,throughput_sps,1193139.45
|
||||
screen,fashion_wide,G0,val_nll,1.981316
|
||||
screen,fashion_wide,G0,val_acc,36.39
|
||||
screen,fashion_wide,G0,gbest_loss,1.98436
|
||||
screen,fashion_wide,G0,gbest_acc,36.05
|
||||
screen,fashion_wide,G0,throughput_sps,540327.57
|
||||
screen,fashion_wide,G5,val_nll,1.945862
|
||||
screen,fashion_wide,G5,val_acc,31.97
|
||||
screen,fashion_wide,G5,gbest_loss,1.938569
|
||||
screen,fashion_wide,G5,gbest_acc,33.3
|
||||
screen,fashion_wide,G5,throughput_sps,807401.18
|
||||
screen,fashion_wide,G6,val_nll,1.975825
|
||||
screen,fashion_wide,G6,val_acc,28.64
|
||||
screen,fashion_wide,G6,gbest_loss,1.960139
|
||||
screen,fashion_wide,G6,gbest_acc,28.1
|
||||
screen,fashion_wide,G6,throughput_sps,735350.44
|
||||
screen,fashion_wide,G8,val_nll,1.987923
|
||||
screen,fashion_wide,G8,val_acc,28.69
|
||||
screen,fashion_wide,G8,gbest_loss,1.976977
|
||||
screen,fashion_wide,G8,gbest_acc,28.45
|
||||
screen,fashion_wide,G8,throughput_sps,865800.87
|
||||
confirm,mnist_compact,G8,val_acc_mean,49.153333
|
||||
confirm,mnist_compact,G8,val_acc_std,1.320656
|
||||
confirm,mnist_compact,G8,val_nll_mean,1.518089
|
||||
confirm,mnist_compact,G8,val_nll_std,0.061523
|
||||
confirm,mnist_compact,G8,wall_time_sec_mean,4.530167
|
||||
confirm,mnist_compact,G6,val_acc_mean,45.966667
|
||||
confirm,mnist_compact,G6,val_acc_std,3.545438
|
||||
confirm,mnist_compact,G6,val_nll_mean,1.642081
|
||||
confirm,mnist_compact,G6,val_nll_std,0.091398
|
||||
confirm,mnist_compact,G6,wall_time_sec_mean,5.2706
|
||||
confirm,mnist_wide,G8,val_acc_mean,41.03
|
||||
confirm,mnist_wide,G8,val_acc_std,4.055083
|
||||
confirm,mnist_wide,G8,val_nll_mean,1.733351
|
||||
confirm,mnist_wide,G8,val_nll_std,0.084331
|
||||
confirm,mnist_wide,G8,wall_time_sec_mean,9.4569
|
||||
confirm,mnist_wide,G5,val_acc_mean,43.86
|
||||
confirm,mnist_wide,G5,val_acc_std,3.512222
|
||||
confirm,mnist_wide,G5,val_nll_mean,1.721259
|
||||
confirm,mnist_wide,G5,val_nll_std,0.098656
|
||||
confirm,mnist_wide,G5,wall_time_sec_mean,9.868467
|
||||
confirm,fashion_compact,G8,val_acc_mean,47.003333
|
||||
confirm,fashion_compact,G8,val_acc_std,5.2259
|
||||
confirm,fashion_compact,G8,val_nll_mean,1.511217
|
||||
confirm,fashion_compact,G8,val_nll_std,0.168628
|
||||
confirm,fashion_compact,G8,wall_time_sec_mean,5.0146
|
||||
confirm,fashion_compact,G6,val_acc_mean,45.616667
|
||||
confirm,fashion_compact,G6,val_acc_std,3.585392
|
||||
confirm,fashion_compact,G6,val_nll_mean,1.584443
|
||||
confirm,fashion_compact,G6,val_nll_std,0.053077
|
||||
confirm,fashion_compact,G6,wall_time_sec_mean,4.839267
|
||||
confirm,fashion_wide,G8,val_acc_mean,41.666667
|
||||
confirm,fashion_wide,G8,val_acc_std,2.269016
|
||||
confirm,fashion_wide,G8,val_nll_mean,1.615937
|
||||
confirm,fashion_wide,G8,val_nll_std,0.036692
|
||||
confirm,fashion_wide,G8,wall_time_sec_mean,9.499333
|
||||
confirm,fashion_wide,G5,val_acc_mean,46.31
|
||||
confirm,fashion_wide,G5,val_acc_std,7.568494
|
||||
confirm,fashion_wide,G5,val_nll_mean,1.525747
|
||||
confirm,fashion_wide,G5,val_nll_std,0.149706
|
||||
confirm,fashion_wide,G5,wall_time_sec_mean,10.038833
|
||||
feasibility,mnist_compact,G8,execution_feasible,True
|
||||
feasibility,mnist_compact,G8,optimization_feasible,True
|
||||
feasibility,mnist_compact,G8,mean_val_nll,1.518089
|
||||
feasibility,mnist_compact,G8,mean_val_acc,49.1533
|
||||
feasibility,mnist_compact,G6,execution_feasible,True
|
||||
feasibility,mnist_compact,G6,optimization_feasible,True
|
||||
feasibility,mnist_compact,G6,mean_val_nll,1.642081
|
||||
feasibility,mnist_compact,G6,mean_val_acc,45.9667
|
||||
feasibility,mnist_wide,G8,execution_feasible,True
|
||||
feasibility,mnist_wide,G8,optimization_feasible,True
|
||||
feasibility,mnist_wide,G8,mean_val_nll,1.733351
|
||||
feasibility,mnist_wide,G8,mean_val_acc,41.03
|
||||
feasibility,mnist_wide,G5,execution_feasible,True
|
||||
feasibility,mnist_wide,G5,optimization_feasible,True
|
||||
feasibility,mnist_wide,G5,mean_val_nll,1.721259
|
||||
feasibility,mnist_wide,G5,mean_val_acc,43.86
|
||||
feasibility,fashion_compact,G8,execution_feasible,True
|
||||
feasibility,fashion_compact,G8,optimization_feasible,True
|
||||
feasibility,fashion_compact,G8,mean_val_nll,1.511217
|
||||
feasibility,fashion_compact,G8,mean_val_acc,47.0033
|
||||
feasibility,fashion_compact,G6,execution_feasible,True
|
||||
feasibility,fashion_compact,G6,optimization_feasible,True
|
||||
feasibility,fashion_compact,G6,mean_val_nll,1.584443
|
||||
feasibility,fashion_compact,G6,mean_val_acc,45.6167
|
||||
feasibility,fashion_wide,G8,execution_feasible,True
|
||||
feasibility,fashion_wide,G8,optimization_feasible,True
|
||||
feasibility,fashion_wide,G8,mean_val_nll,1.615937
|
||||
feasibility,fashion_wide,G8,mean_val_acc,41.6667
|
||||
feasibility,fashion_wide,G5,execution_feasible,True
|
||||
feasibility,fashion_wide,G5,optimization_feasible,True
|
||||
feasibility,fashion_wide,G5,mean_val_nll,1.525747
|
||||
feasibility,fashion_wide,G5,mean_val_acc,46.31
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,21 @@
|
||||
protocol_version,MNIST-PSO-RAW-V6 1.0.0
|
||||
|
||||
--- Phase B Screen Results ---
|
||||
config_id,description,val_nll,val_acc_%,queries,sample_evals,wall_time_sec
|
||||
G0,exact V5 control,1.183117,67.73,4800,9600000,8.4356
|
||||
G1,isolate anisotropic per-tensor scaling,1.437604,62.32,4800,9600000,6.6136
|
||||
G2,isolate nonzero launch velocity,1.296105,60.72,4800,9600000,6.6528
|
||||
G3,isolate mutation,1.130839,68.07,4800,9600000,6.4943
|
||||
G4,velocity x mutation interaction,1.11628,65.76,4800,9600000,6.4837
|
||||
G5,test sufficient bound expansion,0.90151,70.93,4800,9600000,6.76
|
||||
G6,test broader normalized initialization,0.96129,69.56,4800,9600000,6.7408
|
||||
G7,isolate antithetic position coupling against G2,1.291266,58.79,4800,9600000,6.4273
|
||||
G8,retained semantic control (public Optimizer),0.883557,70.99,4800,9600000,8.7632
|
||||
|
||||
--- Phase B Confirmation Aggregates ---
|
||||
config_id,mean_val_acc_%,std_val_acc,mean_val_nll,std_val_nll,num_seeds
|
||||
G0,79.526667,0.576397,0.689687,0.008079,3
|
||||
G1,76.953333,2.490067,0.833759,0.123667,3
|
||||
G5,84.236667,0.903678,0.510297,0.024899,3
|
||||
G6,84.25,0.219317,0.503809,0.015789,3
|
||||
G8,84.916667,0.988804,0.48144,0.029447,3
|
||||
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,73 @@
|
||||
variant_id,phase,split_seed,workload_id,baseline_method,baseline_acc,candidate_acc,acc_gain_pp,baseline_nll,candidate_nll,nll_reduction_fraction,state_ratio,score,pass,is_best_observed,candidate_path,evaluation_path
|
||||
iteration-0001-development,development,20260905,mnist_compact,G8,50.713333,50.723333,0.00999999999999801,1.534298,1.492805,0.0270436381980554,0.4918032786885246,-289.2921035434933,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0001-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0001-development.json
|
||||
iteration-0001-development,development,20260905,mnist_wide,G5,46.656667,46.04,-0.6166669999999996,1.677872,1.675644,0.0013278724479579603,0.5,-289.2921035434933,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0001-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0001-development.json
|
||||
iteration-0001-development,development,20260905,fashion_compact,G8,46.366667,49.576667,3.210000000000001,1.497844,1.400574,0.06494000710354347,0.4918032786885246,-289.2921035434933,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0001-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0001-development.json
|
||||
iteration-0001-development,development,20260905,fashion_wide,G5,48.516667,45.01,-3.506667,1.490304,1.584923,-0.06348973095422142,0.5,-289.2921035434933,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0001-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0001-development.json
|
||||
iteration-0001-development,development,20260906,mnist_compact,G8,46.81,51.663333,4.853332999999999,1.58986,1.436337,0.09656384838916639,0.4918032786885246,-289.2921035434933,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0001-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0001-development.json
|
||||
iteration-0001-development,development,20260906,mnist_wide,G5,44.936667,44.83,-0.10666700000000162,1.665007,1.749135,-0.050527114901018556,0.5,-289.2921035434933,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0001-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0001-development.json
|
||||
iteration-0001-development,development,20260906,fashion_compact,G8,44.193333,48.753333,4.559999999999995,1.553097,1.375751,0.11418861796784104,0.4918032786885246,-289.2921035434933,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0001-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0001-development.json
|
||||
iteration-0001-development,development,20260906,fashion_wide,G5,48.37,41.193333,-7.176666999999995,1.490657,1.707819,-0.14568207173078723,0.5,-289.2921035434933,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0001-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0001-development.json
|
||||
iteration-0002-development,development,20260905,mnist_compact,G8,50.713333,50.723333,0.00999999999999801,1.534298,1.492805,0.0270436381980554,0.4918032786885246,-186.3459040597986,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0002-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0002-development.json
|
||||
iteration-0002-development,development,20260905,mnist_wide,G5,46.656667,45.193333,-1.4633339999999961,1.677872,1.733563,-0.03319144726176963,0.5,-186.3459040597986,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0002-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0002-development.json
|
||||
iteration-0002-development,development,20260905,fashion_compact,G8,46.366667,49.576667,3.210000000000001,1.497844,1.400574,0.06494000710354347,0.4918032786885246,-186.3459040597986,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0002-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0002-development.json
|
||||
iteration-0002-development,development,20260905,fashion_wide,G5,48.516667,47.223333,-1.2933340000000015,1.490304,1.536216,-0.03080713733573818,0.5,-186.3459040597986,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0002-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0002-development.json
|
||||
iteration-0002-development,development,20260906,mnist_compact,G8,46.81,51.663333,4.853332999999999,1.58986,1.436337,0.09656384838916639,0.4918032786885246,-186.3459040597986,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0002-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0002-development.json
|
||||
iteration-0002-development,development,20260906,mnist_wide,G5,44.936667,45.09,0.1533330000000035,1.665007,1.734207,-0.04156138682900441,0.5,-186.3459040597986,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0002-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0002-development.json
|
||||
iteration-0002-development,development,20260906,fashion_compact,G8,44.193333,48.753333,4.559999999999995,1.553097,1.375751,0.11418861796784104,0.4918032786885246,-186.3459040597986,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0002-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0002-development.json
|
||||
iteration-0002-development,development,20260906,fashion_wide,G5,48.37,46.813333,-1.5566669999999974,1.490657,1.475127,0.010418224984016995,0.5,-186.3459040597986,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0002-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0002-development.json
|
||||
iteration-0003-replica1-development,development,20260905,mnist_compact,G8,50.713333,50.723333,0.00999999999999801,1.534298,1.492805,0.0270436381980554,0.4918032786885246,-287.7250973286179,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica1-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica1-development.json
|
||||
iteration-0003-replica1-development,development,20260905,mnist_wide,G5,46.656667,44.923333,-1.7333339999999993,1.677872,1.728906,-0.030415907768888223,0.5,-287.7250973286179,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica1-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica1-development.json
|
||||
iteration-0003-replica1-development,development,20260905,fashion_compact,G8,46.366667,49.576667,3.210000000000001,1.497844,1.400574,0.06494000710354347,0.4918032786885246,-287.7250973286179,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica1-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica1-development.json
|
||||
iteration-0003-replica1-development,development,20260905,fashion_wide,G5,48.516667,47.176667,-1.3399999999999963,1.490304,1.56267,-0.048557878124194744,0.5,-287.7250973286179,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica1-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica1-development.json
|
||||
iteration-0003-replica1-development,development,20260906,mnist_compact,G8,46.81,51.663333,4.853332999999999,1.58986,1.436337,0.09656384838916639,0.4918032786885246,-287.7250973286179,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica1-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica1-development.json
|
||||
iteration-0003-replica1-development,development,20260906,mnist_wide,G5,44.936667,42.113333,-2.8233340000000027,1.665007,1.713134,-0.028904983582651624,0.5,-287.7250973286179,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica1-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica1-development.json
|
||||
iteration-0003-replica1-development,development,20260906,fashion_compact,G8,44.193333,48.753333,4.559999999999995,1.553097,1.375751,0.11418861796784104,0.4918032786885246,-287.7250973286179,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica1-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica1-development.json
|
||||
iteration-0003-replica1-development,development,20260906,fashion_wide,G5,48.37,46.433333,-1.936667,1.490657,1.581386,-0.06086510847230454,0.5,-287.7250973286179,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica1-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica1-development.json
|
||||
iteration-0003-replica2-development,development,20260905,mnist_compact,G8,50.713333,50.723333,0.00999999999999801,1.534298,1.492805,0.0270436381980554,0.4918032786885246,-286.97652369317115,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica2-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica2-development.json
|
||||
iteration-0003-replica2-development,development,20260905,mnist_wide,G5,46.656667,46.01,-0.6466670000000008,1.677872,1.72888,-0.0304004119503752,0.5,-286.97652369317115,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica2-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica2-development.json
|
||||
iteration-0003-replica2-development,development,20260905,fashion_compact,G8,46.366667,49.576667,3.210000000000001,1.497844,1.400574,0.06494000710354347,0.4918032786885246,-286.97652369317115,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica2-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica2-development.json
|
||||
iteration-0003-replica2-development,development,20260905,fashion_wide,G5,48.516667,48.613333,0.09666599999999903,1.490304,1.48049,0.006585233616765431,0.5,-286.97652369317115,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica2-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica2-development.json
|
||||
iteration-0003-replica2-development,development,20260906,mnist_compact,G8,46.81,51.663333,4.853332999999999,1.58986,1.436337,0.09656384838916639,0.4918032786885246,-286.97652369317115,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica2-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica2-development.json
|
||||
iteration-0003-replica2-development,development,20260906,mnist_wide,G5,44.936667,41.556667,-3.3800000000000026,1.665007,1.801698,-0.08209635154687045,0.5,-286.97652369317115,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica2-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica2-development.json
|
||||
iteration-0003-replica2-development,development,20260906,fashion_compact,G8,44.193333,48.753333,4.559999999999995,1.553097,1.375751,0.11418861796784104,0.4918032786885246,-286.97652369317115,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica2-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica2-development.json
|
||||
iteration-0003-replica2-development,development,20260906,fashion_wide,G5,48.37,45.913333,-2.456666999999996,1.490657,1.516614,-0.017413127231817923,0.5,-286.97652369317115,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0003-replica2-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0003-replica2-development.json
|
||||
iteration-0004-development,development,20260905,mnist_compact,G8,50.713333,50.723333,0.00999999999999801,1.534298,1.492805,0.0270436381980554,0.4918032786885246,-390.14004501856266,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0004-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0004-development.json
|
||||
iteration-0004-development,development,20260905,mnist_wide,G5,46.656667,44.763333,-1.8933339999999959,1.677872,1.698999,-0.012591544527830428,0.5,-390.14004501856266,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0004-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0004-development.json
|
||||
iteration-0004-development,development,20260905,fashion_compact,G8,46.366667,49.576667,3.210000000000001,1.497844,1.400574,0.06494000710354347,0.4918032786885246,-390.14004501856266,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0004-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0004-development.json
|
||||
iteration-0004-development,development,20260905,fashion_wide,G5,48.516667,44.816667,-3.6999999999999957,1.490304,1.582588,-0.06192293652838617,0.5,-390.14004501856266,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0004-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0004-development.json
|
||||
iteration-0004-development,development,20260906,mnist_compact,G8,46.81,51.663333,4.853332999999999,1.58986,1.436337,0.09656384838916639,0.4918032786885246,-390.14004501856266,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0004-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0004-development.json
|
||||
iteration-0004-development,development,20260906,mnist_wide,G5,44.936667,38.946667,-5.990000000000002,1.665007,1.819662,-0.09288549537629572,0.5,-390.14004501856266,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0004-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0004-development.json
|
||||
iteration-0004-development,development,20260906,fashion_compact,G8,44.193333,48.753333,4.559999999999995,1.553097,1.375751,0.11418861796784104,0.4918032786885246,-390.14004501856266,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0004-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0004-development.json
|
||||
iteration-0004-development,development,20260906,fashion_wide,G5,48.37,43.9,-4.469999999999999,1.490657,1.658117,-0.11233972671110803,0.5,-390.14004501856266,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0004-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0004-development.json
|
||||
iteration-0005-development,development,20260905,mnist_compact,G8,50.713333,50.723333,0.00999999999999801,1.534298,1.492805,0.0270436381980554,0.4918032786885246,-185.61068572934795,False,True,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0005-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0005-development.json
|
||||
iteration-0005-development,development,20260905,mnist_wide,G5,46.656667,43.723333,-2.933334000000002,1.677872,1.675155,0.0016193130346057866,0.5,-185.61068572934795,False,True,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0005-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0005-development.json
|
||||
iteration-0005-development,development,20260905,fashion_compact,G8,46.366667,49.576667,3.210000000000001,1.497844,1.400574,0.06494000710354347,0.4918032786885246,-185.61068572934795,False,True,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0005-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0005-development.json
|
||||
iteration-0005-development,development,20260905,fashion_wide,G5,48.516667,45.45,-3.0666669999999954,1.490304,1.441667,0.03263562333591002,0.5,-185.61068572934795,False,True,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0005-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0005-development.json
|
||||
iteration-0005-development,development,20260906,mnist_compact,G8,46.81,51.663333,4.853332999999999,1.58986,1.436337,0.09656384838916639,0.4918032786885246,-185.61068572934795,False,True,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0005-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0005-development.json
|
||||
iteration-0005-development,development,20260906,mnist_wide,G5,44.936667,42.543333,-2.393334000000003,1.665007,1.740544,-0.04536737683385127,0.5,-185.61068572934795,False,True,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0005-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0005-development.json
|
||||
iteration-0005-development,development,20260906,fashion_compact,G8,44.193333,48.753333,4.559999999999995,1.553097,1.375751,0.11418861796784104,0.4918032786885246,-185.61068572934795,False,True,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0005-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0005-development.json
|
||||
iteration-0005-development,development,20260906,fashion_wide,G5,48.37,47.65,-0.7199999999999989,1.490657,1.454402,0.02432149045689245,0.5,-185.61068572934795,False,True,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0005-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0005-development.json
|
||||
iteration-0006-development,development,20260905,mnist_compact,G8,50.713333,50.723333,0.00999999999999801,1.534298,1.492805,0.0270436381980554,0.4918032786885246,-186.8633001166316,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0006-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0006-development.json
|
||||
iteration-0006-development,development,20260905,mnist_wide,G5,46.656667,42.006667,-4.649999999999999,1.677872,1.749895,-0.04292520525999596,0.5,-186.8633001166316,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0006-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0006-development.json
|
||||
iteration-0006-development,development,20260905,fashion_compact,G8,46.366667,49.576667,3.210000000000001,1.497844,1.400574,0.06494000710354347,0.4918032786885246,-186.8633001166316,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0006-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0006-development.json
|
||||
iteration-0006-development,development,20260905,fashion_wide,G5,48.516667,49.08,0.5633330000000001,1.490304,1.466855,0.015734373657991962,0.5,-186.8633001166316,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0006-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0006-development.json
|
||||
iteration-0006-development,development,20260906,mnist_compact,G8,46.81,51.663333,4.853332999999999,1.58986,1.436337,0.09656384838916639,0.4918032786885246,-186.8633001166316,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0006-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0006-development.json
|
||||
iteration-0006-development,development,20260906,mnist_wide,G5,44.936667,40.84,-4.0966669999999965,1.665007,1.73617,-0.04274036085133581,0.5,-186.8633001166316,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0006-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0006-development.json
|
||||
iteration-0006-development,development,20260906,fashion_compact,G8,44.193333,48.753333,4.559999999999995,1.553097,1.375751,0.11418861796784104,0.4918032786885246,-186.8633001166316,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0006-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0006-development.json
|
||||
iteration-0006-development,development,20260906,fashion_wide,G5,48.37,46.946667,-1.4233329999999995,1.490657,1.508747,-0.012135588535793386,0.5,-186.8633001166316,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0006-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0006-development.json
|
||||
iteration-0007-development,development,20260905,mnist_compact,G8,50.713333,50.723333,0.00999999999999801,1.534298,1.492805,0.0270436381980554,0.4918032786885246,-391.33742460463645,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0007-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0007-development.json
|
||||
iteration-0007-development,development,20260905,mnist_wide,G5,46.656667,41.05,-5.606667000000002,1.677872,1.794261,-0.06936703157332626,0.5,-391.33742460463645,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0007-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0007-development.json
|
||||
iteration-0007-development,development,20260905,fashion_compact,G8,46.366667,49.576667,3.210000000000001,1.497844,1.400574,0.06494000710354347,0.4918032786885246,-391.33742460463645,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0007-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0007-development.json
|
||||
iteration-0007-development,development,20260905,fashion_wide,G5,48.516667,42.513333,-6.003333999999995,1.490304,1.615206,-0.08380974619943303,0.5,-391.33742460463645,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0007-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0007-development.json
|
||||
iteration-0007-development,development,20260906,mnist_compact,G8,46.81,51.663333,4.853332999999999,1.58986,1.436337,0.09656384838916639,0.4918032786885246,-391.33742460463645,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0007-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0007-development.json
|
||||
iteration-0007-development,development,20260906,mnist_wide,G5,44.936667,40.666667,-4.270000000000003,1.665007,1.764515,-0.059764313303187405,0.5,-391.33742460463645,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0007-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0007-development.json
|
||||
iteration-0007-development,development,20260906,fashion_compact,G8,44.193333,48.753333,4.559999999999995,1.553097,1.375751,0.11418861796784104,0.4918032786885246,-391.33742460463645,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0007-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0007-development.json
|
||||
iteration-0007-development,development,20260906,fashion_wide,G5,48.37,39.916667,-8.453333,1.490657,1.609595,-0.07978897895357565,0.5,-391.33742460463645,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0007-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0007-development.json
|
||||
iteration-0008-development,development,20260905,mnist_compact,G8,50.713333,50.723333,0.00999999999999801,1.534298,1.492805,0.0270436381980554,0.4918032786885246,-390.14811518493707,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0008-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0008-development.json
|
||||
iteration-0008-development,development,20260905,mnist_wide,G5,46.656667,41.14,-5.516666999999998,1.677872,1.748292,-0.041969828449369154,0.5,-390.14811518493707,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0008-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0008-development.json
|
||||
iteration-0008-development,development,20260905,fashion_compact,G8,46.366667,49.576667,3.210000000000001,1.497844,1.400574,0.06494000710354347,0.4918032786885246,-390.14811518493707,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0008-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0008-development.json
|
||||
iteration-0008-development,development,20260905,fashion_wide,G5,48.516667,45.726667,-2.789999999999999,1.490304,1.558163,-0.04553366293051612,0.5,-390.14811518493707,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0008-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0008-development.json
|
||||
iteration-0008-development,development,20260906,mnist_compact,G8,46.81,51.663333,4.853332999999999,1.58986,1.436337,0.09656384838916639,0.4918032786885246,-390.14811518493707,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0008-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0008-development.json
|
||||
iteration-0008-development,development,20260906,mnist_wide,G5,44.936667,39.596667,-5.340000000000003,1.665007,1.763671,-0.059257408527411654,0.5,-390.14811518493707,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0008-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0008-development.json
|
||||
iteration-0008-development,development,20260906,fashion_compact,G8,44.193333,48.753333,4.559999999999995,1.553097,1.375751,0.11418861796784104,0.4918032786885246,-390.14811518493707,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0008-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0008-development.json
|
||||
iteration-0008-development,development,20260906,fashion_wide,G5,48.37,43.246667,-5.123332999999995,1.490657,1.649349,-0.10645775654627461,0.5,-390.14811518493707,False,False,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/candidates/iteration-0008-development.json,.omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z/evaluations/iteration-0008-development.json
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,29 @@
|
||||
Workload,Phase,Method,Accuracy,NLL,Brier,ECE,Margin,WallTimeSeconds,ParameterMultiplier,InferenceMultiplier
|
||||
mnist,validation,reference_single_10e,98.2300,0.060105,0.027982,0.003458,0.969858,0.0000,1.0,1.0
|
||||
mnist,validation,best_single_10e,98.2900,0.054029,0.025526,0.002795,0.971419,0.0000,1.0,1.0
|
||||
mnist,validation,single_50e,98.5200,0.073988,0.024922,0.009463,0.989411,0.0000,1.0,1.0
|
||||
mnist,validation,uniform_ensemble,98.6300,0.046385,0.021527,0.006228,0.965339,0.0000,5.0,5.0
|
||||
mnist,validation,uniform_temperature,98.6300,0.045355,0.021099,0.002518,0.972273,0.0412,5.0,5.0
|
||||
mnist,validation,slsqp_weights,98.6000,0.045902,0.021307,0.005994,0.965581,0.0097,5.0,5.0
|
||||
mnist,validation,pso_weights,98.6100,0.045902,0.021307,0.005893,0.965582,1.8925,5.0,5.0
|
||||
mnist,official_test,reference_single_10e,98.4700,0.044991,0.022366,0.003516,0.970575,0.0000,1.0,1.0
|
||||
mnist,official_test,best_single_10e,98.5500,0.044944,0.022052,0.002515,0.974650,0.0000,1.0,1.0
|
||||
mnist,official_test,single_50e,98.6000,0.062102,0.022912,0.008283,0.988829,0.0000,1.0,1.0
|
||||
mnist,official_test,uniform_ensemble,98.8600,0.036184,0.018010,0.006311,0.968415,0.0000,5.0,5.0
|
||||
mnist,official_test,uniform_temperature,98.8600,0.034129,0.017658,0.003340,0.974869,0.0000,5.0,5.0
|
||||
mnist,official_test,slsqp_weights,98.8300,0.036179,0.017989,0.005818,0.969195,0.0000,5.0,5.0
|
||||
mnist,official_test,pso_weights,98.8300,0.036178,0.017989,0.005817,0.969196,0.0000,5.0,5.0
|
||||
fashion_mnist,validation,reference_single_10e,89.4700,0.303799,0.150671,0.012197,0.808017,0.0000,1.0,1.0
|
||||
fashion_mnist,validation,best_single_10e,89.4700,0.303799,0.150671,0.012197,0.808017,0.0000,1.0,1.0
|
||||
fashion_mnist,validation,single_50e,90.3200,0.289660,0.142115,0.024556,0.869484,0.0000,1.0,1.0
|
||||
fashion_mnist,validation,uniform_ensemble,90.2800,0.286751,0.142881,0.023702,0.795588,0.0000,5.0,5.0
|
||||
fashion_mnist,validation,uniform_temperature,90.2800,0.285048,0.141557,0.011886,0.814383,0.0414,5.0,5.0
|
||||
fashion_mnist,validation,slsqp_weights,90.4200,0.285338,0.142289,0.023705,0.798413,0.0104,5.0,5.0
|
||||
fashion_mnist,validation,pso_weights,90.4200,0.285338,0.142289,0.023661,0.798397,1.8963,5.0,5.0
|
||||
fashion_mnist,official_test,reference_single_10e,88.9000,0.314516,0.161465,0.005320,0.805307,0.0000,1.0,1.0
|
||||
fashion_mnist,official_test,best_single_10e,88.9000,0.314516,0.161465,0.005320,0.805307,0.0000,1.0,1.0
|
||||
fashion_mnist,official_test,single_50e,89.9300,0.302348,0.148289,0.026401,0.865728,0.0000,1.0,1.0
|
||||
fashion_mnist,official_test,uniform_ensemble,89.6500,0.293522,0.151400,0.019111,0.791698,0.0000,5.0,5.0
|
||||
fashion_mnist,official_test,uniform_temperature,89.6500,0.291996,0.150581,0.007597,0.810594,0.0000,5.0,5.0
|
||||
fashion_mnist,official_test,slsqp_weights,89.5400,0.291696,0.150558,0.017186,0.794293,0.0000,5.0,5.0
|
||||
fashion_mnist,official_test,pso_weights,89.5400,0.291700,0.150559,0.017378,0.794274,0.0000,5.0,5.0
|
||||
|
@@ -0,0 +1,683 @@
|
||||
{
|
||||
"protocol_version": "POST-TRAINING-PSO-ENSEMBLE 1.1.0",
|
||||
"config": {
|
||||
"iteration": 1,
|
||||
"archived_iteration0_reference": {
|
||||
"epochs": 50,
|
||||
"queries_per_seed": 1500,
|
||||
"sample_evaluations_per_seed": 15000000,
|
||||
"reason": "wall_time_ratio_gate_exceeded"
|
||||
},
|
||||
"datasets": [
|
||||
"mnist",
|
||||
"fashion_mnist"
|
||||
],
|
||||
"split_seed": 20260904,
|
||||
"search_samples": 50000,
|
||||
"validation_samples": 10000,
|
||||
"pool_seeds": [
|
||||
201,
|
||||
202,
|
||||
203,
|
||||
204,
|
||||
205
|
||||
],
|
||||
"reference_single_seed": 201,
|
||||
"equal_budget_single_epochs": 50,
|
||||
"adam_lr": 0.001,
|
||||
"adam_batch_size": 256,
|
||||
"pso": {
|
||||
"method": "constriction",
|
||||
"evaluation": "full",
|
||||
"renewal": "loss",
|
||||
"particles": 30,
|
||||
"epochs": 30,
|
||||
"swarm_seeds": [
|
||||
301,
|
||||
302,
|
||||
303
|
||||
],
|
||||
"queries_per_seed": 900,
|
||||
"sample_evaluations_per_seed": 9000000,
|
||||
"particle_bounds": [
|
||||
-4.0,
|
||||
4.0
|
||||
],
|
||||
"boundary_strategy": "reflect",
|
||||
"velocity_limit_ratio": 0.1,
|
||||
"initial_position_noise": 0.0
|
||||
},
|
||||
"device": "mps"
|
||||
},
|
||||
"workloads": {
|
||||
"mnist": {
|
||||
"provenance": {
|
||||
"dataset_name": "MNIST",
|
||||
"split_seed": 20260904,
|
||||
"search_samples": 50000,
|
||||
"validation_samples": 10000,
|
||||
"normalization": {
|
||||
"mean": 0.1307128667831421,
|
||||
"std": 0.3081730008125305
|
||||
},
|
||||
"data_fingerprint": "861b07884cffefb9",
|
||||
"split_fingerprint": "258b42dde7da6324"
|
||||
},
|
||||
"training": {
|
||||
"architecture": "CompactCNN",
|
||||
"parameters": 9098,
|
||||
"pool_seeds": [
|
||||
201,
|
||||
202,
|
||||
203,
|
||||
204,
|
||||
205
|
||||
],
|
||||
"pool_epochs_each": 10,
|
||||
"adam_pool_epochs": 50,
|
||||
"adam_lr": 0.001,
|
||||
"adam_batch_size": 256,
|
||||
"adam_pool_wall_time_seconds": 22.098886500985827,
|
||||
"single_50e_epochs": 50,
|
||||
"single_50e_wall_time_seconds": 22.06671220799035,
|
||||
"model_fingerprints": {
|
||||
"201": "046605462c4fc294",
|
||||
"single_50e": "53ff4c61db919c1e",
|
||||
"202": "cd3150799e0db43e",
|
||||
"203": "1feb0f4dd7630ec2",
|
||||
"204": "9dc0606cd8076cb2",
|
||||
"205": "5bc16dc8f86cc869"
|
||||
}
|
||||
},
|
||||
"validation_cache": {
|
||||
"valid": true,
|
||||
"pool_forward_passes": 5,
|
||||
"long_single_forward_passes": 1,
|
||||
"base_cnn_forward_passes_during_optimization": 0,
|
||||
"shape": [
|
||||
5,
|
||||
10000,
|
||||
10
|
||||
],
|
||||
"memory_bytes": 2400000,
|
||||
"wall_time_seconds": 0.09316845799912699
|
||||
},
|
||||
"validation": {
|
||||
"methods": {
|
||||
"reference_single_10e": {
|
||||
"accuracy": 98.23,
|
||||
"nll": 0.060105,
|
||||
"brier": 0.027982,
|
||||
"ece": 0.003458,
|
||||
"margin": 0.969858
|
||||
},
|
||||
"best_single_10e": {
|
||||
"accuracy": 98.29,
|
||||
"nll": 0.054029,
|
||||
"brier": 0.025526,
|
||||
"ece": 0.002795,
|
||||
"margin": 0.971419,
|
||||
"selected_seed": 205
|
||||
},
|
||||
"single_50e": {
|
||||
"accuracy": 98.52,
|
||||
"nll": 0.073988,
|
||||
"brier": 0.024922,
|
||||
"ece": 0.009463,
|
||||
"margin": 0.989411
|
||||
},
|
||||
"uniform_ensemble": {
|
||||
"accuracy": 98.63,
|
||||
"nll": 0.046385,
|
||||
"brier": 0.021527,
|
||||
"ece": 0.006228,
|
||||
"margin": 0.965339
|
||||
},
|
||||
"uniform_temperature": {
|
||||
"fitted_temperature": 0.854572,
|
||||
"wall_time_seconds": 0.04119049999280833,
|
||||
"evaluations": 23,
|
||||
"metrics": {
|
||||
"accuracy": 98.63,
|
||||
"nll": 0.045355,
|
||||
"brier": 0.021099,
|
||||
"ece": 0.002518,
|
||||
"margin": 0.972273
|
||||
}
|
||||
},
|
||||
"slsqp_weights": {
|
||||
"weights": [
|
||||
0.05767137130295378,
|
||||
0.2585033373652759,
|
||||
0.10841769748910005,
|
||||
0.25254869759276993,
|
||||
0.3228588962499004
|
||||
],
|
||||
"evaluations": 23,
|
||||
"wall_time_seconds": 0.009686834004241973,
|
||||
"success": true,
|
||||
"message": "Optimization terminated successfully",
|
||||
"metrics": {
|
||||
"accuracy": 98.6,
|
||||
"nll": 0.045902,
|
||||
"brier": 0.021307,
|
||||
"ece": 0.005994,
|
||||
"margin": 0.965581
|
||||
}
|
||||
},
|
||||
"pso_weights": {
|
||||
"per_seed_runs": [
|
||||
{
|
||||
"seed": 301,
|
||||
"queries": 900,
|
||||
"sample_evaluations": 9000000,
|
||||
"wall_time_seconds": 1.8924723340023775,
|
||||
"metrics": {
|
||||
"accuracy": 98.61,
|
||||
"nll": 0.045902,
|
||||
"brier": 0.021307,
|
||||
"ece": 0.005893,
|
||||
"margin": 0.965582
|
||||
},
|
||||
"weights": [
|
||||
0.057233214378356934,
|
||||
0.2590548098087311,
|
||||
0.10862385481595993,
|
||||
0.25228697061538696,
|
||||
0.3228012025356293
|
||||
]
|
||||
},
|
||||
{
|
||||
"seed": 302,
|
||||
"queries": 900,
|
||||
"sample_evaluations": 9000000,
|
||||
"wall_time_seconds": 1.7105932499980554,
|
||||
"metrics": {
|
||||
"accuracy": 98.6,
|
||||
"nll": 0.045902,
|
||||
"brier": 0.021306,
|
||||
"ece": 0.006183,
|
||||
"margin": 0.965578
|
||||
},
|
||||
"weights": [
|
||||
0.058317527174949646,
|
||||
0.25828817486763,
|
||||
0.10924994200468063,
|
||||
0.252756267786026,
|
||||
0.3213881254196167
|
||||
]
|
||||
},
|
||||
{
|
||||
"seed": 303,
|
||||
"queries": 900,
|
||||
"sample_evaluations": 9000000,
|
||||
"wall_time_seconds": 1.8959678749961313,
|
||||
"metrics": {
|
||||
"accuracy": 98.61,
|
||||
"nll": 0.045902,
|
||||
"brier": 0.021307,
|
||||
"ece": 0.005833,
|
||||
"margin": 0.965583
|
||||
},
|
||||
"weights": [
|
||||
0.056311722844839096,
|
||||
0.2563577890396118,
|
||||
0.11076968908309937,
|
||||
0.25350210070610046,
|
||||
0.3230587840080261
|
||||
]
|
||||
}
|
||||
],
|
||||
"selected_seed": 301,
|
||||
"selected_weights": [
|
||||
0.057233214378356934,
|
||||
0.2590548098087311,
|
||||
0.10862385481595993,
|
||||
0.25228697061538696,
|
||||
0.3228012025356293
|
||||
],
|
||||
"metrics": {
|
||||
"accuracy": 98.61,
|
||||
"nll": 0.045902,
|
||||
"brier": 0.021307,
|
||||
"ece": 0.005893,
|
||||
"margin": 0.965582
|
||||
},
|
||||
"queries_per_seed": 900,
|
||||
"sample_evaluations_per_seed": 9000000,
|
||||
"total_queries": 2700,
|
||||
"total_sample_evaluations": 27000000,
|
||||
"median_one_seed_wall_time_seconds": 1.8924723340023775,
|
||||
"total_wall_time_seconds": 5.499033458996564
|
||||
}
|
||||
}
|
||||
},
|
||||
"official_test_data_loaded_before_freeze": false,
|
||||
"official_test_evaluations_before_freeze": 0,
|
||||
"confirmation": {
|
||||
"official_test_data_loaded": true,
|
||||
"test_cache_counts": {
|
||||
"dataset_loads": 1,
|
||||
"pool_forward_passes": 5,
|
||||
"long_single_forward_passes": 1,
|
||||
"base_cnn_forward_passes_during_optimization": 0,
|
||||
"memory_bytes": 2400000,
|
||||
"wall_time_seconds": 0.11975424998672679
|
||||
},
|
||||
"frozen_methods": {
|
||||
"selected_pso_seed": 301,
|
||||
"selected_pso_weights": [
|
||||
0.057233214378356934,
|
||||
0.2590548098087311,
|
||||
0.10862385481595993,
|
||||
0.25228697061538696,
|
||||
0.3228012025356293
|
||||
],
|
||||
"slsqp_weights": [
|
||||
0.05767137130295378,
|
||||
0.2585033373652759,
|
||||
0.10841769748910005,
|
||||
0.25254869759276993,
|
||||
0.3228588962499004
|
||||
],
|
||||
"fitted_temperature": 0.854572
|
||||
},
|
||||
"methods": {
|
||||
"reference_single_10e": {
|
||||
"accuracy": 98.47,
|
||||
"nll": 0.044991,
|
||||
"brier": 0.022366,
|
||||
"ece": 0.003516,
|
||||
"margin": 0.970575
|
||||
},
|
||||
"best_single_10e": {
|
||||
"accuracy": 98.55,
|
||||
"nll": 0.044944,
|
||||
"brier": 0.022052,
|
||||
"ece": 0.002515,
|
||||
"margin": 0.97465
|
||||
},
|
||||
"single_50e": {
|
||||
"accuracy": 98.6,
|
||||
"nll": 0.062102,
|
||||
"brier": 0.022912,
|
||||
"ece": 0.008283,
|
||||
"margin": 0.988829
|
||||
},
|
||||
"uniform_ensemble": {
|
||||
"accuracy": 98.86,
|
||||
"nll": 0.036184,
|
||||
"brier": 0.01801,
|
||||
"ece": 0.006311,
|
||||
"margin": 0.968415
|
||||
},
|
||||
"uniform_temperature": {
|
||||
"accuracy": 98.86,
|
||||
"nll": 0.034129,
|
||||
"brier": 0.017658,
|
||||
"ece": 0.00334,
|
||||
"margin": 0.974869
|
||||
},
|
||||
"slsqp_weights": {
|
||||
"accuracy": 98.83,
|
||||
"nll": 0.036179,
|
||||
"brier": 0.017989,
|
||||
"ece": 0.005818,
|
||||
"margin": 0.969195
|
||||
},
|
||||
"pso_weights": {
|
||||
"accuracy": 98.83,
|
||||
"nll": 0.036178,
|
||||
"brier": 0.017989,
|
||||
"ece": 0.005817,
|
||||
"margin": 0.969196
|
||||
}
|
||||
},
|
||||
"confirmation_gates": {
|
||||
"all_values_finite": true,
|
||||
"official_test_dataset_loads": 1,
|
||||
"official_test_pool_forward_passes": 5,
|
||||
"official_test_long_single_forward_passes": 1,
|
||||
"maximum_pso_accuracy_regression_vs_uniform_pp": true,
|
||||
"pso_nll_below_reference_single": true,
|
||||
"maximum_pso_nll_regression_vs_equal_budget_single": true,
|
||||
"pass": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"fashion_mnist": {
|
||||
"provenance": {
|
||||
"dataset_name": "FashionMNIST",
|
||||
"split_seed": 20260904,
|
||||
"search_samples": 50000,
|
||||
"validation_samples": 10000,
|
||||
"normalization": {
|
||||
"mean": 0.28573352098464966,
|
||||
"std": 0.35278451442718506
|
||||
},
|
||||
"data_fingerprint": "b56a39e2abf7d9ad",
|
||||
"split_fingerprint": "489c1a04fb44b59a"
|
||||
},
|
||||
"training": {
|
||||
"architecture": "CompactCNN",
|
||||
"parameters": 9098,
|
||||
"pool_seeds": [
|
||||
201,
|
||||
202,
|
||||
203,
|
||||
204,
|
||||
205
|
||||
],
|
||||
"pool_epochs_each": 10,
|
||||
"adam_pool_epochs": 50,
|
||||
"adam_lr": 0.001,
|
||||
"adam_batch_size": 256,
|
||||
"adam_pool_wall_time_seconds": 22.363562541009742,
|
||||
"single_50e_epochs": 50,
|
||||
"single_50e_wall_time_seconds": 22.406685083013144,
|
||||
"model_fingerprints": {
|
||||
"201": "8189d6d2cce8bde8",
|
||||
"single_50e": "5a5b8444219bf301",
|
||||
"202": "a5492b64bfda73cd",
|
||||
"203": "664e02f11e62031f",
|
||||
"204": "be2a23fca4153703",
|
||||
"205": "d3a37cfa6ee482bf"
|
||||
}
|
||||
},
|
||||
"validation_cache": {
|
||||
"valid": true,
|
||||
"pool_forward_passes": 5,
|
||||
"long_single_forward_passes": 1,
|
||||
"base_cnn_forward_passes_during_optimization": 0,
|
||||
"shape": [
|
||||
5,
|
||||
10000,
|
||||
10
|
||||
],
|
||||
"memory_bytes": 2400000,
|
||||
"wall_time_seconds": 0.08030625000537839
|
||||
},
|
||||
"validation": {
|
||||
"methods": {
|
||||
"reference_single_10e": {
|
||||
"accuracy": 89.47,
|
||||
"nll": 0.303799,
|
||||
"brier": 0.150671,
|
||||
"ece": 0.012197,
|
||||
"margin": 0.808017
|
||||
},
|
||||
"best_single_10e": {
|
||||
"accuracy": 89.47,
|
||||
"nll": 0.303799,
|
||||
"brier": 0.150671,
|
||||
"ece": 0.012197,
|
||||
"margin": 0.808017,
|
||||
"selected_seed": 201
|
||||
},
|
||||
"single_50e": {
|
||||
"accuracy": 90.32,
|
||||
"nll": 0.28966,
|
||||
"brier": 0.142115,
|
||||
"ece": 0.024556,
|
||||
"margin": 0.869484
|
||||
},
|
||||
"uniform_ensemble": {
|
||||
"accuracy": 90.28,
|
||||
"nll": 0.286751,
|
||||
"brier": 0.142881,
|
||||
"ece": 0.023702,
|
||||
"margin": 0.795588
|
||||
},
|
||||
"uniform_temperature": {
|
||||
"fitted_temperature": 0.906309,
|
||||
"wall_time_seconds": 0.04135037500236649,
|
||||
"evaluations": 21,
|
||||
"metrics": {
|
||||
"accuracy": 90.28,
|
||||
"nll": 0.285048,
|
||||
"brier": 0.141557,
|
||||
"ece": 0.011886,
|
||||
"margin": 0.814383
|
||||
}
|
||||
},
|
||||
"slsqp_weights": {
|
||||
"weights": [
|
||||
0.23910131729416598,
|
||||
0.2493341982620314,
|
||||
0.18054455195387528,
|
||||
0.021147090032828317,
|
||||
0.30987284245709906
|
||||
],
|
||||
"evaluations": 23,
|
||||
"wall_time_seconds": 0.010389625007519498,
|
||||
"success": true,
|
||||
"message": "Optimization terminated successfully",
|
||||
"metrics": {
|
||||
"accuracy": 90.42,
|
||||
"nll": 0.285338,
|
||||
"brier": 0.142289,
|
||||
"ece": 0.023705,
|
||||
"margin": 0.798413
|
||||
}
|
||||
},
|
||||
"pso_weights": {
|
||||
"per_seed_runs": [
|
||||
{
|
||||
"seed": 301,
|
||||
"queries": 900,
|
||||
"sample_evaluations": 9000000,
|
||||
"wall_time_seconds": 1.8963403329980792,
|
||||
"metrics": {
|
||||
"accuracy": 90.42,
|
||||
"nll": 0.285338,
|
||||
"brier": 0.142289,
|
||||
"ece": 0.023661,
|
||||
"margin": 0.798397
|
||||
},
|
||||
"weights": [
|
||||
0.2385721057653427,
|
||||
0.24956248700618744,
|
||||
0.1805897355079651,
|
||||
0.021864112466573715,
|
||||
0.30941155552864075
|
||||
]
|
||||
},
|
||||
{
|
||||
"seed": 302,
|
||||
"queries": 900,
|
||||
"sample_evaluations": 9000000,
|
||||
"wall_time_seconds": 1.7477522080007475,
|
||||
"metrics": {
|
||||
"accuracy": 90.43,
|
||||
"nll": 0.285338,
|
||||
"brier": 0.142286,
|
||||
"ece": 0.023912,
|
||||
"margin": 0.79841
|
||||
},
|
||||
"weights": [
|
||||
0.2415684461593628,
|
||||
0.2483692765235901,
|
||||
0.18002673983573914,
|
||||
0.02121216244995594,
|
||||
0.30882343649864197
|
||||
]
|
||||
},
|
||||
{
|
||||
"seed": 303,
|
||||
"queries": 900,
|
||||
"sample_evaluations": 9000000,
|
||||
"wall_time_seconds": 2.0081020840007113,
|
||||
"metrics": {
|
||||
"accuracy": 90.41,
|
||||
"nll": 0.285338,
|
||||
"brier": 0.14229,
|
||||
"ece": 0.023601,
|
||||
"margin": 0.798444
|
||||
},
|
||||
"weights": [
|
||||
0.23893120884895325,
|
||||
0.24851834774017334,
|
||||
0.18254171311855316,
|
||||
0.019977180287241936,
|
||||
0.31003159284591675
|
||||
]
|
||||
}
|
||||
],
|
||||
"selected_seed": 301,
|
||||
"selected_weights": [
|
||||
0.2385721057653427,
|
||||
0.24956248700618744,
|
||||
0.1805897355079651,
|
||||
0.021864112466573715,
|
||||
0.30941155552864075
|
||||
],
|
||||
"metrics": {
|
||||
"accuracy": 90.42,
|
||||
"nll": 0.285338,
|
||||
"brier": 0.142289,
|
||||
"ece": 0.023661,
|
||||
"margin": 0.798397
|
||||
},
|
||||
"queries_per_seed": 900,
|
||||
"sample_evaluations_per_seed": 9000000,
|
||||
"total_queries": 2700,
|
||||
"total_sample_evaluations": 27000000,
|
||||
"median_one_seed_wall_time_seconds": 1.8963403329980792,
|
||||
"total_wall_time_seconds": 5.652194624999538
|
||||
}
|
||||
}
|
||||
},
|
||||
"official_test_data_loaded_before_freeze": false,
|
||||
"official_test_evaluations_before_freeze": 0,
|
||||
"confirmation": {
|
||||
"official_test_data_loaded": true,
|
||||
"test_cache_counts": {
|
||||
"dataset_loads": 1,
|
||||
"pool_forward_passes": 5,
|
||||
"long_single_forward_passes": 1,
|
||||
"base_cnn_forward_passes_during_optimization": 0,
|
||||
"memory_bytes": 2400000,
|
||||
"wall_time_seconds": 0.11122124998655636
|
||||
},
|
||||
"frozen_methods": {
|
||||
"selected_pso_seed": 301,
|
||||
"selected_pso_weights": [
|
||||
0.2385721057653427,
|
||||
0.24956248700618744,
|
||||
0.1805897355079651,
|
||||
0.021864112466573715,
|
||||
0.30941155552864075
|
||||
],
|
||||
"slsqp_weights": [
|
||||
0.23910131729416598,
|
||||
0.2493341982620314,
|
||||
0.18054455195387528,
|
||||
0.021147090032828317,
|
||||
0.30987284245709906
|
||||
],
|
||||
"fitted_temperature": 0.906309
|
||||
},
|
||||
"methods": {
|
||||
"reference_single_10e": {
|
||||
"accuracy": 88.9,
|
||||
"nll": 0.314516,
|
||||
"brier": 0.161465,
|
||||
"ece": 0.00532,
|
||||
"margin": 0.805307
|
||||
},
|
||||
"best_single_10e": {
|
||||
"accuracy": 88.9,
|
||||
"nll": 0.314516,
|
||||
"brier": 0.161465,
|
||||
"ece": 0.00532,
|
||||
"margin": 0.805307
|
||||
},
|
||||
"single_50e": {
|
||||
"accuracy": 89.93,
|
||||
"nll": 0.302348,
|
||||
"brier": 0.148289,
|
||||
"ece": 0.026401,
|
||||
"margin": 0.865728
|
||||
},
|
||||
"uniform_ensemble": {
|
||||
"accuracy": 89.65,
|
||||
"nll": 0.293522,
|
||||
"brier": 0.1514,
|
||||
"ece": 0.019111,
|
||||
"margin": 0.791698
|
||||
},
|
||||
"uniform_temperature": {
|
||||
"accuracy": 89.65,
|
||||
"nll": 0.291996,
|
||||
"brier": 0.150581,
|
||||
"ece": 0.007597,
|
||||
"margin": 0.810594
|
||||
},
|
||||
"slsqp_weights": {
|
||||
"accuracy": 89.54,
|
||||
"nll": 0.291696,
|
||||
"brier": 0.150558,
|
||||
"ece": 0.017186,
|
||||
"margin": 0.794293
|
||||
},
|
||||
"pso_weights": {
|
||||
"accuracy": 89.54,
|
||||
"nll": 0.2917,
|
||||
"brier": 0.150559,
|
||||
"ece": 0.017378,
|
||||
"margin": 0.794274
|
||||
}
|
||||
},
|
||||
"confirmation_gates": {
|
||||
"all_values_finite": true,
|
||||
"official_test_dataset_loads": 1,
|
||||
"official_test_pool_forward_passes": 5,
|
||||
"official_test_long_single_forward_passes": 1,
|
||||
"maximum_pso_accuracy_regression_vs_uniform_pp": true,
|
||||
"pso_nll_below_reference_single": true,
|
||||
"maximum_pso_nll_regression_vs_equal_budget_single": true,
|
||||
"pass": true
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"development_pass": true,
|
||||
"development_gates": {
|
||||
"pass": true,
|
||||
"failed_hard_gate_count": 0,
|
||||
"gate_results": {
|
||||
"all_values_finite": true,
|
||||
"validation_pool_forward_passes_exact": true,
|
||||
"optimization_base_model_forward_passes": true,
|
||||
"official_test_data_loaded_before_freeze": true,
|
||||
"slsqp_solver_success": true,
|
||||
"query_and_sample_accounting_exact": true,
|
||||
"maximum_pso_nll_regression_vs_uniform": true,
|
||||
"maximum_pso_accuracy_regression_vs_uniform_pp": true,
|
||||
"pso_nll_below_reference_single": true,
|
||||
"maximum_pso_nll_regression_vs_equal_budget_single": true,
|
||||
"maximum_relative_pso_nll_gap_vs_slsqp": true,
|
||||
"cross_dataset_mean_relative_pso_nll_reduction_vs_uniform_minimum": true,
|
||||
"maximum_median_one_seed_pso_to_pool_training_wall_ratio": true
|
||||
},
|
||||
"issues": []
|
||||
},
|
||||
"policy_frozen": true,
|
||||
"official_test_data_loaded": true,
|
||||
"official_test_evaluations_before_freeze": 0,
|
||||
"post_test_tuning_or_reruns": 0,
|
||||
"resource_totals": {
|
||||
"adam_pool_epochs": 100,
|
||||
"adam_pool_wall_time_seconds": 44.46244904199557,
|
||||
"single_50e_wall_time_seconds": 44.473397291003494,
|
||||
"validation_cache_forward_passes": 12,
|
||||
"pso_total_queries": 5400,
|
||||
"pso_total_sample_evaluations": 54000000,
|
||||
"pso_research_wall_time_seconds": 11.151228083996102,
|
||||
"pso_production_wall_time_seconds": 3.7888126670004567,
|
||||
"pso_to_pool_wall_ratio": 0.0852162664640409,
|
||||
"slsqp_total_evaluations": 46,
|
||||
"slsqp_total_wall_time_seconds": 0.02007645901176147,
|
||||
"official_test_cache_forward_passes": 12
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"evaluator_version": "POST-TRAINING-PSO-ENSEMBLE-EVALUATOR 1.2.0",
|
||||
"pass": true,
|
||||
"score": 22.55299842776338,
|
||||
"development_pass": true,
|
||||
"confirmation_pass": true,
|
||||
"failed_hard_gate_count": 0,
|
||||
"issues": {
|
||||
"schema": [],
|
||||
"config": [],
|
||||
"finite": [],
|
||||
"weights": [],
|
||||
"accounting": [],
|
||||
"leakage": [],
|
||||
"tuning": [],
|
||||
"slsqp": [],
|
||||
"consistency": [],
|
||||
"gates": []
|
||||
},
|
||||
"development_gates": {
|
||||
"all_values_finite": true,
|
||||
"simplex_tolerance": true,
|
||||
"validation_pool_forward_passes_each_dataset": true,
|
||||
"optimization_base_model_forward_passes": true,
|
||||
"official_test_data_loaded_before_freeze": true,
|
||||
"official_test_evaluations_before_freeze": true,
|
||||
"query_and_sample_accounting_exact": true,
|
||||
"maximum_pso_nll_regression_vs_uniform": true,
|
||||
"maximum_pso_accuracy_regression_vs_uniform_pp": true,
|
||||
"pso_nll_below_reference_single": true,
|
||||
"maximum_pso_nll_regression_vs_equal_budget_single": true,
|
||||
"maximum_relative_pso_nll_gap_vs_slsqp": true,
|
||||
"cross_dataset_mean_relative_pso_nll_reduction_vs_uniform_minimum": true,
|
||||
"maximum_median_one_seed_pso_to_pool_training_wall_ratio": true
|
||||
},
|
||||
"confirmation_gates": {
|
||||
"all_values_finite": true,
|
||||
"official_test_dataset_loads_each_dataset": true,
|
||||
"official_test_pool_forward_passes_each_dataset": true,
|
||||
"official_test_long_single_forward_passes_each_dataset": true,
|
||||
"frozen_policy_consistency": true,
|
||||
"maximum_pso_accuracy_regression_vs_uniform_pp": true,
|
||||
"pso_nll_below_reference_single": true,
|
||||
"maximum_pso_nll_regression_vs_equal_budget_single": true,
|
||||
"post_test_tuning_or_reruns": true
|
||||
},
|
||||
"metrics": {
|
||||
"mean_val_relative_nll_reduction_vs_equal_budget_single": 0.197261519715641,
|
||||
"mean_val_accuracy_gain_vs_equal_budget_single_pp": 0.09500000000000597,
|
||||
"mean_test_relative_nll_reduction_vs_equal_budget_single": 0.2263299842776338,
|
||||
"mean_test_accuracy_gain_vs_equal_budget_single_pp": -0.0799999999999983
|
||||
}
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
name: pso
|
||||
channels:
|
||||
- conda-forge
|
||||
- defaults
|
||||
dependencies:
|
||||
- cudatoolkit=11.2
|
||||
- cudnn=8.1.0
|
||||
- pandas=1.5.3
|
||||
- pip=23.0.1
|
||||
- python=3.9.16
|
||||
- tqdm=4.65.0
|
||||
- pip:
|
||||
- numpy==1.25.0
|
||||
- tensorflow==2.11.0
|
||||
- tensorboard==2.11.0
|
||||
+189
-2348
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 186 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 206 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 464 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 268 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 406 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 217 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 281 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 298 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 92 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 162 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 294 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 132 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 126 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 250 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 337 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 232 KiB |
+26
-19
@@ -1,22 +1,29 @@
|
||||
import tensorflow as tf
|
||||
import os
|
||||
from .optimizer import Optimizer as optimizer
|
||||
from .particle import Particle as particle
|
||||
|
||||
__version__ = "1.0.5.1"
|
||||
|
||||
print("pso2keras version : " + __version__)
|
||||
|
||||
gpus = tf.config.experimental.list_physical_devices("GPU")
|
||||
if gpus:
|
||||
try:
|
||||
tf.config.experimental.set_memory_growth(gpus[0], True)
|
||||
except RuntimeError as r:
|
||||
print(r)
|
||||
|
||||
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
|
||||
from ._version import __version__
|
||||
from .optimizer import Optimizer
|
||||
from .particle import Particle
|
||||
from .plugins import (
|
||||
BasePlugin,
|
||||
InitializationPlugin,
|
||||
EvaluationPlugin,
|
||||
MovementPlugin,
|
||||
ConvergencePlugin,
|
||||
RefinementPlugin,
|
||||
PluginMetadata,
|
||||
SwarmState,
|
||||
available_plugins,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"optimizer",
|
||||
"particle",
|
||||
"Optimizer",
|
||||
"Particle",
|
||||
"__version__",
|
||||
"BasePlugin",
|
||||
"InitializationPlugin",
|
||||
"EvaluationPlugin",
|
||||
"MovementPlugin",
|
||||
"ConvergencePlugin",
|
||||
"RefinementPlugin",
|
||||
"PluginMetadata",
|
||||
"SwarmState",
|
||||
"available_plugins",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from importlib.metadata import PackageNotFoundError, version
|
||||
|
||||
try:
|
||||
__version__ = version("pso2keras")
|
||||
except PackageNotFoundError:
|
||||
pyproject_path = Path(__file__).resolve().parent.parent / "pyproject.toml"
|
||||
if pyproject_path.exists():
|
||||
if sys.version_info >= (3, 11):
|
||||
import tomllib
|
||||
else:
|
||||
try:
|
||||
import tomllib # type: ignore
|
||||
except ImportError:
|
||||
import tomli as tomllib # type: ignore
|
||||
|
||||
with pyproject_path.open("rb") as f:
|
||||
data = tomllib.load(f)
|
||||
__version__ = data["project"]["version"]
|
||||
else:
|
||||
__version__ = "0.0.0"
|
||||
@@ -0,0 +1,88 @@
|
||||
import collections
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
class ParameterCodec:
|
||||
"""
|
||||
Private parameter encoder/decoder for flattening and reconstructing PyTorch nn.Module parameters.
|
||||
"""
|
||||
|
||||
def __init__(self, model: nn.Module):
|
||||
if not isinstance(model, nn.Module):
|
||||
raise TypeError("model must be an instance of torch.nn.Module")
|
||||
|
||||
params = list(model.named_parameters())
|
||||
if not params:
|
||||
raise ValueError("model contains zero trainable parameters")
|
||||
|
||||
self.names: list[str] = []
|
||||
self.shapes: list[tuple[int, ...]] = []
|
||||
self.numels: list[int] = []
|
||||
dtypes: set[torch.dtype] = set()
|
||||
|
||||
total_size = 0
|
||||
for name, p in params:
|
||||
if not p.dtype.is_floating_point:
|
||||
raise ValueError(
|
||||
f"Parameter '{name}' has non-floating dtype {p.dtype}. Only floating-point parameters are supported."
|
||||
)
|
||||
self.names.append(name)
|
||||
self.shapes.append(tuple(p.shape))
|
||||
n = p.numel()
|
||||
self.numels.append(n)
|
||||
total_size += n
|
||||
dtypes.add(p.dtype)
|
||||
|
||||
if len(dtypes) > 1:
|
||||
raise ValueError(f"Mixed parameter dtypes found in model: {dtypes}")
|
||||
|
||||
self.dtype: torch.dtype = next(iter(dtypes))
|
||||
self.size: int = total_size
|
||||
|
||||
def encode(self, model: nn.Module) -> torch.Tensor:
|
||||
"""
|
||||
Flattens model parameters into a single detached 1D torch.Tensor.
|
||||
"""
|
||||
params = [p for _, p in model.named_parameters()]
|
||||
return nn.utils.parameters_to_vector(params).detach()
|
||||
|
||||
def apply_vector(self, vector: torch.Tensor, model: nn.Module) -> None:
|
||||
"""
|
||||
Applies a validated 1D parameter vector to model.parameters() in-place without storage aliasing.
|
||||
"""
|
||||
if not isinstance(vector, torch.Tensor) or vector.ndim != 1:
|
||||
raise ValueError("vector must be a 1D torch.Tensor")
|
||||
if vector.numel() != self.size:
|
||||
raise ValueError(
|
||||
f"Vector size {vector.numel()} does not match codec size {self.size}"
|
||||
)
|
||||
if vector.dtype != self.dtype:
|
||||
raise ValueError(
|
||||
f"Vector dtype {vector.dtype} does not match codec dtype {self.dtype}"
|
||||
)
|
||||
|
||||
params = [p for _, p in model.named_parameters()]
|
||||
if params:
|
||||
target_device = params[0].device
|
||||
if vector.device != target_device:
|
||||
vector = vector.to(target_device)
|
||||
|
||||
with torch.no_grad():
|
||||
offset = 0
|
||||
for p, shape, numel in zip(params, self.shapes, self.numels):
|
||||
p.copy_(vector[offset : offset + numel].reshape(shape))
|
||||
offset += numel
|
||||
|
||||
def to_state_dict(
|
||||
self, vector: torch.Tensor, eval_model: nn.Module
|
||||
) -> collections.OrderedDict[str, torch.Tensor]:
|
||||
"""
|
||||
Applies vector to eval_model and returns a defensive CPU-cloned ordered state dict including buffers.
|
||||
"""
|
||||
self.apply_vector(vector, eval_model)
|
||||
state_dict = eval_model.state_dict()
|
||||
res = collections.OrderedDict()
|
||||
for k, v in state_dict.items():
|
||||
res[k] = v.detach().cpu().clone()
|
||||
return res
|
||||
+1608
-690
File diff suppressed because it is too large
Load Diff
+53
-389
@@ -1,409 +1,73 @@
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from tensorflow import keras
|
||||
from typing import Any, Literal
|
||||
import torch
|
||||
from .plugins import FitContext, InitializationPlugin
|
||||
|
||||
|
||||
class Particle:
|
||||
"""
|
||||
Particle Swarm Optimization의 Particle을 구현한 클래스
|
||||
한 파티클의 life cycle은 다음과 같다.
|
||||
1. 초기화
|
||||
2. 손실 함수 계산
|
||||
3. 속도 업데이트
|
||||
4. 가중치 업데이트
|
||||
5. 2번으로 돌아가서 반복
|
||||
Particle Swarm Optimization particle (engine-owned state, plugin-driven movement).
|
||||
Each particle owns position, velocity, personal best score, personal best weights,
|
||||
and monitor state on device.
|
||||
"""
|
||||
|
||||
g_best_score = [np.inf, 0, np.inf]
|
||||
g_best_weights = None
|
||||
count = 0
|
||||
|
||||
MODEL_IS_NONE = "model is None"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: keras.Model,
|
||||
loss: Any = None,
|
||||
index: int,
|
||||
base_vector: torch.Tensor,
|
||||
context: FitContext,
|
||||
init_plugin: InitializationPlugin,
|
||||
negative: bool = False,
|
||||
mutation: float = 0,
|
||||
converge_reset: bool = False,
|
||||
converge_reset_patience: int = 10,
|
||||
converge_reset_monitor: str = "loss",
|
||||
converge_reset_min_delta: float = 0.0001,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
model (keras.models): 학습 및 검증을 위한 모델
|
||||
loss (str|): 손실 함수
|
||||
negative (bool, optional): 음의 가중치 사용 여부 - 전역 탐색 용도(조기 수렴 방지). Defaults to False.
|
||||
mutation (float, optional): 돌연변이 확률. Defaults to 0.
|
||||
converge_reset (bool, optional): 조기 종료 사용 여부. Defaults to False.
|
||||
converge_reset_patience (int, optional): 조기 종료를 위한 기다리는 횟수. Defaults to 10.
|
||||
"""
|
||||
self.set_model(model)
|
||||
self.weights = self._encode(model.get_weights())
|
||||
self.loss = loss
|
||||
|
||||
try:
|
||||
if converge_reset and converge_reset_monitor not in [
|
||||
"acc",
|
||||
"accuracy",
|
||||
"loss",
|
||||
"mse",
|
||||
]:
|
||||
raise ValueError(
|
||||
"converge_reset_monitor must be 'acc' or 'accuracy' or 'loss'"
|
||||
)
|
||||
if converge_reset and converge_reset_min_delta < 0:
|
||||
raise ValueError("converge_reset_min_delta must be positive")
|
||||
if converge_reset and converge_reset_patience < 0:
|
||||
raise ValueError("converge_reset_patience must be positive")
|
||||
except ValueError as e:
|
||||
print(e)
|
||||
exit(1)
|
||||
|
||||
self.velocities = np.zeros(len(self.weights))
|
||||
self.__reset_particle()
|
||||
self.best_weights = self.weights
|
||||
self.index = index
|
||||
self.negative = negative
|
||||
self.mutation = mutation
|
||||
self.local_best_score = [np.inf, 0, np.inf]
|
||||
self.score_history = []
|
||||
self.converge_reset = converge_reset
|
||||
self.converge_reset_patience = converge_reset_patience
|
||||
self.converge_reset_monitor = converge_reset_monitor
|
||||
self.converge_reset_min_delta = converge_reset_min_delta
|
||||
Particle.count += 1
|
||||
|
||||
def __del__(self):
|
||||
del self.model
|
||||
del self.loss
|
||||
del self.velocities
|
||||
del self.negative
|
||||
del self.local_best_score
|
||||
del self.best_weights
|
||||
Particle.count -= 1
|
||||
pos, vel = init_plugin.initialize(index, base_vector, context)
|
||||
self.position: torch.Tensor = pos
|
||||
self.velocity: torch.Tensor = vel
|
||||
|
||||
def set_shape(self, weights: list):
|
||||
"""
|
||||
가중치의 shape을 설정
|
||||
self.personal_best_score: tuple[float, float, float] | None = None
|
||||
self.personal_best_weights: torch.Tensor | None = None
|
||||
self.personal_best_monitor_value: float | None = None
|
||||
|
||||
Args:
|
||||
weights (list): keras model의 가중치
|
||||
"""
|
||||
self.shape = [layer.shape for layer in weights]
|
||||
|
||||
def get_shape(self):
|
||||
return self.shape
|
||||
|
||||
def _encode(self, weights: list):
|
||||
"""
|
||||
가중치를 1차원으로 풀어서 반환
|
||||
|
||||
Args:
|
||||
weights (list) : keras model의 가중치
|
||||
Returns:
|
||||
(numpy array) : 가중치 - 1차원으로 풀어서 반환
|
||||
(list) : 가중치의 원본 shape
|
||||
(list) : 가중치의 원본 shape의 길이
|
||||
"""
|
||||
w_gpu = np.array([])
|
||||
for layer in weights:
|
||||
w_tmp = layer.reshape(-1)
|
||||
w_gpu = np.append(w_gpu, w_tmp)
|
||||
|
||||
return w_gpu
|
||||
|
||||
def _decode(self, weight: np.ndarray):
|
||||
"""
|
||||
_encode 로 인코딩된 가중치를 원본 shape으로 복원
|
||||
파라미터는 encode의 리턴값을 그대로 사용을 권장
|
||||
|
||||
Args:
|
||||
weight (numpy array): 가중치 - 1차원으로 풀어서 반환
|
||||
shape (list): 가중치의 원본 shape
|
||||
length (list): 가중치의 원본 shape의 길이
|
||||
Returns:
|
||||
(list) : 가중치 원본 shape으로 복원
|
||||
"""
|
||||
weights = []
|
||||
start = 0
|
||||
for i in range(len(self.shape)):
|
||||
end = start + np.prod(self.shape[i])
|
||||
w_ = weight[start:end]
|
||||
w_ = np.reshape(w_, self.shape[i])
|
||||
weights.append(w_)
|
||||
start = end
|
||||
|
||||
del start, end, w_
|
||||
del weight
|
||||
|
||||
return weights
|
||||
|
||||
def get_model(self):
|
||||
if self.model is None:
|
||||
raise ValueError(self.MODEL_IS_NONE)
|
||||
|
||||
return self.model
|
||||
|
||||
def set_model(self, model: keras.Model):
|
||||
self.model = model
|
||||
self.set_shape(self.model.get_weights())
|
||||
|
||||
def compile(self):
|
||||
if self.model is None:
|
||||
raise ValueError(self.MODEL_IS_NONE)
|
||||
|
||||
self.model.compile(
|
||||
optimizer="adam",
|
||||
loss=self.loss,
|
||||
metrics=["accuracy", "mse"],
|
||||
)
|
||||
|
||||
def get_weights(self):
|
||||
weights = self._decode(self.weights)
|
||||
|
||||
return weights
|
||||
|
||||
def evaluate(self, x, y):
|
||||
if self.model is None:
|
||||
raise ValueError(self.MODEL_IS_NONE)
|
||||
|
||||
return self.model.evaluate(x, y, verbose=0) # type: ignore
|
||||
|
||||
def get_score(self, x, y, renewal: str = "acc"):
|
||||
"""
|
||||
모델의 성능을 평가하여 점수를 반환
|
||||
|
||||
Args:
|
||||
x (list): 입력 데이터
|
||||
y (list): 출력 데이터
|
||||
renewal (str, optional): 점수 갱신 방식. Defaults to "acc" | "acc" or "loss".
|
||||
|
||||
Returns:
|
||||
(float): 점수
|
||||
"""
|
||||
|
||||
score = self.evaluate(x, y)
|
||||
if renewal == "loss":
|
||||
if score[0] < self.local_best_score[0]:
|
||||
self.local_best_score = score
|
||||
self.best_weights = self.weights
|
||||
elif renewal == "acc":
|
||||
if score[1] > self.local_best_score[1]:
|
||||
self.local_best_score = score
|
||||
self.best_weights = self.weights
|
||||
elif renewal == "mse":
|
||||
if score[2] < self.local_best_score[2]:
|
||||
self.local_best_score = score
|
||||
self.best_weights = self.weights
|
||||
else:
|
||||
raise ValueError("renewal must be 'acc' or 'loss' or 'mse'")
|
||||
|
||||
return score
|
||||
|
||||
def __check_converge_reset(
|
||||
def reset(
|
||||
self,
|
||||
score,
|
||||
monitor: str = "auto",
|
||||
patience: int = 10,
|
||||
min_delta: float = 0.0001,
|
||||
):
|
||||
base_vector: torch.Tensor,
|
||||
context: FitContext,
|
||||
init_plugin: InitializationPlugin,
|
||||
) -> None:
|
||||
"""
|
||||
early stop을 구현한 함수
|
||||
Resets particle position, velocity, and personal best state using the initialization plugin.
|
||||
"""
|
||||
pos, vel = init_plugin.initialize(self.index, base_vector, context)
|
||||
self.position = pos
|
||||
self.velocity = vel
|
||||
self.personal_best_score = None
|
||||
self.personal_best_weights = None
|
||||
self.personal_best_monitor_value = None
|
||||
|
||||
Args:
|
||||
score (float): 현재 점수 [0] - loss, [1] - acc
|
||||
monitor (str, optional): 감시할 점수. Defaults to acc. | "acc" or "loss" or "mse"
|
||||
patience (int, optional): early stop을 위한 기다리는 횟수. Defaults to 10.
|
||||
min_delta (float, optional): early stop을 위한 최소 변화량. Defaults to 0.0001.
|
||||
def apply_boundary_strategy(
|
||||
self,
|
||||
particle_min: float | None,
|
||||
particle_max: float | None,
|
||||
boundary_strategy: str = "clip",
|
||||
) -> None:
|
||||
"""
|
||||
if monitor == "auto":
|
||||
monitor = "acc"
|
||||
if monitor in ["loss"]:
|
||||
self.score_history.append(score[0])
|
||||
elif monitor in ["acc", "accuracy"]:
|
||||
self.score_history.append(score[1])
|
||||
elif monitor in ["mse"]:
|
||||
self.score_history.append(score[2])
|
||||
Applies boundary constraints (clip or reflect) to particle position and velocity.
|
||||
"""
|
||||
if particle_min is not None and particle_max is not None:
|
||||
if boundary_strategy == "reflect":
|
||||
span = float(particle_max - particle_min)
|
||||
shift = self.position - particle_min
|
||||
q = torch.floor(shift / span).to(torch.int64)
|
||||
m = torch.remainder(shift, 2.0 * span)
|
||||
bounded_pos = torch.where(
|
||||
m <= span,
|
||||
particle_min + m,
|
||||
particle_min + (2.0 * span - m),
|
||||
)
|
||||
self.velocity = torch.where(q % 2 != 0, -self.velocity, self.velocity)
|
||||
self.position = bounded_pos
|
||||
else:
|
||||
raise ValueError("monitor must be 'acc' or 'accuracy' or 'loss' or 'mse'")
|
||||
|
||||
if len(self.score_history) > patience:
|
||||
last_scores = self.score_history[-patience:]
|
||||
if max(last_scores) - min(last_scores) < min_delta:
|
||||
return True
|
||||
return False
|
||||
|
||||
def __reset_particle(self):
|
||||
|
||||
self.model = keras.models.model_from_json(self.model.to_json())
|
||||
self.model.compile(
|
||||
optimizer="adam",
|
||||
loss=self.loss,
|
||||
metrics=["accuracy", "mse"],
|
||||
self.position = torch.clamp(
|
||||
self.position, particle_min, particle_max
|
||||
)
|
||||
self.weights = self._encode(self.model.get_weights())
|
||||
rng = np.random.default_rng()
|
||||
self.velocities = rng.uniform(-0.2, 0.2, len(self.weights))
|
||||
|
||||
self.score_history = []
|
||||
|
||||
def _velocity_calculation(self, local_rate, global_rate, w):
|
||||
"""
|
||||
현재 속도 업데이트
|
||||
|
||||
Args:
|
||||
local_rate (float): 지역 최적해의 영향력
|
||||
global_rate (float): 전역 최적해의 영향력
|
||||
w (float): 현재 속도의 영향력 - 관성 | 0.9 ~ 0.4 이 적당
|
||||
"""
|
||||
# 0회차 전역 최적해가 없을 경우 현재 파티클의 최적해로 설정 - 전역최적해의 방향을 0으로 만들기 위함
|
||||
best_particle_weights = (
|
||||
self.best_weights
|
||||
if Particle.g_best_weights is None
|
||||
else Particle.g_best_weights
|
||||
)
|
||||
|
||||
rng = np.random.default_rng(seed=42)
|
||||
r_0 = rng.random()
|
||||
r_1 = rng.random()
|
||||
|
||||
if self.negative:
|
||||
# 지역 최적해와 전역 최적해를 음수로 사용하여 전역 탐색을 유도
|
||||
new_v = (
|
||||
w * self.velocities
|
||||
+ local_rate * r_0 * (self.best_weights - self.weights)
|
||||
- global_rate * r_1 * (best_particle_weights - self.weights)
|
||||
)
|
||||
if (
|
||||
len(self.score_history) > 10
|
||||
and max(self.score_history[-10:]) - min(self.score_history[-10:]) < 0.01
|
||||
):
|
||||
self.__reset_particle()
|
||||
|
||||
else:
|
||||
# 전역 최적해의 acc 가 높을수록 더 빠르게 수렴
|
||||
# 하지만 loss 가 커진 상태에서는 전역 최적해의 영향이
|
||||
new_v = (
|
||||
w * self.velocities
|
||||
+ local_rate
|
||||
* self.local_best_score[1]
|
||||
* r_0
|
||||
* (self.best_weights - self.weights)
|
||||
+ global_rate
|
||||
# * Particle.g_best_score[1]
|
||||
* r_1 * (best_particle_weights - self.weights)
|
||||
)
|
||||
|
||||
if self.mutation != 0.0 and rng.random() < self.mutation:
|
||||
m_v = rng.uniform(-0.2, 0.2, len(self.velocities))
|
||||
new_v = m_v
|
||||
|
||||
self.velocities = new_v
|
||||
|
||||
del r_0, r_1
|
||||
|
||||
def _position_update(self):
|
||||
"""
|
||||
가중치 업데이트
|
||||
"""
|
||||
self.weights = np.add(self.weights, self.velocities)
|
||||
|
||||
self.model.set_weights(self.get_weights())
|
||||
|
||||
def step(self, x, y, local_rate, global_rate, w, renewal: str = "acc"):
|
||||
"""
|
||||
파티클의 한 스텝을 진행합니다.
|
||||
|
||||
Args:
|
||||
x (list): 입력 데이터
|
||||
y (list): 출력 데이터
|
||||
local_rate (float): 지역최적해의 영향력
|
||||
global_rate (float): 전역최적해의 영향력
|
||||
w (float): 관성
|
||||
g_best (list): 전역최적해
|
||||
renewal (str, optional): 최고점수 갱신 방식. Defaults to "acc" | "acc" or "loss"
|
||||
|
||||
Returns:
|
||||
list: 현재 파티클의 점수
|
||||
"""
|
||||
self._velocity_calculation(local_rate, global_rate, w)
|
||||
self._position_update()
|
||||
|
||||
score = self.get_score(x, y, renewal)
|
||||
|
||||
if self.converge_reset and self.__check_converge_reset(
|
||||
score,
|
||||
self.converge_reset_monitor,
|
||||
self.converge_reset_patience,
|
||||
self.converge_reset_min_delta,
|
||||
):
|
||||
self.__reset_particle()
|
||||
score = self.get_score(x, y, renewal)
|
||||
|
||||
while (
|
||||
np.isnan(score[0])
|
||||
or np.isnan(score[1])
|
||||
or np.isnan(score[2])
|
||||
or score[0] == 0
|
||||
or score[1] == 0
|
||||
or score[2] == 0
|
||||
or np.isinf(score[0])
|
||||
or np.isinf(score[1])
|
||||
or np.isinf(score[2])
|
||||
or score[0] > 1000
|
||||
or score[1] > 1
|
||||
or score[2] > 1000
|
||||
):
|
||||
self.__reset_particle()
|
||||
score = self.get_score(x, y, renewal)
|
||||
|
||||
return score
|
||||
|
||||
def get_best_score(self):
|
||||
"""
|
||||
파티클의 최고점수를 반환합니다.
|
||||
|
||||
Returns:
|
||||
float: 최고점수
|
||||
"""
|
||||
return self.local_best_score
|
||||
|
||||
def get_best_weights(self):
|
||||
"""
|
||||
파티클의 최고점수를 받은 가중치를 반환합니다
|
||||
|
||||
Returns:
|
||||
list: 가중치 리스트
|
||||
"""
|
||||
return self._decode(self.best_weights)
|
||||
|
||||
def set_global_score(self):
|
||||
"""전역 최고점수를 현재 파티클의 최고점수로 설정합니다"""
|
||||
Particle.g_best_score = self.local_best_score
|
||||
|
||||
def set_global_weights(self):
|
||||
"""전역 최고점수를 받은 가중치를 현재 파티클의 최고점수를 받은 가중치로 설정합니다"""
|
||||
Particle.g_best_weights = self.best_weights
|
||||
|
||||
def update_global_best(self):
|
||||
"""현재 파티클의 점수와 가중치를 전역 최고점수와 가중치로 설정합니다"""
|
||||
self.set_global_score()
|
||||
self.set_global_weights()
|
||||
|
||||
def check_global_best(self, renewal: str = "loss"):
|
||||
if (
|
||||
(renewal == "loss" and self.local_best_score[0] < Particle.g_best_score[0])
|
||||
or (
|
||||
renewal == "acc" and self.local_best_score[1] > Particle.g_best_score[1]
|
||||
)
|
||||
or (
|
||||
renewal == "mse" and self.local_best_score[2] < Particle.g_best_score[2]
|
||||
)
|
||||
):
|
||||
self.update_global_best()
|
||||
|
||||
|
||||
# 끝
|
||||
|
||||
+1566
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,67 @@
|
||||
[build-system]
|
||||
requires = ["setuptools>=77", "wheel"]
|
||||
build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "pso2keras"
|
||||
version = "4.0.0"
|
||||
description = "Particle Swarm Optimization for PyTorch models"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10,<3.12"
|
||||
license = "MIT"
|
||||
license-files = ["LICENSE"]
|
||||
authors = [
|
||||
{ name = "pieroot", email = "jgbong0306@gmail.com" }
|
||||
]
|
||||
keywords = [
|
||||
"pso",
|
||||
"pytorch",
|
||||
"torch",
|
||||
"optimization",
|
||||
"particle swarm optimization",
|
||||
"pso2keras",
|
||||
]
|
||||
classifiers = [
|
||||
"Programming Language :: Python :: 3 :: Only",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
]
|
||||
dependencies = [
|
||||
"torch>=2.13,<3",
|
||||
"numpy<2",
|
||||
"scikit-learn",
|
||||
"tqdm",
|
||||
"tensorboard",
|
||||
"tomli>=2; python_version < '3.11'",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/jung-geun/PSO"
|
||||
Repository = "https://github.com/jung-geun/PSO"
|
||||
|
||||
[project.optional-dependencies]
|
||||
examples = [
|
||||
"pandas",
|
||||
"ucimlrepo",
|
||||
"torchvision>=0.28,<1",
|
||||
"matplotlib>=3.8,<4",
|
||||
]
|
||||
detection = [
|
||||
"ultralytics==8.4.142",
|
||||
"ensemble-boxes==1.0.9",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=9,<10",
|
||||
"build>=1.3,<2",
|
||||
"twine>=6,<8",
|
||||
]
|
||||
|
||||
[tool.setuptools.packages.find]
|
||||
where = ["."]
|
||||
include = ["pso*"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
python_files = ["test_*.py"]
|
||||
@@ -1,7 +0,0 @@
|
||||
ipython
|
||||
numpy
|
||||
pandas
|
||||
tensorflow==2.15.1
|
||||
tqdm==4.66.4
|
||||
scikit-learn==1.4.2
|
||||
tensorboard==2.15.1
|
||||
@@ -1,41 +0,0 @@
|
||||
from setuptools import find_packages, setup
|
||||
|
||||
import pso
|
||||
|
||||
VERSION = pso.__version__
|
||||
|
||||
|
||||
def get_requirements(path: str):
|
||||
return [l.strip() for l in open(path)]
|
||||
|
||||
|
||||
setup(
|
||||
name="pso2keras",
|
||||
version=VERSION,
|
||||
description="Particle Swarm Optimization on tensorflow package",
|
||||
author="pieroot",
|
||||
author_email="jgbong0306@gmail.com",
|
||||
url="https://github.com/jung-geun/PSO",
|
||||
install_requires=get_requirements("requirements.txt"),
|
||||
packages=find_packages(exclude=[]),
|
||||
keywords=[
|
||||
"pso",
|
||||
"tensorflow",
|
||||
"keras",
|
||||
"optimization",
|
||||
"particle swarm optimization",
|
||||
"pso2keras",
|
||||
],
|
||||
python_requires=">=3.10",
|
||||
package_data={},
|
||||
zip_safe=False,
|
||||
long_description=open("README.md", encoding="UTF8").read(),
|
||||
long_description_content_type="text/markdown",
|
||||
license="MIT",
|
||||
classifiers=[
|
||||
"License :: OSI Approved :: MIT License",
|
||||
"Programming Language :: Python :: 3 :: Only",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
],
|
||||
)
|
||||
+119
-53
@@ -1,72 +1,138 @@
|
||||
import os
|
||||
"""Dry Bean dataset gradient baseline (PyTorch standard backprop optimizer, non-PSO)."""
|
||||
|
||||
from keras.layers import Dense
|
||||
from keras.models import Sequential
|
||||
from keras.utils import to_categorical
|
||||
import copy
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.utils.data import DataLoader, TensorDataset
|
||||
from sklearn.model_selection import train_test_split
|
||||
from tensorflow import keras
|
||||
from sklearn.preprocessing import LabelEncoder
|
||||
from ucimlrepo import fetch_ucirepo
|
||||
|
||||
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
|
||||
os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "true"
|
||||
|
||||
class BeanModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.net = nn.Sequential(
|
||||
nn.Linear(16, 12),
|
||||
nn.ReLU(),
|
||||
nn.Linear(12, 8),
|
||||
nn.ReLU(),
|
||||
nn.Linear(8, 7),
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
return self.net(x)
|
||||
|
||||
|
||||
def make_model():
|
||||
model = Sequential()
|
||||
model.add(Dense(12, input_dim=16, activation="relu"))
|
||||
model.add(Dense(8, activation="relu"))
|
||||
model.add(Dense(7, activation="softmax"))
|
||||
|
||||
return model
|
||||
|
||||
|
||||
def get_data():
|
||||
# fetch dataset
|
||||
def get_data(seed: int = 42):
|
||||
dry_bean_dataset = fetch_ucirepo(id=602)
|
||||
|
||||
# data (as pandas dataframes)
|
||||
X = dry_bean_dataset.data.features
|
||||
y = dry_bean_dataset.data.targets
|
||||
|
||||
x = X.to_numpy()
|
||||
# object to categorical
|
||||
|
||||
x = x.astype("float32")
|
||||
|
||||
y_class = to_categorical(y)
|
||||
|
||||
# metadata
|
||||
# print(dry_bean_dataset.metadata)
|
||||
|
||||
# variable information
|
||||
# print(dry_bean_dataset.variables)
|
||||
|
||||
# print(X.head())
|
||||
# print(y.head())
|
||||
# y_class = to_categorical(y)
|
||||
x = X.to_numpy().astype("float32")
|
||||
encoder = LabelEncoder()
|
||||
y_encoded = encoder.fit_transform(y.values.ravel()).astype("int64")
|
||||
|
||||
x_train, x_test, y_train, y_test = train_test_split(
|
||||
x, y_class, test_size=0.2, random_state=42, shuffle=True
|
||||
x, y_encoded, test_size=0.2, random_state=seed, shuffle=True
|
||||
)
|
||||
return (
|
||||
torch.tensor(x_train, dtype=torch.float32),
|
||||
torch.tensor(x_test, dtype=torch.float32),
|
||||
torch.tensor(y_train, dtype=torch.int64),
|
||||
torch.tensor(y_test, dtype=torch.int64),
|
||||
)
|
||||
return x_train, x_test, y_train, y_test
|
||||
|
||||
|
||||
x_train, x_test, y_train, y_test = get_data()
|
||||
model = make_model()
|
||||
early_stopping = keras.callbacks.EarlyStopping(
|
||||
patience=10, min_delta=0.001, restore_best_weights=True
|
||||
)
|
||||
def get_device() -> torch.device:
|
||||
if (
|
||||
hasattr(torch.backends, "mps")
|
||||
and torch.backends.mps.is_built()
|
||||
and torch.backends.mps.is_available()
|
||||
):
|
||||
return torch.device("mps")
|
||||
elif torch.cuda.is_available():
|
||||
return torch.device("cuda")
|
||||
return torch.device("cpu")
|
||||
|
||||
|
||||
model.compile(
|
||||
loss="sparse_categorical_crossentropy",
|
||||
optimizer="adam",
|
||||
metrics=["accuracy", "mse"],
|
||||
)
|
||||
def main():
|
||||
torch.manual_seed(42)
|
||||
np.random.seed(42)
|
||||
|
||||
model.summary()
|
||||
device = get_device()
|
||||
print(f"Selected device: {device}")
|
||||
|
||||
history = model.fit(
|
||||
x_train, y_train, epochs=150, batch_size=10, callbacks=[early_stopping]
|
||||
)
|
||||
score = model.evaluate(x_test, y_test, verbose=2)
|
||||
x_train, x_test, y_train, y_test = get_data(seed=42)
|
||||
train_dataset = TensorDataset(x_train, y_train)
|
||||
val_dataset = TensorDataset(x_test, y_test)
|
||||
train_loader = DataLoader(train_dataset, batch_size=10, shuffle=True)
|
||||
val_loader = DataLoader(val_dataset, batch_size=10, shuffle=False)
|
||||
|
||||
model = BeanModel().to(device)
|
||||
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
|
||||
best_val_loss = float("inf")
|
||||
best_state = None
|
||||
patience = 10
|
||||
min_delta = 0.001
|
||||
patience_counter = 0
|
||||
|
||||
for epoch in range(150):
|
||||
model.train()
|
||||
for bx, by in train_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
optimizer.zero_grad()
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
model.eval()
|
||||
val_loss = 0.0
|
||||
total = 0
|
||||
with torch.no_grad():
|
||||
for bx, by in val_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
val_loss += loss.item() * bx.size(0)
|
||||
total += bx.size(0)
|
||||
|
||||
val_loss /= total
|
||||
|
||||
if val_loss < best_val_loss - min_delta:
|
||||
best_val_loss = val_loss
|
||||
best_state = copy.deepcopy(model.state_dict())
|
||||
patience_counter = 0
|
||||
else:
|
||||
patience_counter += 1
|
||||
if patience_counter >= patience:
|
||||
break
|
||||
|
||||
if best_state is not None:
|
||||
model.load_state_dict(best_state)
|
||||
|
||||
model.eval()
|
||||
test_loss = 0.0
|
||||
correct = 0
|
||||
total = 0
|
||||
with torch.no_grad():
|
||||
for bx, by in val_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
test_loss += loss.item() * bx.size(0)
|
||||
preds = out.argmax(dim=1)
|
||||
correct += (preds == by).sum().item()
|
||||
total += bx.size(0)
|
||||
|
||||
test_loss /= total
|
||||
test_acc = correct / total
|
||||
print(f"Final test loss: {test_loss:.4f}, accuracy: {test_acc:.4f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+490
@@ -0,0 +1,490 @@
|
||||
"""
|
||||
Command-Line Interface Helpers for PSO Experiments.
|
||||
|
||||
Provides standard argparse argument groups and helper functions for PSO stage selectors
|
||||
(method, initialization, evaluation, convergence, refinement) and common execution parameters.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from typing import Any
|
||||
|
||||
# Supported stage plugin selector options
|
||||
STAGE_SELECTORS: dict[str, list[str]] = {
|
||||
"method": [
|
||||
"original",
|
||||
"inertia",
|
||||
"constriction",
|
||||
"fips",
|
||||
"clpso",
|
||||
"bare_bones",
|
||||
"adaptive_moment",
|
||||
],
|
||||
"initialization": ["model_noise", "uniform"],
|
||||
"evaluation": ["full", "fixed_subset"],
|
||||
"convergence": ["none", "particle_reset", "early_stopping"],
|
||||
"refinement": ["none", "adam"],
|
||||
}
|
||||
|
||||
|
||||
def add_stage_selector_args(
|
||||
parser: argparse.ArgumentParser, defaults: dict[str, Any] | None = None
|
||||
) -> argparse.ArgumentParser:
|
||||
"""Adds the 5 explicit stage selector arguments to an argparse parser."""
|
||||
defaults = defaults or {}
|
||||
|
||||
parser.add_argument(
|
||||
"--method",
|
||||
type=str,
|
||||
choices=STAGE_SELECTORS["method"],
|
||||
default=defaults.get("method", "original"),
|
||||
help="PSO movement stage method (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--initialization",
|
||||
type=str,
|
||||
choices=STAGE_SELECTORS["initialization"],
|
||||
default=defaults.get("initialization", "model_noise"),
|
||||
help="PSO particle initialization stage (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--evaluation",
|
||||
type=str,
|
||||
choices=STAGE_SELECTORS["evaluation"],
|
||||
default=defaults.get("evaluation", "full"),
|
||||
help="PSO objective evaluation stage (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--convergence",
|
||||
type=str,
|
||||
choices=STAGE_SELECTORS["convergence"],
|
||||
default=defaults.get("convergence", "none"),
|
||||
help="PSO convergence behavior stage (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--refinement",
|
||||
type=str,
|
||||
choices=STAGE_SELECTORS["refinement"],
|
||||
default=defaults.get("refinement", "none"),
|
||||
help="PSO post-search refinement stage (default: %(default)s)",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def add_pso_args(
|
||||
parser: argparse.ArgumentParser, defaults: dict[str, Any] | None = None
|
||||
) -> argparse.ArgumentParser:
|
||||
"""Adds stage selectors and common PSO hyperparameter arguments to an argparse parser."""
|
||||
defaults = defaults or {}
|
||||
|
||||
# Add 5 stage selectors
|
||||
add_stage_selector_args(parser, defaults)
|
||||
|
||||
# Core execution and hyperparameter options
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=defaults.get("seed", 42),
|
||||
help="Random seed for reproducibility (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--device",
|
||||
type=str,
|
||||
default=defaults.get("device", None),
|
||||
help="Execution target device (cpu, cuda, mps) (default: auto)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--n-particles",
|
||||
"--particles",
|
||||
dest="n_particles",
|
||||
type=int,
|
||||
default=defaults.get("n_particles", 30),
|
||||
help="Number of swarm particles (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--epochs",
|
||||
type=int,
|
||||
default=defaults.get("epochs", 80),
|
||||
help="Number of PSO optimization epochs (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
"--batch",
|
||||
dest="batch_size",
|
||||
type=int,
|
||||
default=defaults.get("batch_size", None),
|
||||
help="Batch size for objective evaluation (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fitness-size",
|
||||
type=int,
|
||||
default=defaults.get("fitness_size", None),
|
||||
help="Fixed subset sample count (required when evaluation='fixed_subset')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--refinement-epochs",
|
||||
type=int,
|
||||
default=defaults.get("refinement_epochs", 0),
|
||||
help="Refinement epoch count (required when refinement='adam')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--refinement-lr",
|
||||
type=float,
|
||||
default=defaults.get("refinement_lr", 0.001),
|
||||
help="Refinement learning rate for Adam optimizer (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--renewal",
|
||||
type=str,
|
||||
choices=["acc", "loss", "mse"],
|
||||
default=defaults.get("renewal", "loss"),
|
||||
help="Primary metric for global best selection (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
type=str,
|
||||
default=defaults.get("output_dir", None),
|
||||
help="Directory to save model checkpoints and logs",
|
||||
)
|
||||
|
||||
# Optional coefficient overrides
|
||||
parser.add_argument(
|
||||
"--c0",
|
||||
type=float,
|
||||
default=defaults.get("c0", None),
|
||||
help="Cognitive acceleration coefficient override",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--c1",
|
||||
type=float,
|
||||
default=defaults.get("c1", None),
|
||||
help="Social acceleration coefficient override",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--w-min",
|
||||
dest="w_min",
|
||||
type=float,
|
||||
default=defaults.get("w_min", None),
|
||||
help="Minimum inertia weight override",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--w-max",
|
||||
dest="w_max",
|
||||
type=float,
|
||||
default=defaults.get("w_max", None),
|
||||
help="Maximum inertia weight override",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--negative-swarm",
|
||||
type=float,
|
||||
default=defaults.get("negative_swarm", 0.0),
|
||||
help="Negative swarm velocity coefficient (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--mutation-swarm",
|
||||
type=float,
|
||||
default=defaults.get("mutation_swarm", 0.0),
|
||||
help="Swarm mutation probability (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--particle-min",
|
||||
type=float,
|
||||
default=defaults.get("particle_min", None),
|
||||
help="Lower bound for particle position clamping",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--particle-max",
|
||||
type=float,
|
||||
default=defaults.get("particle_max", None),
|
||||
help="Upper bound for particle position clamping",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--velocity-limit-ratio",
|
||||
type=float,
|
||||
default=defaults.get("velocity_limit_ratio", None),
|
||||
help="Maximum velocity limit ratio relative to search domain",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--boundary-strategy",
|
||||
type=str,
|
||||
choices=["clip", "reflect"],
|
||||
default=defaults.get("boundary_strategy", "clip"),
|
||||
help="Position boundary handling strategy (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--initial-position-noise",
|
||||
type=float,
|
||||
default=defaults.get("initial_position_noise", 0.05),
|
||||
help="Initial position noise scale (default: %(default)s)",
|
||||
)
|
||||
# Convergence stage options
|
||||
parser.add_argument(
|
||||
"--convergence-patience",
|
||||
dest="convergence_patience",
|
||||
type=int,
|
||||
default=defaults.get("convergence_patience", 10),
|
||||
help="Convergence reset/early stopping patience epochs (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--convergence-min-delta",
|
||||
dest="convergence_min_delta",
|
||||
type=float,
|
||||
default=defaults.get("convergence_min_delta", 0.0001),
|
||||
help="Minimum improvement delta for convergence (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--convergence-monitor",
|
||||
dest="convergence_monitor",
|
||||
type=str,
|
||||
choices=["loss", "acc", "accuracy", "mse"],
|
||||
default=defaults.get("convergence_monitor", "loss"),
|
||||
help="Metric monitored for convergence (default: %(default)s)",
|
||||
)
|
||||
|
||||
# Adaptive moment options
|
||||
parser.add_argument(
|
||||
"--moment-blend",
|
||||
dest="moment_blend",
|
||||
type=float,
|
||||
default=defaults.get("moment_blend", None),
|
||||
help="Adaptive moment blend factor (default: 0.25 when method='adaptive_moment', else 0.0)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--moment-beta1",
|
||||
dest="moment_beta1",
|
||||
type=float,
|
||||
default=defaults.get("moment_beta1", None),
|
||||
help="Adaptive moment beta1 parameter (default: 0.9 when method='adaptive_moment')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--moment-beta2",
|
||||
dest="moment_beta2",
|
||||
type=float,
|
||||
default=defaults.get("moment_beta2", None),
|
||||
help="Adaptive moment beta2 parameter (default: 0.999 when method='adaptive_moment')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--moment-step-size",
|
||||
dest="moment_step_size",
|
||||
type=float,
|
||||
default=defaults.get("moment_step_size", None),
|
||||
help="Adaptive moment step size (default: 1.0 when method='adaptive_moment')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--moment-epsilon",
|
||||
dest="moment_epsilon",
|
||||
type=float,
|
||||
default=defaults.get("moment_epsilon", None),
|
||||
help="Adaptive moment epsilon parameter (default: 1e-8 when method='adaptive_moment')",
|
||||
)
|
||||
|
||||
# Repeatable method options
|
||||
parser.add_argument(
|
||||
"--method-option",
|
||||
dest="method_options",
|
||||
action="append",
|
||||
metavar="KEY=VALUE",
|
||||
help="Additional key=value option for movement method (repeatable)",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def parse_method_options(options: list[str] | None) -> dict[str, Any]:
|
||||
"""Parses a list of 'KEY=VALUE' strings into a dictionary with typed values."""
|
||||
res: dict[str, Any] = {}
|
||||
if not options:
|
||||
return res
|
||||
for opt in options:
|
||||
if "=" not in opt:
|
||||
raise ValueError(f"Invalid --method-option format '{opt}', expected 'KEY=VALUE'")
|
||||
key, val = opt.split("=", 1)
|
||||
key = key.strip()
|
||||
val = val.strip()
|
||||
val_lower = val.lower()
|
||||
if val_lower == "true":
|
||||
parsed_val: Any = True
|
||||
elif val_lower == "false":
|
||||
parsed_val = False
|
||||
else:
|
||||
try:
|
||||
parsed_val = int(val)
|
||||
except ValueError:
|
||||
try:
|
||||
parsed_val = float(val)
|
||||
except ValueError:
|
||||
parsed_val = val
|
||||
res[key] = parsed_val
|
||||
return res
|
||||
|
||||
|
||||
def build_optimizer_kwargs(
|
||||
args: argparse.Namespace,
|
||||
*,
|
||||
model: Any = None,
|
||||
loss: Any = None,
|
||||
task: str | None = None,
|
||||
inertia_profile: dict[str, float] | None = None,
|
||||
**extra_kwargs: Any,
|
||||
) -> dict[str, Any]:
|
||||
"""Builds Optimizer constructor keyword arguments from parsed CLI arguments.
|
||||
|
||||
Applies method-specific parameter compatibility rules, workload inertia profiles
|
||||
(only when method='inertia'), repeatable method options (--method-option), and
|
||||
adaptive moment flags (only when method='adaptive_moment').
|
||||
"""
|
||||
method = getattr(args, "method", "original")
|
||||
parsed_method_opts = parse_method_options(getattr(args, "method_options", None))
|
||||
|
||||
if method == "inertia":
|
||||
profile = inertia_profile or {}
|
||||
c0 = (
|
||||
parsed_method_opts["c0"]
|
||||
if "c0" in parsed_method_opts
|
||||
else (args.c0 if getattr(args, "c0", None) is not None else profile.get("c0"))
|
||||
)
|
||||
c1 = (
|
||||
parsed_method_opts["c1"]
|
||||
if "c1" in parsed_method_opts
|
||||
else (args.c1 if getattr(args, "c1", None) is not None else profile.get("c1"))
|
||||
)
|
||||
w_min = (
|
||||
parsed_method_opts["w_min"]
|
||||
if "w_min" in parsed_method_opts
|
||||
else (args.w_min if getattr(args, "w_min", None) is not None else profile.get("w_min"))
|
||||
)
|
||||
w_max = (
|
||||
parsed_method_opts["w_max"]
|
||||
if "w_max" in parsed_method_opts
|
||||
else (args.w_max if getattr(args, "w_max", None) is not None else profile.get("w_max"))
|
||||
)
|
||||
elif method in ("original", "constriction", "fips"):
|
||||
c0 = (
|
||||
parsed_method_opts["c0"]
|
||||
if "c0" in parsed_method_opts
|
||||
else getattr(args, "c0", None)
|
||||
)
|
||||
c1 = (
|
||||
parsed_method_opts["c1"]
|
||||
if "c1" in parsed_method_opts
|
||||
else getattr(args, "c1", None)
|
||||
)
|
||||
w_min = None
|
||||
w_max = None
|
||||
elif method == "bare_bones":
|
||||
c0 = None
|
||||
c1 = None
|
||||
w_min = None
|
||||
w_max = None
|
||||
else:
|
||||
c0 = (
|
||||
parsed_method_opts["c0"]
|
||||
if "c0" in parsed_method_opts
|
||||
else getattr(args, "c0", None)
|
||||
)
|
||||
c1 = (
|
||||
parsed_method_opts["c1"]
|
||||
if "c1" in parsed_method_opts
|
||||
else getattr(args, "c1", None)
|
||||
)
|
||||
w_min = (
|
||||
parsed_method_opts["w_min"]
|
||||
if "w_min" in parsed_method_opts
|
||||
else getattr(args, "w_min", None)
|
||||
)
|
||||
w_max = (
|
||||
parsed_method_opts["w_max"]
|
||||
if "w_max" in parsed_method_opts
|
||||
else getattr(args, "w_max", None)
|
||||
)
|
||||
|
||||
if method in ("fips", "clpso", "bare_bones"):
|
||||
neg_swarm = 0.0
|
||||
else:
|
||||
neg_swarm = (
|
||||
float(parsed_method_opts["negative_swarm"])
|
||||
if "negative_swarm" in parsed_method_opts
|
||||
else float(getattr(args, "negative_swarm", 0.0))
|
||||
)
|
||||
|
||||
if method == "bare_bones":
|
||||
mut_swarm = 0.0
|
||||
else:
|
||||
mut_swarm = (
|
||||
float(parsed_method_opts["mutation_swarm"])
|
||||
if "mutation_swarm" in parsed_method_opts
|
||||
else float(getattr(args, "mutation_swarm", 0.0))
|
||||
)
|
||||
|
||||
if method == "bare_bones":
|
||||
vel_ratio = None
|
||||
else:
|
||||
vel_ratio = (
|
||||
parsed_method_opts["velocity_limit_ratio"]
|
||||
if "velocity_limit_ratio" in parsed_method_opts
|
||||
else getattr(args, "velocity_limit_ratio", None)
|
||||
)
|
||||
|
||||
fitness_size = (
|
||||
getattr(args, "fitness_size", None)
|
||||
if getattr(args, "evaluation", None) == "fixed_subset"
|
||||
else None
|
||||
)
|
||||
refinement_epochs = (
|
||||
getattr(args, "refinement_epochs", 0)
|
||||
if getattr(args, "refinement", None) == "adam"
|
||||
else 0
|
||||
)
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": model,
|
||||
"loss": loss,
|
||||
"task": task,
|
||||
"method": method,
|
||||
"initialization": getattr(args, "initialization", "model_noise"),
|
||||
"evaluation": getattr(args, "evaluation", "full"),
|
||||
"convergence": getattr(args, "convergence", "none"),
|
||||
"refinement": getattr(args, "refinement", "none"),
|
||||
"method_options": parsed_method_opts,
|
||||
"n_particles": getattr(args, "n_particles", 30),
|
||||
"c0": c0,
|
||||
"c1": c1,
|
||||
"w_min": w_min,
|
||||
"w_max": w_max,
|
||||
"negative_swarm": neg_swarm,
|
||||
"mutation_swarm": mut_swarm,
|
||||
"particle_min": getattr(args, "particle_min", None),
|
||||
"particle_max": getattr(args, "particle_max", None),
|
||||
"velocity_limit_ratio": vel_ratio,
|
||||
"boundary_strategy": getattr(args, "boundary_strategy", "clip"),
|
||||
"initial_position_noise": getattr(args, "initial_position_noise", 0.05),
|
||||
"seed": getattr(args, "seed", None),
|
||||
"device": getattr(args, "device", None),
|
||||
"fitness_size": fitness_size,
|
||||
"convergence_patience": getattr(args, "convergence_patience", 10),
|
||||
"convergence_min_delta": getattr(args, "convergence_min_delta", 0.0001),
|
||||
"convergence_monitor": getattr(args, "convergence_monitor", "loss"),
|
||||
"refinement_epochs": refinement_epochs,
|
||||
"refinement_lr": getattr(args, "refinement_lr", 0.001),
|
||||
}
|
||||
|
||||
if method == "adaptive_moment":
|
||||
if "moment_blend" in parsed_method_opts:
|
||||
m_blend = float(parsed_method_opts["moment_blend"])
|
||||
elif getattr(args, "moment_blend", None) is not None:
|
||||
m_blend = float(getattr(args, "moment_blend"))
|
||||
else:
|
||||
m_blend = 0.25
|
||||
kwargs["moment_blend"] = m_blend
|
||||
|
||||
for param_name in (
|
||||
"moment_beta1",
|
||||
"moment_beta2",
|
||||
"moment_step_size",
|
||||
"moment_epsilon",
|
||||
):
|
||||
if param_name in parsed_method_opts:
|
||||
kwargs[param_name] = float(parsed_method_opts[param_name])
|
||||
elif getattr(args, param_name, None) is not None:
|
||||
kwargs[param_name] = float(getattr(args, param_name))
|
||||
|
||||
kwargs.update(extra_kwargs)
|
||||
return kwargs
|
||||
Executable
+427
@@ -0,0 +1,427 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
PSO Stage Plugin Method Comparison Tool.
|
||||
|
||||
Compares PSO movement methods (original, inertia, constriction, fips, clpso, bare_bones,
|
||||
adaptive_moment, local_best, quantum)
|
||||
across benchmark datasets (xor, iris, mnist) with reproducible model initialization, dataset splits,
|
||||
and clean console output & JSON result reporting.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from cli import add_pso_args, parse_method_options
|
||||
from pso import Optimizer
|
||||
from pso.plugins import available_plugins
|
||||
|
||||
|
||||
def print_available_methods():
|
||||
"""Prints available movement method plugins and metadata provenance."""
|
||||
movement_plugins = available_plugins("movement")
|
||||
print("Available PSO Movement Method Plugins:")
|
||||
print("=" * 80)
|
||||
for name, meta in movement_plugins.items():
|
||||
print(f" Stage Key : {name}")
|
||||
print(f" Title : {meta.title}")
|
||||
print(f" Source (DOI) : {meta.source or 'N/A'}")
|
||||
print(f" Fidelity : {meta.fidelity}")
|
||||
print(f" Needs Grad : {meta.gradient_required}")
|
||||
print("-" * 80)
|
||||
|
||||
|
||||
def get_xor_workload(seed: int):
|
||||
"""Builds identical XOR dataset and PyTorch model state for a given seed."""
|
||||
torch.manual_seed(seed)
|
||||
x_train = torch.tensor(
|
||||
[[0.0, 0.0], [0.0, 1.0], [1.0, 0.0], [1.0, 1.0]], dtype=torch.float32
|
||||
)
|
||||
y_train = torch.tensor([[0.0], [1.0], [1.0], [0.0]], dtype=torch.float32)
|
||||
|
||||
model = nn.Sequential(
|
||||
nn.Linear(2, 4),
|
||||
nn.Tanh(),
|
||||
nn.Linear(4, 1),
|
||||
)
|
||||
loss_fn = nn.BCEWithLogitsLoss()
|
||||
return x_train, y_train, None, model, loss_fn, "binary"
|
||||
|
||||
|
||||
def get_iris_workload(seed: int):
|
||||
"""Builds identical Iris dataset and PyTorch model state for a given seed."""
|
||||
from sklearn.datasets import load_iris
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
|
||||
torch.manual_seed(seed)
|
||||
iris = load_iris()
|
||||
X = iris.data.astype("float32")
|
||||
y = iris.target.astype("int64")
|
||||
|
||||
x_train, x_test, y_train, y_test = train_test_split(
|
||||
X, y, test_size=0.2, shuffle=True, stratify=y, random_state=seed
|
||||
)
|
||||
scaler = StandardScaler()
|
||||
x_train = scaler.fit_transform(x_train)
|
||||
x_test = scaler.transform(x_test)
|
||||
|
||||
x_tr = torch.tensor(x_train, dtype=torch.float32)
|
||||
y_tr = torch.tensor(y_train, dtype=torch.int64)
|
||||
x_te = torch.tensor(x_test, dtype=torch.float32)
|
||||
y_te = torch.tensor(y_test, dtype=torch.int64)
|
||||
|
||||
model = nn.Sequential(
|
||||
nn.Linear(4, 10),
|
||||
nn.ReLU(),
|
||||
nn.Linear(10, 10),
|
||||
nn.ReLU(),
|
||||
nn.Linear(10, 3),
|
||||
)
|
||||
loss_fn = nn.CrossEntropyLoss()
|
||||
return x_tr, y_tr, (x_te, y_te), model, loss_fn, "multiclass"
|
||||
|
||||
|
||||
def get_mnist_workload(seed: int):
|
||||
"""Builds identical PCA32 MNIST dataset and PyTorch model state for a given seed."""
|
||||
from sklearn.decomposition import PCA
|
||||
from torchvision.datasets import MNIST
|
||||
|
||||
torch.manual_seed(seed)
|
||||
train_ds = MNIST(root="./data", train=True, download=True)
|
||||
test_ds = MNIST(root="./data", train=False, download=True)
|
||||
|
||||
x_tr_raw = (train_ds.data[:3000].float() / 255.0).reshape(3000, -1).numpy()
|
||||
y_tr = train_ds.targets[:3000].long()
|
||||
x_te_raw = (test_ds.data[:1000].float() / 255.0).reshape(1000, -1).numpy()
|
||||
y_te = test_ds.targets[:1000].long()
|
||||
|
||||
pca = PCA(n_components=32, whiten=True, random_state=seed)
|
||||
x_tr_pca = pca.fit_transform(x_tr_raw)
|
||||
x_te_pca = pca.transform(x_te_raw)
|
||||
|
||||
x_tr = torch.tensor(x_tr_pca, dtype=torch.float32)
|
||||
x_te = torch.tensor(x_te_pca, dtype=torch.float32)
|
||||
|
||||
model = nn.Linear(32, 10)
|
||||
loss_fn = nn.CrossEntropyLoss()
|
||||
return x_tr, y_tr, (x_te, y_te), model, loss_fn, "multiclass"
|
||||
|
||||
|
||||
DATASET_LOADERS = {
|
||||
"xor": get_xor_workload,
|
||||
"iris": get_iris_workload,
|
||||
"mnist": get_mnist_workload,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="PSO Stage Plugin Method Comparison Surface"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--list-methods",
|
||||
action="store_true",
|
||||
help="List available movement methods and exit",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dataset",
|
||||
choices=["xor", "iris", "mnist"],
|
||||
default="xor",
|
||||
help="Target dataset workload (default: %(default)s)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--methods",
|
||||
nargs="+",
|
||||
default=["all"],
|
||||
help="Movement method keys to evaluate or 'all' (default: all)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seeds",
|
||||
type=int,
|
||||
nargs="+",
|
||||
default=[42],
|
||||
help="Random seed list (default: 42)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json-path",
|
||||
"--output-json",
|
||||
"--json",
|
||||
dest="json_path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Optional JSON file path to save detailed evaluation metrics",
|
||||
)
|
||||
|
||||
# Add standard stage selector and hyperparameter options
|
||||
add_pso_args(parser, defaults={"n_particles": 20, "epochs": 30})
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.list_methods:
|
||||
print_available_methods()
|
||||
sys.exit(0)
|
||||
|
||||
# Determine movement methods to test
|
||||
available_m_plugins = available_plugins("movement")
|
||||
if "all" in args.methods or "ALL" in args.methods:
|
||||
methods_to_test = list(available_m_plugins.keys())
|
||||
else:
|
||||
methods_to_test = []
|
||||
for m in args.methods:
|
||||
if m not in available_m_plugins:
|
||||
raise ValueError(
|
||||
f"Unknown movement method '{m}'. Available: {list(available_m_plugins.keys())}"
|
||||
)
|
||||
methods_to_test.append(m)
|
||||
|
||||
loader = DATASET_LOADERS[args.dataset]
|
||||
eval_stage = args.evaluation
|
||||
fitness_size = args.fitness_size if eval_stage == "fixed_subset" else None
|
||||
|
||||
if eval_stage == "fixed_subset" and fitness_size is None:
|
||||
if args.dataset == "xor":
|
||||
fitness_size = 4
|
||||
elif args.dataset == "iris":
|
||||
fitness_size = 100
|
||||
elif args.dataset == "mnist":
|
||||
fitness_size = 2000
|
||||
refine_stage = args.refinement
|
||||
refinement_epochs = args.refinement_epochs if refine_stage == "adam" else 0
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
parsed_method_opts = parse_method_options(args.method_options)
|
||||
has_val = args.dataset != "xor"
|
||||
|
||||
print("=" * 80)
|
||||
print(f"PSO Movement Method Comparison on '{args.dataset}' Dataset")
|
||||
print(
|
||||
f"Stages: initialization='{args.initialization}', evaluation='{eval_stage}', "
|
||||
f"convergence='{args.convergence}', refinement='{refine_stage}'"
|
||||
)
|
||||
print(
|
||||
f"Parameters: particles={args.n_particles}, epochs={args.epochs}, seeds={args.seeds}"
|
||||
)
|
||||
print("=" * 80)
|
||||
if has_val:
|
||||
print(
|
||||
f"{'Method':<18} {'Seed':<6} {'Val Loss':<12} {'Val Accuracy':<12} {'Val MSE':<12} {'Time (s)':<10}"
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"{'Method':<18} {'Seed':<6} {'Loss':<12} {'Accuracy':<12} {'MSE':<12} {'Time (s)':<10}"
|
||||
)
|
||||
print("-" * 80)
|
||||
|
||||
for method_key in methods_to_test:
|
||||
for seed in args.seeds:
|
||||
x_tr, y_tr, val_data, model, loss_fn, task = loader(seed)
|
||||
|
||||
eff_fitness_size = (
|
||||
min(fitness_size, x_tr.shape[0])
|
||||
if fitness_size is not None
|
||||
else None
|
||||
)
|
||||
|
||||
c0 = args.c0
|
||||
c1 = args.c1
|
||||
w_min = args.w_min
|
||||
w_max = args.w_max
|
||||
|
||||
neg_swarm = (
|
||||
args.negative_swarm
|
||||
if method_key not in ("fips", "clpso", "bare_bones")
|
||||
else 0.0
|
||||
)
|
||||
mut_swarm = args.mutation_swarm if method_key != "bare_bones" else 0.0
|
||||
vel_ratio = (
|
||||
args.velocity_limit_ratio if method_key != "bare_bones" else None
|
||||
)
|
||||
|
||||
kwargs: dict[str, Any] = {
|
||||
"model": model,
|
||||
"loss": loss_fn,
|
||||
"task": task,
|
||||
"method": method_key,
|
||||
"initialization": args.initialization,
|
||||
"evaluation": eval_stage,
|
||||
"convergence": args.convergence,
|
||||
"refinement": refine_stage,
|
||||
"method_options": parsed_method_opts,
|
||||
"n_particles": args.n_particles,
|
||||
"c0": c0,
|
||||
"c1": c1,
|
||||
"w_min": w_min,
|
||||
"w_max": w_max,
|
||||
"negative_swarm": neg_swarm,
|
||||
"mutation_swarm": mut_swarm,
|
||||
"particle_min": args.particle_min,
|
||||
"particle_max": args.particle_max,
|
||||
"velocity_limit_ratio": vel_ratio,
|
||||
"boundary_strategy": args.boundary_strategy,
|
||||
"initial_position_noise": args.initial_position_noise,
|
||||
"seed": seed,
|
||||
"device": args.device,
|
||||
"fitness_size": eff_fitness_size,
|
||||
"convergence_patience": args.convergence_patience,
|
||||
"convergence_min_delta": args.convergence_min_delta,
|
||||
"convergence_monitor": args.convergence_monitor,
|
||||
"refinement_epochs": refinement_epochs,
|
||||
"refinement_lr": args.refinement_lr,
|
||||
}
|
||||
|
||||
if method_key == "adaptive_moment":
|
||||
if args.moment_blend is not None:
|
||||
kwargs["moment_blend"] = args.moment_blend
|
||||
elif "moment_blend" in parsed_method_opts:
|
||||
kwargs["moment_blend"] = float(parsed_method_opts["moment_blend"])
|
||||
else:
|
||||
kwargs["moment_blend"] = 0.25
|
||||
|
||||
if args.moment_beta1 is not None:
|
||||
kwargs["moment_beta1"] = args.moment_beta1
|
||||
if args.moment_beta2 is not None:
|
||||
kwargs["moment_beta2"] = args.moment_beta2
|
||||
if args.moment_step_size is not None:
|
||||
kwargs["moment_step_size"] = args.moment_step_size
|
||||
if args.moment_epsilon is not None:
|
||||
kwargs["moment_epsilon"] = args.moment_epsilon
|
||||
|
||||
opt = Optimizer(**kwargs)
|
||||
|
||||
start_t = time.perf_counter()
|
||||
best_score = opt.fit(
|
||||
x_tr,
|
||||
y_tr,
|
||||
epochs=args.epochs,
|
||||
batch_size=args.batch_size,
|
||||
fitness_size=eff_fitness_size,
|
||||
renewal=args.renewal,
|
||||
validation_data=val_data,
|
||||
output_dir=None,
|
||||
refinement_epochs=refinement_epochs,
|
||||
refinement_lr=args.refinement_lr,
|
||||
)
|
||||
elapsed_t = time.perf_counter() - start_t
|
||||
|
||||
tr_loss, tr_acc, tr_mse = best_score
|
||||
if val_data is not None:
|
||||
val_x, val_y = val_data
|
||||
val_score = opt.evaluate(val_x, val_y)
|
||||
eval_loss, eval_acc, eval_mse = val_score
|
||||
score_src = "validation"
|
||||
else:
|
||||
eval_loss, eval_acc, eval_mse = tr_loss, tr_acc, tr_mse
|
||||
score_src = "training"
|
||||
|
||||
results.append(
|
||||
{
|
||||
"method": method_key,
|
||||
"seed": seed,
|
||||
"score_source": score_src,
|
||||
"train_loss": tr_loss,
|
||||
"train_accuracy": tr_acc,
|
||||
"train_mse": tr_mse,
|
||||
"eval_loss": eval_loss,
|
||||
"eval_accuracy": eval_acc,
|
||||
"eval_mse": eval_mse,
|
||||
"loss": eval_loss,
|
||||
"accuracy": eval_acc,
|
||||
"mse": eval_mse,
|
||||
"elapsed_time": elapsed_t,
|
||||
}
|
||||
)
|
||||
|
||||
print(
|
||||
f"{method_key:<18} {seed:<6} {eval_loss:<12.6f} {eval_acc:<12.6f} {eval_mse:<12.6f} {elapsed_t:<10.4f}"
|
||||
)
|
||||
|
||||
print("-" * 80)
|
||||
print("\nAggregate Summary (Mean across seeds):")
|
||||
print("=" * 80)
|
||||
if has_val:
|
||||
print(
|
||||
f"{'Method':<18} {'Mean Val Loss':<14} {'Mean Val Acc':<14} {'Mean Val MSE':<14} {'Mean Time (s)':<12}"
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"{'Method':<18} {'Mean Loss':<12} {'Mean Acc':<12} {'Mean MSE':<12} {'Mean Time (s)':<12}"
|
||||
)
|
||||
print("-" * 80)
|
||||
|
||||
summary_list: list[dict[str, Any]] = []
|
||||
for method_key in methods_to_test:
|
||||
method_runs = [r for r in results if r["method"] == method_key]
|
||||
if not method_runs:
|
||||
continue
|
||||
n_runs = len(method_runs)
|
||||
mean_tr_loss = sum(r["train_loss"] for r in method_runs) / n_runs
|
||||
mean_tr_acc = sum(r["train_accuracy"] for r in method_runs) / n_runs
|
||||
mean_tr_mse = sum(r["train_mse"] for r in method_runs) / n_runs
|
||||
mean_eval_loss = sum(r["eval_loss"] for r in method_runs) / n_runs
|
||||
mean_eval_acc = sum(r["eval_accuracy"] for r in method_runs) / n_runs
|
||||
mean_eval_mse = sum(r["eval_mse"] for r in method_runs) / n_runs
|
||||
mean_time = sum(r["elapsed_time"] for r in method_runs) / n_runs
|
||||
score_src = method_runs[0]["score_source"]
|
||||
|
||||
summary_entry = {
|
||||
"method": method_key,
|
||||
"title": available_m_plugins[method_key].title,
|
||||
"source": available_m_plugins[method_key].source,
|
||||
"score_source": score_src,
|
||||
"mean_train_loss": mean_tr_loss,
|
||||
"mean_train_accuracy": mean_tr_acc,
|
||||
"mean_train_mse": mean_tr_mse,
|
||||
"mean_eval_loss": mean_eval_loss,
|
||||
"mean_eval_accuracy": mean_eval_acc,
|
||||
"mean_eval_mse": mean_eval_mse,
|
||||
"mean_loss": mean_eval_loss,
|
||||
"mean_accuracy": mean_eval_acc,
|
||||
"mean_mse": mean_eval_mse,
|
||||
"mean_elapsed_time": mean_time,
|
||||
"runs": n_runs,
|
||||
}
|
||||
summary_list.append(summary_entry)
|
||||
|
||||
if has_val:
|
||||
print(
|
||||
f"{method_key:<18} {mean_eval_loss:<14.6f} {mean_eval_acc:<14.6f} {mean_eval_mse:<14.6f} {mean_time:<12.4f}"
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"{method_key:<18} {mean_eval_loss:<12.6f} {mean_eval_acc:<12.6f} {mean_eval_mse:<12.6f} {mean_time:<12.4f}"
|
||||
)
|
||||
print("=" * 80)
|
||||
|
||||
if args.json_path:
|
||||
payload = {
|
||||
"dataset": args.dataset,
|
||||
"score_source": "validation" if has_val else "training",
|
||||
"selectors": {
|
||||
"initialization": args.initialization,
|
||||
"evaluation": eval_stage,
|
||||
"convergence": args.convergence,
|
||||
"refinement": refine_stage,
|
||||
},
|
||||
"parameters": {
|
||||
"n_particles": args.n_particles,
|
||||
"epochs": args.epochs,
|
||||
"batch_size": args.batch_size,
|
||||
"fitness_size": fitness_size,
|
||||
"refinement_epochs": refinement_epochs,
|
||||
"refinement_lr": args.refinement_lr,
|
||||
"seeds": args.seeds,
|
||||
},
|
||||
"results": results,
|
||||
"summary": summary_list,
|
||||
}
|
||||
with open(args.json_path, "w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, indent=2)
|
||||
print(f"\nSaved comparison results JSON to: {args.json_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,932 @@
|
||||
"""
|
||||
MNIST Deep Accuracy Study: Architecture vs Optimizer Profiles
|
||||
|
||||
Evaluates official MNIST (60,000 train / 10,000 test) across:
|
||||
1. Architecture Lane: Raw Linear, Raw MLP, Compact CNN under standard full-data Adam.
|
||||
2. Optimizer Lane: Adam-Only, PSO-Only (adaptive_moment on 2k subset), and Hybrid (PSO warm start + Adam fine-tuning) on Compact CNN.
|
||||
|
||||
Contract:
|
||||
- Official 60k train / 10k test split with train-only statistics normalization (no PCA).
|
||||
- Avoid BatchNorm/Dropout so PSO and eval semantics match.
|
||||
- Seed model construction identically per seed to preserve explicit initial state fingerprint across lanes.
|
||||
- Fixed no-scheduler contract for Adam with CrossEntropyLoss and lr=1e-3.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import datetime
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
# Ensure test/ directory is in sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from benchmark_suite import (
|
||||
calc_stats,
|
||||
compute_data_fingerprint,
|
||||
compute_model_fingerprint,
|
||||
extract_plugin_metadata,
|
||||
get_hardware_provenance,
|
||||
resolve_execution_device,
|
||||
save_json_atomic,
|
||||
sync_device,
|
||||
)
|
||||
from pso import Optimizer, __version__ as pso_version
|
||||
|
||||
DEEP_ACCURACY_PROTOCOL_VERSION = "1.0.0"
|
||||
|
||||
|
||||
# ==========================================
|
||||
# Data Preparation (No PCA, Raw 1x28x28)
|
||||
# ==========================================
|
||||
|
||||
def prepare_deep_accuracy_mnist_data() -> Tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
str,
|
||||
Dict[str, Any],
|
||||
]:
|
||||
"""
|
||||
Loads official torchvision MNIST dataset (60,000 train / 10,000 test).
|
||||
Normalizes images using train-only mean and std (no PCA).
|
||||
Validates sample counts and label range [0, 9].
|
||||
Returns (x_train, y_train, x_test, y_test, data_fingerprint, provenance_dict).
|
||||
"""
|
||||
from torchvision.datasets import MNIST
|
||||
|
||||
cache_dir = Path("result/cache")
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
train_dataset = MNIST(root=str(cache_dir), train=True, download=True)
|
||||
test_dataset = MNIST(root=str(cache_dir), train=False, download=True)
|
||||
|
||||
n_train = len(train_dataset.data)
|
||||
n_test = len(test_dataset.data)
|
||||
if n_train != 60000:
|
||||
raise ValueError(f"Expected 60,000 training samples; got {n_train}")
|
||||
if n_test != 10000:
|
||||
raise ValueError(f"Expected 10,000 test samples; got {n_test}")
|
||||
|
||||
y_train = train_dataset.targets.long()
|
||||
y_test = test_dataset.targets.long()
|
||||
|
||||
min_tr, max_tr = int(y_train.min()), int(y_train.max())
|
||||
min_te, max_te = int(y_test.min()), int(y_test.max())
|
||||
if min_tr != 0 or max_tr != 9:
|
||||
raise ValueError(f"Train label range must be [0, 9]; got [{min_tr}, {max_tr}]")
|
||||
if min_te != 0 or max_te != 9:
|
||||
raise ValueError(f"Test label range must be [0, 9]; got [{min_te}, {max_te}]")
|
||||
|
||||
x_train_raw = train_dataset.data.float() / 255.0 # (60000, 28, 28)
|
||||
x_test_raw = test_dataset.data.float() / 255.0 # (10000, 28, 28)
|
||||
|
||||
# Compute normalization statistics from TRAIN split only
|
||||
mean_val = float(x_train_raw.mean())
|
||||
std_val = float(x_train_raw.std())
|
||||
|
||||
x_train_norm = ((x_train_raw - mean_val) / std_val).unsqueeze(1) # (60000, 1, 28, 28)
|
||||
x_test_norm = ((x_test_raw - mean_val) / std_val).unsqueeze(1) # (10000, 1, 28, 28)
|
||||
|
||||
data_fp = compute_data_fingerprint(x_train_norm, x_test_norm, y_train, y_test)
|
||||
normalization_provenance = {
|
||||
"input_shape": [1, 28, 28],
|
||||
"pca": False,
|
||||
"raw_inputs": True,
|
||||
"normalization_scope": "official_train_split_60000_only",
|
||||
"train_mean": round(mean_val, 6),
|
||||
"train_std": round(std_val, 6),
|
||||
"train_samples": 60000,
|
||||
"test_samples": 10000,
|
||||
}
|
||||
return x_train_norm, y_train, x_test_norm, y_test, data_fp, normalization_provenance
|
||||
|
||||
|
||||
# ==========================================
|
||||
# Architectures (No BatchNorm / No Dropout)
|
||||
# ==========================================
|
||||
|
||||
def count_parameters(model: nn.Module) -> int:
|
||||
return sum(p.numel() for p in model.parameters() if p.requires_grad)
|
||||
|
||||
|
||||
def make_raw_linear(seed: int = 41) -> nn.Module:
|
||||
torch.manual_seed(seed)
|
||||
return nn.Sequential(
|
||||
nn.Flatten(),
|
||||
nn.Linear(784, 10),
|
||||
)
|
||||
|
||||
|
||||
def make_raw_mlp(seed: int = 41) -> nn.Module:
|
||||
torch.manual_seed(seed)
|
||||
return nn.Sequential(
|
||||
nn.Flatten(),
|
||||
nn.Linear(784, 128),
|
||||
nn.ReLU(),
|
||||
nn.Linear(128, 64),
|
||||
nn.ReLU(),
|
||||
nn.Linear(64, 10),
|
||||
)
|
||||
|
||||
|
||||
class CompactCNN(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.conv1 = nn.Conv2d(1, 8, kernel_size=3, padding=1)
|
||||
self.relu1 = nn.ReLU()
|
||||
self.pool1 = nn.MaxPool2d(2, 2)
|
||||
self.conv2 = nn.Conv2d(8, 16, kernel_size=3, padding=1)
|
||||
self.relu2 = nn.ReLU()
|
||||
self.pool2 = nn.MaxPool2d(2, 2)
|
||||
self.flatten = nn.Flatten()
|
||||
self.fc = nn.Linear(784, 10)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if x.dim() == 2 and x.shape[1] == 784:
|
||||
x = x.view(-1, 1, 28, 28)
|
||||
out = self.pool1(self.relu1(self.conv1(x)))
|
||||
out = self.pool2(self.relu2(self.conv2(out)))
|
||||
out = self.flatten(out)
|
||||
return self.fc(out)
|
||||
|
||||
|
||||
def make_compact_cnn(seed: int = 41) -> nn.Module:
|
||||
torch.manual_seed(seed)
|
||||
return CompactCNN()
|
||||
|
||||
|
||||
# ==========================================
|
||||
# Evaluation & Training Routines
|
||||
# ==========================================
|
||||
|
||||
def evaluate_model_on_test(
|
||||
model: nn.Module,
|
||||
x_test: torch.Tensor,
|
||||
y_test: torch.Tensor,
|
||||
device: torch.device,
|
||||
batch_size: int = 512,
|
||||
) -> Tuple[float, float]:
|
||||
"""
|
||||
Evaluates model on test data without gradients.
|
||||
Returns (test_loss, test_accuracy).
|
||||
"""
|
||||
model.to(device)
|
||||
model.eval()
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
total_loss = 0.0
|
||||
correct = 0
|
||||
total = 0
|
||||
|
||||
with torch.no_grad():
|
||||
n_test = x_test.shape[0]
|
||||
for i in range(0, n_test, batch_size):
|
||||
bx = x_test[i : i + batch_size].to(device)
|
||||
by = y_test[i : i + batch_size].to(device)
|
||||
outputs = model(bx)
|
||||
loss = criterion(outputs, by)
|
||||
total_loss += loss.item() * bx.size(0)
|
||||
preds = outputs.argmax(dim=1)
|
||||
correct += (preds == by).sum().item()
|
||||
total += bx.size(0)
|
||||
|
||||
avg_loss = total_loss / total if total > 0 else 0.0
|
||||
accuracy = correct / total if total > 0 else 0.0
|
||||
return float(avg_loss), float(accuracy)
|
||||
|
||||
|
||||
def train_adam_routine(
|
||||
model: nn.Module,
|
||||
x_train: torch.Tensor,
|
||||
y_train: torch.Tensor,
|
||||
x_test: torch.Tensor,
|
||||
y_test: torch.Tensor,
|
||||
epochs: int,
|
||||
batch_size: int,
|
||||
lr: float,
|
||||
seed: int,
|
||||
device: torch.device,
|
||||
) -> Tuple[List[Dict[str, Any]], float, float, float]:
|
||||
"""
|
||||
Standard full-data Adam training routine with CrossEntropyLoss, Adam lr, fixed no-scheduler contract.
|
||||
Returns (history, final_test_loss, final_test_acc, elapsed_sec).
|
||||
"""
|
||||
model.to(device)
|
||||
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
|
||||
generator = torch.Generator().manual_seed(seed)
|
||||
n_train = x_train.shape[0]
|
||||
|
||||
history = []
|
||||
init_loss, init_acc = evaluate_model_on_test(model, x_test, y_test, device)
|
||||
history.append({"epoch": 0, "test_loss": round(init_loss, 6), "test_acc": round(init_acc, 6)})
|
||||
|
||||
sync_device(device)
|
||||
t0 = time.time()
|
||||
for epoch in range(1, epochs + 1):
|
||||
model.train()
|
||||
perm = torch.randperm(n_train, generator=generator)
|
||||
for i in range(0, n_train, batch_size):
|
||||
indices = perm[i : i + batch_size]
|
||||
bx = x_train[indices].to(device)
|
||||
by = y_train[indices].to(device)
|
||||
optimizer.zero_grad()
|
||||
outputs = model(bx)
|
||||
loss = criterion(outputs, by)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
test_loss, test_acc = evaluate_model_on_test(model, x_test, y_test, device)
|
||||
history.append({
|
||||
"epoch": epoch,
|
||||
"test_loss": round(test_loss, 6),
|
||||
"test_acc": round(test_acc, 6),
|
||||
})
|
||||
|
||||
sync_device(device)
|
||||
elapsed = time.time() - t0
|
||||
|
||||
final_loss = history[-1]["test_loss"]
|
||||
final_acc = history[-1]["test_acc"]
|
||||
return history, final_loss, final_acc, elapsed
|
||||
|
||||
|
||||
def run_pso_routine(
|
||||
model: nn.Module,
|
||||
x_train: torch.Tensor,
|
||||
y_train: torch.Tensor,
|
||||
x_test: torch.Tensor,
|
||||
y_test: torch.Tensor,
|
||||
pso_epochs: int,
|
||||
n_particles: int,
|
||||
fitness_size: int,
|
||||
seed: int,
|
||||
device: torch.device,
|
||||
) -> Tuple[nn.Module, Dict[str, float], float, float, Dict[str, Any]]:
|
||||
"""
|
||||
PSO-only adaptive_moment on fixed train subset without gradient refinement.
|
||||
Returns (best_model, fitness_score_dict, test_loss, test_acc, pso_metadata).
|
||||
"""
|
||||
model.to(device)
|
||||
loss_fn = nn.CrossEntropyLoss()
|
||||
|
||||
opt = Optimizer(
|
||||
model=model,
|
||||
loss=loss_fn,
|
||||
task="multiclass",
|
||||
method="adaptive_moment",
|
||||
evaluation="fixed_subset",
|
||||
fitness_size=fitness_size,
|
||||
n_particles=n_particles,
|
||||
c0=1.49618,
|
||||
c1=1.49618,
|
||||
w_min=0.7298,
|
||||
w_max=0.7298,
|
||||
particle_min=-3.0,
|
||||
particle_max=3.0,
|
||||
boundary_strategy="reflect",
|
||||
velocity_limit_ratio=0.025,
|
||||
mutation_swarm=0.02,
|
||||
initialization="model_noise",
|
||||
initial_position_noise=0.05,
|
||||
moment_blend=0.06,
|
||||
moment_step_size=0.5,
|
||||
moment_beta1=0.9,
|
||||
seed=seed,
|
||||
device=device,
|
||||
)
|
||||
|
||||
t0 = time.time()
|
||||
opt.fit(x_train, y_train, epochs=pso_epochs)
|
||||
sync_device(device)
|
||||
elapsed = time.time() - t0
|
||||
|
||||
best_model = opt.get_best_model()
|
||||
best_score_tuple = opt.get_best_score()
|
||||
fitness_score = {
|
||||
"subset_loss": round(float(best_score_tuple[0]), 6),
|
||||
"subset_acc": round(float(best_score_tuple[1]), 6),
|
||||
"subset_mse": round(float(best_score_tuple[2]), 6),
|
||||
}
|
||||
|
||||
test_loss, test_acc = evaluate_model_on_test(best_model, x_test, y_test, device)
|
||||
plugin_meta = extract_plugin_metadata(opt)
|
||||
|
||||
pso_meta = {
|
||||
"elapsed_sec": round(elapsed, 4),
|
||||
"particles": n_particles,
|
||||
"pso_epochs": pso_epochs,
|
||||
"fitness_size": fitness_size,
|
||||
"plugins": plugin_meta,
|
||||
}
|
||||
return best_model, fitness_score, float(test_loss), float(test_acc), pso_meta
|
||||
|
||||
|
||||
# ==========================================
|
||||
# Output Generation (CSV, Plot)
|
||||
# ==========================================
|
||||
|
||||
def save_csv_records(records: List[Dict[str, Any]], csv_path: Path):
|
||||
csv_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fieldnames = [
|
||||
"lane",
|
||||
"profile_or_arch",
|
||||
"seed",
|
||||
"model_name",
|
||||
"param_count",
|
||||
"initial_test_acc",
|
||||
"final_test_acc",
|
||||
"final_test_loss",
|
||||
"subset_fitness_acc",
|
||||
"subset_fitness_loss",
|
||||
"pso_epochs",
|
||||
"adam_epochs",
|
||||
"elapsed_sec",
|
||||
"model_fingerprint",
|
||||
"data_fingerprint",
|
||||
]
|
||||
with open(csv_path, "w", newline="", encoding="utf-8") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
for r in records:
|
||||
writer.writerow({
|
||||
"lane": r.get("lane"),
|
||||
"profile_or_arch": r.get("profile_or_arch"),
|
||||
"seed": r.get("seed"),
|
||||
"model_name": r.get("model_name"),
|
||||
"param_count": r.get("param_count"),
|
||||
"initial_test_acc": r.get("initial_test_acc"),
|
||||
"final_test_acc": r.get("final_test_acc"),
|
||||
"final_test_loss": r.get("final_test_loss"),
|
||||
"subset_fitness_acc": r.get("subset_fitness_acc"),
|
||||
"subset_fitness_loss": r.get("subset_fitness_loss"),
|
||||
"pso_epochs": r.get("pso_epochs"),
|
||||
"adam_epochs": r.get("adam_epochs"),
|
||||
"elapsed_sec": r.get("elapsed_sec"),
|
||||
"model_fingerprint": r.get("model_fingerprint"),
|
||||
"data_fingerprint": r.get("data_fingerprint"),
|
||||
})
|
||||
|
||||
|
||||
def render_plots(
|
||||
arch_summary: Dict[str, Dict[str, float]],
|
||||
opt_summary: Dict[str, Dict[str, float]],
|
||||
figure_path: Path,
|
||||
):
|
||||
figure_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 6))
|
||||
|
||||
if arch_summary:
|
||||
arch_labels = list(arch_summary.keys())
|
||||
arch_means = [arch_summary[k]["mean"] * 100 for k in arch_labels]
|
||||
arch_sds = [arch_summary[k]["std"] * 100 for k in arch_labels]
|
||||
arch_display = {
|
||||
"raw_linear": "Raw Linear",
|
||||
"raw_mlp": "Raw MLP",
|
||||
"compact_cnn": "Compact CNN",
|
||||
}
|
||||
|
||||
x_arch = np.arange(len(arch_labels))
|
||||
ax1.bar(x_arch, arch_means, yerr=arch_sds, capsize=5, color="#56B4E9", edgecolor="black", alpha=0.85)
|
||||
ax1.axhline(98.0, color="red", linestyle="--", linewidth=1.5, label="98% Target")
|
||||
ax1.set_xticks(x_arch)
|
||||
ax1.set_xticklabels([arch_display[k] for k in arch_labels], rotation=15)
|
||||
ax1.set_ylabel("Final Test Accuracy (%)")
|
||||
ax1.set_title("Architecture Lane (Full-Data Adam)")
|
||||
ax1.set_ylim(0, 110)
|
||||
ax1.set_axisbelow(True)
|
||||
ax1.grid(axis="y", linestyle=":", alpha=0.6)
|
||||
ax1.legend(loc="lower left")
|
||||
|
||||
for i, (m, sd) in enumerate(zip(arch_means, arch_sds)):
|
||||
ax1.text(
|
||||
i,
|
||||
105.0 if m >= 90.0 else min(m + sd + 1.5, 102.5),
|
||||
f"{m:.2f}%",
|
||||
ha="center",
|
||||
va="bottom",
|
||||
fontsize=9,
|
||||
fontweight="bold",
|
||||
bbox={"facecolor": "white", "edgecolor": "none", "alpha": 0.9, "pad": 1.5},
|
||||
)
|
||||
|
||||
if opt_summary:
|
||||
opt_labels = list(opt_summary.keys())
|
||||
opt_means = [opt_summary[k]["mean"] * 100 for k in opt_labels]
|
||||
opt_sds = [opt_summary[k]["std"] * 100 for k in opt_labels]
|
||||
opt_display = {
|
||||
"adam_only": "Adam Only",
|
||||
"pso_only": "PSO Only",
|
||||
"hybrid": "PSO → Adam",
|
||||
}
|
||||
|
||||
x_opt = np.arange(len(opt_labels))
|
||||
colors = ["#009E73", "#E69F00", "#CC79A7"]
|
||||
ax2.bar(x_opt, opt_means, yerr=opt_sds, capsize=5, color=colors[:len(opt_labels)], edgecolor="black", alpha=0.85)
|
||||
ax2.axhline(98.0, color="red", linestyle="--", linewidth=1.5, label="98% Target")
|
||||
ax2.set_xticks(x_opt)
|
||||
ax2.set_xticklabels([opt_display[k] for k in opt_labels], rotation=15)
|
||||
ax2.set_ylabel("Final Test Accuracy (%)")
|
||||
ax2.set_title("Optimizer Lane (Compact CNN)")
|
||||
ax2.set_ylim(0, 110)
|
||||
ax2.set_axisbelow(True)
|
||||
ax2.grid(axis="y", linestyle=":", alpha=0.6)
|
||||
ax2.legend(loc="lower left")
|
||||
|
||||
for i, (m, sd) in enumerate(zip(opt_means, opt_sds)):
|
||||
ax2.text(
|
||||
i,
|
||||
105.0 if m >= 90.0 else min(m + sd + 1.5, 102.5),
|
||||
f"{m:.2f}%",
|
||||
ha="center",
|
||||
va="bottom",
|
||||
fontsize=9,
|
||||
fontweight="bold",
|
||||
bbox={"facecolor": "white", "edgecolor": "none", "alpha": 0.9, "pad": 1.5},
|
||||
)
|
||||
|
||||
plt.suptitle("MNIST Deep Accuracy Study: Architectures & Optimizer Profiles", fontsize=14, fontweight="bold")
|
||||
plt.tight_layout()
|
||||
fig.savefig(figure_path, dpi=300)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
# ==========================================
|
||||
# Main CLI & Runner
|
||||
# ==========================================
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="MNIST Deep Accuracy Study: Architecture vs Optimizer Profiles"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seeds",
|
||||
type=str,
|
||||
default="101,102,103",
|
||||
help="Comma-separated random seeds (default: 101,102,103)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--adam-epochs",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Full-data Adam training epochs (default: 10)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pso-epochs",
|
||||
type=int,
|
||||
default=40,
|
||||
help="PSO swarm optimization epochs (default: 40)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--particles",
|
||||
"--n-particles",
|
||||
type=int,
|
||||
default=30,
|
||||
dest="particles",
|
||||
help="Number of PSO particles (default: 30)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--fitness-size",
|
||||
type=int,
|
||||
default=2000,
|
||||
help="Fixed train subset fitness size for PSO (default: 2000)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-size",
|
||||
type=int,
|
||||
default=256,
|
||||
help="Batch size for Adam DataLoader (default: 256)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lr",
|
||||
"--learning-rate",
|
||||
type=float,
|
||||
default=1e-3,
|
||||
dest="lr",
|
||||
help="Learning rate for Adam optimizer (default: 1e-3)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--device",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Target PyTorch device ('cpu', 'cuda', 'mps'; default: auto-detect)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json-path",
|
||||
type=str,
|
||||
default="benchmark_results/pso_v4_deep_accuracy.json",
|
||||
help="JSON result output path (default: benchmark_results/pso_v4_deep_accuracy.json)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--csv-path",
|
||||
type=str,
|
||||
default="benchmark_results/pso_v4_deep_accuracy.csv",
|
||||
help="CSV result output path (default: benchmark_results/pso_v4_deep_accuracy.csv)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--figure-path",
|
||||
"--plot-path",
|
||||
type=str,
|
||||
default="history_plt/pso_v4_deep_accuracy.png",
|
||||
dest="figure_path",
|
||||
help="Figure output path (default: history_plt/pso_v4_deep_accuracy.png)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lanes",
|
||||
"--profiles",
|
||||
type=str,
|
||||
choices=["all", "architectures", "optimizers"],
|
||||
default="all",
|
||||
help="Lanes/profiles to evaluate (choices: all, architectures, optimizers; default: all)",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
seed_list = [int(s.strip()) for s in args.seeds.split(",") if s.strip()]
|
||||
if not seed_list:
|
||||
raise ValueError("--seeds must contain at least one integer")
|
||||
if len(seed_list) != len(set(seed_list)):
|
||||
raise ValueError("--seeds must not contain duplicates")
|
||||
if any(seed < 0 for seed in seed_list):
|
||||
raise ValueError("--seeds values must be non-negative")
|
||||
for name, value in (
|
||||
("--adam-epochs", args.adam_epochs),
|
||||
("--pso-epochs", args.pso_epochs),
|
||||
("--particles", args.particles),
|
||||
("--fitness-size", args.fitness_size),
|
||||
("--batch-size", args.batch_size),
|
||||
):
|
||||
if value <= 0:
|
||||
raise ValueError(f"{name} must be positive")
|
||||
if not math.isfinite(args.lr) or args.lr <= 0.0:
|
||||
raise ValueError("--lr must be a positive finite number")
|
||||
|
||||
device = resolve_execution_device(args.device)
|
||||
|
||||
json_path = Path(args.json_path)
|
||||
csv_path = Path(args.csv_path)
|
||||
figure_path = Path(args.figure_path)
|
||||
|
||||
print(f"=== Starting MNIST Deep Accuracy Study (Protocol v{DEEP_ACCURACY_PROTOCOL_VERSION}) ===")
|
||||
print(f"Device: {device} | Seeds: {seed_list} | Adam Epochs: {args.adam_epochs} | PSO Epochs: {args.pso_epochs}")
|
||||
print(f"Particles: {args.particles} | Fitness Size: {args.fitness_size} | Batch Size: {args.batch_size} | LR: {args.lr}")
|
||||
|
||||
# Prepare data
|
||||
x_train, y_train, x_test, y_test, data_fp, norm_provenance = prepare_deep_accuracy_mnist_data()
|
||||
print(f"Data Loaded: Train {x_train.shape[0]} / Test {x_test.shape[0]} | Fingerprint: {data_fp[:12]}...")
|
||||
|
||||
hardware_prov = get_hardware_provenance(device)
|
||||
all_csv_records: List[Dict[str, Any]] = []
|
||||
architecture_lane_runs: List[Dict[str, Any]] = []
|
||||
optimizer_lane_runs: List[Dict[str, Any]] = []
|
||||
|
||||
# Map architecture factories
|
||||
arch_factories = {
|
||||
"raw_linear": ("Raw Linear (784->10)", make_raw_linear),
|
||||
"raw_mlp": ("Raw MLP (784->128->64->10)", make_raw_mlp),
|
||||
"compact_cnn": ("Compact CNN (9,098 params)", make_compact_cnn),
|
||||
}
|
||||
|
||||
try:
|
||||
# ----------------------------------------------------
|
||||
# 1. Architecture Lane
|
||||
# ----------------------------------------------------
|
||||
compact_cnn_adam_cache: Dict[int, Dict[str, Any]] = {}
|
||||
|
||||
if args.lanes in ("all", "architectures"):
|
||||
print("\n--- Running Architecture Lane (Full-Data Adam) ---")
|
||||
for arch_key, (arch_name, factory) in arch_factories.items():
|
||||
for s in seed_list:
|
||||
# Construct base model and record initial fingerprint
|
||||
model = factory(seed=s)
|
||||
p_count = count_parameters(model)
|
||||
init_fp = compute_model_fingerprint(model)
|
||||
|
||||
init_loss, init_acc = evaluate_model_on_test(model, x_test, y_test, device)
|
||||
|
||||
print(f"[Arch: {arch_key} | Seed: {s}] Params: {p_count} | Init Acc: {init_acc*100:.2f}% | Training Adam...")
|
||||
|
||||
history, final_loss, final_acc, elapsed = train_adam_routine(
|
||||
model=model,
|
||||
x_train=x_train,
|
||||
y_train=y_train,
|
||||
x_test=x_test,
|
||||
y_test=y_test,
|
||||
epochs=args.adam_epochs,
|
||||
batch_size=args.batch_size,
|
||||
lr=args.lr,
|
||||
seed=s,
|
||||
device=device,
|
||||
)
|
||||
|
||||
run_record = {
|
||||
"lane": "architecture",
|
||||
"profile_or_arch": arch_key,
|
||||
"seed": s,
|
||||
"model_name": arch_name,
|
||||
"param_count": p_count,
|
||||
"initial_test_acc": round(init_acc, 6),
|
||||
"final_test_acc": round(final_acc, 6),
|
||||
"final_test_loss": round(final_loss, 6),
|
||||
"subset_fitness_acc": None,
|
||||
"subset_fitness_loss": None,
|
||||
"pso_epochs": 0,
|
||||
"adam_epochs": args.adam_epochs,
|
||||
"elapsed_sec": round(elapsed, 4),
|
||||
"model_fingerprint": init_fp,
|
||||
"data_fingerprint": data_fp,
|
||||
"epoch_history": history,
|
||||
}
|
||||
architecture_lane_runs.append(run_record)
|
||||
all_csv_records.append(run_record)
|
||||
|
||||
if arch_key == "compact_cnn":
|
||||
compact_cnn_adam_cache[s] = run_record
|
||||
|
||||
print(f" -> Final Test Acc: {final_acc*100:.2f}% | Loss: {final_loss:.4f} | Time: {elapsed:.2f}s")
|
||||
|
||||
# ----------------------------------------------------
|
||||
# 2. Optimizer Lane (Compact CNN)
|
||||
# ----------------------------------------------------
|
||||
if args.lanes in ("all", "optimizers"):
|
||||
print("\n--- Running Optimizer Lane (Compact CNN) ---")
|
||||
profiles = ["adam_only", "pso_only", "hybrid"]
|
||||
|
||||
for prof in profiles:
|
||||
for s in seed_list:
|
||||
# Construct Compact CNN with seed s to ensure same base initial state
|
||||
model_base = make_compact_cnn(seed=s)
|
||||
p_count = count_parameters(model_base)
|
||||
init_fp = compute_model_fingerprint(model_base)
|
||||
init_loss, init_acc = evaluate_model_on_test(model_base, x_test, y_test, device)
|
||||
|
||||
if prof == "adam_only":
|
||||
if s in compact_cnn_adam_cache:
|
||||
# Explicitly reuse record from Architecture Lane
|
||||
cached = compact_cnn_adam_cache[s]
|
||||
run_record = {
|
||||
"lane": "optimizer",
|
||||
"profile_or_arch": "adam_only",
|
||||
"seed": s,
|
||||
"model_name": "Compact CNN (Adam-Only)",
|
||||
"param_count": p_count,
|
||||
"initial_test_acc": cached["initial_test_acc"],
|
||||
"final_test_acc": cached["final_test_acc"],
|
||||
"final_test_loss": cached["final_test_loss"],
|
||||
"subset_fitness_acc": None,
|
||||
"subset_fitness_loss": None,
|
||||
"pso_epochs": 0,
|
||||
"adam_epochs": args.adam_epochs,
|
||||
"elapsed_sec": cached["elapsed_sec"],
|
||||
"model_fingerprint": init_fp,
|
||||
"data_fingerprint": data_fp,
|
||||
"reused_from_architecture_lane": True,
|
||||
"epoch_history": cached["epoch_history"],
|
||||
}
|
||||
print(f"[Opt: adam_only | Seed: {s}] Reused from Architecture Lane | Final Acc: {cached['final_test_acc']*100:.2f}%")
|
||||
else:
|
||||
print(f"[Opt: adam_only | Seed: {s}] Training Adam...")
|
||||
history, final_loss, final_acc, elapsed = train_adam_routine(
|
||||
model=model_base,
|
||||
x_train=x_train,
|
||||
y_train=y_train,
|
||||
x_test=x_test,
|
||||
y_test=y_test,
|
||||
epochs=args.adam_epochs,
|
||||
batch_size=args.batch_size,
|
||||
lr=args.lr,
|
||||
seed=s,
|
||||
device=device,
|
||||
)
|
||||
run_record = {
|
||||
"lane": "optimizer",
|
||||
"profile_or_arch": "adam_only",
|
||||
"seed": s,
|
||||
"model_name": "Compact CNN (Adam-Only)",
|
||||
"param_count": p_count,
|
||||
"initial_test_acc": round(init_acc, 6),
|
||||
"final_test_acc": round(final_acc, 6),
|
||||
"final_test_loss": round(final_loss, 6),
|
||||
"subset_fitness_acc": None,
|
||||
"subset_fitness_loss": None,
|
||||
"pso_epochs": 0,
|
||||
"adam_epochs": args.adam_epochs,
|
||||
"elapsed_sec": round(elapsed, 4),
|
||||
"model_fingerprint": init_fp,
|
||||
"data_fingerprint": data_fp,
|
||||
"reused_from_architecture_lane": False,
|
||||
"epoch_history": history,
|
||||
}
|
||||
print(f" -> Final Test Acc: {final_acc*100:.2f}% | Loss: {final_loss:.4f} | Time: {elapsed:.2f}s")
|
||||
|
||||
optimizer_lane_runs.append(run_record)
|
||||
all_csv_records.append(run_record)
|
||||
|
||||
elif prof == "pso_only":
|
||||
print(
|
||||
f"[Opt: pso_only | Seed: {s}] Running PSO Adaptive Moment "
|
||||
f"on {args.fitness_size:,} fixed-subset samples..."
|
||||
)
|
||||
best_model, fitness_score, pso_test_loss, pso_test_acc, pso_meta = run_pso_routine(
|
||||
model=model_base,
|
||||
x_train=x_train,
|
||||
y_train=y_train,
|
||||
x_test=x_test,
|
||||
y_test=y_test,
|
||||
pso_epochs=args.pso_epochs,
|
||||
n_particles=args.particles,
|
||||
fitness_size=args.fitness_size,
|
||||
seed=s,
|
||||
device=device,
|
||||
)
|
||||
|
||||
run_record = {
|
||||
"lane": "optimizer",
|
||||
"profile_or_arch": "pso_only",
|
||||
"seed": s,
|
||||
"model_name": "Compact CNN (PSO-Only)",
|
||||
"param_count": p_count,
|
||||
"initial_test_acc": round(init_acc, 6),
|
||||
"final_test_acc": round(pso_test_acc, 6),
|
||||
"final_test_loss": round(pso_test_loss, 6),
|
||||
"subset_fitness_acc": fitness_score["subset_acc"],
|
||||
"subset_fitness_loss": fitness_score["subset_loss"],
|
||||
"pso_epochs": args.pso_epochs,
|
||||
"adam_epochs": 0,
|
||||
"elapsed_sec": pso_meta["elapsed_sec"],
|
||||
"model_fingerprint": init_fp,
|
||||
"data_fingerprint": data_fp,
|
||||
"pso_metadata": pso_meta,
|
||||
}
|
||||
optimizer_lane_runs.append(run_record)
|
||||
all_csv_records.append(run_record)
|
||||
|
||||
print(f" -> Fitness Subset Acc: {fitness_score['subset_acc']*100:.2f}% | Full Test Acc: {pso_test_acc*100:.2f}% | Time: {pso_meta['elapsed_sec']:.2f}s")
|
||||
|
||||
elif prof == "hybrid":
|
||||
print(f"[Opt: hybrid | Seed: {s}] Running Hybrid (PSO Warm Start + Adam Fine-Tuning)...")
|
||||
# 1. PSO Warm Start
|
||||
t_hyb_start = time.time()
|
||||
pso_best_model, fitness_score, pso_test_loss, pso_test_acc, pso_meta = run_pso_routine(
|
||||
model=model_base,
|
||||
x_train=x_train,
|
||||
y_train=y_train,
|
||||
x_test=x_test,
|
||||
y_test=y_test,
|
||||
pso_epochs=args.pso_epochs,
|
||||
n_particles=args.particles,
|
||||
fitness_size=args.fitness_size,
|
||||
seed=s,
|
||||
device=device,
|
||||
)
|
||||
|
||||
# 2. Continue with full-data Adam
|
||||
post_adam_history, final_test_loss, final_test_acc, adam_elapsed = train_adam_routine(
|
||||
model=pso_best_model,
|
||||
x_train=x_train,
|
||||
y_train=y_train,
|
||||
x_test=x_test,
|
||||
y_test=y_test,
|
||||
epochs=args.adam_epochs,
|
||||
batch_size=args.batch_size,
|
||||
lr=args.lr,
|
||||
seed=s,
|
||||
device=device,
|
||||
)
|
||||
hyb_total_elapsed = time.time() - t_hyb_start
|
||||
|
||||
run_record = {
|
||||
"lane": "optimizer",
|
||||
"profile_or_arch": "hybrid",
|
||||
"seed": s,
|
||||
"model_name": "Compact CNN (Hybrid)",
|
||||
"param_count": p_count,
|
||||
"initial_test_acc": round(init_acc, 6),
|
||||
"final_test_acc": round(final_test_acc, 6),
|
||||
"final_test_loss": round(final_test_loss, 6),
|
||||
"subset_fitness_acc": fitness_score["subset_acc"],
|
||||
"subset_fitness_loss": fitness_score["subset_loss"],
|
||||
"pso_epochs": args.pso_epochs,
|
||||
"adam_epochs": args.adam_epochs,
|
||||
"elapsed_sec": round(hyb_total_elapsed, 4),
|
||||
"model_fingerprint": init_fp,
|
||||
"data_fingerprint": data_fp,
|
||||
"post_pso_test_acc": round(pso_test_acc, 6),
|
||||
"post_pso_test_loss": round(pso_test_loss, 6),
|
||||
"post_adam_history": post_adam_history,
|
||||
"efficiency_label": "HYBRID_GETS_EXTRA_WORK_UNFAIR_EFFICIENCY_COMPARISON",
|
||||
}
|
||||
optimizer_lane_runs.append(run_record)
|
||||
all_csv_records.append(run_record)
|
||||
|
||||
print(f" -> Post-PSO Test Acc: {pso_test_acc*100:.2f}% | Final Hybrid Test Acc: {final_test_acc*100:.2f}% | Time: {hyb_total_elapsed:.2f}s")
|
||||
|
||||
# Compute summary statistics
|
||||
arch_summary: Dict[str, Dict[str, float]] = {}
|
||||
for arch_key in arch_factories.keys():
|
||||
vals = [r["final_test_acc"] for r in architecture_lane_runs if r["profile_or_arch"] == arch_key]
|
||||
if vals:
|
||||
arch_summary[arch_key] = calc_stats(vals)
|
||||
|
||||
opt_summary: Dict[str, Dict[str, float]] = {}
|
||||
for prof in ["adam_only", "pso_only", "hybrid"]:
|
||||
vals = [r["final_test_acc"] for r in optimizer_lane_runs if r["profile_or_arch"] == prof]
|
||||
if vals:
|
||||
opt_summary[prof] = calc_stats(vals)
|
||||
|
||||
# Structure complete JSON output
|
||||
result_payload = {
|
||||
"protocol_version": DEEP_ACCURACY_PROTOCOL_VERSION,
|
||||
"pso_version": pso_version,
|
||||
"timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"completed": True,
|
||||
"error": None,
|
||||
"hardware_provenance": hardware_prov,
|
||||
"data_provenance": norm_provenance,
|
||||
"data_fingerprint": data_fp,
|
||||
"configuration": {
|
||||
"seeds": seed_list,
|
||||
"adam_epochs": args.adam_epochs,
|
||||
"pso_epochs": args.pso_epochs,
|
||||
"particles": args.particles,
|
||||
"fitness_size": args.fitness_size,
|
||||
"batch_size": args.batch_size,
|
||||
"lr": args.lr,
|
||||
"device": str(device),
|
||||
},
|
||||
"summaries": {
|
||||
"architecture_lane": arch_summary,
|
||||
"optimizer_lane": opt_summary,
|
||||
},
|
||||
"architecture_lane_runs": architecture_lane_runs,
|
||||
"optimizer_lane_runs": optimizer_lane_runs,
|
||||
}
|
||||
|
||||
# Save JSON output atomically
|
||||
save_json_atomic(result_payload, json_path)
|
||||
print(f"\nSaved JSON results to {json_path}")
|
||||
|
||||
# Save CSV records and render plot only after all runs succeed
|
||||
save_csv_records(all_csv_records, csv_path)
|
||||
print(f"Saved CSV records to {csv_path}")
|
||||
|
||||
render_plots(arch_summary, opt_summary, figure_path)
|
||||
print(f"Saved summary figure to {figure_path}")
|
||||
|
||||
# Print final summary table
|
||||
print("\n========================================================")
|
||||
print(" FINAL SUMMARY TABLE ")
|
||||
print("========================================================")
|
||||
if arch_summary:
|
||||
print("Architecture Lane (Full-Data Adam):")
|
||||
for arch_key, stats in arch_summary.items():
|
||||
print(f" - {arch_key:15s}: Mean Acc = {stats['mean']*100:6.2f}% ± {stats['std']*100:5.2f}% (Median: {stats['median']*100:.2f}%)")
|
||||
if opt_summary:
|
||||
print("\nOptimizer Lane (Compact CNN):")
|
||||
for prof, stats in opt_summary.items():
|
||||
print(f" - {prof:15s}: Mean Acc = {stats['mean']*100:6.2f}% ± {stats['std']*100:5.2f}% (Median: {stats['median']*100:.2f}%)")
|
||||
print("========================================================\n")
|
||||
|
||||
except Exception as e:
|
||||
error_payload = {
|
||||
"protocol_version": DEEP_ACCURACY_PROTOCOL_VERSION,
|
||||
"pso_version": pso_version,
|
||||
"timestamp": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
"completed": False,
|
||||
"error": str(e),
|
||||
"hardware_provenance": hardware_prov,
|
||||
"configuration": {
|
||||
"seeds": seed_list,
|
||||
"adam_epochs": args.adam_epochs,
|
||||
"pso_epochs": args.pso_epochs,
|
||||
"particles": args.particles,
|
||||
"fitness_size": args.fitness_size,
|
||||
"batch_size": args.batch_size,
|
||||
"lr": args.lr,
|
||||
"device": str(device),
|
||||
},
|
||||
"architecture_lane_runs": architecture_lane_runs,
|
||||
"optimizer_lane_runs": optimizer_lane_runs,
|
||||
}
|
||||
save_json_atomic(error_payload, json_path)
|
||||
print(f"\n[ERROR] Study failed: {e}")
|
||||
print(f"Saved failure audit record to {json_path}")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
+1464
File diff suppressed because it is too large
Load Diff
+91
-51
@@ -1,71 +1,111 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
from keras.layers import Dense
|
||||
from keras.models import Sequential
|
||||
from keras.utils import to_categorical
|
||||
import argparse
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from sklearn.datasets import load_digits
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
|
||||
from pso import optimizer
|
||||
|
||||
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
|
||||
from pso import Optimizer
|
||||
from cli import add_pso_args, build_optimizer_kwargs
|
||||
|
||||
|
||||
def make_model():
|
||||
model = Sequential()
|
||||
model.add(Dense(12, input_dim=64, activation="relu"))
|
||||
model.add(Dense(10, activation="relu"))
|
||||
model.add(Dense(10, activation="softmax"))
|
||||
|
||||
return model
|
||||
def make_model(seed: int = 42):
|
||||
torch.manual_seed(seed)
|
||||
return nn.Sequential(
|
||||
nn.Linear(64, 12),
|
||||
nn.ReLU(),
|
||||
nn.Linear(12, 10),
|
||||
nn.ReLU(),
|
||||
nn.Linear(10, 10),
|
||||
)
|
||||
|
||||
|
||||
def get_data():
|
||||
def get_data(seed: int = 42):
|
||||
digits = load_digits()
|
||||
X = digits.data
|
||||
y = digits.target
|
||||
|
||||
x = X.astype("float32")
|
||||
|
||||
y_class = to_categorical(y)
|
||||
x = digits.data.astype("float32")
|
||||
y = digits.target.astype("int64")
|
||||
|
||||
x_train, x_test, y_train, y_test = train_test_split(
|
||||
x, y_class, test_size=0.2, random_state=42, shuffle=True
|
||||
x, y, test_size=0.2, random_state=seed, shuffle=True
|
||||
)
|
||||
scaler = StandardScaler()
|
||||
x_train = scaler.fit_transform(x_train)
|
||||
x_test = scaler.transform(x_test)
|
||||
|
||||
return (
|
||||
torch.tensor(x_train, dtype=torch.float32),
|
||||
torch.tensor(x_test, dtype=torch.float32),
|
||||
torch.tensor(y_train, dtype=torch.int64),
|
||||
torch.tensor(y_test, dtype=torch.int64),
|
||||
)
|
||||
return x_train, x_test, y_train, y_test
|
||||
|
||||
|
||||
x_train, x_test, y_train, y_test = get_data()
|
||||
model = make_model()
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="PSO Digits Benchmark Script")
|
||||
add_pso_args(
|
||||
parser,
|
||||
defaults={
|
||||
"method": "original",
|
||||
"initialization": "model_noise",
|
||||
"evaluation": "fixed_subset",
|
||||
"convergence": "particle_reset",
|
||||
"refinement": "adam",
|
||||
"n_particles": 30,
|
||||
"c0": None,
|
||||
"c1": None,
|
||||
"w_min": None,
|
||||
"w_max": None,
|
||||
"negative_swarm": 0.0,
|
||||
"mutation_swarm": 0.1,
|
||||
"particle_min": -3.0,
|
||||
"particle_max": 3.0,
|
||||
"velocity_limit_ratio": 0.1,
|
||||
"boundary_strategy": "reflect",
|
||||
"seed": 42,
|
||||
"epochs": 80,
|
||||
"batch_size": 200,
|
||||
"fitness_size": 1000,
|
||||
"renewal": "loss",
|
||||
"output_dir": "output/digits",
|
||||
"refinement_epochs": 10,
|
||||
"refinement_lr": 0.001,
|
||||
},
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
digits_pso = optimizer(
|
||||
model,
|
||||
loss="categorical_crossentropy",
|
||||
n_particles=300,
|
||||
c0=0.5,
|
||||
c1=0.3,
|
||||
w_min=0.2,
|
||||
w_max=0.9,
|
||||
negative_swarm=0,
|
||||
mutation_swarm=0.1,
|
||||
convergence_reset=True,
|
||||
convergence_reset_patience=10,
|
||||
convergence_reset_monitor="loss",
|
||||
convergence_reset_min_delta=0.001,
|
||||
)
|
||||
x_train, x_test, y_train, y_test = get_data(seed=args.seed)
|
||||
model = make_model(seed=args.seed)
|
||||
|
||||
digits_pso.fit(
|
||||
fitness_size = args.fitness_size if args.evaluation == "fixed_subset" else None
|
||||
refinement_epochs = args.refinement_epochs if args.refinement == "adam" else 0
|
||||
|
||||
kwargs = build_optimizer_kwargs(
|
||||
args,
|
||||
model=model,
|
||||
loss=nn.CrossEntropyLoss(),
|
||||
task="multiclass",
|
||||
inertia_profile={"c0": 0.5, "c1": 0.3, "w_min": 0.2, "w_max": 0.9},
|
||||
)
|
||||
digits_pso = Optimizer(**kwargs)
|
||||
|
||||
print(f"Optimizer device: {digits_pso.device}")
|
||||
|
||||
best_score = digits_pso.fit(
|
||||
x_train,
|
||||
y_train,
|
||||
epochs=500,
|
||||
validate_data=(x_test, y_test),
|
||||
log=2,
|
||||
epochs=args.epochs,
|
||||
batch_size=args.batch_size,
|
||||
fitness_size=fitness_size,
|
||||
renewal=args.renewal,
|
||||
validation_data=(x_test, y_test),
|
||||
output_dir=args.output_dir,
|
||||
save_info=True,
|
||||
renewal="loss",
|
||||
log_name="digits",
|
||||
)
|
||||
refinement_epochs=refinement_epochs,
|
||||
refinement_lr=args.refinement_lr,
|
||||
)
|
||||
|
||||
print("Done!")
|
||||
print(f"Done! Best score: {best_score}")
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
import tensorflow as tf
|
||||
from sklearn.datasets import load_digits
|
||||
from sklearn.model_selection import train_test_split
|
||||
from tensorflow import keras
|
||||
from tensorflow.keras import layers
|
||||
from tensorflow.keras.layers import Dense
|
||||
from tensorflow.keras.models import Sequential
|
||||
from tensorflow.keras.utils import to_categorical
|
||||
|
||||
|
||||
gpus = tf.config.experimental.list_physical_devices("GPU")
|
||||
if gpus:
|
||||
try:
|
||||
tf.config.experimental.set_memory_growth(gpus[0], True)
|
||||
except RuntimeError as r:
|
||||
print(r)
|
||||
|
||||
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
|
||||
|
||||
|
||||
def make_model():
|
||||
model = Sequential()
|
||||
model.add(Dense(12, input_dim=64, activation="relu"))
|
||||
model.add(Dense(12, activation="relu"))
|
||||
model.add(Dense(10, activation="softmax"))
|
||||
|
||||
return model
|
||||
|
||||
|
||||
def get_data():
|
||||
digits = load_digits()
|
||||
X = digits.data
|
||||
y = digits.target
|
||||
|
||||
x = X.astype("float32")
|
||||
|
||||
y_class = to_categorical(y)
|
||||
|
||||
x_train, x_test, y_train, y_test = train_test_split(
|
||||
x, y_class, test_size=0.2, random_state=42, shuffle=True
|
||||
)
|
||||
return x_train, x_test, y_train, y_test
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
model = make_model()
|
||||
x_train, x_test, y_train, y_test = get_data()
|
||||
|
||||
callbacks = [
|
||||
tf.keras.callbacks.EarlyStopping(
|
||||
monitor="val_loss", patience=10, restore_best_weights=True
|
||||
)
|
||||
]
|
||||
|
||||
print(x_train.shape, y_train.shape)
|
||||
|
||||
model.compile(
|
||||
optimizer="adam",
|
||||
loss="categorical_crossentropy",
|
||||
metrics=["accuracy", "mse"],
|
||||
)
|
||||
|
||||
print(model.summary())
|
||||
|
||||
history = model.fit(
|
||||
x_train,
|
||||
y_train,
|
||||
epochs=500,
|
||||
batch_size=32,
|
||||
verbose=1,
|
||||
validation_data=(x_test, y_test),
|
||||
callbacks=callbacks,
|
||||
)
|
||||
|
||||
print("Done!")
|
||||
|
||||
sys.exit(0)
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Digits dataset gradient baseline (PyTorch standard backprop optimizer, non-PSO)."""
|
||||
|
||||
import copy
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.utils.data import DataLoader, TensorDataset
|
||||
from sklearn.datasets import load_digits
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
|
||||
class DigitsModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.net = nn.Sequential(
|
||||
nn.Linear(64, 12),
|
||||
nn.ReLU(),
|
||||
nn.Linear(12, 12),
|
||||
nn.ReLU(),
|
||||
nn.Linear(12, 10),
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
return self.net(x)
|
||||
|
||||
|
||||
def get_data(seed: int = 42):
|
||||
digits = load_digits()
|
||||
X = digits.data.astype("float32")
|
||||
y = digits.target.astype("int64")
|
||||
|
||||
x_train, x_test, y_train, y_test = train_test_split(
|
||||
X, y, test_size=0.2, random_state=seed, shuffle=True
|
||||
)
|
||||
return (
|
||||
torch.tensor(x_train, dtype=torch.float32),
|
||||
torch.tensor(x_test, dtype=torch.float32),
|
||||
torch.tensor(y_train, dtype=torch.int64),
|
||||
torch.tensor(y_test, dtype=torch.int64),
|
||||
)
|
||||
|
||||
|
||||
def get_device() -> torch.device:
|
||||
if (
|
||||
hasattr(torch.backends, "mps")
|
||||
and torch.backends.mps.is_built()
|
||||
and torch.backends.mps.is_available()
|
||||
):
|
||||
return torch.device("mps")
|
||||
elif torch.cuda.is_available():
|
||||
return torch.device("cuda")
|
||||
return torch.device("cpu")
|
||||
|
||||
|
||||
def main():
|
||||
torch.manual_seed(42)
|
||||
np.random.seed(42)
|
||||
|
||||
device = get_device()
|
||||
print(f"Selected device: {device}")
|
||||
|
||||
x_train, x_test, y_train, y_test = get_data(seed=42)
|
||||
train_loader = DataLoader(
|
||||
TensorDataset(x_train, y_train), batch_size=32, shuffle=True
|
||||
)
|
||||
val_loader = DataLoader(
|
||||
TensorDataset(x_test, y_test), batch_size=32, shuffle=False
|
||||
)
|
||||
|
||||
model = DigitsModel().to(device)
|
||||
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
|
||||
best_val_loss = float("inf")
|
||||
best_state = None
|
||||
patience = 10
|
||||
patience_counter = 0
|
||||
|
||||
for epoch in range(500):
|
||||
model.train()
|
||||
for bx, by in train_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
optimizer.zero_grad()
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
model.eval()
|
||||
val_loss = 0.0
|
||||
total = 0
|
||||
with torch.no_grad():
|
||||
for bx, by in val_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
val_loss += loss.item() * bx.size(0)
|
||||
total += bx.size(0)
|
||||
|
||||
val_loss /= total
|
||||
|
||||
if val_loss < best_val_loss:
|
||||
best_val_loss = val_loss
|
||||
best_state = copy.deepcopy(model.state_dict())
|
||||
patience_counter = 0
|
||||
else:
|
||||
patience_counter += 1
|
||||
if patience_counter >= patience:
|
||||
break
|
||||
|
||||
if best_state is not None:
|
||||
model.load_state_dict(best_state)
|
||||
|
||||
model.eval()
|
||||
test_loss = 0.0
|
||||
correct = 0
|
||||
total = 0
|
||||
with torch.no_grad():
|
||||
for bx, by in val_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
test_loss += loss.item() * bx.size(0)
|
||||
preds = out.argmax(dim=1)
|
||||
correct += (preds == by).sum().item()
|
||||
total += bx.size(0)
|
||||
|
||||
test_loss /= total
|
||||
test_acc = correct / total
|
||||
print(f"Final test loss: {test_loss:.4f}, accuracy: {test_acc:.4f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,673 @@
|
||||
"""
|
||||
Adaptive Moment 120-Particle x 240-Epoch MNIST Convergence & Trajectory Analysis
|
||||
|
||||
Evaluates whether the published 120-particle x 80-epoch MNIST PCA32 Adaptive Moment run
|
||||
benefits from continued optimization up to 240 epochs or enters a generalization plateau.
|
||||
|
||||
Predeclared Contract Criteria:
|
||||
1. Exact Replay Verification at Epoch 80 (seeds 71-75):
|
||||
Per-seed test accuracy absolute delta vs baseline <= 0.005 (0.5%p).
|
||||
2. Primary Diagnostic Endpoints: Epochs 80, 120, 160, 200, 240.
|
||||
3. Classifications:
|
||||
- Training Still Improving: Mean global-best fitness loss falls >= 1% from epoch 80 to 240.
|
||||
- Meaningful Held-Out Gain: Mean test accuracy at epoch 240 rises >= 1 percentage point vs epoch 80.
|
||||
- Overfitting Signal: Training loss improves but epoch 240 test accuracy falls >= 1 point.
|
||||
- Early Stagnation: Training loss improves < 1% and absolute test change remains < 1 point.
|
||||
- Generalization Plateau: Training loss improves >= 1% while absolute test gain remains < 1 point.
|
||||
- Late Plateau Diagnostic: 200->240 training-loss improvement < 1% AND absolute test-accuracy change < 0.5 points.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import datetime
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
# Ensure test/ directory is in sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from benchmark_suite import (
|
||||
calc_stats,
|
||||
compute_model_fingerprint,
|
||||
get_hardware_provenance,
|
||||
make_mnist_model,
|
||||
resolve_execution_device,
|
||||
save_json_atomic,
|
||||
sync_device,
|
||||
)
|
||||
from pso import Optimizer, __version__ as pso_version
|
||||
from reproduce_scaling import (
|
||||
REPLAY_SEEDS,
|
||||
REPLAY_TOLERANCE,
|
||||
prepare_full_pca_data,
|
||||
validate_and_load_baseline,
|
||||
)
|
||||
from tuning_suite import TUNING_PROTOCOL_VERSION
|
||||
|
||||
EPOCH_CONVERGENCE_PROTOCOL_VERSION = "1.0.0"
|
||||
CHECKPOINT_EPOCHS = [20, 40, 60, 80, 100, 120, 140, 160, 180, 200, 220, 240]
|
||||
POST80_LOSS_REDUCTION_THRESHOLD = 0.01
|
||||
TEST_ACCURACY_GAIN_THRESHOLD = 0.01
|
||||
OVERFITTING_ACCURACY_DROP_THRESHOLD = -0.01
|
||||
LATE_LOSS_REDUCTION_THRESHOLD = 0.01
|
||||
LATE_ACCURACY_CHANGE_THRESHOLD = 0.005
|
||||
|
||||
|
||||
def run_epoch_convergence_analysis(
|
||||
baseline_path: Path,
|
||||
output_json_path: Path,
|
||||
output_csv_path: Path,
|
||||
figure_path: Path,
|
||||
device_str: str | None = None,
|
||||
) -> bool:
|
||||
device = resolve_execution_device(device_str)
|
||||
hw_provenance = get_hardware_provenance(device)
|
||||
|
||||
# 1. Validate and load baseline
|
||||
baseline_data, baseline_records, winner_cfg, expected_fp = validate_and_load_baseline(
|
||||
baseline_path
|
||||
)
|
||||
if baseline_data.get("device") != device.type:
|
||||
raise ValueError(
|
||||
f"Exact trajectory extension requires baseline device "
|
||||
f"{baseline_data.get('device')!r}; got {device.type!r}."
|
||||
)
|
||||
if baseline_data.get("pso_version") != pso_version:
|
||||
raise ValueError(
|
||||
f"Exact trajectory extension requires pso version "
|
||||
f"{baseline_data.get('pso_version')!r}; got {pso_version!r}."
|
||||
)
|
||||
if baseline_data.get("torch_version") != torch.__version__:
|
||||
raise ValueError(
|
||||
f"Exact trajectory extension requires torch version "
|
||||
f"{baseline_data.get('torch_version')!r}; got {torch.__version__!r}."
|
||||
)
|
||||
|
||||
base_rec_by_seed = {r["seed"]: r for r in baseline_records}
|
||||
|
||||
# 2. Prepare full PCA dataset
|
||||
x_full_tr, y_train_3000, x_full_test, y_test_1000, data_fp = prepare_full_pca_data()
|
||||
if data_fp != expected_fp:
|
||||
raise ValueError(
|
||||
f"Data fingerprint mismatch: computed {data_fp}, baseline expected {expected_fp}"
|
||||
)
|
||||
|
||||
opt_kwargs = winner_cfg.to_optimizer_kwargs(quick=False)
|
||||
n_particles = 120
|
||||
target_epochs = 240
|
||||
batch_size = 1000
|
||||
|
||||
runs: List[Dict[str, Any]] = []
|
||||
flat_csv_rows: List[Dict[str, Any]] = []
|
||||
fidelity_passed = True
|
||||
seed_fidelity_deltas: Dict[int, float] = {}
|
||||
|
||||
# 3. Process seeds 71-75
|
||||
for seed in sorted(REPLAY_SEEDS):
|
||||
base_rec = base_rec_by_seed[seed]
|
||||
expected_model_fp = base_rec["model_fingerprint"]
|
||||
baseline_test_acc = float(base_rec["test_acc"])
|
||||
|
||||
# --- Untimed 2-Epoch Warmup Phase ---
|
||||
warmup_model = make_mnist_model(seed=seed)
|
||||
warmup_loss = nn.CrossEntropyLoss()
|
||||
warmup_opt = Optimizer(
|
||||
model=warmup_model,
|
||||
loss=warmup_loss,
|
||||
task="multiclass",
|
||||
n_particles=n_particles,
|
||||
seed=seed,
|
||||
device=device,
|
||||
**opt_kwargs,
|
||||
)
|
||||
warmup_opt.fit(
|
||||
x_full_tr,
|
||||
y_train_3000,
|
||||
epochs=2,
|
||||
batch_size=batch_size,
|
||||
renewal="loss",
|
||||
)
|
||||
sync_device(device)
|
||||
del warmup_opt, warmup_model, warmup_loss
|
||||
|
||||
# --- Timed 240-Epoch Continuous Trajectory ---
|
||||
model = make_mnist_model(seed=seed)
|
||||
model_fp = compute_model_fingerprint(model)
|
||||
fp_match = (model_fp == expected_model_fp)
|
||||
if not fp_match:
|
||||
print(
|
||||
f"[WARNING] Seed {seed} model fingerprint mismatch: "
|
||||
f"got {model_fp}, expected {expected_model_fp}"
|
||||
)
|
||||
fidelity_passed = False
|
||||
|
||||
loss_inst = nn.CrossEntropyLoss()
|
||||
opt = Optimizer(
|
||||
model=model,
|
||||
loss=loss_inst,
|
||||
task="multiclass",
|
||||
n_particles=n_particles,
|
||||
seed=seed,
|
||||
device=device,
|
||||
**opt_kwargs,
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir_str:
|
||||
output_dir = Path(temp_dir_str)
|
||||
|
||||
sync_device(device)
|
||||
t0 = time.perf_counter()
|
||||
train_loss_final, train_acc_final, train_mse_final = opt.fit(
|
||||
x_full_tr,
|
||||
y_train_3000,
|
||||
epochs=target_epochs,
|
||||
batch_size=batch_size,
|
||||
renewal="loss",
|
||||
output_dir=output_dir,
|
||||
log_format="csv",
|
||||
checkpoint_interval=20,
|
||||
)
|
||||
sync_device(device)
|
||||
t1 = time.perf_counter()
|
||||
fit_time_sec = t1 - t0
|
||||
|
||||
# Read full epoch history from history.csv
|
||||
history_csv_path = output_dir / "history.csv"
|
||||
epoch_history = []
|
||||
with open(history_csv_path, "r", encoding="utf-8") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
epoch_history.append({
|
||||
"epoch": int(row["epoch"]),
|
||||
"loss": float(row["loss"]),
|
||||
"accuracy": float(row["accuracy"]),
|
||||
"mse": float(row["mse"]),
|
||||
})
|
||||
|
||||
# Track training global-best improvement epochs
|
||||
prev_best_loss = float("inf")
|
||||
improvement_count = 0
|
||||
last_improvement_epoch = 1
|
||||
for row in epoch_history:
|
||||
ep_num = row["epoch"]
|
||||
l_val = row["loss"]
|
||||
if l_val < prev_best_loss:
|
||||
improvement_count += 1
|
||||
last_improvement_epoch = ep_num
|
||||
prev_best_loss = l_val
|
||||
|
||||
# Read and evaluate checkpoints
|
||||
checkpoints: List[Dict[str, Any]] = []
|
||||
ckpt_dir = output_dir / "checkpoints"
|
||||
epoch80_test_acc = None
|
||||
|
||||
for ep in CHECKPOINT_EPOCHS:
|
||||
ckpt_path = ckpt_dir / f"epoch-{ep}.pt"
|
||||
if not ckpt_path.exists():
|
||||
raise FileNotFoundError(f"Missing checkpoint file: {ckpt_path}")
|
||||
|
||||
payload = torch.load(ckpt_path, map_location=device, weights_only=True)
|
||||
ckpt_train_loss, ckpt_train_acc, ckpt_train_mse = payload["score"]
|
||||
|
||||
# Load checkpoint state_dict into opt.eval_model and evaluate on held-out test tensor
|
||||
opt.eval_model.load_state_dict(payload["model_state_dict"])
|
||||
opt._global_best_weights = opt.codec.encode(opt.eval_model)
|
||||
test_loss, test_acc, test_mse = opt.evaluate(x_full_test, y_test_1000)
|
||||
|
||||
ckpt_record = {
|
||||
"epoch": ep,
|
||||
"train_loss": float(ckpt_train_loss),
|
||||
"train_acc": float(ckpt_train_acc),
|
||||
"train_mse": float(ckpt_train_mse),
|
||||
"test_loss": float(test_loss),
|
||||
"test_acc": float(test_acc),
|
||||
"test_mse": float(test_mse),
|
||||
}
|
||||
checkpoints.append(ckpt_record)
|
||||
|
||||
if ep == 80:
|
||||
epoch80_test_acc = float(test_acc)
|
||||
|
||||
flat_csv_rows.append({
|
||||
"seed": seed,
|
||||
"epoch": ep,
|
||||
"train_loss": float(ckpt_train_loss),
|
||||
"train_acc": float(ckpt_train_acc),
|
||||
"train_mse": float(ckpt_train_mse),
|
||||
"test_loss": float(test_loss),
|
||||
"test_acc": float(test_acc),
|
||||
"test_mse": float(test_mse),
|
||||
"fit_time_sec": round(fit_time_sec, 4),
|
||||
})
|
||||
|
||||
# Fidelity check at epoch 80 vs baseline
|
||||
assert epoch80_test_acc is not None
|
||||
delta_ep80 = abs(epoch80_test_acc - baseline_test_acc)
|
||||
seed_fidelity_deltas[seed] = delta_ep80
|
||||
|
||||
if delta_ep80 > REPLAY_TOLERANCE:
|
||||
print(
|
||||
f"[WARNING] Seed {seed} epoch 80 test accuracy delta {delta_ep80:.6f} "
|
||||
f"exceeds tolerance {REPLAY_TOLERANCE} (actual={epoch80_test_acc:.4f}, baseline={baseline_test_acc:.4f})"
|
||||
)
|
||||
fidelity_passed = False
|
||||
|
||||
runs.append({
|
||||
"seed": seed,
|
||||
"model_fingerprint": model_fp,
|
||||
"expected_model_fingerprint": expected_model_fp,
|
||||
"fingerprint_matched": fp_match,
|
||||
"baseline_epoch80_test_acc": baseline_test_acc,
|
||||
"epoch80_test_acc": epoch80_test_acc,
|
||||
"epoch80_abs_delta": delta_ep80,
|
||||
"fit_time_sec": round(fit_time_sec, 4),
|
||||
"improvement_count": improvement_count,
|
||||
"last_improvement_epoch": last_improvement_epoch,
|
||||
"checkpoints": checkpoints,
|
||||
})
|
||||
|
||||
max_ep80_delta = max(seed_fidelity_deltas.values()) if seed_fidelity_deltas else 0.0
|
||||
|
||||
# 4. Aggregations and Statistical Summaries
|
||||
checkpoint_stats: Dict[int, Dict[str, Any]] = {}
|
||||
for ep in CHECKPOINT_EPOCHS:
|
||||
ep_train_losses = [next(c["train_loss"] for c in r["checkpoints"] if c["epoch"] == ep) for r in runs]
|
||||
ep_train_accs = [next(c["train_acc"] for c in r["checkpoints"] if c["epoch"] == ep) for r in runs]
|
||||
ep_train_mses = [next(c["train_mse"] for c in r["checkpoints"] if c["epoch"] == ep) for r in runs]
|
||||
|
||||
ep_test_losses = [next(c["test_loss"] for c in r["checkpoints"] if c["epoch"] == ep) for r in runs]
|
||||
ep_test_accs = [next(c["test_acc"] for c in r["checkpoints"] if c["epoch"] == ep) for r in runs]
|
||||
ep_test_mses = [next(c["test_mse"] for c in r["checkpoints"] if c["epoch"] == ep) for r in runs]
|
||||
|
||||
checkpoint_stats[ep] = {
|
||||
"train_loss": calc_stats(ep_train_losses),
|
||||
"train_acc": calc_stats(ep_train_accs),
|
||||
"train_mse": calc_stats(ep_train_mses),
|
||||
"test_loss": calc_stats(ep_test_losses),
|
||||
"test_acc": calc_stats(ep_test_accs),
|
||||
"test_mse": calc_stats(ep_test_mses),
|
||||
}
|
||||
|
||||
# Endpoint paired deltas (80->240 and 200->240)
|
||||
deltas_80_to_240_train_rel = []
|
||||
deltas_80_to_240_test_acc = []
|
||||
|
||||
deltas_200_to_240_train_rel = []
|
||||
deltas_200_to_240_test_acc = []
|
||||
deltas_200_to_240_test_acc_abs = []
|
||||
|
||||
for r in runs:
|
||||
ckpt_map = {c["epoch"]: c for c in r["checkpoints"]}
|
||||
|
||||
# 80 to 240
|
||||
tl80 = ckpt_map[80]["train_loss"]
|
||||
tl240 = ckpt_map[240]["train_loss"]
|
||||
rel_red_80_240 = (tl80 - tl240) / tl80 if tl80 > 0 else 0.0
|
||||
deltas_80_to_240_train_rel.append(rel_red_80_240)
|
||||
|
||||
ta80 = ckpt_map[80]["test_acc"]
|
||||
ta240 = ckpt_map[240]["test_acc"]
|
||||
acc_delta_80_240 = ta240 - ta80
|
||||
deltas_80_to_240_test_acc.append(acc_delta_80_240)
|
||||
|
||||
# 200 to 240
|
||||
tl200 = ckpt_map[200]["train_loss"]
|
||||
rel_red_200_240 = (tl200 - tl240) / tl200 if tl200 > 0 else 0.0
|
||||
deltas_200_to_240_train_rel.append(rel_red_200_240)
|
||||
|
||||
ta200 = ckpt_map[200]["test_acc"]
|
||||
acc_delta_200_240 = ta240 - ta200
|
||||
acc_abs_change_200_240 = abs(ta240 - ta200)
|
||||
deltas_200_to_240_test_acc.append(acc_delta_200_240)
|
||||
deltas_200_to_240_test_acc_abs.append(acc_abs_change_200_240)
|
||||
|
||||
# Calculate overall mean metrics for classifications
|
||||
mean_train_loss_80 = checkpoint_stats[80]["train_loss"]["mean"]
|
||||
mean_train_loss_200 = checkpoint_stats[200]["train_loss"]["mean"]
|
||||
mean_train_loss_240 = checkpoint_stats[240]["train_loss"]["mean"]
|
||||
|
||||
mean_test_acc_80 = checkpoint_stats[80]["test_acc"]["mean"]
|
||||
mean_test_acc_200 = checkpoint_stats[200]["test_acc"]["mean"]
|
||||
mean_test_acc_240 = checkpoint_stats[240]["test_acc"]["mean"]
|
||||
|
||||
rel_train_loss_reduction_80_240 = (
|
||||
(mean_train_loss_80 - mean_train_loss_240) / mean_train_loss_80
|
||||
)
|
||||
test_acc_gain_80_240 = mean_test_acc_240 - mean_test_acc_80
|
||||
|
||||
rel_train_loss_reduction_200_240 = (
|
||||
(mean_train_loss_200 - mean_train_loss_240) / mean_train_loss_200
|
||||
)
|
||||
abs_test_acc_change_200_240 = abs(mean_test_acc_240 - mean_test_acc_200)
|
||||
|
||||
# 5. Shared Contract Predeclared Classifications
|
||||
post_80_training_converging = bool(
|
||||
rel_train_loss_reduction_80_240 >= POST80_LOSS_REDUCTION_THRESHOLD
|
||||
)
|
||||
meaningful_held_out_gain = bool(
|
||||
test_acc_gain_80_240 >= TEST_ACCURACY_GAIN_THRESHOLD
|
||||
)
|
||||
overfitting_signal = bool(
|
||||
post_80_training_converging
|
||||
and test_acc_gain_80_240 <= OVERFITTING_ACCURACY_DROP_THRESHOLD
|
||||
)
|
||||
early_stagnation = bool(
|
||||
not post_80_training_converging
|
||||
and abs(test_acc_gain_80_240) < TEST_ACCURACY_GAIN_THRESHOLD
|
||||
)
|
||||
generalization_plateau = bool(
|
||||
post_80_training_converging
|
||||
and not meaningful_held_out_gain
|
||||
and not overfitting_signal
|
||||
)
|
||||
late_plateau_diagnostic = bool(
|
||||
rel_train_loss_reduction_200_240 < LATE_LOSS_REDUCTION_THRESHOLD
|
||||
and abs_test_acc_change_200_240 < LATE_ACCURACY_CHANGE_THRESHOLD
|
||||
)
|
||||
|
||||
if meaningful_held_out_gain:
|
||||
summary_verdict = (
|
||||
f"Training beyond epoch 80 continues to improve held-out test accuracy by "
|
||||
f"{test_acc_gain_80_240 * 100:.2f} percentage points "
|
||||
f"(from {mean_test_acc_80 * 100:.2f}% to {mean_test_acc_240 * 100:.2f}%)."
|
||||
)
|
||||
elif overfitting_signal:
|
||||
summary_verdict = (
|
||||
f"Training beyond epoch 80 exhibits overfitting: training loss falls by "
|
||||
f"{rel_train_loss_reduction_80_240 * 100:.2f}% while held-out test accuracy drops by "
|
||||
f"{abs(test_acc_gain_80_240) * 100:.2f} percentage points."
|
||||
)
|
||||
elif early_stagnation:
|
||||
summary_verdict = (
|
||||
f"Optimization has effectively stagnated after epoch 80: training loss falls by only "
|
||||
f"{rel_train_loss_reduction_80_240 * 100:.2f}% and held-out accuracy changes by "
|
||||
f"{test_acc_gain_80_240 * 100:+.2f} percentage points through epoch 240."
|
||||
)
|
||||
elif generalization_plateau:
|
||||
summary_verdict = (
|
||||
f"Training loss continues to improve after epoch 80, but held-out accuracy plateaus: "
|
||||
f"{test_acc_gain_80_240 * 100:+.2f} percentage points "
|
||||
f"(from {mean_test_acc_80 * 100:.2f}% to {mean_test_acc_240 * 100:.2f}%)."
|
||||
)
|
||||
else:
|
||||
summary_verdict = (
|
||||
"The fixed endpoint criteria are inconclusive; inspect the paired checkpoint "
|
||||
"trajectory before extending the epoch horizon."
|
||||
)
|
||||
|
||||
last_improvement_epochs_dict = {r["seed"]: r["last_improvement_epoch"] for r in runs}
|
||||
paired_endpoint_deltas = []
|
||||
for r in runs:
|
||||
ckpt_map = {c["epoch"]: c for c in r["checkpoints"]}
|
||||
paired_endpoint_deltas.append(
|
||||
{
|
||||
"seed": r["seed"],
|
||||
"train_loss_relative_reduction_80_to_240": (
|
||||
ckpt_map[80]["train_loss"] - ckpt_map[240]["train_loss"]
|
||||
)
|
||||
/ ckpt_map[80]["train_loss"],
|
||||
"test_accuracy_delta_80_to_240": (
|
||||
ckpt_map[240]["test_acc"] - ckpt_map[80]["test_acc"]
|
||||
),
|
||||
"train_loss_relative_reduction_200_to_240": (
|
||||
ckpt_map[200]["train_loss"] - ckpt_map[240]["train_loss"]
|
||||
)
|
||||
/ ckpt_map[200]["train_loss"],
|
||||
"test_accuracy_delta_200_to_240": (
|
||||
ckpt_map[240]["test_acc"] - ckpt_map[200]["test_acc"]
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# Build final payload
|
||||
payload = {
|
||||
"epoch_convergence_protocol_version": EPOCH_CONVERGENCE_PROTOCOL_VERSION,
|
||||
"tuning_protocol_version": TUNING_PROTOCOL_VERSION,
|
||||
"pso_version": pso_version,
|
||||
"torch_version": torch.__version__,
|
||||
"hardware": hw_provenance,
|
||||
"device": device.type,
|
||||
"timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"baseline_path": str(baseline_path),
|
||||
"data_fingerprint": data_fp,
|
||||
"candidate_label": winner_cfg.candidate_label,
|
||||
"config": {
|
||||
**opt_kwargs,
|
||||
"n_particles": n_particles,
|
||||
"epochs": target_epochs,
|
||||
"batch_size": batch_size,
|
||||
"renewal": "loss",
|
||||
"checkpoint_interval": 20,
|
||||
},
|
||||
"contract_criteria": {
|
||||
"primary_endpoints": [80, 120, 160, 200, 240],
|
||||
"post_80_convergence_threshold_loss_reduction": POST80_LOSS_REDUCTION_THRESHOLD,
|
||||
"meaningful_gain_threshold_test_acc": TEST_ACCURACY_GAIN_THRESHOLD,
|
||||
"overfitting_threshold_test_acc": OVERFITTING_ACCURACY_DROP_THRESHOLD,
|
||||
"late_plateau_200_240_loss_threshold": LATE_LOSS_REDUCTION_THRESHOLD,
|
||||
"late_plateau_200_240_acc_threshold": LATE_ACCURACY_CHANGE_THRESHOLD,
|
||||
"replay_tolerance": REPLAY_TOLERANCE,
|
||||
},
|
||||
"fidelity_validation": {
|
||||
"replay_seeds": sorted(REPLAY_SEEDS),
|
||||
"max_epoch80_test_acc_delta": round(max_ep80_delta, 6),
|
||||
"tolerance": REPLAY_TOLERANCE,
|
||||
"passed": fidelity_passed,
|
||||
},
|
||||
"predeclared_classifications": {
|
||||
"post_80_training_converging": post_80_training_converging,
|
||||
"meaningful_held_out_gain": meaningful_held_out_gain,
|
||||
"overfitting_signal": overfitting_signal,
|
||||
"early_stagnation": early_stagnation,
|
||||
"generalization_plateau": generalization_plateau,
|
||||
"late_plateau_diagnostic": late_plateau_diagnostic,
|
||||
"summary_verdict": summary_verdict,
|
||||
},
|
||||
"summary": {
|
||||
"epochs": CHECKPOINT_EPOCHS,
|
||||
"checkpoint_stats": {str(ep): stats for ep, stats in checkpoint_stats.items()},
|
||||
"paired_deltas": {
|
||||
"80_to_240": {
|
||||
"train_loss_rel_reduction": calc_stats(deltas_80_to_240_train_rel),
|
||||
"test_acc_delta": calc_stats(deltas_80_to_240_test_acc),
|
||||
},
|
||||
"200_to_240": {
|
||||
"train_loss_rel_reduction": calc_stats(deltas_200_to_240_train_rel),
|
||||
"test_acc_delta": calc_stats(deltas_200_to_240_test_acc),
|
||||
"test_acc_abs_change": calc_stats(deltas_200_to_240_test_acc_abs),
|
||||
},
|
||||
},
|
||||
"paired_endpoint_deltas_by_seed": paired_endpoint_deltas,
|
||||
"last_training_best_improvement_epochs": last_improvement_epochs_dict,
|
||||
},
|
||||
"runs": runs,
|
||||
"completed": True,
|
||||
"valid": fidelity_passed,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
# Write output JSON
|
||||
save_json_atomic(payload, output_json_path)
|
||||
print(f"Saved analysis JSON to {output_json_path}")
|
||||
|
||||
# Write output CSV
|
||||
output_csv_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with open(output_csv_path, "w", newline="", encoding="utf-8") as f:
|
||||
fieldnames = [
|
||||
"seed",
|
||||
"epoch",
|
||||
"train_loss",
|
||||
"train_acc",
|
||||
"train_mse",
|
||||
"test_loss",
|
||||
"test_acc",
|
||||
"test_mse",
|
||||
"fit_time_sec",
|
||||
]
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(flat_csv_rows)
|
||||
print(f"Saved flat checkpoint CSV to {output_csv_path}")
|
||||
|
||||
# 6. Render Figure
|
||||
render_convergence_figure(CHECKPOINT_EPOCHS, runs, checkpoint_stats, figure_path)
|
||||
print(f"Rendered plot to {figure_path}")
|
||||
|
||||
# Print summary block
|
||||
print("\n" + "=" * 70)
|
||||
print("EPOCH CONVERGENCE & TRAJECTORY ANALYSIS RESULTS")
|
||||
print("=" * 70)
|
||||
print(f"Device: {device.type} | Seeds: {sorted(REPLAY_SEEDS)}")
|
||||
print(f"Fidelity Replay Check (Epoch 80 <= {REPLAY_TOLERANCE}): Max Delta = {max_ep80_delta:.6f} -> Passed: {fidelity_passed}")
|
||||
print("-" * 70)
|
||||
print("Predeclared Classifications (Epoch 80 -> 240):")
|
||||
print(f" Post-80 Training Converging (Loss Drop >= 1%): {post_80_training_converging} ({rel_train_loss_reduction_80_240 * 100:.2f}%)")
|
||||
print(f" Meaningful Held-Out Gain (Acc Rise >= 1%p): {meaningful_held_out_gain} ({test_acc_gain_80_240 * 100:+.2f}%p)")
|
||||
print(f" Overfitting Signal: {overfitting_signal}")
|
||||
print(f" Early Stagnation: {early_stagnation}")
|
||||
print(f" Generalization Plateau: {generalization_plateau}")
|
||||
print(f" Late Plateau Diagnostic (200 -> 240): {late_plateau_diagnostic}")
|
||||
print("-" * 70)
|
||||
print(f"Verdict: {summary_verdict}")
|
||||
print("=" * 70 + "\n")
|
||||
|
||||
return fidelity_passed
|
||||
|
||||
|
||||
def render_convergence_figure(
|
||||
epochs: List[int],
|
||||
runs: List[Dict[str, Any]],
|
||||
checkpoint_stats: Dict[int, Dict[str, Any]],
|
||||
figure_path: Path,
|
||||
):
|
||||
figure_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5.5))
|
||||
|
||||
mean_train_loss = [checkpoint_stats[ep]["train_loss"]["mean"] for ep in epochs]
|
||||
std_train_loss = [checkpoint_stats[ep]["train_loss"]["std"] for ep in epochs]
|
||||
|
||||
mean_test_acc = [checkpoint_stats[ep]["test_acc"]["mean"] for ep in epochs]
|
||||
std_test_acc = [checkpoint_stats[ep]["test_acc"]["std"] for ep in epochs]
|
||||
|
||||
epochs_arr = np.array(epochs)
|
||||
mean_tl_arr = np.array(mean_train_loss)
|
||||
std_tl_arr = np.array(std_train_loss)
|
||||
|
||||
mean_ta_arr = np.array(mean_test_acc)
|
||||
std_ta_arr = np.array(std_test_acc)
|
||||
|
||||
# Subplot 1: Training Loss
|
||||
for r in runs:
|
||||
r_epochs = [c["epoch"] for c in r["checkpoints"]]
|
||||
r_losses = [c["train_loss"] for c in r["checkpoints"]]
|
||||
ax1.plot(r_epochs, r_losses, color="#1f77b4", alpha=0.25, linestyle=":", linewidth=1.2)
|
||||
|
||||
ax1.plot(epochs_arr, mean_tl_arr, marker="o", color="#1f77b4", linewidth=2.2, label="Mean Training Loss")
|
||||
ax1.fill_between(
|
||||
epochs_arr,
|
||||
mean_tl_arr - std_tl_arr,
|
||||
mean_tl_arr + std_tl_arr,
|
||||
color="#1f77b4",
|
||||
alpha=0.15,
|
||||
label="±1 Std Dev",
|
||||
)
|
||||
ax1.axvline(80, color="#d62728", linestyle="--", linewidth=1.5, label="Baseline Horizon (Epoch 80)")
|
||||
ax1.set_xlabel("Epoch", fontsize=11)
|
||||
ax1.set_ylabel("Global-Best Training Loss", fontsize=11)
|
||||
ax1.set_title("Global-Best Training Loss Trajectory", fontsize=12, fontweight="bold")
|
||||
ax1.grid(True, alpha=0.3)
|
||||
ax1.legend(loc="upper right", frameon=True)
|
||||
|
||||
# Subplot 2: Held-Out Test Accuracy
|
||||
for r in runs:
|
||||
r_epochs = [c["epoch"] for c in r["checkpoints"]]
|
||||
r_accs = [c["test_acc"] for c in r["checkpoints"]]
|
||||
ax2.plot(r_epochs, r_accs, color="#2ca02c", alpha=0.25, linestyle=":", linewidth=1.2)
|
||||
|
||||
ax2.plot(epochs_arr, mean_ta_arr, marker="s", color="#2ca02c", linewidth=2.2, label="Mean Test Accuracy")
|
||||
ax2.fill_between(
|
||||
epochs_arr,
|
||||
mean_ta_arr - std_ta_arr,
|
||||
mean_ta_arr + std_ta_arr,
|
||||
color="#2ca02c",
|
||||
alpha=0.15,
|
||||
label="±1 Std Dev",
|
||||
)
|
||||
ax2.axvline(80, color="#d62728", linestyle="--", linewidth=1.5, label="Baseline Horizon (Epoch 80)")
|
||||
ax2.set_xlabel("Epoch", fontsize=11)
|
||||
ax2.set_ylabel("Held-Out Test Accuracy", fontsize=11)
|
||||
ax2.set_title("Held-Out Test Accuracy Trajectory", fontsize=12, fontweight="bold")
|
||||
ax2.grid(True, alpha=0.3)
|
||||
ax2.legend(loc="lower right", frameon=True)
|
||||
|
||||
fig.suptitle(
|
||||
"Adaptive Moment 120-Particle MNIST 240-Epoch Convergence & Trajectory Analysis",
|
||||
fontsize=13,
|
||||
fontweight="bold",
|
||||
)
|
||||
plt.tight_layout(rect=(0.0, 0.0, 1.0, 0.95))
|
||||
plt.savefig(figure_path, dpi=300, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Adaptive Moment 120-Particle x 240-Epoch MNIST Convergence Analysis"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--baseline-json",
|
||||
type=Path,
|
||||
default=Path("benchmark_results/pso_v4_tuning.json"),
|
||||
help="Path to baseline tuning JSON file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-json",
|
||||
type=Path,
|
||||
default=Path("benchmark_results/pso_v4_epoch_convergence.json"),
|
||||
help="Path to output analysis JSON file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-csv",
|
||||
type=Path,
|
||||
default=Path("benchmark_results/pso_v4_epoch_convergence.csv"),
|
||||
help="Path to output checkpoint CSV file",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--figure",
|
||||
type=Path,
|
||||
default=Path("history_plt/pso_v4_epoch_convergence.png"),
|
||||
help="Path to output convergence PNG figure",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--device",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Execution device (mps, cuda, cpu; default: auto-detect)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
passed = run_epoch_convergence_analysis(
|
||||
baseline_path=args.baseline_json,
|
||||
output_json_path=args.output_json,
|
||||
output_csv_path=args.output_csv,
|
||||
figure_path=args.figure,
|
||||
device_str=args.device,
|
||||
)
|
||||
|
||||
if not passed:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,539 @@
|
||||
"""
|
||||
Strict Pareto Evaluator for Heavy Task PSO Autoresearch.
|
||||
|
||||
Evaluates candidate equalized signed-hash subspace experiment artifacts against
|
||||
baseline heavy task benchmark results (benchmark_results/pso_v6_heavy_tasks.json).
|
||||
|
||||
Baseline Policy:
|
||||
- mnist_compact: G8
|
||||
- mnist_wide: G5
|
||||
- fashion_compact: G8
|
||||
- fashion_wide: G5
|
||||
|
||||
Evaluates 7 Hard Gates:
|
||||
1. Finite Metrics (all validation metrics finite across runs)
|
||||
2. Test-Sealed (zero official test data loaded & 0 test evaluations)
|
||||
3. Config Matched (12 particles, 80 epochs, 10k subset, seeds 101-103)
|
||||
4. State Ratio Boundary (max state ratio <= 0.5 across all workloads)
|
||||
5. Workload Accuracy Regression Boundary (each workload acc regression <= 1.0 pp)
|
||||
6. Workload NLL Regression Boundary (each workload NLL regression <= 5.0%)
|
||||
7. Worst-Workload (mnist_wide) Improvement (acc gain >= 2.0 pp OR NLL reduction >= 5.0%)
|
||||
|
||||
Numeric Score:
|
||||
Score = mean_rel_nll_reduction_pct + mean_acc_gain_pp + 10 * log2(1 / max_state_ratio) - 100 * failed_gate_count
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
||||
|
||||
# Ensure test directory and repo root are in Python path
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent if Path(__file__).resolve().parent.name != "PSO" else Path(__file__).resolve().parent
|
||||
TEST_DIR = REPO_ROOT / "test"
|
||||
if str(TEST_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TEST_DIR))
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from benchmark_suite import save_json_atomic
|
||||
from heavy_pso_autoresearch import compute_latent_dim, compute_core_swarm_state_bytes
|
||||
EVALUATOR_VERSION = "EVALUATE-HEAVY-AUTORESEARCH 1.0.0"
|
||||
|
||||
BASELINE_POLICY: Dict[str, str] = {
|
||||
"mnist_compact": "G8",
|
||||
"mnist_wide": "G5",
|
||||
"fashion_compact": "G8",
|
||||
"fashion_wide": "G5",
|
||||
}
|
||||
|
||||
EXPECTED_SEEDS = [101, 102, 103]
|
||||
EXPECTED_PARTICLES = 12
|
||||
EXPECTED_EPOCHS = 80
|
||||
EXPECTED_SUBSET_SIZE = 10000
|
||||
EXPECTED_WORKLOADS = ["mnist_compact", "mnist_wide", "fashion_compact", "fashion_wide"]
|
||||
|
||||
|
||||
def evaluate_heavy_autoresearch(
|
||||
baseline_path: Union[str, Path],
|
||||
candidate_path: Union[str, Path],
|
||||
output_path: Optional[Union[str, Path]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Evaluates a candidate experiment artifact against baseline heavy task results.
|
||||
Validates schemas, configurations, seeds, test-sealed constraints, finiteness,
|
||||
calculates every hard gate and numeric score, selects the winning candidate,
|
||||
and returns a structured evaluator payload.
|
||||
"""
|
||||
baseline_path = Path(baseline_path)
|
||||
candidate_path = Path(candidate_path)
|
||||
|
||||
if not baseline_path.is_file():
|
||||
raise FileNotFoundError(f"Baseline artifact not found at '{baseline_path}'")
|
||||
if not candidate_path.is_file():
|
||||
raise FileNotFoundError(f"Candidate artifact not found at '{candidate_path}'")
|
||||
|
||||
with open(baseline_path, "r", encoding="utf-8") as f:
|
||||
baseline_data = json.load(f)
|
||||
|
||||
with open(candidate_path, "r", encoding="utf-8") as f:
|
||||
candidate_data = json.load(f)
|
||||
|
||||
WORKLOAD_TOTAL_DIMS: Dict[str, int] = {
|
||||
"mnist_compact": 9098,
|
||||
"mnist_wide": 55338,
|
||||
"fashion_compact": 9098,
|
||||
"fashion_wide": 55338,
|
||||
}
|
||||
|
||||
if (
|
||||
baseline_data.get("official_test_data_loaded") is not False
|
||||
or baseline_data.get("official_test_evaluations") != 0
|
||||
):
|
||||
raise ValueError("Baseline artifact must explicitly seal official test data.")
|
||||
|
||||
# Validate baseline JSON schema & confirmation results
|
||||
if "confirmation_results" not in baseline_data:
|
||||
raise KeyError("Baseline JSON missing top-level 'confirmation_results' key")
|
||||
|
||||
baseline_confirm = baseline_data["confirmation_results"]
|
||||
baseline_metrics: Dict[str, Dict[str, float]] = {}
|
||||
baseline_core_bytes: Dict[str, int] = {}
|
||||
|
||||
for wl in EXPECTED_WORKLOADS:
|
||||
if wl not in baseline_confirm:
|
||||
raise KeyError(f"Baseline confirmation results missing workload '{wl}'")
|
||||
selected_method = BASELINE_POLICY[wl]
|
||||
if selected_method not in baseline_confirm[wl]:
|
||||
raise KeyError(
|
||||
f"Baseline confirmation results for '{wl}' missing policy method '{selected_method}'"
|
||||
)
|
||||
entry = baseline_confirm[wl][selected_method]
|
||||
stats = entry.get("stats", {}) if isinstance(entry, dict) else {}
|
||||
val_nll_mean = float(stats.get("val_nll", {}).get("mean", float("nan")))
|
||||
val_acc_mean = float(stats.get("val_acc", {}).get("mean", float("nan")))
|
||||
if not (math.isfinite(val_nll_mean) and math.isfinite(val_acc_mean)):
|
||||
raise ValueError(f"Baseline metrics for '{wl}' method '{selected_method}' contain NaN or non-finite value")
|
||||
|
||||
baseline_metrics[wl] = {
|
||||
"val_nll": val_nll_mean,
|
||||
"val_acc": val_acc_mean,
|
||||
"val_brier": float(stats.get("val_brier", {}).get("mean", 0.0) or 0.0),
|
||||
"val_ece": float(stats.get("val_ece", {}).get("mean", 0.0) or 0.0),
|
||||
"method_id": selected_method,
|
||||
}
|
||||
|
||||
bytes_list = []
|
||||
if (
|
||||
isinstance(entry, dict)
|
||||
and isinstance(entry.get("per_seed_runs"), list)
|
||||
):
|
||||
bytes_list = [
|
||||
int(run["core_swarm_state_bytes"])
|
||||
for run in entry["per_seed_runs"]
|
||||
if (
|
||||
isinstance(run, dict)
|
||||
and "core_swarm_state_bytes" in run
|
||||
and math.isfinite(float(run["core_swarm_state_bytes"]))
|
||||
)
|
||||
]
|
||||
if bytes_list and len(set(bytes_list)) != 1:
|
||||
raise ValueError(f"Baseline core-state bytes vary across seeds for '{wl}'.")
|
||||
if bytes_list:
|
||||
b_bytes = bytes_list[0]
|
||||
else:
|
||||
states = 5 * EXPECTED_PARTICLES + (1 if selected_method == "G8" else 0)
|
||||
b_bytes = states * WORKLOAD_TOTAL_DIMS[wl] * 4
|
||||
baseline_core_bytes[wl] = b_bytes
|
||||
|
||||
# Validate candidate JSON schema
|
||||
if "candidate_runs" not in candidate_data:
|
||||
raise KeyError("Candidate JSON missing top-level 'candidate_runs' key")
|
||||
|
||||
candidate_runs = candidate_data["candidate_runs"]
|
||||
if not isinstance(candidate_runs, dict) or len(candidate_runs) == 0:
|
||||
raise ValueError("Candidate JSON 'candidate_runs' must be a non-empty dictionary")
|
||||
|
||||
# Global candidate test-sealed checks
|
||||
top_test_loaded_present = "official_test_data_loaded" in candidate_data
|
||||
top_test_loaded_val = candidate_data.get("official_test_data_loaded")
|
||||
top_test_evals_present = "official_test_evaluations" in candidate_data
|
||||
top_test_evals_val = candidate_data.get("official_test_evaluations")
|
||||
|
||||
top_test_sealed = (
|
||||
top_test_loaded_present
|
||||
and top_test_loaded_val is False
|
||||
and top_test_evals_present
|
||||
and top_test_evals_val == 0
|
||||
)
|
||||
|
||||
candidate_evaluations: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
for cand_id, wl_map in candidate_runs.items():
|
||||
gate_finite = True
|
||||
gate_test_sealed = bool(top_test_sealed)
|
||||
gate_config_matched = True
|
||||
|
||||
if not isinstance(wl_map, dict):
|
||||
gate_config_matched = False
|
||||
gate_finite = False
|
||||
wl_map = {}
|
||||
|
||||
if set(wl_map) != set(EXPECTED_WORKLOADS):
|
||||
gate_config_matched = False
|
||||
|
||||
for wl in EXPECTED_WORKLOADS:
|
||||
if wl not in wl_map:
|
||||
gate_config_matched = False
|
||||
gate_finite = False
|
||||
|
||||
derived_ratios: Dict[str, float] = {}
|
||||
per_workload_deltas: Dict[str, Dict[str, float]] = {}
|
||||
acc_regressions_valid = True
|
||||
nll_regressions_valid = True
|
||||
|
||||
for wl in EXPECTED_WORKLOADS:
|
||||
if wl not in wl_map or not isinstance(wl_map[wl], dict):
|
||||
acc_regressions_valid = False
|
||||
nll_regressions_valid = False
|
||||
continue
|
||||
|
||||
wl_entry = wl_map[wl]
|
||||
p_val = wl_entry.get("particles")
|
||||
e_val = wl_entry.get("epochs")
|
||||
sub_val = wl_entry.get("subset_size")
|
||||
seeds_val = wl_entry.get("seeds")
|
||||
|
||||
if (
|
||||
p_val != EXPECTED_PARTICLES
|
||||
or e_val != EXPECTED_EPOCHS
|
||||
or sub_val != EXPECTED_SUBSET_SIZE
|
||||
or seeds_val != EXPECTED_SEEDS
|
||||
):
|
||||
gate_config_matched = False
|
||||
|
||||
per_seed = wl_entry.get("per_seed_runs")
|
||||
if not isinstance(per_seed, list) or len(per_seed) != len(EXPECTED_SEEDS):
|
||||
gate_config_matched = False
|
||||
gate_finite = False
|
||||
actual_seeds = []
|
||||
else:
|
||||
actual_seeds = [
|
||||
s_run.get("seed") for s_run in per_seed if isinstance(s_run, dict) and "seed" in s_run
|
||||
]
|
||||
if actual_seeds != EXPECTED_SEEDS:
|
||||
gate_config_matched = False
|
||||
|
||||
expected_queries = EXPECTED_PARTICLES * EXPECTED_EPOCHS
|
||||
expected_samples = expected_queries * EXPECTED_SUBSET_SIZE
|
||||
for s_run in per_seed:
|
||||
if (
|
||||
not isinstance(s_run, dict)
|
||||
or s_run.get("total_queries") != expected_queries
|
||||
or s_run.get("total_sample_evaluations") != expected_samples
|
||||
):
|
||||
gate_config_matched = False
|
||||
|
||||
stats = wl_entry.get("stats") if isinstance(wl_entry.get("stats"), dict) else {}
|
||||
val_nll_dict = stats.get("val_nll") if isinstance(stats.get("val_nll"), dict) else {}
|
||||
val_acc_dict = stats.get("val_acc") if isinstance(stats.get("val_acc"), dict) else {}
|
||||
|
||||
c_nll_raw = val_nll_dict.get("mean")
|
||||
c_acc_raw = val_acc_dict.get("mean")
|
||||
|
||||
if (
|
||||
c_nll_raw is None
|
||||
or c_acc_raw is None
|
||||
or not isinstance(c_nll_raw, (int, float))
|
||||
or not isinstance(c_acc_raw, (int, float))
|
||||
or not (math.isfinite(float(c_nll_raw)) and math.isfinite(float(c_acc_raw)))
|
||||
):
|
||||
gate_finite = False
|
||||
c_nll = float("nan")
|
||||
c_acc = float("nan")
|
||||
else:
|
||||
c_nll = float(c_nll_raw)
|
||||
c_acc = float(c_acc_raw)
|
||||
|
||||
if isinstance(per_seed, list):
|
||||
for s_run in per_seed:
|
||||
if not isinstance(s_run, dict):
|
||||
gate_finite = False
|
||||
gate_config_matched = False
|
||||
continue
|
||||
if "official_test_evaluations" not in s_run or s_run["official_test_evaluations"] != 0:
|
||||
gate_test_sealed = False
|
||||
if s_run.get("is_finite") is not True:
|
||||
gate_finite = False
|
||||
|
||||
s_nll = s_run.get("val_selected_loss")
|
||||
s_acc = s_run.get("val_selected_acc")
|
||||
g_nll = s_run.get("gbest_loss")
|
||||
g_acc = s_run.get("gbest_acc")
|
||||
w_time = s_run.get("wall_time_sec")
|
||||
val_m = s_run.get("val_metrics")
|
||||
|
||||
val_m_ok = (
|
||||
isinstance(val_m, dict)
|
||||
and len(val_m) > 0
|
||||
and all(
|
||||
isinstance(v, (int, float)) and math.isfinite(float(v))
|
||||
for v in val_m.values()
|
||||
)
|
||||
)
|
||||
|
||||
scalars_for_s_run = [s_nll, s_acc, g_nll, g_acc, w_time]
|
||||
s_scalars_ok = all(
|
||||
v is not None and isinstance(v, (int, float)) and math.isfinite(float(v))
|
||||
for v in scalars_for_s_run
|
||||
)
|
||||
|
||||
if not (val_m_ok and s_scalars_ok):
|
||||
gate_finite = False
|
||||
|
||||
expected_total_dim = WORKLOAD_TOTAL_DIMS[wl]
|
||||
total_dim = wl_entry.get("total_dim")
|
||||
raw_ratio = wl_entry.get("ratio")
|
||||
if (
|
||||
total_dim != expected_total_dim
|
||||
or not isinstance(raw_ratio, (int, float))
|
||||
or not (0.0 < float(raw_ratio) <= 1.0)
|
||||
):
|
||||
gate_config_matched = False
|
||||
total_dim = expected_total_dim
|
||||
raw_ratio = 1.0
|
||||
|
||||
latent_dim = compute_latent_dim(total_dim, float(raw_ratio))
|
||||
if wl_entry.get("latent_dim") != latent_dim:
|
||||
gate_config_matched = False
|
||||
|
||||
analytical_cand_bytes = compute_core_swarm_state_bytes(EXPECTED_PARTICLES, latent_dim)
|
||||
b_bytes = baseline_core_bytes[wl]
|
||||
|
||||
derived_ratio = float(analytical_cand_bytes) / float(b_bytes)
|
||||
derived_ratios[wl] = derived_ratio
|
||||
|
||||
if wl_entry.get("core_swarm_state_bytes") != analytical_cand_bytes:
|
||||
gate_config_matched = False
|
||||
|
||||
if isinstance(per_seed, list):
|
||||
for s_run in per_seed:
|
||||
if isinstance(s_run, dict) and s_run.get("core_swarm_state_bytes") != analytical_cand_bytes:
|
||||
gate_config_matched = False
|
||||
|
||||
reported_state_ratio = wl_entry.get("state_ratio")
|
||||
if (
|
||||
not isinstance(reported_state_ratio, (int, float))
|
||||
or not math.isfinite(float(reported_state_ratio))
|
||||
or not math.isclose(
|
||||
float(reported_state_ratio),
|
||||
derived_ratio,
|
||||
rel_tol=1e-5,
|
||||
abs_tol=1e-5,
|
||||
)
|
||||
):
|
||||
gate_config_matched = False
|
||||
|
||||
b_metrics = baseline_metrics[wl]
|
||||
b_nll = b_metrics["val_nll"]
|
||||
b_acc = b_metrics["val_acc"]
|
||||
|
||||
if math.isfinite(c_nll) and math.isfinite(c_acc) and math.isfinite(b_nll) and math.isfinite(b_acc):
|
||||
nll_delta = c_nll - b_nll
|
||||
rel_nll_reduction_pct = ((b_nll - c_nll) / b_nll) * 100.0 if b_nll > 0 else 0.0
|
||||
acc_gain_pp = c_acc - b_acc
|
||||
|
||||
per_workload_deltas[wl] = {
|
||||
"state_ratio": derived_ratio,
|
||||
"baseline_nll": b_nll,
|
||||
"candidate_nll": c_nll,
|
||||
"nll_delta": nll_delta,
|
||||
"rel_nll_reduction_pct": rel_nll_reduction_pct,
|
||||
"baseline_acc": b_acc,
|
||||
"candidate_acc": c_acc,
|
||||
"acc_gain_pp": acc_gain_pp,
|
||||
}
|
||||
|
||||
if acc_gain_pp < -1.0:
|
||||
acc_regressions_valid = False
|
||||
if rel_nll_reduction_pct < -5.0:
|
||||
nll_regressions_valid = False
|
||||
else:
|
||||
acc_regressions_valid = False
|
||||
nll_regressions_valid = False
|
||||
gate_finite = False
|
||||
|
||||
if derived_ratios and len(derived_ratios) == len(EXPECTED_WORKLOADS):
|
||||
max_state_ratio = max(derived_ratios.values())
|
||||
else:
|
||||
max_state_ratio = 1.0
|
||||
|
||||
gate_state_ratio = bool(0.0 < max_state_ratio <= 0.5 and math.isfinite(max_state_ratio))
|
||||
gate_acc_regression = bool(acc_regressions_valid)
|
||||
gate_nll_regression = bool(nll_regressions_valid)
|
||||
|
||||
if "mnist_wide" in per_workload_deltas:
|
||||
mw_delta = per_workload_deltas["mnist_wide"]
|
||||
mw_acc_gain = mw_delta["acc_gain_pp"]
|
||||
mw_nll_red = mw_delta["rel_nll_reduction_pct"]
|
||||
gate_baseline_worst_improvement = bool((mw_acc_gain >= 2.0) or (mw_nll_red >= 5.0))
|
||||
else:
|
||||
gate_baseline_worst_improvement = False
|
||||
|
||||
gates = {
|
||||
"gate_finite": bool(gate_finite),
|
||||
"gate_test_sealed": bool(gate_test_sealed),
|
||||
"gate_config_matched": bool(gate_config_matched),
|
||||
"gate_state_ratio": bool(gate_state_ratio),
|
||||
"gate_acc_regression": bool(gate_acc_regression),
|
||||
"gate_nll_regression": bool(gate_nll_regression),
|
||||
"gate_baseline_worst_improvement": bool(gate_baseline_worst_improvement),
|
||||
}
|
||||
|
||||
failed_gates = [g_name for g_name, g_pass in gates.items() if not g_pass]
|
||||
failed_gate_count = len(failed_gates)
|
||||
cand_pass = bool(failed_gate_count == 0)
|
||||
|
||||
if len(per_workload_deltas) == len(EXPECTED_WORKLOADS):
|
||||
mean_rel_nll_reduction_pct = float(
|
||||
sum(d["rel_nll_reduction_pct"] for d in per_workload_deltas.values())
|
||||
/ len(per_workload_deltas)
|
||||
)
|
||||
mean_acc_gain_pp = float(
|
||||
sum(d["acc_gain_pp"] for d in per_workload_deltas.values())
|
||||
/ len(per_workload_deltas)
|
||||
)
|
||||
else:
|
||||
mean_rel_nll_reduction_pct = 0.0
|
||||
mean_acc_gain_pp = 0.0
|
||||
|
||||
if 0.0 < max_state_ratio <= 1.0 and math.isfinite(max_state_ratio):
|
||||
state_efficiency_bonus = 10.0 * math.log2(1.0 / max_state_ratio)
|
||||
else:
|
||||
state_efficiency_bonus = 0.0
|
||||
|
||||
score = (
|
||||
mean_rel_nll_reduction_pct
|
||||
+ mean_acc_gain_pp
|
||||
+ state_efficiency_bonus
|
||||
- (100.0 * failed_gate_count)
|
||||
)
|
||||
|
||||
if not math.isfinite(score):
|
||||
score = -100.0 * max(1, failed_gate_count)
|
||||
|
||||
candidate_evaluations[cand_id] = {
|
||||
"candidate_id": cand_id,
|
||||
"pass": cand_pass,
|
||||
"score": float(score),
|
||||
"failed_gates": failed_gates,
|
||||
"failed_gate_count": failed_gate_count,
|
||||
"gate_details": gates,
|
||||
"per_workload": per_workload_deltas,
|
||||
"state_ratios": derived_ratios,
|
||||
"summary_metrics": {
|
||||
"mean_rel_nll_reduction_pct": mean_rel_nll_reduction_pct,
|
||||
"mean_acc_gain_pp": mean_acc_gain_pp,
|
||||
"max_state_ratio": max_state_ratio,
|
||||
"state_efficiency_bonus": state_efficiency_bonus,
|
||||
},
|
||||
}
|
||||
|
||||
passing_cand_ids = [
|
||||
c_id for c_id, c_eval in candidate_evaluations.items() if c_eval["pass"]
|
||||
]
|
||||
|
||||
if passing_cand_ids:
|
||||
selected_candidate_id = max(
|
||||
passing_cand_ids, key=lambda c_id: candidate_evaluations[c_id]["score"]
|
||||
)
|
||||
overall_pass = True
|
||||
else:
|
||||
selected_candidate_id = max(
|
||||
candidate_evaluations.keys(),
|
||||
key=lambda c_id: candidate_evaluations[c_id]["score"],
|
||||
)
|
||||
overall_pass = False
|
||||
|
||||
selected_eval = candidate_evaluations[selected_candidate_id]
|
||||
|
||||
evaluator_output = {
|
||||
"pass": overall_pass,
|
||||
"score": float(selected_eval["score"]),
|
||||
"selected_candidate_id": selected_candidate_id,
|
||||
"evaluator_version": EVALUATOR_VERSION,
|
||||
"source_paths": {
|
||||
"baseline_path": str(baseline_path),
|
||||
"candidate_path": str(candidate_path),
|
||||
},
|
||||
"baseline_policy": BASELINE_POLICY,
|
||||
"candidate_evaluations": candidate_evaluations,
|
||||
"per_workload_deltas": selected_eval["per_workload"],
|
||||
"state_ratios": selected_eval["state_ratios"],
|
||||
}
|
||||
|
||||
if output_path is not None:
|
||||
save_json_atomic(evaluator_output, Path(output_path))
|
||||
|
||||
return evaluator_output
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Strict Pareto Evaluator for Heavy Task PSO Autoresearch"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--baseline",
|
||||
type=str,
|
||||
default="benchmark_results/pso_v6_heavy_tasks.json",
|
||||
help="Path to baseline heavy tasks JSON artifact",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--candidate",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to candidate heavy autoresearch JSON artifact",
|
||||
)
|
||||
parser.add_argument(
|
||||
"candidate_pos",
|
||||
nargs="?",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Positional candidate JSON path fallback",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Optional path to write evaluator result JSON",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main():
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
cand_path = args.candidate or args.candidate_pos
|
||||
if not cand_path:
|
||||
parser.error("Candidate JSON path must be supplied via --candidate or positional argument.")
|
||||
|
||||
out_path = Path(args.output) if args.output else None
|
||||
|
||||
result = evaluate_heavy_autoresearch(
|
||||
baseline_path=args.baseline,
|
||||
candidate_path=cand_path,
|
||||
output_path=out_path,
|
||||
)
|
||||
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,741 @@
|
||||
"""Strict evaluator for the Heavy PSO cross-split robustness mission.
|
||||
|
||||
Every candidate is compared with a baseline rerun on the same train/validation
|
||||
partition and swarm seeds. Official test data must remain sealed. Development
|
||||
may qualify a policy for one-shot confirmation, but mission ``pass`` is true
|
||||
only when both phases satisfy the frozen evaluator contract.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
TEST_DIR = REPO_ROOT / "test"
|
||||
if str(TEST_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TEST_DIR))
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from benchmark_suite import save_json_atomic
|
||||
from heavy_pso_autoresearch import compute_core_swarm_state_bytes, compute_latent_dim
|
||||
|
||||
EVALUATOR_VERSION = "HEAVY-PSO-CROSS-SPLIT-EVALUATOR 1.0.0"
|
||||
EXPECTED_PARTICLES = 12
|
||||
EXPECTED_EPOCHS = 80
|
||||
EXPECTED_SUBSET_SIZE = 10000
|
||||
EXPECTED_QUERIES = EXPECTED_PARTICLES * EXPECTED_EPOCHS
|
||||
EXPECTED_SAMPLES = EXPECTED_QUERIES * EXPECTED_SUBSET_SIZE
|
||||
EXPECTED_DEV_SPLIT_SEEDS = [20260905, 20260906]
|
||||
EXPECTED_DEV_SWARM_SEEDS = [101, 102, 103]
|
||||
EXPECTED_CONF_SPLIT_SEEDS = [20260907]
|
||||
EXPECTED_CONF_SWARM_SEEDS = [111, 112, 113]
|
||||
WORKLOADS = ["mnist_compact", "mnist_wide", "fashion_compact", "fashion_wide"]
|
||||
BASELINE_METHODS = {
|
||||
"mnist_compact": "G8",
|
||||
"mnist_wide": "G5",
|
||||
"fashion_compact": "G8",
|
||||
"fashion_wide": "G5",
|
||||
}
|
||||
TOTAL_DIMS = {
|
||||
"mnist_compact": 9098,
|
||||
"mnist_wide": 55338,
|
||||
"fashion_compact": 9098,
|
||||
"fashion_wide": 55338,
|
||||
}
|
||||
PHASE_SPECS = {
|
||||
"development": (EXPECTED_DEV_SPLIT_SEEDS, EXPECTED_DEV_SWARM_SEEDS),
|
||||
"confirmation": (EXPECTED_CONF_SPLIT_SEEDS, EXPECTED_CONF_SWARM_SEEDS),
|
||||
}
|
||||
|
||||
|
||||
def load_artifact(path: Path) -> Dict[str, Any]:
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"Artifact file not found: {path}")
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"Artifact at {path} must be a JSON object")
|
||||
return data
|
||||
|
||||
|
||||
def _is_finite_number(value: Any) -> bool:
|
||||
return (
|
||||
isinstance(value, (int, float))
|
||||
and not isinstance(value, bool)
|
||||
and math.isfinite(float(value))
|
||||
)
|
||||
|
||||
|
||||
def _mean(values: Sequence[float]) -> float:
|
||||
return float(math.fsum(values) / len(values))
|
||||
|
||||
|
||||
def _append_issue(issues: Dict[str, List[str]], category: str, message: str) -> None:
|
||||
issues[category].append(message)
|
||||
|
||||
|
||||
def _validate_stats(
|
||||
entry: Dict[str, Any],
|
||||
per_seed_runs: List[Dict[str, Any]],
|
||||
label: str,
|
||||
issues: Dict[str, List[str]],
|
||||
) -> Optional[Tuple[float, float]]:
|
||||
stats = entry.get("stats")
|
||||
if not isinstance(stats, dict):
|
||||
_append_issue(issues, "schema", f"{label}: missing stats object")
|
||||
return None
|
||||
|
||||
try:
|
||||
acc_mean = stats["val_acc"]["mean"]
|
||||
nll_mean = stats["val_nll"]["mean"]
|
||||
except (KeyError, TypeError):
|
||||
_append_issue(issues, "schema", f"{label}: missing val_acc/val_nll means")
|
||||
return None
|
||||
|
||||
if not (_is_finite_number(acc_mean) and _is_finite_number(nll_mean)):
|
||||
_append_issue(issues, "finite", f"{label}: non-finite aggregate metrics")
|
||||
return None
|
||||
|
||||
if len(per_seed_runs) > 0:
|
||||
run_accs: List[float] = []
|
||||
run_nlls: List[float] = []
|
||||
for run in per_seed_runs:
|
||||
if not isinstance(run, dict):
|
||||
continue
|
||||
seed_label = f"{label}/seed={run.get('seed')}"
|
||||
|
||||
if "val_selected_acc" not in run or run.get("val_selected_acc") is None:
|
||||
_append_issue(issues, "schema", f"{seed_label}: missing val_selected_acc")
|
||||
else:
|
||||
val_acc = run["val_selected_acc"]
|
||||
if not isinstance(val_acc, (int, float)) or isinstance(val_acc, bool):
|
||||
_append_issue(issues, "schema", f"{seed_label}: non-numeric val_selected_acc")
|
||||
elif not math.isfinite(float(val_acc)):
|
||||
_append_issue(issues, "finite", f"{seed_label}: non-finite val_selected_acc")
|
||||
else:
|
||||
run_accs.append(float(val_acc))
|
||||
|
||||
if "val_selected_loss" not in run or run.get("val_selected_loss") is None:
|
||||
_append_issue(issues, "schema", f"{seed_label}: missing val_selected_loss")
|
||||
else:
|
||||
val_nll = run["val_selected_loss"]
|
||||
if not isinstance(val_nll, (int, float)) or isinstance(val_nll, bool):
|
||||
_append_issue(issues, "schema", f"{seed_label}: non-numeric val_selected_loss")
|
||||
elif not math.isfinite(float(val_nll)):
|
||||
_append_issue(issues, "finite", f"{seed_label}: non-finite val_selected_loss")
|
||||
else:
|
||||
run_nlls.append(float(val_nll))
|
||||
|
||||
if len(run_accs) == len(per_seed_runs):
|
||||
if not math.isclose(float(acc_mean), _mean(run_accs), rel_tol=1e-6, abs_tol=1e-5):
|
||||
_append_issue(issues, "schema", f"{label}: val_acc mean disagrees with per-seed runs")
|
||||
if len(run_nlls) == len(per_seed_runs):
|
||||
if not math.isclose(float(nll_mean), _mean(run_nlls), rel_tol=1e-6, abs_tol=1e-5):
|
||||
_append_issue(issues, "schema", f"{label}: val_nll mean disagrees with per-seed runs")
|
||||
return float(acc_mean), float(nll_mean)
|
||||
|
||||
|
||||
def _validate_runs(
|
||||
entry: Dict[str, Any],
|
||||
expected_seeds: List[int],
|
||||
expected_state_bytes: int,
|
||||
label: str,
|
||||
candidate: bool,
|
||||
issues: Dict[str, List[str]],
|
||||
) -> Tuple[List[Dict[str, Any]], int, int]:
|
||||
runs = entry.get("per_seed_runs")
|
||||
if not isinstance(runs, list) or len(runs) != len(expected_seeds):
|
||||
_append_issue(issues, "schema", f"{label}: expected {len(expected_seeds)} per-seed runs")
|
||||
return [], 0, 0
|
||||
|
||||
if [run.get("seed") if isinstance(run, dict) else None for run in runs] != expected_seeds:
|
||||
_append_issue(issues, "config", f"{label}: per-seed run order/content does not match {expected_seeds}")
|
||||
|
||||
total_queries = 0
|
||||
total_samples = 0
|
||||
numeric_fields = (
|
||||
"val_selected_loss",
|
||||
"val_selected_acc",
|
||||
"gbest_loss",
|
||||
"gbest_acc",
|
||||
"wall_time_sec",
|
||||
"optimization_wall_time_sec",
|
||||
"validation_wall_time_sec",
|
||||
"throughput_samples_per_sec",
|
||||
)
|
||||
|
||||
valid_runs: List[Dict[str, Any]] = []
|
||||
for run in runs:
|
||||
if not isinstance(run, dict):
|
||||
_append_issue(issues, "schema", f"{label}: non-object seed record")
|
||||
continue
|
||||
seed_label = f"{label}/seed={run.get('seed')}"
|
||||
for field in numeric_fields:
|
||||
if not _is_finite_number(run.get(field)):
|
||||
_append_issue(issues, "finite", f"{seed_label}: missing/non-finite {field}")
|
||||
|
||||
val_metrics = run.get("val_metrics")
|
||||
if (
|
||||
not isinstance(val_metrics, dict)
|
||||
or not val_metrics
|
||||
or any(not _is_finite_number(value) for value in val_metrics.values())
|
||||
):
|
||||
_append_issue(issues, "finite", f"{seed_label}: missing/non-finite val_metrics")
|
||||
|
||||
if run.get("official_test_evaluations") != 0:
|
||||
_append_issue(issues, "test", f"{seed_label}: official_test_evaluations must be 0")
|
||||
if candidate and run.get("is_finite") is not True:
|
||||
_append_issue(issues, "finite", f"{seed_label}: candidate is_finite must be true")
|
||||
if run.get("total_queries") != EXPECTED_QUERIES:
|
||||
_append_issue(issues, "accounting", f"{seed_label}: total_queries must be {EXPECTED_QUERIES}")
|
||||
if run.get("total_sample_evaluations") != EXPECTED_SAMPLES:
|
||||
_append_issue(issues, "accounting", f"{seed_label}: total_sample_evaluations must be {EXPECTED_SAMPLES}")
|
||||
if run.get("core_swarm_state_bytes") != expected_state_bytes:
|
||||
_append_issue(issues, "state", f"{seed_label}: incorrect core_swarm_state_bytes")
|
||||
|
||||
if isinstance(run.get("total_queries"), int):
|
||||
total_queries += run["total_queries"]
|
||||
if isinstance(run.get("total_sample_evaluations"), int):
|
||||
total_samples += run["total_sample_evaluations"]
|
||||
valid_runs.append(run)
|
||||
|
||||
return valid_runs, total_queries, total_samples
|
||||
|
||||
|
||||
def _validate_artifact(artifact: Dict[str, Any], phase: str) -> Dict[str, Any]:
|
||||
expected_splits, expected_seeds = PHASE_SPECS[phase]
|
||||
issues: Dict[str, List[str]] = {
|
||||
"schema": [],
|
||||
"test": [],
|
||||
"finite": [],
|
||||
"provenance": [],
|
||||
"config": [],
|
||||
"accounting": [],
|
||||
"state": [],
|
||||
}
|
||||
cells: List[Dict[str, Any]] = []
|
||||
state_ratios: Dict[str, float] = {}
|
||||
expected_split_keys = {str(seed) for seed in expected_splits}
|
||||
|
||||
if not isinstance(artifact, dict):
|
||||
_append_issue(issues, "schema", f"{phase}: artifact must be an object")
|
||||
return {
|
||||
"issues": issues,
|
||||
"cells": cells,
|
||||
"state_ratios": state_ratios,
|
||||
"max_state_ratio": 1.0,
|
||||
"policy_signature": None,
|
||||
}
|
||||
|
||||
if artifact.get("phase") != phase:
|
||||
_append_issue(issues, "schema", f"{phase}: phase field mismatch")
|
||||
if artifact.get("split_seeds") != expected_splits:
|
||||
_append_issue(issues, "config", f"{phase}: split_seeds must be {expected_splits}")
|
||||
if artifact.get("swarm_seeds") != expected_seeds:
|
||||
_append_issue(issues, "config", f"{phase}: swarm_seeds must be {expected_seeds}")
|
||||
if artifact.get("official_test_data_loaded") is not False:
|
||||
_append_issue(issues, "test", f"{phase}: official_test_data_loaded must be false")
|
||||
if artifact.get("official_test_evaluations") != 0:
|
||||
_append_issue(issues, "test", f"{phase}: official_test_evaluations must be 0")
|
||||
|
||||
candidate_config = artifact.get("candidate_config")
|
||||
if not isinstance(candidate_config, dict):
|
||||
_append_issue(issues, "schema", f"{phase}: missing candidate_config")
|
||||
candidate_config = {}
|
||||
for field, expected in (
|
||||
("particles", EXPECTED_PARTICLES),
|
||||
("epochs", EXPECTED_EPOCHS),
|
||||
("subset_size", EXPECTED_SUBSET_SIZE),
|
||||
):
|
||||
if candidate_config.get(field) != expected:
|
||||
_append_issue(issues, "config", f"{phase}: candidate_config.{field} must be {expected}")
|
||||
|
||||
workload_config = artifact.get("workloads")
|
||||
if not isinstance(workload_config, dict) or set(workload_config) != set(WORKLOADS):
|
||||
_append_issue(issues, "schema", f"{phase}: workloads metadata must contain exactly {WORKLOADS}")
|
||||
workload_config = {}
|
||||
|
||||
splits = artifact.get("splits")
|
||||
if not isinstance(splits, dict) or set(splits) != expected_split_keys:
|
||||
_append_issue(issues, "schema", f"{phase}: splits must contain exactly {sorted(expected_split_keys)}")
|
||||
splits = splits if isinstance(splits, dict) else {}
|
||||
|
||||
observed_runs = 0
|
||||
observed_queries = 0
|
||||
observed_samples = 0
|
||||
|
||||
for split_seed in expected_splits:
|
||||
split_key = str(split_seed)
|
||||
split_entry = splits.get(split_key)
|
||||
if not isinstance(split_entry, dict):
|
||||
_append_issue(issues, "schema", f"{phase}/{split_key}: missing split object")
|
||||
continue
|
||||
if split_entry.get("split_seed") != split_seed:
|
||||
_append_issue(issues, "provenance", f"{phase}/{split_key}: split_seed mismatch")
|
||||
|
||||
baselines = split_entry.get("baselines")
|
||||
candidates = split_entry.get("candidates")
|
||||
if not isinstance(baselines, dict) or set(baselines) != set(WORKLOADS):
|
||||
_append_issue(issues, "schema", f"{phase}/{split_key}: baseline workloads incomplete")
|
||||
baselines = baselines if isinstance(baselines, dict) else {}
|
||||
if not isinstance(candidates, dict) or set(candidates) != set(WORKLOADS):
|
||||
_append_issue(issues, "schema", f"{phase}/{split_key}: candidate workloads incomplete")
|
||||
candidates = candidates if isinstance(candidates, dict) else {}
|
||||
|
||||
for workload in WORKLOADS:
|
||||
baseline = baselines.get(workload)
|
||||
candidate_entry = candidates.get(workload)
|
||||
label = f"{phase}/{split_key}/{workload}"
|
||||
if not isinstance(baseline, dict) or not isinstance(candidate_entry, dict):
|
||||
_append_issue(issues, "schema", f"{label}: missing baseline or candidate entry")
|
||||
continue
|
||||
|
||||
if baseline.get("method_id") != BASELINE_METHODS[workload]:
|
||||
_append_issue(issues, "config", f"{label}: wrong baseline method")
|
||||
for mode, entry in (("baseline", baseline), ("candidate", candidate_entry)):
|
||||
if entry.get("workload_id") != workload:
|
||||
_append_issue(issues, "schema", f"{label}/{mode}: workload_id mismatch")
|
||||
if entry.get("split_seed") != split_seed:
|
||||
_append_issue(issues, "provenance", f"{label}/{mode}: split_seed mismatch")
|
||||
if entry.get("particles") != EXPECTED_PARTICLES:
|
||||
_append_issue(issues, "config", f"{label}/{mode}: particles mismatch")
|
||||
if entry.get("epochs") != EXPECTED_EPOCHS:
|
||||
_append_issue(issues, "config", f"{label}/{mode}: epochs mismatch")
|
||||
if entry.get("subset_size") != EXPECTED_SUBSET_SIZE:
|
||||
_append_issue(issues, "config", f"{label}/{mode}: subset_size mismatch")
|
||||
if entry.get("seeds") != expected_seeds:
|
||||
_append_issue(issues, "config", f"{label}/{mode}: seeds mismatch")
|
||||
|
||||
fingerprints = (
|
||||
baseline.get("split_fingerprint"),
|
||||
candidate_entry.get("split_fingerprint"),
|
||||
baseline.get("data_fingerprint"),
|
||||
candidate_entry.get("data_fingerprint"),
|
||||
)
|
||||
if any(not isinstance(value, str) or not value for value in fingerprints):
|
||||
_append_issue(issues, "provenance", f"{label}: fingerprints must be non-empty strings")
|
||||
elif fingerprints[0] != fingerprints[1] or fingerprints[2] != fingerprints[3]:
|
||||
_append_issue(issues, "provenance", f"{label}: baseline/candidate fingerprints differ")
|
||||
|
||||
total_dim = TOTAL_DIMS[workload]
|
||||
baseline_states = 5 * EXPECTED_PARTICLES + (1 if BASELINE_METHODS[workload] == "G8" else 0)
|
||||
expected_baseline_bytes = baseline_states * total_dim * 4
|
||||
raw_ratio = candidate_entry.get("ratio")
|
||||
if not _is_finite_number(raw_ratio) or not (0.0 < float(raw_ratio) <= 1.0):
|
||||
_append_issue(issues, "state", f"{label}: invalid candidate ratio")
|
||||
expected_candidate_bytes = -1
|
||||
else:
|
||||
expected_latent_dim = compute_latent_dim(total_dim, float(raw_ratio))
|
||||
expected_candidate_bytes = compute_core_swarm_state_bytes(EXPECTED_PARTICLES, expected_latent_dim)
|
||||
if candidate_entry.get("total_dim") != total_dim:
|
||||
_append_issue(issues, "state", f"{label}: total_dim mismatch")
|
||||
if candidate_entry.get("latent_dim") != expected_latent_dim:
|
||||
_append_issue(issues, "state", f"{label}: latent_dim mismatch")
|
||||
if candidate_entry.get("core_swarm_state_bytes") != expected_candidate_bytes:
|
||||
_append_issue(issues, "state", f"{label}: candidate state bytes mismatch")
|
||||
if candidate_entry.get("baseline_core_swarm_state_bytes") != expected_baseline_bytes:
|
||||
_append_issue(issues, "state", f"{label}: candidate baseline state bytes mismatch")
|
||||
ratio = expected_candidate_bytes / expected_baseline_bytes
|
||||
state_ratios[workload] = max(state_ratios.get(workload, 0.0), ratio)
|
||||
if not _is_finite_number(candidate_entry.get("state_ratio")) or not math.isclose(
|
||||
float(candidate_entry.get("state_ratio", -1.0)), ratio, rel_tol=1e-6, abs_tol=1e-6
|
||||
):
|
||||
_append_issue(issues, "state", f"{label}: reported state_ratio mismatch")
|
||||
|
||||
baseline_runs, b_queries, b_samples = _validate_runs(
|
||||
baseline,
|
||||
expected_seeds,
|
||||
expected_baseline_bytes,
|
||||
f"{label}/baseline",
|
||||
False,
|
||||
issues,
|
||||
)
|
||||
candidate_runs, c_queries, c_samples = _validate_runs(
|
||||
candidate_entry,
|
||||
expected_seeds,
|
||||
expected_candidate_bytes,
|
||||
f"{label}/candidate",
|
||||
True,
|
||||
issues,
|
||||
)
|
||||
observed_runs += len(baseline_runs) + len(candidate_runs)
|
||||
observed_queries += b_queries + c_queries
|
||||
observed_samples += b_samples + c_samples
|
||||
|
||||
baseline_stats = _validate_stats(baseline, baseline_runs, f"{label}/baseline", issues)
|
||||
candidate_stats = _validate_stats(candidate_entry, candidate_runs, f"{label}/candidate", issues)
|
||||
if baseline_stats is not None and candidate_stats is not None:
|
||||
baseline_acc, baseline_nll = baseline_stats
|
||||
candidate_acc, candidate_nll = candidate_stats
|
||||
nll_reduction = (
|
||||
(baseline_nll - candidate_nll) / baseline_nll
|
||||
if baseline_nll > 0.0
|
||||
else float("nan")
|
||||
)
|
||||
if not math.isfinite(nll_reduction):
|
||||
_append_issue(issues, "finite", f"{label}: NLL reduction is non-finite")
|
||||
else:
|
||||
cells.append(
|
||||
{
|
||||
"phase": phase,
|
||||
"split_seed": split_seed,
|
||||
"workload_id": workload,
|
||||
"baseline_acc": baseline_acc,
|
||||
"candidate_acc": candidate_acc,
|
||||
"baseline_nll": baseline_nll,
|
||||
"candidate_nll": candidate_nll,
|
||||
"acc_gain_pp": candidate_acc - baseline_acc,
|
||||
"nll_reduction_fraction": nll_reduction,
|
||||
}
|
||||
)
|
||||
|
||||
expected_runs = len(expected_splits) * len(WORKLOADS) * len(expected_seeds) * 2
|
||||
if observed_runs != expected_runs:
|
||||
_append_issue(issues, "accounting", f"{phase}: observed {observed_runs} runs, expected {expected_runs}")
|
||||
resources = artifact.get("resource_totals")
|
||||
if not isinstance(resources, dict):
|
||||
_append_issue(issues, "accounting", f"{phase}: missing resource_totals")
|
||||
resources = {}
|
||||
if resources.get("total_runs") != observed_runs:
|
||||
_append_issue(issues, "accounting", f"{phase}: total_runs does not match records")
|
||||
if resources.get("total_queries") != observed_queries:
|
||||
_append_issue(issues, "accounting", f"{phase}: total_queries does not match records")
|
||||
if resources.get("total_samples_evaluated") != observed_samples:
|
||||
_append_issue(issues, "accounting", f"{phase}: total_samples_evaluated does not match records")
|
||||
if resources.get("official_test_evaluations") != 0:
|
||||
_append_issue(issues, "test", f"{phase}: resource official_test_evaluations must be 0")
|
||||
|
||||
max_state_ratio = max(state_ratios.values(), default=1.0)
|
||||
policy_signature = {
|
||||
"candidate_config": candidate_config,
|
||||
"workloads": workload_config,
|
||||
}
|
||||
return {
|
||||
"issues": issues,
|
||||
"cells": cells,
|
||||
"state_ratios": state_ratios,
|
||||
"max_state_ratio": max_state_ratio,
|
||||
"policy_signature": policy_signature,
|
||||
}
|
||||
|
||||
|
||||
def evaluate_heavy_cross_split(
|
||||
development_artifact: Dict[str, Any],
|
||||
confirmation_artifact: Optional[Dict[str, Any]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
dev_result = _validate_artifact(development_artifact, "development")
|
||||
conf_result = (
|
||||
_validate_artifact(confirmation_artifact, "confirmation")
|
||||
if confirmation_artifact is not None
|
||||
else None
|
||||
)
|
||||
|
||||
gates: Dict[str, Dict[str, Any]] = {}
|
||||
failed_gates: List[str] = []
|
||||
development_gate_names: List[str] = []
|
||||
|
||||
def record_gate(
|
||||
name: str,
|
||||
passed: bool,
|
||||
observed: Any,
|
||||
expected: Any,
|
||||
details: str,
|
||||
development_gate: bool = False,
|
||||
) -> None:
|
||||
gates[name] = {
|
||||
"pass": bool(passed),
|
||||
"observed": observed,
|
||||
"expected": expected,
|
||||
"details": details,
|
||||
}
|
||||
if not passed:
|
||||
failed_gates.append(name)
|
||||
if development_gate:
|
||||
development_gate_names.append(name)
|
||||
|
||||
all_results = [dev_result] + ([conf_result] if conf_result is not None else [])
|
||||
record_gate(
|
||||
"schema_and_phase_seeds",
|
||||
all(not result["issues"]["schema"] for result in all_results),
|
||||
[message for result in all_results for message in result["issues"]["schema"]][:10],
|
||||
"complete artifacts with exact declared phase/split/swarm seeds",
|
||||
"Missing evidence is rejected rather than defaulted",
|
||||
True,
|
||||
)
|
||||
record_gate(
|
||||
"official_test_sealed",
|
||||
all(not result["issues"]["test"] for result in all_results),
|
||||
[message for result in all_results for message in result["issues"]["test"]][:10],
|
||||
"loaded=false and evaluations=0 at artifact, resource, and run levels",
|
||||
"Official test data must never be loaded or evaluated",
|
||||
True,
|
||||
)
|
||||
record_gate(
|
||||
"all_runs_finite",
|
||||
all(not result["issues"]["finite"] for result in all_results),
|
||||
[message for result in all_results for message in result["issues"]["finite"]][:10],
|
||||
"all aggregate and per-run validation metrics and times finite",
|
||||
"Every recorded validation metric must be finite",
|
||||
True,
|
||||
)
|
||||
record_gate(
|
||||
"split_and_fingerprint_matched",
|
||||
all(not result["issues"]["provenance"] for result in all_results),
|
||||
[message for result in all_results for message in result["issues"]["provenance"]][:10],
|
||||
"non-empty matching split/data fingerprints and split seeds per baseline/candidate cell",
|
||||
"Every delta must use a baseline rerun on the identical partition",
|
||||
True,
|
||||
)
|
||||
|
||||
policy_matches = conf_result is None or (
|
||||
dev_result["policy_signature"] == conf_result["policy_signature"]
|
||||
)
|
||||
config_issues = [message for result in all_results for message in result["issues"]["config"]]
|
||||
if not policy_matches:
|
||||
config_issues.append("confirmation candidate policy differs from the frozen development policy")
|
||||
record_gate(
|
||||
"configuration_and_policy_matched",
|
||||
not config_issues,
|
||||
config_issues[:10],
|
||||
"12p x 80e x fixed10k, exact seeds, matching baseline, identical frozen candidate policy",
|
||||
"Confirmation cannot change the development-selected policy",
|
||||
True,
|
||||
)
|
||||
record_gate(
|
||||
"query_and_sample_accounting_exact",
|
||||
all(not result["issues"]["accounting"] for result in all_results),
|
||||
[message for result in all_results for message in result["issues"]["accounting"]][:10],
|
||||
f"{EXPECTED_QUERIES} queries and {EXPECTED_SAMPLES} sample evaluations per run",
|
||||
"Per-run and aggregate accounting must agree exactly",
|
||||
True,
|
||||
)
|
||||
|
||||
max_state_ratio = max(result["max_state_ratio"] for result in all_results)
|
||||
state_issues = [message for result in all_results for message in result["issues"]["state"]]
|
||||
state_ok = not state_issues and 0.0 < max_state_ratio <= 0.5 + 1e-12
|
||||
record_gate(
|
||||
"maximum_state_ratio_each_workload",
|
||||
state_ok,
|
||||
{"max_state_ratio": max_state_ratio, "issues": state_issues[:10]},
|
||||
"analytically verified state ratio <= 0.5 for every cell",
|
||||
"No absent or reported-only state evidence is accepted",
|
||||
True,
|
||||
)
|
||||
|
||||
dev_cells = dev_result["cells"]
|
||||
conf_cells = conf_result["cells"] if conf_result is not None else []
|
||||
all_cells = dev_cells + conf_cells
|
||||
expected_dev_cells = len(EXPECTED_DEV_SPLIT_SEEDS) * len(WORKLOADS)
|
||||
complete_dev_cells = len(dev_cells) == expected_dev_cells
|
||||
|
||||
def nonregression(cells: List[Dict[str, Any]]) -> Tuple[bool, float, float]:
|
||||
if not cells:
|
||||
return False, float("inf"), float("inf")
|
||||
max_acc_regression = max(-cell["acc_gain_pp"] for cell in cells)
|
||||
max_nll_regression = max(-cell["nll_reduction_fraction"] for cell in cells)
|
||||
return (
|
||||
max_acc_regression <= 1.0 + 1e-12 and max_nll_regression <= 0.05 + 1e-12,
|
||||
max_acc_regression,
|
||||
max_nll_regression,
|
||||
)
|
||||
|
||||
all_nonreg, max_acc_reg, max_nll_reg = nonregression(all_cells)
|
||||
record_gate(
|
||||
"maximum_accuracy_regression_percentage_points_each_split_workload",
|
||||
bool(all_cells) and max_acc_reg <= 1.0 + 1e-12,
|
||||
max_acc_reg,
|
||||
"<= 1.0 pp",
|
||||
"No evaluated split-workload cell may regress accuracy by more than 1 pp",
|
||||
True,
|
||||
)
|
||||
record_gate(
|
||||
"maximum_nll_regression_fraction_each_split_workload",
|
||||
bool(all_cells) and max_nll_reg <= 0.05 + 1e-12,
|
||||
max_nll_reg,
|
||||
"<= 0.05",
|
||||
"No evaluated split-workload cell may regress NLL by more than 5%",
|
||||
True,
|
||||
)
|
||||
dev_nonreg, dev_acc_reg, dev_nll_reg = nonregression(dev_cells)
|
||||
|
||||
dev_acc_mean = _mean([cell["acc_gain_pp"] for cell in dev_cells]) if dev_cells else float("-inf")
|
||||
dev_nll_mean = _mean([cell["nll_reduction_fraction"] for cell in dev_cells]) if dev_cells else float("-inf")
|
||||
record_gate(
|
||||
"development_grand_mean_accuracy_gain_minimum_pp",
|
||||
complete_dev_cells and dev_acc_mean >= 0.0,
|
||||
dev_acc_mean,
|
||||
">= 0.0 pp",
|
||||
"Development grand mean accuracy must not regress",
|
||||
True,
|
||||
)
|
||||
record_gate(
|
||||
"development_grand_mean_nll_reduction_minimum_fraction",
|
||||
complete_dev_cells and dev_nll_mean >= 0.0,
|
||||
dev_nll_mean,
|
||||
">= 0.0",
|
||||
"Development grand mean NLL must not regress",
|
||||
True,
|
||||
)
|
||||
|
||||
dev_mw = [cell for cell in dev_cells if cell["workload_id"] == "mnist_wide"]
|
||||
dev_mw_acc = _mean([cell["acc_gain_pp"] for cell in dev_mw]) if dev_mw else float("-inf")
|
||||
dev_mw_nll = _mean([cell["nll_reduction_fraction"] for cell in dev_mw]) if dev_mw else float("-inf")
|
||||
record_gate(
|
||||
"development_mnist_wide_improvement",
|
||||
len(dev_mw) == len(EXPECTED_DEV_SPLIT_SEEDS) and (dev_mw_acc >= 2.0 or dev_mw_nll >= 0.05),
|
||||
{"accuracy_gain_pp": dev_mw_acc, "nll_reduction_fraction": dev_mw_nll},
|
||||
"mean accuracy gain >=2pp OR mean NLL reduction >=5%",
|
||||
"The prior worst baseline workload must materially improve across development partitions",
|
||||
True,
|
||||
)
|
||||
|
||||
development_pass = (
|
||||
all(not messages for messages in dev_result["issues"].values())
|
||||
and 0.0 < dev_result["max_state_ratio"] <= 0.5 + 1e-12
|
||||
and complete_dev_cells
|
||||
and dev_nonreg
|
||||
and dev_acc_mean >= 0.0
|
||||
and dev_nll_mean >= 0.0
|
||||
and len(dev_mw) == len(EXPECTED_DEV_SPLIT_SEEDS)
|
||||
and (dev_mw_acc >= 2.0 or dev_mw_nll >= 0.05)
|
||||
)
|
||||
if confirmation_artifact is None:
|
||||
record_gate(
|
||||
"confirmation_executed",
|
||||
False,
|
||||
"not executed",
|
||||
"one exact confirmation artifact after development_pass",
|
||||
"A development pass only qualifies the frozen policy for one-shot confirmation",
|
||||
)
|
||||
else:
|
||||
record_gate(
|
||||
"confirmation_executed",
|
||||
True,
|
||||
{"split_seeds": confirmation_artifact.get("split_seeds"), "swarm_seeds": confirmation_artifact.get("swarm_seeds")},
|
||||
{"split_seeds": EXPECTED_CONF_SPLIT_SEEDS, "swarm_seeds": EXPECTED_CONF_SWARM_SEEDS},
|
||||
"Confirmation evidence is evaluated only with the exact sealed phase contract",
|
||||
)
|
||||
|
||||
expected_conf_cells = len(EXPECTED_CONF_SPLIT_SEEDS) * len(WORKLOADS)
|
||||
complete_conf_cells = len(conf_cells) == expected_conf_cells
|
||||
conf_nonreg, conf_acc_reg, conf_nll_reg = nonregression(conf_cells)
|
||||
record_gate(
|
||||
"confirmation_per_cell_non_regression",
|
||||
complete_conf_cells and conf_nonreg,
|
||||
{"cells": len(conf_cells), "max_acc_regression_pp": conf_acc_reg, "max_nll_regression_fraction": conf_nll_reg},
|
||||
"4 cells; accuracy regression <=1pp and NLL regression <=5% in each",
|
||||
"The sealed partition must remain safe workload by workload",
|
||||
)
|
||||
|
||||
conf_acc_mean = _mean([cell["acc_gain_pp"] for cell in conf_cells]) if conf_cells else float("-inf")
|
||||
conf_nll_mean = _mean([cell["nll_reduction_fraction"] for cell in conf_cells]) if conf_cells else float("-inf")
|
||||
record_gate(
|
||||
"confirmation_grand_mean_accuracy_gain_minimum_pp",
|
||||
complete_conf_cells and conf_acc_mean >= 1.5,
|
||||
conf_acc_mean,
|
||||
">= 1.5 pp",
|
||||
"One-shot confirmation must retain the predeclared accuracy effect",
|
||||
)
|
||||
record_gate(
|
||||
"confirmation_grand_mean_nll_reduction_minimum_fraction",
|
||||
complete_conf_cells and conf_nll_mean >= 0.02,
|
||||
conf_nll_mean,
|
||||
">= 0.02",
|
||||
"One-shot confirmation must retain the predeclared NLL effect",
|
||||
)
|
||||
|
||||
conf_mw = [cell for cell in conf_cells if cell["workload_id"] == "mnist_wide"]
|
||||
conf_mw_acc = _mean([cell["acc_gain_pp"] for cell in conf_mw]) if conf_mw else float("-inf")
|
||||
conf_mw_nll = _mean([cell["nll_reduction_fraction"] for cell in conf_mw]) if conf_mw else float("-inf")
|
||||
record_gate(
|
||||
"confirmation_mnist_wide_improvement",
|
||||
len(conf_mw) == 1 and (conf_mw_acc >= 1.0 or conf_mw_nll >= 0.03),
|
||||
{"accuracy_gain_pp": conf_mw_acc, "nll_reduction_fraction": conf_mw_nll},
|
||||
"accuracy gain >=1pp OR NLL reduction >=3%",
|
||||
"The prior worst workload must improve on the sealed partition",
|
||||
)
|
||||
|
||||
combined_mw = dev_mw + conf_mw
|
||||
combined_mw_acc = _mean([cell["acc_gain_pp"] for cell in combined_mw]) if combined_mw else float("-inf")
|
||||
combined_mw_nll = _mean([cell["nll_reduction_fraction"] for cell in combined_mw]) if combined_mw else float("-inf")
|
||||
record_gate(
|
||||
"combined_mnist_wide_improvement",
|
||||
len(combined_mw) == 3 and (combined_mw_acc >= 2.0 or combined_mw_nll >= 0.05),
|
||||
{"accuracy_gain_pp": combined_mw_acc, "nll_reduction_fraction": combined_mw_nll},
|
||||
"three-split mean accuracy gain >=2pp OR NLL reduction >=5%",
|
||||
"The material worst-workload improvement must hold across all new partitions",
|
||||
)
|
||||
|
||||
score_cells = all_cells if confirmation_artifact is not None else dev_cells
|
||||
mean_acc_gain = _mean([cell["acc_gain_pp"] for cell in score_cells]) if score_cells else 0.0
|
||||
mean_nll_reduction = _mean([cell["nll_reduction_fraction"] for cell in score_cells]) if score_cells else 0.0
|
||||
score_gate_names = list(gates) if confirmation_artifact is not None else development_gate_names
|
||||
score_failed_gates = sum(not gates[name]["pass"] for name in score_gate_names)
|
||||
state_points = 10.0 * math.log2(1.0 / max_state_ratio) if 0.0 < max_state_ratio <= 1.0 else 0.0
|
||||
score = (
|
||||
100.0 * mean_nll_reduction
|
||||
+ mean_acc_gain
|
||||
+ state_points
|
||||
- 100.0 * score_failed_gates
|
||||
)
|
||||
mission_pass = confirmation_artifact is not None and not failed_gates
|
||||
|
||||
return {
|
||||
"pass": bool(mission_pass),
|
||||
"development_pass": bool(development_pass),
|
||||
"eligible_for_confirmation": bool(development_pass and confirmation_artifact is None),
|
||||
"score": float(score),
|
||||
"evaluator_version": EVALUATOR_VERSION,
|
||||
"failed_hard_gate_count": len(failed_gates),
|
||||
"failed_gates": failed_gates,
|
||||
"score_failed_gate_count": score_failed_gates,
|
||||
"gates": gates,
|
||||
"score_components": {
|
||||
"mean_relative_nll_reduction_pct": 100.0 * mean_nll_reduction,
|
||||
"mean_accuracy_gain_pp": mean_acc_gain,
|
||||
"state_efficiency_points": state_points,
|
||||
"gate_penalty_points": 100.0 * score_failed_gates,
|
||||
"max_state_ratio": max_state_ratio,
|
||||
},
|
||||
"summary_metrics": {
|
||||
"development_cells": len(dev_cells),
|
||||
"confirmation_cells": len(conf_cells),
|
||||
"development_grand_mean_accuracy_gain_pp": dev_acc_mean,
|
||||
"development_grand_mean_nll_reduction_fraction": dev_nll_mean,
|
||||
"development_mnist_wide_accuracy_gain_pp": dev_mw_acc,
|
||||
"development_mnist_wide_nll_reduction_fraction": dev_mw_nll,
|
||||
},
|
||||
"cell_metrics": all_cells,
|
||||
"state_ratios": {
|
||||
"development": dev_result["state_ratios"],
|
||||
"confirmation": conf_result["state_ratios"] if conf_result is not None else None,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Strict Heavy PSO cross-split evaluator")
|
||||
parser.add_argument("--development", required=True, help="Development artifact JSON")
|
||||
parser.add_argument("--confirmation", default=None, help="Optional one-shot confirmation artifact JSON")
|
||||
parser.add_argument("--output", default=None, help="Optional evaluation JSON output")
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = build_parser().parse_args()
|
||||
development = load_artifact(Path(args.development))
|
||||
confirmation = load_artifact(Path(args.confirmation)) if args.confirmation else None
|
||||
result = evaluate_heavy_cross_split(development, confirmation)
|
||||
if args.output:
|
||||
save_json_atomic(result, Path(args.output))
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,974 @@
|
||||
"""Independent evaluator for the post-training model-convergence protocol.
|
||||
|
||||
This module is deliberately data-only: it reads JSON manifests, result records and
|
||||
stored prediction records. It never imports an adapter, optional detection
|
||||
package, dataset, checkpoint, or model. A result is useful only when all of the
|
||||
frozen matrix and sealing invariants can be demonstrated from the saved evidence.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable, Mapping, Sequence
|
||||
|
||||
EVALUATOR_VERSION = "POST-TRAINING-MODEL-CONVERGENCE-EVALUATOR 1.0.0"
|
||||
PROTOCOL_VERSION = "post-training-model-convergence-1.0.0"
|
||||
WORKLOADS = ("cifar10_resnet18", "cifar10_resnet50", "voc_yolo11n")
|
||||
CLASSIFICATION_WORKLOADS = WORKLOADS[:2]
|
||||
DETECTION_WORKLOAD = WORKLOADS[2]
|
||||
BASE_SEEDS = (501, 502, 503)
|
||||
SWARM_SEEDS = (601, 602, 603)
|
||||
SPLIT_SEED = 20260908
|
||||
PROJECTION_SEED = 20260909
|
||||
BOOTSTRAP_SEED = 20260910
|
||||
PARTICLES = 12
|
||||
PRIMARY_GENERATIONS = 60
|
||||
ENSEMBLE_GENERATIONS = 20
|
||||
PRIMARY_QUERIES = PARTICLES * PRIMARY_GENERATIONS
|
||||
ENSEMBLE_QUERIES = PARTICLES * ENSEMBLE_GENERATIONS
|
||||
PRIMARY_RUNS = len(WORKLOADS) * len(BASE_SEEDS) * len(SWARM_SEEDS)
|
||||
ENSEMBLE_RUNS = len(WORKLOADS) * len(SWARM_SEEDS)
|
||||
TOTAL_PSO_QUERIES = PRIMARY_RUNS * PRIMARY_QUERIES + ENSEMBLE_RUNS * ENSEMBLE_QUERIES
|
||||
OBJECTIVE_SAMPLES = {"cifar10_resnet18": 1024, "cifar10_resnet50": 1024, "voc_yolo11n": 512}
|
||||
TOTAL_CANDIDATE_SAMPLES = sum(
|
||||
(len(BASE_SEEDS) * len(SWARM_SEEDS) * PRIMARY_QUERIES + len(SWARM_SEEDS) * ENSEMBLE_QUERIES)
|
||||
* OBJECTIVE_SAMPLES[w] for w in WORKLOADS
|
||||
)
|
||||
BOOTSTRAP_RESAMPLES = 2000
|
||||
BOOTSTRAP_ALPHA = 0.05 / 6.0
|
||||
|
||||
ISSUE_CATEGORIES = (
|
||||
"schema", "provenance", "matrix", "accounting", "seal", "selection",
|
||||
"finite", "metrics", "plateau", "overfit", "leakage", "bootstrap", "gates",
|
||||
)
|
||||
|
||||
|
||||
def _issue(issues: dict[str, list[str]], category: str, message: str) -> None:
|
||||
issues.setdefault(category, []).append(message)
|
||||
|
||||
|
||||
def _finite(value: Any) -> bool:
|
||||
return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(float(value))
|
||||
|
||||
|
||||
def _walk_nonfinite(value: Any, path: str = "") -> list[str]:
|
||||
out: list[str] = []
|
||||
if isinstance(value, float) and not math.isfinite(value):
|
||||
out.append(path or "$")
|
||||
elif isinstance(value, Mapping):
|
||||
for key, item in value.items():
|
||||
out.extend(_walk_nonfinite(item, f"{path}.{key}" if path else str(key)))
|
||||
elif isinstance(value, (list, tuple)):
|
||||
for i, item in enumerate(value):
|
||||
out.extend(_walk_nonfinite(item, f"{path}[{i}]"))
|
||||
return out
|
||||
|
||||
|
||||
def _json(path: Path) -> Any:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
return json.load(handle)
|
||||
|
||||
|
||||
def _canonical(value: Any) -> bytes:
|
||||
return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||
|
||||
|
||||
def _sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _atomic_json(path: Path, value: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, name = tempfile.mkstemp(prefix=f".{path.name}.", dir=str(path.parent))
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(value, handle, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(name, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(name)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def _number(value: Any, *keys: str) -> float | None:
|
||||
if isinstance(value, Mapping):
|
||||
for key in keys:
|
||||
candidate = value.get(key)
|
||||
if _finite(candidate):
|
||||
return float(candidate)
|
||||
for candidate in value.values():
|
||||
found = _number(candidate, *keys)
|
||||
if found is not None:
|
||||
return found
|
||||
elif isinstance(value, (list, tuple)):
|
||||
for candidate in value:
|
||||
found = _number(candidate, *keys)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _int(value: Any, *keys: str) -> int | None:
|
||||
number = _number(value, *keys)
|
||||
if number is None or not number.is_integer():
|
||||
return None
|
||||
return int(number)
|
||||
|
||||
|
||||
def _same(a: Any, b: Any, tol: float = 1e-9) -> bool:
|
||||
return _finite(a) and _finite(b) and math.isclose(float(a), float(b), rel_tol=tol, abs_tol=tol)
|
||||
|
||||
|
||||
def _resolve_json(root: Path, value: Any) -> Any:
|
||||
"""Resolve a saved JSON prediction reference without opening model/data files."""
|
||||
if isinstance(value, Mapping):
|
||||
for key in ("path", "file", "artifact", "prediction_artifact", "predictions_path"):
|
||||
ref = value.get(key)
|
||||
if isinstance(ref, str) and ref.lower().endswith((".json", ".jsonl", ".pt")):
|
||||
return _resolve_json(root, ref)
|
||||
return value
|
||||
if isinstance(value, str) and value.lower().endswith((".json", ".jsonl", ".pt")):
|
||||
path = (root / value).resolve()
|
||||
if root.resolve() not in path.parents:
|
||||
raise ValueError(f"prediction artifact escapes run root: {value}")
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(path)
|
||||
if path.suffix == ".jsonl":
|
||||
return [_json_line for _json_line in (json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()) if _json_line]
|
||||
if path.suffix == ".pt":
|
||||
try:
|
||||
import torch
|
||||
return torch.load(path, map_location="cpu", weights_only=True)
|
||||
except (ImportError, OSError, RuntimeError, TypeError, ValueError) as exc:
|
||||
raise ValueError(f"unable to load weights-only prediction artifact: {path}: {exc}") from exc
|
||||
return _json(path)
|
||||
return value
|
||||
|
||||
|
||||
def _hash_manifest(root: Path, manifest: Mapping[str, Any], issues: dict[str, list[str]]) -> bool:
|
||||
good = True
|
||||
if manifest.get("protocol_version") != PROTOCOL_VERSION:
|
||||
_issue(issues, "provenance", f"frozen manifest protocol mismatch: {manifest.get('protocol_version')!r}")
|
||||
good = False
|
||||
if manifest.get("state") != "frozen":
|
||||
_issue(issues, "seal", f"frozen manifest state must be 'frozen', got {manifest.get('state')!r}")
|
||||
good = False
|
||||
declared = manifest.get("manifest_hash")
|
||||
payload = {key: manifest[key] for key in ("protocol_version", "config", "artifacts", "state") if key in manifest}
|
||||
if not isinstance(declared, str) or hashlib.sha256(_canonical(payload)).hexdigest() != declared:
|
||||
_issue(issues, "seal", "frozen_manifest.json self-hash mismatch")
|
||||
good = False
|
||||
artifacts = manifest.get("artifacts")
|
||||
if not isinstance(artifacts, Mapping) or not artifacts:
|
||||
_issue(issues, "schema", "frozen manifest requires a non-empty artifacts map")
|
||||
return False
|
||||
for name, expected in artifacts.items():
|
||||
if not isinstance(name, str) or not isinstance(expected, str) or len(expected) != 64:
|
||||
_issue(issues, "schema", f"invalid frozen artifact hash declaration: {name!r}")
|
||||
good = False
|
||||
continue
|
||||
path = (root / name).resolve()
|
||||
if root.resolve() not in path.parents:
|
||||
_issue(issues, "seal", f"frozen artifact escapes run root: {name}")
|
||||
good = False
|
||||
elif not path.is_file():
|
||||
_issue(issues, "seal", f"frozen artifact is missing: {name}")
|
||||
good = False
|
||||
elif _sha256(path) != expected:
|
||||
_issue(issues, "seal", f"frozen artifact hash drift: {name}")
|
||||
good = False
|
||||
return good
|
||||
|
||||
|
||||
def _config_checks(config: Any, issues: dict[str, list[str]], workload_id: str | None = None) -> None:
|
||||
if not isinstance(config, Mapping):
|
||||
_issue(issues, "schema", "missing or non-object config")
|
||||
return
|
||||
expected: dict[str, Any] = {
|
||||
"protocol_version": PROTOCOL_VERSION, "split_seed": SPLIT_SEED,
|
||||
"projection_seed": PROJECTION_SEED, "bootstrap_seed": BOOTSTRAP_SEED,
|
||||
"particle_count": PARTICLES, "pso_generations": PRIMARY_GENERATIONS,
|
||||
"residual_dimension": 64, "residual_bound": 1.0, "initial_radius": 0.25,
|
||||
"objective_checkpoints": [0, 10, 20, 30, 40, 50, 60],
|
||||
"base_seeds": list(BASE_SEEDS), "swarm_seeds": list(SWARM_SEEDS),
|
||||
}
|
||||
for key, value in expected.items():
|
||||
got = config.get(key)
|
||||
if got != value and not (isinstance(got, tuple) and list(got) == value):
|
||||
_issue(issues, "provenance", f"config.{key} must be {value!r}, got {got!r}")
|
||||
ids = config.get("workload_ids")
|
||||
if ids is not None and sorted(ids) != sorted(WORKLOADS):
|
||||
_issue(issues, "matrix", f"config.workload_ids must be {list(WORKLOADS)!r}")
|
||||
if workload_id and config.get("workload_id") not in (None, workload_id):
|
||||
_issue(issues, "matrix", f"config.workload_id disagrees with {workload_id}")
|
||||
|
||||
|
||||
def _counter(value: Any, keys: Sequence[str]) -> int | None:
|
||||
if not isinstance(value, Mapping):
|
||||
return None
|
||||
for key in keys:
|
||||
candidate = value.get(key)
|
||||
if isinstance(candidate, int) and not isinstance(candidate, bool):
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def _check_leakage(result: Mapping[str, Any], issues: dict[str, list[str]], workload: str) -> None:
|
||||
leakage = result.get("leakage_counters")
|
||||
if not isinstance(leakage, Mapping):
|
||||
_issue(issues, "schema", f"{workload}: missing leakage_counters")
|
||||
leakage = {}
|
||||
loaded_before = leakage.get("official_test_data_loaded_before_freeze", leakage.get("test_data_loaded_before_freeze"))
|
||||
if loaded_before is not False:
|
||||
_issue(issues, "leakage", f"{workload}: official test data must be explicitly marked not loaded before freeze")
|
||||
evaluated_before = leakage.get("official_test_evaluations_before_freeze", leakage.get("test_evaluations_before_freeze", leakage.get("official_test_forward_passes_before_freeze")))
|
||||
if evaluated_before != 0:
|
||||
_issue(issues, "leakage", f"{workload}: official test exposure before freeze must be explicitly zero")
|
||||
for key in ("official_test_data_loaded_before_freeze", "official_test_evaluations_before_freeze"):
|
||||
if key in result and ((key.endswith("freeze") and result[key] not in (False, 0))):
|
||||
_issue(issues, "leakage", f"{workload}: contradictory top-level {key}")
|
||||
construction = _counter(leakage, ("official_test_construction", "official_test_dataset_construction"))
|
||||
if construction != 1:
|
||||
_issue(issues, "leakage", f"{workload}: official test construction must equal one, got {construction}")
|
||||
forwards = _counter(leakage, ("official_test_forward_passes", "official_test_evaluations"))
|
||||
if forwards is None or forwards < 1:
|
||||
_issue(issues, "leakage", f"{workload}: official test forward/evaluation ledger must be positive, got {forwards}")
|
||||
confirmation = result.get("confirmation")
|
||||
if not isinstance(confirmation, Mapping):
|
||||
_issue(issues, "schema", f"{workload}: missing confirmation record")
|
||||
return
|
||||
for key in ("second_confirmation", "confirmation_repeated", "post_test_tuning", "post_test_reruns"):
|
||||
if confirmation.get(key) not in (None, False, 0, []):
|
||||
_issue(issues, "leakage", f"{workload}: forbidden repeated confirmation/tuning flag {key}")
|
||||
for key in ("official_test_data_loaded_before_freeze", "official_test_evaluations_before_freeze"):
|
||||
if key in confirmation and confirmation[key] not in (False, 0):
|
||||
_issue(issues, "leakage", f"{workload}: confirmation contradicts sealed pre-freeze {key}")
|
||||
|
||||
|
||||
def _record_queries(record: Mapping[str, Any]) -> tuple[int | None, int | None]:
|
||||
counters = record.get("counters") if isinstance(record.get("counters"), Mapping) else record
|
||||
queries = _int(counters, "objective_queries", "queries", "query_count", "total_queries", "evaluated_queries")
|
||||
samples = _int(counters, "objective_samples", "samples", "sample_evaluations", "candidate_sample_evaluations", "total_sample_evaluations")
|
||||
return queries, samples
|
||||
|
||||
|
||||
def _seed_from_key(key: Any) -> int | None:
|
||||
try:
|
||||
text = str(key)
|
||||
if text.isdigit():
|
||||
return int(text)
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _collect_cells(node: Any, base: int | None = None, swarm: int | None = None) -> list[dict[str, Any]]:
|
||||
"""Collect cells from either explicit records or base->swarm mappings."""
|
||||
found: list[dict[str, Any]] = []
|
||||
if isinstance(node, Mapping):
|
||||
b = _int(node, "base_seed") if _int(node, "base_seed") is not None else base
|
||||
s = _int(node, "swarm_seed") if _int(node, "swarm_seed") is not None else swarm
|
||||
if s is None:
|
||||
s = _int(node, "seed")
|
||||
if b is not None and s is not None and any(k in node for k in ("counters", "objective_queries", "objective_samples", "queries", "samples", "generation", "endpoint", "metrics")):
|
||||
item = dict(node); item.setdefault("base_seed", b); item.setdefault("swarm_seed", s); found.append(item)
|
||||
for key, value in node.items():
|
||||
key_seed = _seed_from_key(key)
|
||||
if key_seed in BASE_SEEDS:
|
||||
found.extend(_collect_cells(value, key_seed, s))
|
||||
elif key_seed in SWARM_SEEDS:
|
||||
found.extend(_collect_cells(value, b, key_seed))
|
||||
elif key not in {"base_seed", "swarm_seed"}:
|
||||
found.extend(_collect_cells(value, b, s))
|
||||
elif isinstance(node, list):
|
||||
for value in node:
|
||||
found.extend(_collect_cells(value, base, swarm))
|
||||
return found
|
||||
|
||||
|
||||
def _collect_base_cells(node: Any, base: int | None = None) -> list[dict[str, Any]]:
|
||||
"""Collect one record per base seed from flattened or base->record schemas."""
|
||||
found: list[dict[str, Any]] = []
|
||||
if isinstance(node, Mapping):
|
||||
b = _int(node, "base_seed") if _int(node, "base_seed") is not None else base
|
||||
if b is not None and any(k in node for k in ("updates", "gradient_updates", "counters", "metrics", "checkpoint", "endpoint", "objective")):
|
||||
item = dict(node); item.setdefault("base_seed", b); found.append(item)
|
||||
for key, value in node.items():
|
||||
key_seed = _seed_from_key(key)
|
||||
if key_seed in BASE_SEEDS: found.extend(_collect_base_cells(value, key_seed))
|
||||
elif key not in {"base_seed", "swarm_seed"}: found.extend(_collect_base_cells(value, b))
|
||||
elif isinstance(node, list):
|
||||
for value in node: found.extend(_collect_base_cells(value, base))
|
||||
return found
|
||||
|
||||
|
||||
def _method_base_cells(result: Mapping[str, Any], method: str) -> list[dict[str, Any]]:
|
||||
arms = result.get("arms")
|
||||
return _collect_base_cells(arms.get(method)) if isinstance(arms, Mapping) and method in arms else []
|
||||
|
||||
|
||||
def _method_evidence(value: Any, method: str, family: str, root: Path) -> bool:
|
||||
"""Require non-empty metric or prediction evidence below an exact method key."""
|
||||
if isinstance(value, Mapping):
|
||||
for key, item in value.items():
|
||||
normalized = str(key).lower().replace("-", "_")
|
||||
if normalized == method:
|
||||
if _prediction_records(item, root) is not None: return True
|
||||
if isinstance(item, Mapping) and (_number(item, "nll", "loss", "accuracy", "map50_95", "map50", "mAP50-95") is not None): return True
|
||||
if _method_evidence(item, method, family, root): return True
|
||||
if _method_evidence(item, method, family, root): return True
|
||||
elif isinstance(value, list):
|
||||
return bool(value) and any(_method_evidence(item, method, family, root) for item in value)
|
||||
return False
|
||||
|
||||
|
||||
def _require_confirmation_methods(result: Mapping[str, Any], workload: str, family: str, root: Path, issues: dict[str, list[str]]) -> None:
|
||||
confirmation = result.get("confirmation")
|
||||
required = {"feature_pso", "feature_random", "feature_adam", "head_adam"}
|
||||
required |= ({"uniform", "uniform_temperature", "slsqp_weights", "ensemble_pso"} if family == "classification" else {"uniform_wbf", "ensemble_pso", "ensemble_random"})
|
||||
for method in sorted(required):
|
||||
if not _method_evidence(confirmation, method, family, root):
|
||||
_issue(issues, "metrics", f"{workload}: confirmation lacks predictions/metrics for required method {method}")
|
||||
|
||||
def _method_cells(result: Mapping[str, Any], method: str) -> list[dict[str, Any]]:
|
||||
arms = result.get("arms")
|
||||
if not isinstance(arms, Mapping):
|
||||
return []
|
||||
value = arms.get(method)
|
||||
return _collect_cells(value) if value is not None else []
|
||||
|
||||
|
||||
def _verify_matrix(result: Mapping[str, Any], workload: str, issues: dict[str, list[str]]) -> dict[str, Any]:
|
||||
arms = result.get("arms")
|
||||
if not isinstance(arms, Mapping):
|
||||
_issue(issues, "schema", f"{workload}: arms must be an object")
|
||||
arms = {}
|
||||
required = {"feature_pso", "feature_random", "feature_adam", "head_adam"}
|
||||
missing = required - set(arms)
|
||||
if missing:
|
||||
_issue(issues, "matrix", f"{workload}: missing arms {sorted(missing)}")
|
||||
stats: dict[str, Any] = {"primary_queries": 0, "primary_random_queries": 0, "ensemble_queries": 0, "primary_samples": 0, "primary_random_samples": 0, "ensemble_samples": 0, "cells": {}}
|
||||
for method in ("feature_pso", "feature_random"):
|
||||
cells = _method_cells(result, method)
|
||||
stats["cells"][method] = len(cells)
|
||||
expected = len(BASE_SEEDS) * len(SWARM_SEEDS)
|
||||
if len(cells) != expected:
|
||||
_issue(issues, "matrix", f"{workload}: {method} requires {expected} base/swarm cells, got {len(cells)}")
|
||||
seen: set[tuple[int, int]] = set()
|
||||
for cell in cells:
|
||||
key = (_int(cell, "base_seed") or -1, _int(cell, "swarm_seed") or -1)
|
||||
if key in seen or key[0] not in BASE_SEEDS or key[1] not in SWARM_SEEDS:
|
||||
_issue(issues, "matrix", f"{workload}: invalid or duplicate {method} cell {key}")
|
||||
seen.add(key)
|
||||
queries, samples = _record_queries(cell)
|
||||
if queries != PRIMARY_QUERIES:
|
||||
_issue(issues, "accounting", f"{workload}: {method} {key} queries must be {PRIMARY_QUERIES}, got {queries}")
|
||||
if samples != PRIMARY_QUERIES * OBJECTIVE_SAMPLES[workload]:
|
||||
_issue(issues, "accounting", f"{workload}: {method} {key} samples must be {PRIMARY_QUERIES * OBJECTIVE_SAMPLES[workload]}, got {samples}")
|
||||
if method == "feature_pso":
|
||||
if queries is not None: stats["primary_queries"] += queries
|
||||
if samples is not None: stats["primary_samples"] += samples
|
||||
else:
|
||||
if queries is not None: stats["primary_random_queries"] += queries
|
||||
if samples is not None: stats["primary_random_samples"] += samples
|
||||
for method in ("feature_adam", "head_adam"):
|
||||
cells = _method_base_cells(result, method); stats["cells"][method] = len(cells)
|
||||
if len(cells) != len(BASE_SEEDS): _issue(issues, "matrix", f"{workload}: {method} requires exactly three base cells, got {len(cells)}")
|
||||
seen = set()
|
||||
for cell in cells:
|
||||
seed = _int(cell, "base_seed")
|
||||
if seed in seen or seed not in BASE_SEEDS: _issue(issues, "matrix", f"{workload}: invalid or duplicate {method} base cell {seed}")
|
||||
if seed is not None: seen.add(seed)
|
||||
|
||||
ensemble = result.get("ensemble")
|
||||
if not isinstance(ensemble, Mapping):
|
||||
_issue(issues, "schema", f"{workload}: ensemble must be an object")
|
||||
ensemble = {}
|
||||
required_ensemble = ("uniform", "uniform_temperature", "slsqp_weights", "ensemble_pso") if workload in CLASSIFICATION_WORKLOADS else ("uniform_wbf", "ensemble_pso", "ensemble_random")
|
||||
for method in required_ensemble:
|
||||
if method not in ensemble or ensemble.get(method) in (None, {}, []): _issue(issues, "matrix", f"{workload}: missing required ensemble method {method}")
|
||||
ens_pso = ensemble.get("ensemble_pso", ensemble.get("pso"))
|
||||
cells = _collect_cells(ens_pso) if ens_pso is not None else []
|
||||
stats["cells"]["ensemble_pso"] = len(cells)
|
||||
if len(cells) != len(SWARM_SEEDS):
|
||||
_issue(issues, "matrix", f"{workload}: ensemble_pso requires three swarm cells, got {len(cells)}")
|
||||
seen_swarm: set[int] = set()
|
||||
for cell in cells:
|
||||
seed = _int(cell, "swarm_seed", "seed")
|
||||
if seed in seen_swarm or seed not in SWARM_SEEDS:
|
||||
_issue(issues, "matrix", f"{workload}: invalid or duplicate ensemble swarm cell {seed}")
|
||||
if seed is not None: seen_swarm.add(seed)
|
||||
queries, samples = _record_queries(cell)
|
||||
if queries != ENSEMBLE_QUERIES:
|
||||
_issue(issues, "accounting", f"{workload}: ensemble_pso queries must be {ENSEMBLE_QUERIES}, got {queries}")
|
||||
if samples != ENSEMBLE_QUERIES * OBJECTIVE_SAMPLES[workload]:
|
||||
_issue(issues, "accounting", f"{workload}: ensemble_pso samples must be {ENSEMBLE_QUERIES * OBJECTIVE_SAMPLES[workload]}, got {samples}")
|
||||
if queries is not None: stats["ensemble_queries"] += queries
|
||||
if samples is not None: stats["ensemble_samples"] += samples
|
||||
if workload == DETECTION_WORKLOAD:
|
||||
random_cells = _collect_cells(ensemble.get("ensemble_random")) if ensemble.get("ensemble_random") is not None else []
|
||||
stats["cells"]["ensemble_random"] = len(random_cells)
|
||||
if len(random_cells) != len(SWARM_SEEDS): _issue(issues, "matrix", f"{workload}: ensemble_random requires three swarm cells, got {len(random_cells)}")
|
||||
seen_random = set()
|
||||
for cell in random_cells:
|
||||
seed = _int(cell, "swarm_seed", "seed")
|
||||
if seed in seen_random or seed not in SWARM_SEEDS: _issue(issues, "matrix", f"{workload}: invalid or duplicate ensemble_random seed {seed}")
|
||||
if seed is not None: seen_random.add(seed)
|
||||
return stats
|
||||
|
||||
|
||||
def _verify_selection(result: Mapping[str, Any], workload: str, issues: dict[str, list[str]]) -> None:
|
||||
selection = result.get("development_selection")
|
||||
if not isinstance(selection, Mapping):
|
||||
_issue(issues, "selection", f"{workload}: missing development_selection")
|
||||
return
|
||||
selected = selection.get("primary", selection.get("feature_pso", selection.get("selected")))
|
||||
if not isinstance(selected, Mapping):
|
||||
_issue(issues, "selection", f"{workload}: missing primary selected endpoints")
|
||||
return
|
||||
for base in BASE_SEEDS:
|
||||
entry = selected.get(str(base), selected.get(base))
|
||||
if not isinstance(entry, Mapping):
|
||||
_issue(issues, "selection", f"{workload}: no selected endpoint for base seed {base}")
|
||||
continue
|
||||
swarm = _int(entry, "swarm_seed", "seed")
|
||||
generation = _int(entry, "generation", "final_generation")
|
||||
if swarm not in SWARM_SEEDS:
|
||||
_issue(issues, "selection", f"{workload}: selected seed {base} has invalid swarm {swarm}")
|
||||
if generation != PRIMARY_GENERATIONS:
|
||||
_issue(issues, "selection", f"{workload}: selected endpoint {base} is not final generation 60")
|
||||
matches = [c for c in _method_cells(result, "feature_pso") if _int(c, "base_seed") == base and _int(c, "swarm_seed") == swarm]
|
||||
if not matches:
|
||||
_issue(issues, "selection", f"{workload}: selected endpoint {base}/{swarm} is not a feature_pso cell")
|
||||
elif entry.get("vector_hash") and matches[0].get("vector_hash") and entry["vector_hash"] != matches[0]["vector_hash"]:
|
||||
_issue(issues, "selection", f"{workload}: selected vector hash drift for base {base}")
|
||||
|
||||
|
||||
def _prediction_records(value: Any, root: Path) -> list[dict[str, Any]] | None:
|
||||
try: value = _resolve_json(root, value)
|
||||
except (OSError, ValueError, json.JSONDecodeError): return None
|
||||
if isinstance(value, Mapping):
|
||||
if "probabilities" in value and "targets" in value:
|
||||
probabilities, targets = value["probabilities"], value["targets"]
|
||||
for attr in ("detach", "cpu"):
|
||||
if hasattr(probabilities, attr): probabilities = getattr(probabilities, attr)()
|
||||
if hasattr(targets, attr): targets = getattr(targets, attr)()
|
||||
if hasattr(probabilities, "tolist"): probabilities = probabilities.tolist()
|
||||
if hasattr(targets, "tolist"): targets = targets.tolist()
|
||||
if isinstance(probabilities, (list, tuple)) and isinstance(targets, (list, tuple)) and len(probabilities) == len(targets):
|
||||
return [{"probabilities": list(probability), "target": int(target)} for probability, target in zip(probabilities, targets)]
|
||||
return None
|
||||
for key in ("records", "predictions", "images", "examples", "data"):
|
||||
if key in value:
|
||||
got = _prediction_records(value[key], root)
|
||||
if got is not None: return got
|
||||
return None
|
||||
if isinstance(value, list) and all(isinstance(x, Mapping) for x in value):
|
||||
return [dict(x) for x in value]
|
||||
return None
|
||||
|
||||
|
||||
def _find_prediction_sets(value: Any, root: Path, prefix: str = "$") -> dict[str, list[dict[str, Any]]]:
|
||||
found: dict[str, list[dict[str, Any]]] = {}
|
||||
if isinstance(value, Mapping):
|
||||
for key, item in value.items():
|
||||
name = f"{prefix}.{key}"
|
||||
if "prediction" in str(key).lower() or str(key).lower() in {"base", "pso", "feature_pso", "test"}:
|
||||
records = _prediction_records(item, root)
|
||||
if records is not None: found[name] = records
|
||||
found.update(_find_prediction_sets(item, root, name))
|
||||
elif isinstance(value, list) and value and isinstance(value[0], Mapping):
|
||||
records = _prediction_records(value, root)
|
||||
if records is not None: found[prefix] = records
|
||||
return found
|
||||
|
||||
|
||||
def classification_metrics(records: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
|
||||
"""Recompute unrounded NLL and accuracy from per-image probabilities/targets."""
|
||||
probs: list[list[float]] = []; targets: list[int] = []
|
||||
for i, record in enumerate(records):
|
||||
p = record.get("probabilities", record.get("probs", record.get("prob")))
|
||||
target = record.get("target", record.get("label", record.get("class_id")))
|
||||
if not isinstance(p, (list, tuple)) or not p or not isinstance(target, int) or isinstance(target, bool) or target < 0 or target >= len(p):
|
||||
raise ValueError(f"invalid classification prediction at index {i}")
|
||||
vals = [float(x) for x in p]
|
||||
if not all(math.isfinite(x) and x >= 0.0 for x in vals) or not math.isclose(math.fsum(vals), 1.0, rel_tol=1e-6, abs_tol=1e-6):
|
||||
raise ValueError(f"probabilities must be finite and sum to one at index {i}")
|
||||
probs.append(vals); targets.append(target)
|
||||
nll = math.fsum(
|
||||
-math.log(max(p[t], 1e-300)) for p, t in zip(probs, targets)
|
||||
) / len(probs)
|
||||
predictions = [
|
||||
max(range(len(p)), key=p.__getitem__) for p in probs
|
||||
]
|
||||
accuracy = sum(
|
||||
prediction == target
|
||||
for prediction, target in zip(predictions, targets)
|
||||
) / len(probs)
|
||||
brier = math.fsum(
|
||||
math.fsum(
|
||||
(probability - float(index == target)) ** 2
|
||||
for index, probability in enumerate(p)
|
||||
)
|
||||
for p, target in zip(probs, targets)
|
||||
) / len(probs)
|
||||
confidences = [p[prediction] for p, prediction in zip(probs, predictions)]
|
||||
ece = 0.0
|
||||
for bin_index in range(15):
|
||||
lower = bin_index / 15.0
|
||||
upper = (bin_index + 1) / 15.0
|
||||
members = [
|
||||
index
|
||||
for index, confidence in enumerate(confidences)
|
||||
if lower <= confidence <= upper
|
||||
if bin_index == 14 or confidence < upper
|
||||
]
|
||||
if members:
|
||||
bin_accuracy = math.fsum(
|
||||
predictions[index] == targets[index] for index in members
|
||||
) / len(members)
|
||||
bin_confidence = math.fsum(
|
||||
confidences[index] for index in members
|
||||
) / len(members)
|
||||
ece += (
|
||||
abs(bin_accuracy - bin_confidence)
|
||||
* len(members)
|
||||
/ len(probs)
|
||||
)
|
||||
return {
|
||||
"n": len(probs),
|
||||
"nll": nll,
|
||||
"accuracy": accuracy,
|
||||
"brier": brier,
|
||||
"ece15": ece,
|
||||
"probabilities": probs,
|
||||
"targets": targets,
|
||||
}
|
||||
|
||||
|
||||
def _box(record: Mapping[str, Any]) -> tuple[float, float, float, float] | None:
|
||||
value = record.get("box", record.get("bbox", record.get("xyxy")))
|
||||
if not isinstance(value, (list, tuple)) or len(value) != 4 or not all(_finite(x) for x in value): return None
|
||||
x1, y1, x2, y2 = map(float, value)
|
||||
return (x1, y1, x2, y2) if x2 >= x1 and y2 >= y1 else None
|
||||
|
||||
|
||||
def _iou(a: Sequence[float], b: Sequence[float]) -> float:
|
||||
x1, y1 = max(a[0], b[0]), max(a[1], b[1]); x2, y2 = min(a[2], b[2]), min(a[3], b[3])
|
||||
inter = max(0.0, x2 - x1) * max(0.0, y2 - y1)
|
||||
area_a = max(0.0, a[2] - a[0]) * max(0.0, a[3] - a[1]); area_b = max(0.0, b[2] - b[0]) * max(0.0, b[3] - b[1])
|
||||
return inter / (area_a + area_b - inter) if area_a + area_b - inter > 0 else 0.0
|
||||
|
||||
|
||||
def _interp_ap_101(recall: Sequence[float], precision: Sequence[float]) -> float:
|
||||
"""Ultralytics compute_ap: precision envelope and 101-point trapezoid."""
|
||||
mrec = [0.0, *map(float, recall), 1.0]
|
||||
mpre = [1.0, *map(float, precision), 0.0]
|
||||
for i in range(len(mpre) - 2, -1, -1): mpre[i] = max(mpre[i], mpre[i + 1])
|
||||
values: list[float] = []
|
||||
for k in range(101):
|
||||
x = k / 100.0; j = 0
|
||||
while j + 1 < len(mrec) and mrec[j + 1] <= x: j += 1
|
||||
if j + 1 >= len(mrec): values.append(mpre[-1]); continue
|
||||
span = mrec[j + 1] - mrec[j]
|
||||
values.append(mpre[j] if span <= 0 else mpre[j] + (mpre[j + 1] - mpre[j]) * (x - mrec[j]) / span)
|
||||
return sum((values[i] + values[i + 1]) * 0.5 / 100.0 for i in range(100))
|
||||
|
||||
|
||||
def detection_metrics(records: Sequence[Mapping[str, Any]], class_count: int | None = None) -> dict[str, Any]:
|
||||
"""Recompute Ultralytics-style whole-dataset AP at IoU .50:.95.
|
||||
|
||||
Matching is performed independently per image. Candidate matches are sorted
|
||||
by IoU and deduplicated by prediction and ground truth, as in
|
||||
DetectionValidator.process_batch; AP then uses the pinned 101-point
|
||||
interpolated trapezoid.
|
||||
"""
|
||||
thresholds = [0.50 + 0.05 * i for i in range(10)]
|
||||
parsed: list[tuple[list[dict[str, Any]], list[dict[str, Any]]]] = []; max_class = -1
|
||||
for i, image in enumerate(records):
|
||||
predictions = image.get("predictions", image.get("detections", image.get("pred", [])))
|
||||
truth = image.get("ground_truth", image.get("targets", image.get("gt", image.get("labels", []))))
|
||||
if not isinstance(predictions, list) or not isinstance(truth, list): raise ValueError(f"invalid detection image record {i}")
|
||||
pp: list[dict[str, Any]] = []; gg: list[dict[str, Any]] = []
|
||||
for item in predictions:
|
||||
if not isinstance(item, Mapping) or _box(item) is None: raise ValueError(f"invalid detection prediction {i}")
|
||||
cls = item.get("class_id", item.get("class", item.get("cls"))); score = item.get("score", item.get("confidence", item.get("conf")))
|
||||
if not isinstance(cls, int) or isinstance(cls, bool) or not _finite(score): raise ValueError(f"invalid detection prediction fields {i}")
|
||||
pp.append({"box": _box(item), "class_id": cls, "score": float(score)}); max_class = max(max_class, cls)
|
||||
for item in truth:
|
||||
if not isinstance(item, Mapping) or _box(item) is None: raise ValueError(f"invalid ground truth {i}")
|
||||
cls = item.get("class_id", item.get("class", item.get("cls")))
|
||||
if not isinstance(cls, int) or isinstance(cls, bool): raise ValueError(f"invalid ground truth class {i}")
|
||||
gg.append({"box": _box(item), "class_id": cls}); max_class = max(max_class, cls)
|
||||
parsed.append((pp, gg))
|
||||
present = sorted({g["class_id"] for _, gt in parsed for g in gt})
|
||||
classes = present if class_count is None else [c for c in range(class_count) if c in present]
|
||||
if not classes: classes = list(range(class_count or (max_class + 1))) or [0]
|
||||
aps: dict[str, list[float]] = {}; precision50: list[float] = []; recall50: list[float] = []
|
||||
for cls in classes:
|
||||
gt_count = sum(sum(x["class_id"] == cls for x in gt) for _, gt in parsed); class_aps: list[float] = []
|
||||
for threshold in thresholds:
|
||||
true_by_image: list[list[bool]] = []
|
||||
for preds, gt in parsed:
|
||||
candidates = []
|
||||
for pi, pred in enumerate(preds):
|
||||
if pred["class_id"] != cls: continue
|
||||
for gi, target in enumerate(gt):
|
||||
if target["class_id"] == cls:
|
||||
overlap = _iou(pred["box"], target["box"])
|
||||
if overlap >= threshold: candidates.append((overlap, pi, gi))
|
||||
candidates.sort(key=lambda x: -x[0]); used_pred: set[int] = set(); used_gt: set[int] = set(); matched: set[int] = set()
|
||||
for overlap, pi, gi in candidates:
|
||||
if pi not in used_pred and gi not in used_gt: used_pred.add(pi); used_gt.add(gi); matched.add(pi)
|
||||
true_by_image.append([i in matched for i in range(len(preds))])
|
||||
ranked = sorted(((pred["score"], hit) for (preds, _), hits in zip(parsed, true_by_image) for pred, hit in zip(preds, hits) if pred["class_id"] == cls), key=lambda x: -x[0])
|
||||
tp=[]; fp=[]; ctp=cfp=0
|
||||
for _, hit in ranked:
|
||||
ctp += int(hit); cfp += int(not hit); tp.append(ctp); fp.append(cfp)
|
||||
if gt_count == 0 or not ranked:
|
||||
# Ultralytics ap_per_class skips classes with no predictions; AP is zero.
|
||||
class_aps.append(0.0); continue
|
||||
recalls = [x / gt_count for x in tp]; precisions = [x / max(x + y, 1) for x, y in zip(tp, fp)]
|
||||
class_aps.append(_interp_ap_101(recalls, precisions))
|
||||
if threshold == 0.5:
|
||||
precision50.append(precisions[-1] if precisions else 0.0); recall50.append(recalls[-1] if recalls else 0.0)
|
||||
aps[str(cls)] = class_aps
|
||||
map50 = math.fsum(v[0] for v in aps.values()) / len(aps); map5095 = math.fsum(x for values in aps.values() for x in values) / (len(aps) * 10)
|
||||
return {"per_class_ap": aps, "n": len(records), "map50": map50, "map50_95": map5095,
|
||||
"precision": math.fsum(precision50) / len(precision50) if precision50 else 0.0,
|
||||
"recall": math.fsum(recall50) / len(recall50) if recall50 else 0.0,
|
||||
"ground_truth": sum(len(gt) for _, gt in parsed), "predictions": sum(len(preds) for preds, _ in parsed)}
|
||||
|
||||
def _metric_from_record(record: Any, family: str) -> dict[str, float] | None:
|
||||
if not isinstance(record, Mapping): return None
|
||||
keys = ("nll", "loss") if family == "classification" else ("map50_95", "map50-95", "mAP50-95", "map5095")
|
||||
primary = _number(record, *keys); accuracy = _number(record, "accuracy", "acc")
|
||||
map50 = _number(record, "map50", "mAP50")
|
||||
if family == "classification" and primary is not None and accuracy is not None: return {"nll": primary, "accuracy": accuracy}
|
||||
if family == "detection" and primary is not None: return {"map50_95": primary, **({"map50": map50} if map50 is not None else {})}
|
||||
return None
|
||||
|
||||
|
||||
def _quantile(values: Sequence[float], q: float) -> float:
|
||||
ordered = sorted(float(x) for x in values)
|
||||
if not ordered: raise ValueError("cannot quantile an empty sequence")
|
||||
position = (len(ordered) - 1) * q; lower = int(math.floor(position)); upper = min(lower + 1, len(ordered) - 1)
|
||||
return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower)
|
||||
|
||||
|
||||
def _record_identity(record: Mapping[str, Any], index: int) -> Any:
|
||||
for key in ("image_id", "id", "key", "filename", "path", "index"):
|
||||
if key in record: return (key, str(record[key]))
|
||||
return ("position", index)
|
||||
|
||||
|
||||
def _record_ground_truth(record: Mapping[str, Any]) -> Any:
|
||||
return record.get("ground_truth", record.get("targets", record.get("gt", record.get("labels", []))))
|
||||
|
||||
|
||||
def _bootstrap_alignment(pairs: Sequence[tuple[Sequence[Mapping[str, Any]], Sequence[Mapping[str, Any]]]], family: str) -> tuple[bool, str]:
|
||||
if not pairs or any(len(a) != len(b) or not a for a, b in pairs): return False, "incomplete or length-mismatched paired records"
|
||||
reference_ids = [_record_identity(x, i) for i, x in enumerate(pairs[0][0])]
|
||||
reference_gt = [_record_ground_truth(x) for x in pairs[0][0]]
|
||||
for pair_index, (base, pso) in enumerate(pairs):
|
||||
if [_record_identity(x, i) for i, x in enumerate(base)] != reference_ids or [_record_identity(x, i) for i, x in enumerate(pso)] != reference_ids:
|
||||
return False, f"pair {pair_index} image IDs/order differ"
|
||||
if family == "classification":
|
||||
base_targets = [x.get("target", x.get("label", x.get("class_id"))) for x in base]
|
||||
pso_targets = [x.get("target", x.get("label", x.get("class_id"))) for x in pso]
|
||||
ref_targets = [x.get("target", x.get("label", x.get("class_id"))) for x in pairs[0][0]]
|
||||
if base_targets != ref_targets or pso_targets != ref_targets: return False, f"pair {pair_index} targets differ"
|
||||
elif [_canonical(_record_ground_truth(x)) for x in base] != [_canonical(x) for x in reference_gt] or [_canonical(_record_ground_truth(x)) for x in pso] != [_canonical(x) for x in reference_gt]:
|
||||
return False, f"pair {pair_index} ground truth differs"
|
||||
return True, "aligned"
|
||||
|
||||
|
||||
def _bootstrap_from_records(pairs: Sequence[tuple[Sequence[Mapping[str, Any]], Sequence[Mapping[str, Any]]]], family: str) -> dict[str, Any]:
|
||||
aligned, reason = _bootstrap_alignment(pairs, family)
|
||||
if not aligned: return {"available": False, "seed": BOOTSTRAP_SEED, "resamples": BOOTSTRAP_RESAMPLES, "reason": reason}
|
||||
rng = random.Random(BOOTSTRAP_SEED); stats: list[float] = []
|
||||
if family == "classification":
|
||||
parsed = [(classification_metrics(a), classification_metrics(b)) for a, b in pairs]
|
||||
by_class: dict[int, list[int]] = {}
|
||||
for i, target in enumerate(parsed[0][0]["targets"]): by_class.setdefault(target, []).append(i)
|
||||
for _ in range(BOOTSTRAP_RESAMPLES):
|
||||
# One class-stratified draw is shared by every paired base/PSO model.
|
||||
indices = [rng.choice(class_indices) for class_indices in by_class.values() for _ in class_indices]
|
||||
deltas = []
|
||||
for base, pso in parsed:
|
||||
b_nll = math.fsum(-math.log(max(base["probabilities"][i][base["targets"][i]], 1e-300)) for i in indices) / len(indices)
|
||||
p_nll = math.fsum(-math.log(max(pso["probabilities"][i][pso["targets"][i]], 1e-300)) for i in indices) / len(indices)
|
||||
deltas.append((b_nll - p_nll) / b_nll if b_nll else 0.0)
|
||||
stats.append(math.fsum(deltas) / len(deltas))
|
||||
else:
|
||||
for _ in range(BOOTSTRAP_RESAMPLES):
|
||||
# One whole-image draw is shared by every paired base/PSO model.
|
||||
indices = [rng.randrange(len(pairs[0][0])) for _ in pairs[0][0]]
|
||||
deltas = []
|
||||
for base, pso in pairs:
|
||||
b = detection_metrics([base[i] for i in indices])["map50_95"]
|
||||
p = detection_metrics([pso[i] for i in indices])["map50_95"]
|
||||
deltas.append(p - b)
|
||||
stats.append(math.fsum(deltas) / len(deltas))
|
||||
lo = _quantile(stats, BOOTSTRAP_ALPHA); hi = _quantile(stats, 1.0 - BOOTSTRAP_ALPHA)
|
||||
return {"available": True, "seed": BOOTSTRAP_SEED, "resamples": BOOTSTRAP_RESAMPLES, "alpha": BOOTSTRAP_ALPHA, "lower": lo, "upper": hi, "statistic": math.fsum(stats) / len(stats), "excludes_zero": lo > 0 or hi < 0}
|
||||
|
||||
def _prediction_pairs(result: Mapping[str, Any], root: Path, family: str) -> list[tuple[int, list[dict[str, Any]], list[dict[str, Any]]]]:
|
||||
"""Find one test base/selected pair per frozen base seed by explicit names."""
|
||||
sets = _find_prediction_sets(result.get("confirmation", {}), root)
|
||||
out: list[tuple[int, list[dict[str, Any]], list[dict[str, Any]]]] = []
|
||||
for seed in BASE_SEEDS:
|
||||
candidates = [(name, records) for name, records in sets.items() if str(seed) in name]
|
||||
base = next((records for name, records in candidates if any(x in name.lower() for x in ("base", "frozen"))), None)
|
||||
pso = next((records for name, records in candidates if any(x in name.lower() for x in ("feature_pso", "selected", "pso")) and "ensemble" not in name.lower()), None)
|
||||
if base is not None and pso is not None: out.append((seed, base, pso))
|
||||
return out
|
||||
|
||||
|
||||
def _audit_series(value: Any) -> list[dict[str, Any]]:
|
||||
found: list[dict[str, Any]] = []
|
||||
if isinstance(value, Mapping):
|
||||
for item in value.values(): found.extend(_audit_series(item))
|
||||
elif isinstance(value, list) and value and all(isinstance(item, Mapping) for item in value):
|
||||
if any("epoch" in item or "step" in item for item in value) and any(_number(item, "loss", "audit_loss") is not None for item in value):
|
||||
found.append({"records": value})
|
||||
else:
|
||||
for item in value: found.extend(_audit_series(item))
|
||||
return found
|
||||
|
||||
|
||||
def _plateau_flags(result: Mapping[str, Any], workload: str, family: str, issues: dict[str, list[str]]) -> dict[str, Any]:
|
||||
baselines = result.get("baselines", {})
|
||||
if not isinstance(baselines, Mapping): return {"available": False, "passed": False}
|
||||
per_seed: dict[str, bool] = {}; details: dict[str, Any] = {}
|
||||
for seed in BASE_SEEDS:
|
||||
entry = baselines.get(str(seed), baselines.get(seed))
|
||||
series = _audit_series(entry)
|
||||
records = series[0]["records"] if series else []
|
||||
records = records[-11:] if len(records) >= 11 else []
|
||||
losses = [_number(x, "loss", "audit_loss") for x in records]
|
||||
metric_keys = ("accuracy", "primary_metric", "selection_accuracy") if family == "classification" else ("map50_95", "mAP50-95", "primary_metric", "selection_metric")
|
||||
metrics = [_number(x, *metric_keys) for x in records]
|
||||
loss_ok = len(losses) == 11 and all(x is not None for x in losses)
|
||||
metric_ok = len(metrics) == 11 and all(x is not None for x in metrics)
|
||||
if loss_ok:
|
||||
mean = math.fsum(float(x) for x in losses) / len(losses)
|
||||
loss_ok = (max(losses) - min(losses)) / max(abs(mean), 1e-12) <= 0.01
|
||||
if metric_ok: metric_ok = max(metrics) - min(metrics) <= 0.005
|
||||
passed = bool(loss_ok and metric_ok); per_seed[str(seed)] = passed
|
||||
details[str(seed)] = {"loss_ok": bool(loss_ok), "metric_ok": bool(metric_ok), "observations": len(records)}
|
||||
declared = _number(entry, "baseline_plateau") if isinstance(entry, Mapping) else None
|
||||
if isinstance(entry, Mapping) and "baseline_plateau" in entry and bool(entry["baseline_plateau"]) != passed:
|
||||
_issue(issues, "plateau", f"{workload}: baseline seed {seed} inflated/incorrect plateau flag")
|
||||
return {"available": bool(per_seed), "per_seed": per_seed, "details": details, "passed": bool(per_seed) and all(per_seed.values())}
|
||||
|
||||
|
||||
def _workload_gates(result: Mapping[str, Any], workload: str, family: str, root: Path, issues: dict[str, list[str]]) -> dict[str, Any]:
|
||||
pairs = _prediction_pairs(result, root, family)
|
||||
per_seed: dict[str, Any] = {}
|
||||
pair_records: list[tuple[Sequence[Mapping[str, Any]], Sequence[Mapping[str, Any]]]] = []
|
||||
for seed, base_records, pso_records in pairs:
|
||||
try:
|
||||
base = classification_metrics(base_records) if family == "classification" else detection_metrics(base_records, 20)
|
||||
pso = classification_metrics(pso_records) if family == "classification" else detection_metrics(pso_records, 20)
|
||||
except (ValueError, ZeroDivisionError) as exc:
|
||||
_issue(issues, "metrics", f"{workload}: test pair {seed} cannot be recomputed: {exc}"); continue
|
||||
pair_records.append((base_records, pso_records))
|
||||
if family == "classification": per_seed[str(seed)] = {"base": {"nll": base["nll"], "accuracy": base["accuracy"]}, "pso": {"nll": pso["nll"], "accuracy": pso["accuracy"]}, "relative_nll_reduction": (base["nll"] - pso["nll"]) / base["nll"], "accuracy_delta": pso["accuracy"] - base["accuracy"]}
|
||||
else: per_seed[str(seed)] = {"base": {"map50_95": base["map50_95"], "map50": base["map50"]}, "pso": {"map50_95": pso["map50_95"], "map50": pso["map50"]}, "map50_95_delta": pso["map50_95"] - base["map50_95"], "map50_delta": pso["map50"] - base["map50"]}
|
||||
ci = _bootstrap_from_records(pair_records, family)
|
||||
# Confirmation predictions and the paired CI are required integrity evidence.
|
||||
# A numerically negative CI is a valid scientific result; an unavailable CI
|
||||
# means the frozen/confirmed artifact set is incomplete or tampered.
|
||||
if len(pairs) != len(BASE_SEEDS):
|
||||
_issue(issues, "metrics", f"{workload}: required confirmation pairs are incomplete ({len(pairs)}/{len(BASE_SEEDS)})")
|
||||
if not ci.get("available", False):
|
||||
_issue(issues, "metrics", f"{workload}: required paired bootstrap unavailable: {ci.get('reason', 'missing prediction evidence')}")
|
||||
if family == "classification" and per_seed:
|
||||
reductions = [x["relative_nll_reduction"] for x in per_seed.values()]; acc_deltas = [x["accuracy_delta"] for x in per_seed.values()]
|
||||
gate = len(reductions) == 3 and math.fsum(reductions) / 3 >= 0.01 and math.fsum(acc_deltas) / 3 >= -0.002 and min(acc_deltas) >= -0.005 and sum(x > 0 for x in reductions) >= 2 and ci.get("excludes_zero", False)
|
||||
elif family == "detection" and per_seed:
|
||||
deltas = [x["map50_95_delta"] for x in per_seed.values()]
|
||||
gate = len(deltas) == 3 and math.fsum(deltas) / 3 >= 0.005 and min(deltas) >= -0.005 and sum(x > 0 for x in deltas) >= 2 and ci.get("excludes_zero", False)
|
||||
else: gate = False
|
||||
if len(pairs) != len(BASE_SEEDS): _issue(issues, "bootstrap", f"{workload}: missing complete official-test base/feature_pso prediction pairs ({len(pairs)}/3)")
|
||||
return {"available": bool(per_seed), "per_seed": per_seed, "bootstrap": ci, "generalization_pass": bool(gate)}
|
||||
|
||||
|
||||
def _verify_hashes(result: Mapping[str, Any], root: Path, issues: dict[str, list[str]], workload: str) -> None:
|
||||
declared = result.get("artifact_hashes")
|
||||
if not isinstance(declared, Mapping):
|
||||
_issue(issues, "schema", f"{workload}: missing artifact_hashes")
|
||||
return
|
||||
for name, expected in declared.items():
|
||||
if not isinstance(name, str) or not isinstance(expected, str):
|
||||
_issue(issues, "seal", f"{workload}: malformed artifact hash entry")
|
||||
continue
|
||||
path = (root / name).resolve()
|
||||
if root.resolve() not in path.parents or not path.is_file():
|
||||
_issue(issues, "seal", f"{workload}: missing/escaping artifact {name}")
|
||||
elif _sha256(path) != expected:
|
||||
_issue(issues, "seal", f"{workload}: artifact hash drift {name}")
|
||||
|
||||
|
||||
def _compare_development_snapshot(development: Any, current: Mapping[str, Any], workload: str, issues: dict[str, list[str]]) -> None:
|
||||
"""Ensure confirmation did not alter any sealed development decision/state."""
|
||||
if not isinstance(development, Mapping):
|
||||
_issue(issues, "schema", f"{workload}: development_result.json must contain an object")
|
||||
return
|
||||
development_leakage = development.get("leakage_counters")
|
||||
if not isinstance(development_leakage, Mapping):
|
||||
_issue(issues, "leakage", f"{workload}: development snapshot lacks leakage_counters")
|
||||
else:
|
||||
loaded = development_leakage.get("official_test_data_loaded_before_freeze", development_leakage.get("test_data_loaded_before_freeze"))
|
||||
evaluated = development_leakage.get("official_test_evaluations_before_freeze", development_leakage.get("test_evaluations_before_freeze", development_leakage.get("official_test_forward_passes_before_freeze")))
|
||||
if loaded is not False or evaluated != 0:
|
||||
_issue(issues, "leakage", f"{workload}: development snapshot records pre-freeze official-test exposure")
|
||||
for key in ("official_test_construction", "official_test_dataset_construction", "official_test_forward_passes", "official_test_evaluations"):
|
||||
if key in development_leakage and development_leakage[key] != 0:
|
||||
_issue(issues, "leakage", f"{workload}: development snapshot {key} must be zero")
|
||||
development_confirmation = development.get("confirmation")
|
||||
if development_confirmation not in ({}, None):
|
||||
_issue(issues, "leakage", f"{workload}: development snapshot confirmation must be empty")
|
||||
fields = ("config", "manifests", "provenance", "baselines", "arms", "ensemble", "development_selection", "integrity", "resource_ledger", "artifact_hashes")
|
||||
for field in fields:
|
||||
if field not in development:
|
||||
_issue(issues, "seal", f"{workload}: development_result.json missing sealed field {field}")
|
||||
continue
|
||||
if field not in current:
|
||||
_issue(issues, "seal", f"{workload}: current result missing sealed field {field}")
|
||||
continue
|
||||
if field == "artifact_hashes":
|
||||
# Confirmation may add test prediction hashes. Every development hash
|
||||
# must nevertheless remain present and byte-identical.
|
||||
old_hashes = development[field]; new_hashes = current[field]
|
||||
if not isinstance(old_hashes, Mapping) or not isinstance(new_hashes, Mapping):
|
||||
_issue(issues, "seal", f"{workload}: artifact_hashes changed shape across confirmation")
|
||||
else:
|
||||
for name, value in old_hashes.items():
|
||||
if new_hashes.get(name) != value: _issue(issues, "seal", f"{workload}: sealed artifact hash drift for {name}")
|
||||
elif field == "integrity":
|
||||
old_integrity = development[field]; new_integrity = current[field]
|
||||
if not isinstance(old_integrity, Mapping) or not isinstance(new_integrity, Mapping):
|
||||
if old_integrity != new_integrity: _issue(issues, "seal", f"{workload}: pre-confirmation integrity changed")
|
||||
else:
|
||||
for name, value in old_integrity.items():
|
||||
if new_integrity.get(name) != value: _issue(issues, "seal", f"{workload}: pre-confirmation integrity field changed: {name}")
|
||||
elif _canonical(development[field]) != _canonical(current[field]):
|
||||
_issue(issues, "seal", f"{workload}: sealed pre-confirmation field changed: {field}")
|
||||
|
||||
def _evaluate_workload(result: Any, root: Path, workload: str, issues: dict[str, list[str]], development: Any = None) -> dict[str, Any]:
|
||||
if not isinstance(result, Mapping):
|
||||
_issue(issues, "schema", f"{workload}: result must be an object"); return {"workload_id": workload, "valid": False}
|
||||
if result.get("workload_id") != workload: _issue(issues, "matrix", f"{workload}: workload_id mismatch")
|
||||
if development is not None: _compare_development_snapshot(development, result, workload, issues)
|
||||
family = "detection" if workload == DETECTION_WORKLOAD else "classification"
|
||||
if result.get("family") != family: _issue(issues, "matrix", f"{workload}: family must be {family}")
|
||||
for key in ("manifests", "provenance", "baselines", "arms", "ensemble", "development_selection", "confirmation", "integrity", "leakage_counters", "resource_ledger", "artifact_hashes"):
|
||||
if key not in result: _issue(issues, "schema", f"{workload}: missing top-level {key}")
|
||||
_config_checks(result.get("config"), issues, workload); _check_leakage(result, issues, workload); _verify_hashes(result, root, issues, workload)
|
||||
accounting = _verify_matrix(result, workload, issues); _verify_selection(result, workload, issues); _require_confirmation_methods(result, workload, family, root, issues)
|
||||
if result.get("integrity", {}).get("confirmed") is False if isinstance(result.get("integrity"), Mapping) else False:
|
||||
_issue(issues, "seal", f"{workload}: integrity declares confirmation failure")
|
||||
prediction_sets = _find_prediction_sets(result.get("confirmation", {}), root)
|
||||
recomputed: dict[str, Any] = {}
|
||||
for name, records in prediction_sets.items():
|
||||
try: recomputed[name] = classification_metrics(records) if family == "classification" else detection_metrics(records, 20)
|
||||
except (ValueError, ZeroDivisionError) as exc: _issue(issues, "metrics", f"{workload}: invalid stored predictions at {name}: {exc}")
|
||||
# Check every explicitly stored metric that has a corresponding recomputation.
|
||||
for name, metric in recomputed.items():
|
||||
if "pso" in name.lower() and isinstance(metric, Mapping):
|
||||
stored = _metric_from_record(result.get("confirmation"), family)
|
||||
if stored and family == "classification" and not (_same(stored.get("nll"), metric.get("nll"), 1e-7) and _same(stored.get("accuracy"), metric.get("accuracy"), 1e-7)):
|
||||
_issue(issues, "metrics", f"{workload}: stored classification metric disagrees with probabilities at {name}")
|
||||
plateau = _plateau_flags(result, workload, family, issues)
|
||||
gates = _workload_gates(result, workload, family, root, issues)
|
||||
objective = _number(result.get("development_selection"), "objective_improvement", "relative_objective_improvement")
|
||||
selection_metric = _number(result.get("development_selection"), "selection_metric", "selection_nll", "selection_map50_95")
|
||||
overfit = bool(objective is not None and objective >= 0.01 and selection_metric is not None and ((family == "classification" and selection_metric > 0.01) or (family == "detection" and selection_metric < -0.005)))
|
||||
if overfit: _issue(issues, "overfit", f"{workload}: objective improvement conflicts with held-out/selection metric")
|
||||
return {"workload_id": workload, "family": family, "valid": True, "accounting": accounting, "recomputed": recomputed, "plateau": plateau, "gates": gates, "overfit_signal": overfit}
|
||||
|
||||
|
||||
def evaluate_run(run_root: str | os.PathLike[str]) -> dict[str, Any]:
|
||||
"""Evaluate one frozen run, returning findings even when artifacts are malformed."""
|
||||
root = Path(run_root); issues = {key: [] for key in ISSUE_CATEGORIES}; workloads: dict[str, Any] = {}
|
||||
manifest: Any = None
|
||||
try: manifest = _json(root / "frozen_manifest.json")
|
||||
except (OSError, ValueError, json.JSONDecodeError) as exc: _issue(issues, "schema", f"cannot load frozen_manifest.json: {exc}")
|
||||
if isinstance(manifest, Mapping):
|
||||
for location in _walk_nonfinite(manifest): _issue(issues, "finite", f"non-finite value in frozen manifest at {location}")
|
||||
_hash_manifest(root, manifest, issues); _config_checks(manifest.get("config"), issues)
|
||||
frozen_config = manifest.get("config") if isinstance(manifest.get("config"), Mapping) else {}
|
||||
if sorted(frozen_config.get("workload_ids", ())) != sorted(WORKLOADS): _issue(issues, "matrix", "frozen manifest does not seal all three workloads")
|
||||
for workload in WORKLOADS:
|
||||
path = root / "workloads" / workload / "result.json"
|
||||
development_path = root / "workloads" / workload / "development_result.json"
|
||||
if isinstance(manifest, Mapping):
|
||||
manifest_artifacts = manifest.get("artifacts", {})
|
||||
expected_development = f"workloads/{workload}/development_result.json"
|
||||
if not isinstance(manifest_artifacts, Mapping) or expected_development not in manifest_artifacts:
|
||||
_issue(issues, "seal", f"{workload}: frozen manifest must seal {expected_development}")
|
||||
try: result = _json(path)
|
||||
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
_issue(issues, "schema", f"{workload}: cannot load result.json: {exc}"); continue
|
||||
try: development = _json(development_path)
|
||||
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
_issue(issues, "seal", f"{workload}: cannot load development_result.json: {exc}"); development = None
|
||||
for location in _walk_nonfinite(result): _issue(issues, "finite", f"{workload}: non-finite value at {location}")
|
||||
if development is not None:
|
||||
for location in _walk_nonfinite(development): _issue(issues, "finite", f"{workload}: non-finite development value at {location}")
|
||||
workloads[workload] = _evaluate_workload(result, root, workload, issues, development)
|
||||
# Cross-workload exact accounting is intentionally independent of stored totals.
|
||||
totals = {key: sum(int(w.get("accounting", {}).get(key, 0)) for w in workloads.values()) for key in ("primary_queries", "primary_random_queries", "ensemble_queries", "primary_samples", "primary_random_samples", "ensemble_samples")}
|
||||
totals["pso_queries"] = totals["primary_queries"] + totals["ensemble_queries"]
|
||||
totals["candidate_samples"] = totals["primary_samples"] + totals["ensemble_samples"]
|
||||
if totals["pso_queries"] != TOTAL_PSO_QUERIES: _issue(issues, "accounting", f"total PSO queries must be {TOTAL_PSO_QUERIES}, got {totals['pso_queries']}")
|
||||
if totals["candidate_samples"] != TOTAL_CANDIDATE_SAMPLES: _issue(issues, "accounting", f"total candidate-sample evaluations must be {TOTAL_CANDIDATE_SAMPLES}, got {totals['candidate_samples']}")
|
||||
# A success flag is never consumed; it is checked against independently observed integrity.
|
||||
integrity_ok = not any(issues[key] for key in ("schema", "provenance", "matrix", "accounting", "seal", "selection", "finite", "leakage", "metrics"))
|
||||
payload = {"evaluator_version": EVALUATOR_VERSION, "protocol_version": PROTOCOL_VERSION, "run_root": str(root), "pass": integrity_ok, "integrity_pass": integrity_ok, "workloads": workloads, "accounting": {**totals, "expected_pso_queries": TOTAL_PSO_QUERIES, "expected_candidate_samples": TOTAL_CANDIDATE_SAMPLES}, "issues": issues, "issue_counts": {key: len(value) for key, value in issues.items()}}
|
||||
return payload
|
||||
|
||||
|
||||
def build_cli_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Evaluate frozen post-training model-convergence artifacts")
|
||||
parser.add_argument("--run-root", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, default=None)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = build_cli_parser().parse_args(argv); payload = evaluate_run(args.run_root)
|
||||
destination = args.output or args.run_root / "evaluation.json"
|
||||
try: _atomic_json(destination, payload)
|
||||
except OSError as exc:
|
||||
print(f"evaluator output failed: {exc}", file=sys.stderr); return 2
|
||||
print(json.dumps(payload, sort_keys=True, separators=(",", ":")))
|
||||
return 0 if payload["pass"] else 1
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BASE_SEEDS", "BOOTSTRAP_ALPHA", "BOOTSTRAP_RESAMPLES", "BOOTSTRAP_SEED", "CLASSIFICATION_WORKLOADS",
|
||||
"DETECTION_WORKLOAD", "EVALUATOR_VERSION", "ENSEMBLE_QUERIES", "TOTAL_CANDIDATE_SAMPLES", "TOTAL_PSO_QUERIES",
|
||||
"WORKLOADS", "classification_metrics", "detection_metrics", "evaluate_run", "main", "build_cli_parser",
|
||||
]
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+87
-64
@@ -1,87 +1,110 @@
|
||||
# %%
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import numpy as np
|
||||
import tensorflow as tf
|
||||
from keras.datasets import fashion_mnist
|
||||
from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
|
||||
from keras.models import Sequential
|
||||
|
||||
from pso import optimizer
|
||||
|
||||
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
|
||||
from pso import Optimizer
|
||||
from cli import add_pso_args, build_optimizer_kwargs
|
||||
|
||||
|
||||
def get_data():
|
||||
(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()
|
||||
def get_data(seed: int = 42):
|
||||
from sklearn.decomposition import PCA
|
||||
from torchvision.datasets import FashionMNIST
|
||||
|
||||
x_train, x_test = x_train / 255.0, x_test / 255.0
|
||||
x_train = x_train.reshape((60000, 28, 28, 1))
|
||||
x_test = x_test.reshape((10000, 28, 28, 1))
|
||||
train_dataset = FashionMNIST(root="./data", train=True, download=True)
|
||||
test_dataset = FashionMNIST(root="./data", train=False, download=True)
|
||||
|
||||
y_train, y_test = tf.one_hot(y_train, 10), tf.one_hot(y_test, 10)
|
||||
x_train_raw = (train_dataset.data[:3000].float() / 255.0).reshape(3000, -1).numpy()
|
||||
y_train = train_dataset.targets[:3000].long()
|
||||
|
||||
x_train, x_test = tf.convert_to_tensor(x_train), tf.convert_to_tensor(x_test)
|
||||
y_train, y_test = tf.convert_to_tensor(y_train), tf.convert_to_tensor(y_test)
|
||||
x_test_raw = (test_dataset.data[:1000].float() / 255.0).reshape(1000, -1).numpy()
|
||||
y_test = test_dataset.targets[:1000].long()
|
||||
|
||||
print(f"x_train : {x_train[0].shape} | y_train : {y_train[0].shape}")
|
||||
print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}")
|
||||
pca = PCA(n_components=32, whiten=True, random_state=seed)
|
||||
x_train_pca = pca.fit_transform(x_train_raw)
|
||||
x_test_pca = pca.transform(x_test_raw)
|
||||
|
||||
x_train = torch.tensor(x_train_pca, dtype=torch.float32)
|
||||
x_test = torch.tensor(x_test_pca, dtype=torch.float32)
|
||||
|
||||
print(f"x_train : {x_train.shape} | y_train : {y_train.shape}")
|
||||
print(f"x_test : {x_test.shape} | y_test : {y_test.shape}")
|
||||
|
||||
return x_train, y_train, x_test, y_test
|
||||
|
||||
|
||||
def make_model():
|
||||
model = Sequential()
|
||||
model.add(
|
||||
Conv2D(32, kernel_size=(5, 5), activation="relu", input_shape=(28, 28, 1))
|
||||
def make_model(seed: int = 42):
|
||||
torch.manual_seed(seed)
|
||||
return nn.Linear(32, 10)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="PSO Fashion-MNIST Benchmark Script")
|
||||
add_pso_args(
|
||||
parser,
|
||||
defaults={
|
||||
"method": "original",
|
||||
"initialization": "model_noise",
|
||||
"evaluation": "fixed_subset",
|
||||
"convergence": "particle_reset",
|
||||
"refinement": "adam",
|
||||
"n_particles": 30,
|
||||
"c0": None,
|
||||
"c1": None,
|
||||
"w_min": None,
|
||||
"w_max": None,
|
||||
"negative_swarm": 0.0,
|
||||
"mutation_swarm": 0.05,
|
||||
"particle_min": -3.0,
|
||||
"particle_max": 3.0,
|
||||
"velocity_limit_ratio": 0.1,
|
||||
"boundary_strategy": "reflect",
|
||||
"seed": 42,
|
||||
"epochs": 80,
|
||||
"batch_size": 1000,
|
||||
"fitness_size": 2000,
|
||||
"renewal": "loss",
|
||||
"output_dir": "output/fashion_mnist",
|
||||
"checkpoint_interval": 25,
|
||||
"refinement_epochs": 10,
|
||||
"refinement_lr": 0.001,
|
||||
},
|
||||
)
|
||||
model.add(MaxPooling2D(pool_size=(2, 2)))
|
||||
model.add(Conv2D(64, kernel_size=(3, 3), activation="relu"))
|
||||
model.add(MaxPooling2D(pool_size=(2, 2)))
|
||||
model.add(Flatten())
|
||||
model.add(Dropout(0.25))
|
||||
model.add(Dense(256, activation="relu"))
|
||||
model.add(Dense(128, activation="relu"))
|
||||
model.add(Dense(10, activation="softmax"))
|
||||
args = parser.parse_args()
|
||||
|
||||
return model
|
||||
model = make_model(seed=args.seed)
|
||||
x_train, y_train, x_test, y_test = get_data(seed=args.seed)
|
||||
|
||||
fitness_size = args.fitness_size if args.evaluation == "fixed_subset" else None
|
||||
refinement_epochs = args.refinement_epochs if args.refinement == "adam" else 0
|
||||
|
||||
# %%
|
||||
model = make_model()
|
||||
x_train, y_train, x_test, y_test = get_data()
|
||||
kwargs = build_optimizer_kwargs(
|
||||
args,
|
||||
model=model,
|
||||
loss=nn.CrossEntropyLoss(),
|
||||
task="multiclass",
|
||||
inertia_profile={"c0": 0.7, "c1": 0.5, "w_min": 0.1, "w_max": 0.8},
|
||||
)
|
||||
pso_fashion = Optimizer(**kwargs)
|
||||
|
||||
print(f"Optimizer device: {pso_fashion.device}")
|
||||
|
||||
pso_mnist = optimizer(
|
||||
model,
|
||||
loss="categorical_crossentropy",
|
||||
n_particles=200,
|
||||
c0=0.7,
|
||||
c1=0.5,
|
||||
w_min=0.1,
|
||||
w_max=0.8,
|
||||
negative_swarm=0.0,
|
||||
mutation_swarm=0.05,
|
||||
convergence_reset=True,
|
||||
convergence_reset_patience=10,
|
||||
convergence_reset_monitor="loss",
|
||||
)
|
||||
|
||||
best_score = pso_mnist.fit(
|
||||
best_score = pso_fashion.fit(
|
||||
x_train,
|
||||
y_train,
|
||||
epochs=1000,
|
||||
epochs=args.epochs,
|
||||
batch_size=args.batch_size,
|
||||
fitness_size=fitness_size,
|
||||
renewal=args.renewal,
|
||||
validation_data=(x_test, y_test),
|
||||
output_dir=args.output_dir,
|
||||
checkpoint_interval=25,
|
||||
save_info=True,
|
||||
log=2,
|
||||
log_name="fashion_mnist",
|
||||
renewal="loss",
|
||||
check_point=25,
|
||||
batch_size=5000,
|
||||
)
|
||||
refinement_epochs=refinement_epochs,
|
||||
refinement_lr=args.refinement_lr,
|
||||
)
|
||||
|
||||
print("Done!")
|
||||
print(f"Done! Best score: {best_score}")
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -1,115 +0,0 @@
|
||||
from keras.models import Sequential
|
||||
from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
|
||||
from keras.datasets import mnist, fashion_mnist
|
||||
from keras.utils import to_categorical
|
||||
# from tensorflow.data.Dataset import from_tensor_slices
|
||||
import tensorflow as tf
|
||||
import os
|
||||
|
||||
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
|
||||
|
||||
|
||||
gpus = tf.config.experimental.list_physical_devices("GPU")
|
||||
if gpus:
|
||||
try:
|
||||
tf.config.experimental.set_memory_growth(gpus[0], True)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
finally:
|
||||
del gpus
|
||||
|
||||
|
||||
def get_data():
|
||||
(x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()
|
||||
print(f"y_train : {y_train[0]} | y_test : {y_test[0]}")
|
||||
|
||||
x_train, x_test = x_train / 255.0, x_test / 255.0
|
||||
x_train = x_train.reshape((60000, 28, 28, 1))
|
||||
x_test = x_test.reshape((10000, 28, 28, 1))
|
||||
|
||||
print(f"x_train : {x_train[0].shape} | y_train : {y_train[0].shape}")
|
||||
print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}")
|
||||
|
||||
return x_train, y_train, x_test, y_test
|
||||
|
||||
|
||||
class _batch_generator:
|
||||
def __init__(self, x, y, batch_size: int = 32):
|
||||
self.batch_size = batch_size
|
||||
self.index = 0
|
||||
self.x = x
|
||||
self.y = y
|
||||
self.setBatchSize(batch_size)
|
||||
|
||||
def next(self):
|
||||
self.index += 1
|
||||
if self.index >= self.max_index:
|
||||
self.index = 0
|
||||
return self.dataset[self.index][0], self.dataset[self.index][1]
|
||||
|
||||
def getMaxIndex(self):
|
||||
return self.max_index
|
||||
|
||||
def getIndex(self):
|
||||
return self.index
|
||||
|
||||
def setIndex(self, index):
|
||||
self.index = index
|
||||
|
||||
def getBatchSize(self):
|
||||
return self.batch_size
|
||||
|
||||
def setBatchSize(self, batch_size):
|
||||
self.batch_size = batch_size
|
||||
self.dataset = list(
|
||||
tf.data.Dataset.from_tensor_slices(
|
||||
(self.x, self.y)).batch(batch_size)
|
||||
)
|
||||
self.max_index = len(self.dataset)
|
||||
|
||||
def getDataset(self):
|
||||
return self.dataset
|
||||
|
||||
|
||||
def make_model():
|
||||
model = Sequential()
|
||||
model.add(
|
||||
Conv2D(32, kernel_size=(5, 5), activation="sigmoid",
|
||||
input_shape=(28, 28, 1))
|
||||
)
|
||||
model.add(MaxPooling2D(pool_size=(2, 2)))
|
||||
model.add(Conv2D(64, kernel_size=(3, 3), activation="sigmoid"))
|
||||
model.add(MaxPooling2D(pool_size=(2, 2)))
|
||||
model.add(Flatten())
|
||||
model.add(Dropout(0.25))
|
||||
model.add(Dense(128, activation="sigmoid"))
|
||||
model.add(Dense(10, activation="softmax"))
|
||||
|
||||
return model
|
||||
|
||||
|
||||
model = make_model()
|
||||
x_train, y_train, x_test, y_test = get_data()
|
||||
print(x_train.shape)
|
||||
y_train = tf.one_hot(y_train, 10)
|
||||
y_test = tf.one_hot(y_test, 10)
|
||||
|
||||
dataset = _batch_generator(x_train, y_train, 32)
|
||||
|
||||
model.compile(optimizer="adam", loss="mse", metrics=["accuracy"])
|
||||
|
||||
count = 0
|
||||
|
||||
while count < 100:
|
||||
x_batch, y_batch = dataset.next()
|
||||
count += 1
|
||||
print("Training model...")
|
||||
model.fit(x_batch, y_batch, epochs=1, batch_size=1, verbose=1)
|
||||
|
||||
print(count)
|
||||
print(f"Max index : {dataset.getMaxIndex()}")
|
||||
|
||||
print("Evaluating model...")
|
||||
model.evaluate(x_test, y_test, verbose=2)
|
||||
|
||||
weights = model.get_weights()
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Fashion-MNIST dataset gradient baseline (PyTorch standard backprop optimizer, non-PSO)."""
|
||||
|
||||
import copy
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
|
||||
class FashionMNISTModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.conv1 = nn.Conv2d(1, 32, kernel_size=5)
|
||||
self.sig1 = nn.Sigmoid()
|
||||
self.pool1 = nn.MaxPool2d(2, 2)
|
||||
|
||||
self.conv2 = nn.Conv2d(32, 64, kernel_size=3)
|
||||
self.sig2 = nn.Sigmoid()
|
||||
self.pool2 = nn.MaxPool2d(2, 2)
|
||||
|
||||
self.drop = nn.Dropout(0.25)
|
||||
self.fc1 = nn.Linear(64 * 5 * 5, 128)
|
||||
self.sig3 = nn.Sigmoid()
|
||||
self.fc2 = nn.Linear(128, 10)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.pool1(self.sig1(self.conv1(x)))
|
||||
x = self.pool2(self.sig2(self.conv2(x)))
|
||||
x = torch.flatten(x, 1)
|
||||
x = self.drop(x)
|
||||
x = self.sig3(self.fc1(x))
|
||||
x = self.fc2(x)
|
||||
return x
|
||||
|
||||
|
||||
def get_data(download: bool = True):
|
||||
from torchvision import datasets, transforms
|
||||
|
||||
transform = transforms.ToTensor()
|
||||
train_dataset = datasets.FashionMNIST(
|
||||
root="./data", train=True, transform=transform, download=download
|
||||
)
|
||||
test_dataset = datasets.FashionMNIST(
|
||||
root="./data", train=False, transform=transform, download=download
|
||||
)
|
||||
return train_dataset, test_dataset
|
||||
|
||||
|
||||
def get_device() -> torch.device:
|
||||
if (
|
||||
hasattr(torch.backends, "mps")
|
||||
and torch.backends.mps.is_built()
|
||||
and torch.backends.mps.is_available()
|
||||
):
|
||||
return torch.device("mps")
|
||||
elif torch.cuda.is_available():
|
||||
return torch.device("cuda")
|
||||
return torch.device("cpu")
|
||||
|
||||
|
||||
def main():
|
||||
torch.manual_seed(42)
|
||||
np.random.seed(42)
|
||||
|
||||
device = get_device()
|
||||
print(f"Selected device: {device}")
|
||||
|
||||
train_dataset, test_dataset = get_data(download=True)
|
||||
train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
|
||||
test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False)
|
||||
|
||||
model = FashionMNISTModel().to(device)
|
||||
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
|
||||
best_val_loss = float("inf")
|
||||
best_state = None
|
||||
|
||||
for epoch in range(10):
|
||||
model.train()
|
||||
for bx, by in train_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
optimizer.zero_grad()
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
model.eval()
|
||||
val_loss = 0.0
|
||||
total = 0
|
||||
with torch.no_grad():
|
||||
for bx, by in test_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
val_loss += loss.item() * bx.size(0)
|
||||
total += bx.size(0)
|
||||
|
||||
val_loss /= total
|
||||
|
||||
if val_loss < best_val_loss:
|
||||
best_val_loss = val_loss
|
||||
best_state = copy.deepcopy(model.state_dict())
|
||||
|
||||
if best_state is not None:
|
||||
model.load_state_dict(best_state)
|
||||
|
||||
model.eval()
|
||||
test_loss = 0.0
|
||||
correct = 0
|
||||
total = 0
|
||||
with torch.no_grad():
|
||||
for bx, by in test_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
test_loss += loss.item() * bx.size(0)
|
||||
preds = out.argmax(dim=1)
|
||||
correct += (preds == by).sum().item()
|
||||
total += bx.size(0)
|
||||
|
||||
test_loss /= total
|
||||
test_acc = correct / total
|
||||
print(f"Final test loss: {test_loss:.4f}, accuracy: {test_acc:.4f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,707 @@
|
||||
"""
|
||||
Full-MNIST 60k/10k Particle Swarm Optimization Trajectory Analysis
|
||||
|
||||
Evaluates the selected Adaptive Moment PSO configuration on full official MNIST (60,000 train / 10,000 test)
|
||||
across 120 particles for 240 continuous epochs, scoring all 60,000 training examples for every particle
|
||||
at every epoch.
|
||||
|
||||
Predeclared Diagnostic Criteria:
|
||||
1. Post-80 Training Convergence: Mean training loss falls >= 1% from epoch 80 to epoch 240.
|
||||
2. Epoch 240 Test Gain: Mean test accuracy at epoch 240 rises >= 1 percentage point vs epoch 80.
|
||||
3. Late Plateau (200->240): Training loss improvement < 1% AND absolute test accuracy change < 0.5 percentage points.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import datetime
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
# Ensure test/ directory is in sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from benchmark_suite import (
|
||||
calc_stats,
|
||||
extract_plugin_metadata,
|
||||
compute_data_fingerprint,
|
||||
compute_model_fingerprint,
|
||||
get_hardware_provenance,
|
||||
make_mnist_model,
|
||||
resolve_execution_device,
|
||||
save_json_atomic,
|
||||
sync_device,
|
||||
)
|
||||
from pso import Optimizer, __version__ as pso_version
|
||||
from reproduce_scaling import validate_and_load_baseline
|
||||
|
||||
FULL_MNIST_PROTOCOL_VERSION = "1.0.0"
|
||||
|
||||
|
||||
def prepare_full_mnist_data() -> Tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
str,
|
||||
Dict[str, Any],
|
||||
]:
|
||||
"""
|
||||
Load official torchvision MNIST full train (60,000) and test (10,000).
|
||||
Normalize pixels, flatten to 784, fit PCA(n_components=32, whiten=True, random_state=42)
|
||||
on train only, transform test. Validate 60,000/10,000 sample counts and label range [0, 9].
|
||||
"""
|
||||
from torchvision.datasets import MNIST
|
||||
from sklearn.decomposition import PCA
|
||||
|
||||
cache_dir = Path("result/cache")
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
train_dataset = MNIST(root=str(cache_dir), train=True, download=True)
|
||||
test_dataset = MNIST(root=str(cache_dir), train=False, download=True)
|
||||
|
||||
n_train = len(train_dataset.data)
|
||||
n_test = len(test_dataset.data)
|
||||
if n_train != 60000:
|
||||
raise ValueError(f"Expected 60,000 training samples; got {n_train}")
|
||||
if n_test != 10000:
|
||||
raise ValueError(f"Expected 10,000 test samples; got {n_test}")
|
||||
|
||||
y_train_60000 = train_dataset.targets.long()
|
||||
y_test_10000 = test_dataset.targets.long()
|
||||
|
||||
min_tr_lbl, max_tr_lbl = int(y_train_60000.min()), int(y_train_60000.max())
|
||||
min_te_lbl, max_te_lbl = int(y_test_10000.min()), int(y_test_10000.max())
|
||||
|
||||
if min_tr_lbl != 0 or max_tr_lbl != 9:
|
||||
raise ValueError(f"Train label range must be [0, 9]; got [{min_tr_lbl}, {max_tr_lbl}]")
|
||||
if min_te_lbl != 0 or max_te_lbl != 9:
|
||||
raise ValueError(f"Test label range must be [0, 9]; got [{min_te_lbl}, {max_te_lbl}]")
|
||||
|
||||
x_train_raw = (train_dataset.data.float() / 255.0).reshape(60000, -1).numpy()
|
||||
x_test_raw = (test_dataset.data.float() / 255.0).reshape(10000, -1).numpy()
|
||||
|
||||
pca = PCA(n_components=32, whiten=True, random_state=42)
|
||||
x_full_tr = torch.tensor(pca.fit_transform(x_train_raw), dtype=torch.float32)
|
||||
x_full_test = torch.tensor(pca.transform(x_test_raw), dtype=torch.float32)
|
||||
|
||||
data_fp = compute_data_fingerprint(x_full_tr, x_full_test, y_train_60000, y_test_10000)
|
||||
pca_provenance = {
|
||||
"n_components": 32,
|
||||
"whiten": True,
|
||||
"random_state": 42,
|
||||
"fit_scope": "official_train_split_60000_only",
|
||||
"train_samples_fit": 60000,
|
||||
"test_samples_transformed": 10000,
|
||||
"explained_variance_ratio_sum": float(np.sum(pca.explained_variance_ratio_)),
|
||||
}
|
||||
return (
|
||||
x_full_tr,
|
||||
y_train_60000,
|
||||
x_full_test,
|
||||
y_test_10000,
|
||||
data_fp,
|
||||
pca_provenance,
|
||||
)
|
||||
|
||||
|
||||
def render_trajectory_plot(
|
||||
checkpoint_stats: Dict[int, Dict[str, Any]],
|
||||
subset_comparison: Dict[str, Any],
|
||||
figure_path: Path,
|
||||
):
|
||||
"""
|
||||
Render a readable two-panel mean +/- SD trajectory plot for training loss and test accuracy,
|
||||
with optional dashed subset-study mean comparison and epoch 80 marker.
|
||||
"""
|
||||
figure_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
eps = sorted(list(checkpoint_stats.keys()))
|
||||
tr_loss_mean = [checkpoint_stats[ep]["train_loss"]["mean"] for ep in eps]
|
||||
tr_loss_std = [checkpoint_stats[ep]["train_loss"]["std"] for ep in eps]
|
||||
te_acc_mean = [checkpoint_stats[ep]["test_acc"]["mean"] for ep in eps]
|
||||
te_acc_std = [checkpoint_stats[ep]["test_acc"]["std"] for ep in eps]
|
||||
|
||||
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 8), sharex=True)
|
||||
|
||||
# Panel 1: Training Loss
|
||||
ax1.plot(eps, tr_loss_mean, "o-", color="tab:blue", linewidth=2, label="Full MNIST Train Loss (Mean)")
|
||||
ax1.fill_between(
|
||||
eps,
|
||||
np.array(tr_loss_mean) - np.array(tr_loss_std),
|
||||
np.array(tr_loss_mean) + np.array(tr_loss_std),
|
||||
color="tab:blue",
|
||||
alpha=0.2,
|
||||
label="±1 SD",
|
||||
)
|
||||
|
||||
if "matching_epoch_deltas" in subset_comparison:
|
||||
sub_deltas = subset_comparison["matching_epoch_deltas"]
|
||||
sub_eps = sorted([ep for ep in eps if str(ep) in sub_deltas or ep in sub_deltas])
|
||||
if sub_eps:
|
||||
sub_tr_loss = [sub_deltas.get(str(ep), sub_deltas.get(ep, {}))["subset_study_train_loss_mean"] for ep in sub_eps]
|
||||
ax1.plot(sub_eps, sub_tr_loss, "--", color="gray", alpha=0.8, label="Subset Study (2k sample) Train Loss")
|
||||
|
||||
if 80 in eps:
|
||||
ax1.axvline(80, color="red", linestyle=":", label="Epoch 80 Marker")
|
||||
|
||||
ax1.set_ylabel("Training Loss")
|
||||
ax1.set_title("Full MNIST (60,000 Train / 10,000 Test) PSO Trajectory (120 Particles, AM)")
|
||||
ax1.grid(True, linestyle="--", alpha=0.5)
|
||||
ax1.legend(loc="upper right")
|
||||
|
||||
# Panel 2: Test Accuracy
|
||||
ax2.plot(eps, te_acc_mean, "s-", color="tab:green", linewidth=2, label="Full MNIST Test Accuracy (Mean)")
|
||||
ax2.fill_between(
|
||||
eps,
|
||||
np.array(te_acc_mean) - np.array(te_acc_std),
|
||||
np.array(te_acc_mean) + np.array(te_acc_std),
|
||||
color="tab:green",
|
||||
alpha=0.2,
|
||||
label="±1 SD",
|
||||
)
|
||||
|
||||
if "matching_epoch_deltas" in subset_comparison:
|
||||
sub_deltas = subset_comparison["matching_epoch_deltas"]
|
||||
sub_eps = sorted([ep for ep in eps if str(ep) in sub_deltas or ep in sub_deltas])
|
||||
if sub_eps:
|
||||
sub_te_acc = [sub_deltas.get(str(ep), sub_deltas.get(ep, {}))["subset_study_test_acc_mean"] for ep in sub_eps]
|
||||
ax2.plot(sub_eps, sub_te_acc, "--", color="gray", alpha=0.8, label="Subset Study (2k sample) Test Acc")
|
||||
|
||||
if 80 in eps:
|
||||
ax2.axvline(80, color="red", linestyle=":", label="Epoch 80 Marker")
|
||||
|
||||
ax2.set_xlabel("Epoch")
|
||||
ax2.set_ylabel("Test Accuracy")
|
||||
ax2.grid(True, linestyle="--", alpha=0.5)
|
||||
ax2.legend(loc="lower right")
|
||||
|
||||
plt.tight_layout()
|
||||
fig.savefig(figure_path, dpi=300)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def run_full_mnist_study(
|
||||
baseline_path: Path,
|
||||
subset_study_path: Path,
|
||||
output_json_path: Path,
|
||||
output_csv_path: Path,
|
||||
figure_path: Path,
|
||||
device_str: str = "auto",
|
||||
epochs: int = 240,
|
||||
seeds: List[int] = None,
|
||||
) -> bool:
|
||||
if seeds is None:
|
||||
seeds = [71, 72, 73, 74, 75]
|
||||
if not isinstance(epochs, int) or isinstance(epochs, bool) or epochs <= 0:
|
||||
raise ValueError("epochs must be a positive integer")
|
||||
if epochs % 20 != 0:
|
||||
raise ValueError("epochs must be a multiple of 20 so every final checkpoint exists")
|
||||
if not seeds or len(seeds) != len(set(seeds)):
|
||||
raise ValueError("seeds must be a non-empty list of unique integers")
|
||||
if any(isinstance(seed, bool) or not isinstance(seed, int) or seed < 0 for seed in seeds):
|
||||
raise ValueError("every seed must be a non-negative integer")
|
||||
|
||||
|
||||
dev_input = None if device_str == "auto" else device_str
|
||||
device = resolve_execution_device(dev_input)
|
||||
hw_provenance = get_hardware_provenance(device)
|
||||
|
||||
# 1. Validate baseline JSON & load winner config
|
||||
baseline_data, _baseline_records, winner_cfg, _expected_fp = validate_and_load_baseline(
|
||||
baseline_path
|
||||
)
|
||||
|
||||
if baseline_data.get("device") != device.type:
|
||||
raise ValueError(
|
||||
f"Baseline device mismatch: baseline requires {baseline_data.get('device')!r}, got {device.type!r}"
|
||||
)
|
||||
if baseline_data.get("pso_version") != pso_version:
|
||||
raise ValueError(
|
||||
f"PSO version mismatch: baseline requires {baseline_data.get('pso_version')!r}, got {pso_version!r}"
|
||||
)
|
||||
if baseline_data.get("torch_version") != torch.__version__:
|
||||
raise ValueError(
|
||||
f"Torch version mismatch: baseline requires {baseline_data.get('torch_version')!r}, got {torch.__version__!r}"
|
||||
)
|
||||
|
||||
# 2. Prepare Full MNIST PCA Data (60k train / 10k test)
|
||||
(
|
||||
x_full_tr,
|
||||
y_train_60000,
|
||||
x_full_test,
|
||||
y_test_10000,
|
||||
data_fp,
|
||||
pca_provenance,
|
||||
) = prepare_full_mnist_data()
|
||||
|
||||
opt_kwargs = winner_cfg.to_optimizer_kwargs(quick=False)
|
||||
opt_kwargs["evaluation"] = "full"
|
||||
opt_kwargs.pop("fitness_size", None)
|
||||
|
||||
n_particles = 120
|
||||
target_epochs = epochs
|
||||
batch_size = 60000
|
||||
checkpoint_interval = 20
|
||||
|
||||
ckpt_epochs = [ep for ep in range(checkpoint_interval, target_epochs + 1, checkpoint_interval)]
|
||||
if not ckpt_epochs or ckpt_epochs[-1] != target_epochs:
|
||||
if target_epochs not in ckpt_epochs:
|
||||
ckpt_epochs.append(target_epochs)
|
||||
ckpt_epochs = sorted(list(set(ckpt_epochs)))
|
||||
|
||||
runs: List[Dict[str, Any]] = []
|
||||
flat_csv_rows: List[Dict[str, Any]] = []
|
||||
plugin_meta: Dict[str, Any] | None = None
|
||||
|
||||
# 3. Seed Runs
|
||||
for seed in sorted(seeds):
|
||||
# Warmup Phase (2 epochs full eval)
|
||||
warmup_model = make_mnist_model(seed=seed)
|
||||
warmup_loss = nn.CrossEntropyLoss()
|
||||
warmup_opt = Optimizer(
|
||||
model=warmup_model,
|
||||
loss=warmup_loss,
|
||||
task="multiclass",
|
||||
n_particles=n_particles,
|
||||
seed=seed,
|
||||
device=device,
|
||||
**opt_kwargs,
|
||||
)
|
||||
warmup_opt.fit(
|
||||
x_full_tr,
|
||||
y_train_60000,
|
||||
epochs=2,
|
||||
batch_size=batch_size,
|
||||
renewal="loss",
|
||||
)
|
||||
sync_device(device)
|
||||
del warmup_opt, warmup_model, warmup_loss
|
||||
|
||||
# Timed Continuous Trajectory
|
||||
model = make_mnist_model(seed=seed)
|
||||
model_fp = compute_model_fingerprint(model)
|
||||
loss_inst = nn.CrossEntropyLoss()
|
||||
opt = Optimizer(
|
||||
model=model,
|
||||
loss=loss_inst,
|
||||
task="multiclass",
|
||||
n_particles=n_particles,
|
||||
seed=seed,
|
||||
device=device,
|
||||
**opt_kwargs,
|
||||
)
|
||||
current_plugin_meta = extract_plugin_metadata(opt)
|
||||
if plugin_meta is None:
|
||||
plugin_meta = current_plugin_meta
|
||||
elif current_plugin_meta != plugin_meta:
|
||||
raise RuntimeError("Resolved plugin metadata changed across seeds")
|
||||
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir_str:
|
||||
output_dir = Path(temp_dir_str)
|
||||
sync_device(device)
|
||||
t0 = time.perf_counter()
|
||||
train_loss_final, train_acc_final, train_mse_final = opt.fit(
|
||||
x_full_tr,
|
||||
y_train_60000,
|
||||
epochs=target_epochs,
|
||||
batch_size=batch_size,
|
||||
renewal="loss",
|
||||
output_dir=output_dir,
|
||||
log_format="csv",
|
||||
checkpoint_interval=checkpoint_interval,
|
||||
)
|
||||
sync_device(device)
|
||||
t1 = time.perf_counter()
|
||||
fit_time_sec = t1 - t0
|
||||
|
||||
if not (math.isfinite(train_loss_final) and math.isfinite(train_acc_final) and math.isfinite(train_mse_final)):
|
||||
raise RuntimeError(f"Seed {seed} final metrics non-finite: loss={train_loss_final}, acc={train_acc_final}")
|
||||
|
||||
history_csv_path = output_dir / "history.csv"
|
||||
epoch_history = []
|
||||
if history_csv_path.exists():
|
||||
with open(history_csv_path, "r", encoding="utf-8") as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
epoch_history.append({
|
||||
"epoch": int(row["epoch"]),
|
||||
"loss": float(row["loss"]),
|
||||
"accuracy": float(row["accuracy"]),
|
||||
"mse": float(row["mse"]),
|
||||
})
|
||||
|
||||
prev_best_loss = float("inf")
|
||||
improvement_count = 0
|
||||
last_improvement_epoch = 1
|
||||
for row in epoch_history:
|
||||
ep_num = row["epoch"]
|
||||
l_val = row["loss"]
|
||||
if l_val < prev_best_loss:
|
||||
improvement_count += 1
|
||||
last_improvement_epoch = ep_num
|
||||
prev_best_loss = l_val
|
||||
|
||||
checkpoints: List[Dict[str, Any]] = []
|
||||
ckpt_dir = output_dir / "checkpoints"
|
||||
for ep in ckpt_epochs:
|
||||
ckpt_path = ckpt_dir / f"epoch-{ep}.pt"
|
||||
if not ckpt_path.exists():
|
||||
raise FileNotFoundError(f"Missing checkpoint file: {ckpt_path}")
|
||||
payload = torch.load(ckpt_path, map_location=device, weights_only=True)
|
||||
ckpt_tr_loss, ckpt_tr_acc, ckpt_tr_mse = payload["score"]
|
||||
if not (math.isfinite(ckpt_tr_loss) and math.isfinite(ckpt_tr_acc) and math.isfinite(ckpt_tr_mse)):
|
||||
raise RuntimeError(f"Seed {seed} epoch {ep} score non-finite: {payload['score']}")
|
||||
|
||||
opt.eval_model.load_state_dict(payload["model_state_dict"])
|
||||
opt._global_best_weights = opt.codec.encode(opt.eval_model)
|
||||
test_loss, test_acc, test_mse = opt.evaluate(x_full_test, y_test_10000)
|
||||
|
||||
if not (math.isfinite(test_loss) and math.isfinite(test_acc) and math.isfinite(test_mse)):
|
||||
raise RuntimeError(f"Seed {seed} epoch {ep} test metrics non-finite: loss={test_loss}, acc={test_acc}")
|
||||
|
||||
ckpt_record = {
|
||||
"epoch": ep,
|
||||
"train_loss": float(ckpt_tr_loss),
|
||||
"train_acc": float(ckpt_tr_acc),
|
||||
"train_mse": float(ckpt_tr_mse),
|
||||
"test_loss": float(test_loss),
|
||||
"test_acc": float(test_acc),
|
||||
"test_mse": float(test_mse),
|
||||
}
|
||||
checkpoints.append(ckpt_record)
|
||||
|
||||
flat_csv_rows.append({
|
||||
"seed": seed,
|
||||
"epoch": ep,
|
||||
"train_loss": float(ckpt_tr_loss),
|
||||
"train_acc": float(ckpt_tr_acc),
|
||||
"train_mse": float(ckpt_tr_mse),
|
||||
"test_loss": float(test_loss),
|
||||
"test_acc": float(test_acc),
|
||||
"test_mse": float(test_mse),
|
||||
"fit_time_sec": round(fit_time_sec, 4),
|
||||
})
|
||||
|
||||
runs.append({
|
||||
"seed": seed,
|
||||
"model_fingerprint": model_fp,
|
||||
"fit_time_sec": round(fit_time_sec, 4),
|
||||
"improvement_count": improvement_count,
|
||||
"last_improvement_epoch": last_improvement_epoch,
|
||||
"checkpoints": checkpoints,
|
||||
"completed": True,
|
||||
"error": None,
|
||||
"plugins": current_plugin_meta,
|
||||
"epoch_history": epoch_history,
|
||||
})
|
||||
|
||||
# 4. Aggregations & Statistics
|
||||
checkpoint_stats: Dict[int, Dict[str, Any]] = {}
|
||||
for ep in ckpt_epochs:
|
||||
ep_train_losses = [next(c["train_loss"] for c in r["checkpoints"] if c["epoch"] == ep) for r in runs]
|
||||
ep_train_accs = [next(c["train_acc"] for c in r["checkpoints"] if c["epoch"] == ep) for r in runs]
|
||||
ep_train_mses = [next(c["train_mse"] for c in r["checkpoints"] if c["epoch"] == ep) for r in runs]
|
||||
|
||||
ep_test_losses = [next(c["test_loss"] for c in r["checkpoints"] if c["epoch"] == ep) for r in runs]
|
||||
ep_test_accs = [next(c["test_acc"] for c in r["checkpoints"] if c["epoch"] == ep) for r in runs]
|
||||
ep_test_mses = [next(c["test_mse"] for c in r["checkpoints"] if c["epoch"] == ep) for r in runs]
|
||||
|
||||
checkpoint_stats[ep] = {
|
||||
"train_loss": calc_stats(ep_train_losses),
|
||||
"train_acc": calc_stats(ep_train_accs),
|
||||
"train_mse": calc_stats(ep_train_mses),
|
||||
"test_loss": calc_stats(ep_test_losses),
|
||||
"test_acc": calc_stats(ep_test_accs),
|
||||
"test_mse": calc_stats(ep_test_mses),
|
||||
}
|
||||
|
||||
# Endpoint paired deltas (80->240 and 200->240 if available)
|
||||
paired_deltas: Dict[str, Any] = {}
|
||||
paired_endpoint_deltas_by_seed: List[Dict[str, Any]] = []
|
||||
|
||||
has_80 = 80 in checkpoint_stats
|
||||
has_200 = 200 in checkpoint_stats
|
||||
has_240 = 240 in checkpoint_stats
|
||||
|
||||
if has_80 and has_240:
|
||||
deltas_80_to_240_train_rel = []
|
||||
deltas_80_to_240_test_acc = []
|
||||
for r in runs:
|
||||
c80 = next(c for c in r["checkpoints"] if c["epoch"] == 80)
|
||||
c240 = next(c for c in r["checkpoints"] if c["epoch"] == 240)
|
||||
tl80, tl240 = c80["train_loss"], c240["train_loss"]
|
||||
ta80, ta240 = c80["test_acc"], c240["test_acc"]
|
||||
|
||||
rel_red = (tl80 - tl240) / tl80 if tl80 > 0 else 0.0
|
||||
acc_delta = ta240 - ta80
|
||||
deltas_80_to_240_train_rel.append(rel_red)
|
||||
deltas_80_to_240_test_acc.append(acc_delta)
|
||||
|
||||
paired_rec = {
|
||||
"seed": r["seed"],
|
||||
"train_loss_relative_reduction_80_to_240": rel_red,
|
||||
"test_accuracy_delta_80_to_240": acc_delta,
|
||||
}
|
||||
if has_200:
|
||||
c200 = next(c for c in r["checkpoints"] if c["epoch"] == 200)
|
||||
tl200, ta200 = c200["train_loss"], c200["test_acc"]
|
||||
rel_red_200 = (tl200 - tl240) / tl200 if tl200 > 0 else 0.0
|
||||
acc_delta_200 = ta240 - ta200
|
||||
paired_rec["train_loss_relative_reduction_200_to_240"] = rel_red_200
|
||||
paired_rec["test_accuracy_delta_200_to_240"] = acc_delta_200
|
||||
paired_endpoint_deltas_by_seed.append(paired_rec)
|
||||
|
||||
paired_deltas["80_to_240"] = {
|
||||
"train_loss_rel_reduction": calc_stats(deltas_80_to_240_train_rel),
|
||||
"test_acc_delta": calc_stats(deltas_80_to_240_test_acc),
|
||||
}
|
||||
|
||||
if has_200 and has_240:
|
||||
deltas_200_to_240_train_rel = []
|
||||
deltas_200_to_240_test_acc = []
|
||||
deltas_200_to_240_test_acc_abs = []
|
||||
for r in runs:
|
||||
c200 = next(c for c in r["checkpoints"] if c["epoch"] == 200)
|
||||
c240 = next(c for c in r["checkpoints"] if c["epoch"] == 240)
|
||||
tl200, tl240 = c200["train_loss"], c240["train_loss"]
|
||||
ta200, ta240 = c200["test_acc"], c240["test_acc"]
|
||||
rel_red_200 = (tl200 - tl240) / tl200 if tl200 > 0 else 0.0
|
||||
acc_delta_200 = ta240 - ta200
|
||||
acc_abs_200 = abs(ta240 - ta200)
|
||||
deltas_200_to_240_train_rel.append(rel_red_200)
|
||||
deltas_200_to_240_test_acc.append(acc_delta_200)
|
||||
deltas_200_to_240_test_acc_abs.append(acc_abs_200)
|
||||
|
||||
paired_deltas["200_to_240"] = {
|
||||
"train_loss_rel_reduction": calc_stats(deltas_200_to_240_train_rel),
|
||||
"test_acc_delta": calc_stats(deltas_200_to_240_test_acc),
|
||||
"test_acc_abs_change": calc_stats(deltas_200_to_240_test_acc_abs),
|
||||
}
|
||||
|
||||
# Predeclared Diagnostics
|
||||
diagnostics = {}
|
||||
if has_80 and has_240:
|
||||
mean_tl80 = checkpoint_stats[80]["train_loss"]["mean"]
|
||||
mean_tl240 = checkpoint_stats[240]["train_loss"]["mean"]
|
||||
mean_ta80 = checkpoint_stats[80]["test_acc"]["mean"]
|
||||
mean_ta240 = checkpoint_stats[240]["test_acc"]["mean"]
|
||||
|
||||
post80_train_improvement_pct = (mean_tl80 - mean_tl240) / mean_tl80 if mean_tl80 > 0 else 0.0
|
||||
epoch240_test_gain = mean_ta240 - mean_ta80
|
||||
|
||||
diagnostics["post80_train_improvement_pct"] = round(post80_train_improvement_pct, 6)
|
||||
diagnostics["post80_train_improvement_passed"] = bool(post80_train_improvement_pct >= 0.01)
|
||||
|
||||
diagnostics["epoch240_test_gain"] = round(epoch240_test_gain, 6)
|
||||
diagnostics["epoch240_test_gain_passed"] = bool(epoch240_test_gain >= 0.01)
|
||||
|
||||
if has_200:
|
||||
mean_tl200 = checkpoint_stats[200]["train_loss"]["mean"]
|
||||
mean_ta200 = checkpoint_stats[200]["test_acc"]["mean"]
|
||||
late_rel_red = (mean_tl200 - mean_tl240) / mean_tl200 if mean_tl200 > 0 else 0.0
|
||||
late_acc_abs = abs(mean_ta240 - mean_ta200)
|
||||
|
||||
is_late_plateau = bool(late_rel_red < 0.01 and late_acc_abs < 0.005)
|
||||
diagnostics["late_plateau_200_240"] = is_late_plateau
|
||||
diagnostics["late_plateau_train_loss_rel_reduction_200_240"] = round(late_rel_red, 6)
|
||||
diagnostics["late_plateau_test_acc_abs_change_200_240"] = round(late_acc_abs, 6)
|
||||
|
||||
# 5. Descriptive Comparison vs Subset Study
|
||||
subset_comparison: Dict[str, Any] = {}
|
||||
if subset_study_path.exists():
|
||||
try:
|
||||
with open(subset_study_path, "r", encoding="utf-8") as f:
|
||||
sub_json_data = json.load(f)
|
||||
sub_ckpt_stats = sub_json_data.get("summary", {}).get("checkpoint_stats", {})
|
||||
matching_stats = {}
|
||||
for ep in ckpt_epochs:
|
||||
ep_str = str(ep)
|
||||
if ep_str in sub_ckpt_stats:
|
||||
sub_e = sub_ckpt_stats[ep_str]
|
||||
full_tr_l = checkpoint_stats[ep]["train_loss"]["mean"]
|
||||
sub_tr_l = sub_e["train_loss"]["mean"]
|
||||
full_te_a = checkpoint_stats[ep]["test_acc"]["mean"]
|
||||
sub_te_a = sub_e["test_acc"]["mean"]
|
||||
|
||||
matching_stats[ep_str] = {
|
||||
"full_mnist_train_loss_mean": full_tr_l,
|
||||
"subset_study_train_loss_mean": sub_tr_l,
|
||||
"train_loss_delta_full_minus_subset": round(full_tr_l - sub_tr_l, 6),
|
||||
"full_mnist_test_acc_mean": full_te_a,
|
||||
"subset_study_test_acc_mean": sub_te_a,
|
||||
"test_acc_delta_full_minus_subset": round(full_te_a - sub_te_a, 6),
|
||||
}
|
||||
|
||||
subset_comparison = {
|
||||
"subset_study_path": str(subset_study_path),
|
||||
"disclaimer": (
|
||||
"The full and subset studies optimize different training objectives and the evaluation "
|
||||
"plugin changes RNG consumption (all 60,000 train samples versus a fixed 2,000-sample "
|
||||
"fitness subset). Matching-seed and matching-epoch comparisons are descriptive, not an "
|
||||
"exact paired causal isolation."
|
||||
),
|
||||
"matching_epoch_deltas": matching_stats,
|
||||
}
|
||||
except Exception as err:
|
||||
subset_comparison = {"error": f"Failed to parse subset study JSON: {err}"}
|
||||
|
||||
# 6. Save Artifacts
|
||||
# JSON output
|
||||
out_dict = {
|
||||
"full_mnist_protocol_version": FULL_MNIST_PROTOCOL_VERSION,
|
||||
"pso_version": pso_version,
|
||||
"torch_version": torch.__version__,
|
||||
"hardware": hw_provenance,
|
||||
"device": device.type,
|
||||
"timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"baseline_path": str(baseline_path),
|
||||
"subset_study_path": str(subset_study_path),
|
||||
"sample_counts": {
|
||||
"train_samples": 60000,
|
||||
"test_samples": 10000,
|
||||
},
|
||||
"pca_provenance": pca_provenance,
|
||||
"data_fingerprint": data_fp,
|
||||
"fitness_evaluation_contract": {
|
||||
"selector": "full",
|
||||
"fitness_size": None,
|
||||
"train_samples_per_particle_per_epoch": 60000,
|
||||
"particle_evaluations": n_particles * target_epochs * len(seeds),
|
||||
"particle_sample_evaluations": (
|
||||
n_particles * target_epochs * len(seeds) * 60000
|
||||
),
|
||||
"test_samples_per_checkpoint": 10000,
|
||||
},
|
||||
"candidate_label": winner_cfg.candidate_label,
|
||||
"config": {
|
||||
**opt_kwargs,
|
||||
"n_particles": n_particles,
|
||||
"epochs": target_epochs,
|
||||
"batch_size": batch_size,
|
||||
"renewal": "loss",
|
||||
"checkpoint_interval": checkpoint_interval,
|
||||
},
|
||||
"plugins": plugin_meta,
|
||||
"timing_scope": "fit_only_after_full-evaluation_two-epoch_warmup",
|
||||
"diagnostics": diagnostics,
|
||||
"descriptive_subset_comparison": subset_comparison,
|
||||
"summary": {
|
||||
"epochs": ckpt_epochs,
|
||||
"checkpoint_stats": {str(k): v for k, v in checkpoint_stats.items()},
|
||||
"paired_deltas": paired_deltas,
|
||||
"paired_endpoint_deltas_by_seed": paired_endpoint_deltas_by_seed,
|
||||
},
|
||||
"runs": runs,
|
||||
"completed": True,
|
||||
"valid": True,
|
||||
"error": None,
|
||||
}
|
||||
save_json_atomic(out_dict, output_json_path)
|
||||
|
||||
# CSV output
|
||||
output_csv_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fieldnames = [
|
||||
"seed",
|
||||
"epoch",
|
||||
"train_loss",
|
||||
"train_acc",
|
||||
"train_mse",
|
||||
"test_loss",
|
||||
"test_acc",
|
||||
"test_mse",
|
||||
"fit_time_sec",
|
||||
]
|
||||
with open(output_csv_path, "w", encoding="utf-8", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
for r in flat_csv_rows:
|
||||
writer.writerow(r)
|
||||
|
||||
# Plot output
|
||||
render_trajectory_plot(checkpoint_stats, subset_comparison, figure_path)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Full-MNIST 60k/10k Particle Swarm Optimization Trajectory Analysis"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--baseline-json",
|
||||
type=Path,
|
||||
default=Path("benchmark_results/pso_v4_tuning.json"),
|
||||
help="Path to baseline tuning JSON (default: benchmark_results/pso_v4_tuning.json)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--subset-study-json",
|
||||
type=Path,
|
||||
default=Path("benchmark_results/pso_v4_epoch_convergence.json"),
|
||||
help="Path to subset epoch convergence JSON (default: benchmark_results/pso_v4_epoch_convergence.json)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-json",
|
||||
type=Path,
|
||||
default=Path("benchmark_results/pso_v4_full_mnist.json"),
|
||||
help="Path to output JSON (default: benchmark_results/pso_v4_full_mnist.json)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-csv",
|
||||
type=Path,
|
||||
default=Path("benchmark_results/pso_v4_full_mnist.csv"),
|
||||
help="Path to output CSV (default: benchmark_results/pso_v4_full_mnist.csv)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--figure",
|
||||
type=Path,
|
||||
default=Path("history_plt/pso_v4_full_mnist.png"),
|
||||
help="Path to output plot figure PNG (default: history_plt/pso_v4_full_mnist.png)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--device",
|
||||
type=str,
|
||||
default="auto",
|
||||
help="Execution device: auto, mps, cpu, cuda (default: auto)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--epochs",
|
||||
type=int,
|
||||
default=240,
|
||||
help="Number of PSO training epochs (default: 240)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seeds",
|
||||
type=str,
|
||||
default="71,72,73,74,75",
|
||||
help="Comma-separated random seeds (default: 71,72,73,74,75)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
seed_list = [int(s.strip()) for s in args.seeds.split(",") if s.strip()]
|
||||
|
||||
run_full_mnist_study(
|
||||
baseline_path=args.baseline_json,
|
||||
subset_study_path=args.subset_study_json,
|
||||
output_json_path=args.output_json,
|
||||
output_csv_path=args.output_csv,
|
||||
figure_path=args.figure,
|
||||
device_str=args.device,
|
||||
epochs=args.epochs,
|
||||
seeds=seed_list,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,335 @@
|
||||
"""
|
||||
Heavy PSO Cross-Split Experiment Runner.
|
||||
|
||||
Protocol Version: HEAVY-PSO-CROSS-SPLIT 1.0.0
|
||||
|
||||
Runs matching baseline and candidate PSO experiments across development or confirmation
|
||||
data splits under sealed official test conditions (official_test_evaluations = 0).
|
||||
|
||||
Phase Specifications:
|
||||
- Development: split_seeds = (20260905, 20260906), swarm_seeds = (101, 102, 103)
|
||||
- Confirmation: split_seeds = (20260907,), swarm_seeds = (111, 112, 113)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
# Ensure test directory and repo root are in Python path
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent if Path(__file__).resolve().parent.name != "PSO" else Path(__file__).resolve().parent
|
||||
TEST_DIR = REPO_ROOT / "test"
|
||||
if str(TEST_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TEST_DIR))
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from benchmark_suite import (
|
||||
get_hardware_provenance,
|
||||
resolve_execution_device,
|
||||
save_json_atomic,
|
||||
)
|
||||
from heavy_pso_autoresearch import (
|
||||
DEFAULT_GEOMETRY_MULTIPLIER,
|
||||
parse_projection_seed_arg,
|
||||
parse_projection_scope_arg,
|
||||
validate_projection_scope_config,
|
||||
get_effective_projection_scope,
|
||||
run_heavy_pso_autoresearch,
|
||||
)
|
||||
from heavy_task_feasibility import (
|
||||
WORKLOADS,
|
||||
run_heavy_task_confirm,
|
||||
)
|
||||
from pso import __version__ as pso_version
|
||||
|
||||
PROTOCOL_VERSION = "HEAVY-PSO-CROSS-SPLIT 1.0.0"
|
||||
|
||||
PHASE_CONFIGS = {
|
||||
"development": {
|
||||
"split_seeds": [20260905, 20260906],
|
||||
"swarm_seeds": [101, 102, 103],
|
||||
},
|
||||
"confirmation": {
|
||||
"split_seeds": [20260907],
|
||||
"swarm_seeds": [111, 112, 113],
|
||||
},
|
||||
}
|
||||
|
||||
WORKLOAD_BASELINE_METHODS = {
|
||||
"mnist_compact": "G8",
|
||||
"mnist_wide": "G5",
|
||||
"fashion_compact": "G8",
|
||||
"fashion_wide": "G5",
|
||||
}
|
||||
|
||||
FROZEN_PROJECTION_SEEDS = {
|
||||
"mnist_compact": 1800044939,
|
||||
"mnist_wide": 592157828,
|
||||
"fashion_compact": 1363313651,
|
||||
"fashion_wide": 189641451,
|
||||
}
|
||||
|
||||
|
||||
def run_heavy_pso_cross_split(
|
||||
phase: str = "development",
|
||||
ratio: float = 0.5,
|
||||
geometry_policy: str = "baseline_aligned",
|
||||
projection_scope: Union[str, Dict[str, str]] = "global",
|
||||
projection_seed_mode: str = "explicit",
|
||||
projection_seed: Optional[Union[int, Dict[str, int]]] = None,
|
||||
geometry_multiplier: float = DEFAULT_GEOMETRY_MULTIPLIER,
|
||||
particles: int = 12,
|
||||
epochs: int = 80,
|
||||
subset_size: int = 10000,
|
||||
device_str: Optional[str] = None,
|
||||
cache_dir: Optional[Path] = None,
|
||||
output_path: Optional[Path] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Runs cross-split evaluation for development or confirmation phase.
|
||||
Enforces exact split and swarm seed contracts for each phase.
|
||||
Reruns matching baseline and candidate models per split seed.
|
||||
"""
|
||||
projection_scope = parse_projection_scope_arg(projection_scope)
|
||||
validate_projection_scope_config(projection_scope)
|
||||
if phase not in PHASE_CONFIGS:
|
||||
raise ValueError(
|
||||
f"Invalid phase '{phase}'. Must be one of {list(PHASE_CONFIGS.keys())}"
|
||||
)
|
||||
|
||||
phase_spec = PHASE_CONFIGS[phase]
|
||||
split_seeds = phase_spec["split_seeds"]
|
||||
swarm_seeds = phase_spec["swarm_seeds"]
|
||||
|
||||
if projection_seed_mode == "explicit" and projection_seed is None:
|
||||
projection_seed = dict(FROZEN_PROJECTION_SEEDS)
|
||||
|
||||
start_time = time.time()
|
||||
device = resolve_execution_device(device_str)
|
||||
hardware_info = get_hardware_provenance(device)
|
||||
|
||||
if cache_dir is None:
|
||||
cache_dir = REPO_ROOT / "result" / "cache"
|
||||
|
||||
splits_payload: Dict[str, Any] = {}
|
||||
total_runs = 0
|
||||
total_queries = 0
|
||||
total_samples_evaluated = 0
|
||||
|
||||
for split_seed in split_seeds:
|
||||
# 1. Baseline runs for this split seed
|
||||
selected_baseline_methods = {
|
||||
wl_id: [WORKLOAD_BASELINE_METHODS[wl_id]] for wl_id in WORKLOADS
|
||||
}
|
||||
baseline_res = run_heavy_task_confirm(
|
||||
workloads=WORKLOADS,
|
||||
selected_methods=selected_baseline_methods,
|
||||
particles=particles,
|
||||
epochs=epochs,
|
||||
seeds=swarm_seeds,
|
||||
split_seed=split_seed,
|
||||
device=device,
|
||||
cache_dir=cache_dir,
|
||||
)
|
||||
|
||||
# 2. Candidate runs for this split seed
|
||||
candidate_res = run_heavy_pso_autoresearch(
|
||||
ratios=[ratio],
|
||||
particles=particles,
|
||||
epochs=epochs,
|
||||
subset_size=subset_size,
|
||||
seeds=swarm_seeds,
|
||||
geometry_policy=geometry_policy,
|
||||
device_str=device_str,
|
||||
cache_dir=cache_dir,
|
||||
split_seed=split_seed,
|
||||
projection_scope=projection_scope,
|
||||
projection_seed_mode=projection_seed_mode,
|
||||
projection_seed=projection_seed,
|
||||
geometry_multiplier=geometry_multiplier,
|
||||
)
|
||||
|
||||
# Extract candidate ratio payload
|
||||
candidate_ratio_runs = list(candidate_res["candidate_runs"].values())[0]
|
||||
|
||||
baselines_split: Dict[str, Any] = {}
|
||||
candidates_split: Dict[str, Any] = {}
|
||||
dataset_fingerprints: Dict[str, str] = {}
|
||||
split_fingerprints: Dict[str, str] = {}
|
||||
|
||||
for wl_id in WORKLOADS:
|
||||
b_method = WORKLOAD_BASELINE_METHODS[wl_id]
|
||||
b_entry = baseline_res[wl_id][b_method]
|
||||
baselines_split[wl_id] = b_entry
|
||||
|
||||
c_entry = candidate_ratio_runs[wl_id]
|
||||
candidates_split[wl_id] = c_entry
|
||||
|
||||
dataset_name = WORKLOADS[wl_id].dataset_name
|
||||
dataset_fingerprints[dataset_name] = c_entry["data_fingerprint"]
|
||||
split_fingerprints[dataset_name] = c_entry["split_fingerprint"]
|
||||
|
||||
# Resource accumulation
|
||||
for r in b_entry["per_seed_runs"]:
|
||||
total_runs += 1
|
||||
total_queries += r["total_queries"]
|
||||
total_samples_evaluated += r["total_sample_evaluations"]
|
||||
for r in c_entry["per_seed_runs"]:
|
||||
total_runs += 1
|
||||
total_queries += r["total_queries"]
|
||||
total_samples_evaluated += r["total_sample_evaluations"]
|
||||
|
||||
splits_payload[str(split_seed)] = {
|
||||
"split_seed": split_seed,
|
||||
"data_fingerprints": dataset_fingerprints,
|
||||
"split_fingerprints": split_fingerprints,
|
||||
"baselines": baselines_split,
|
||||
"candidates": candidates_split,
|
||||
}
|
||||
|
||||
payload = {
|
||||
"version": PROTOCOL_VERSION,
|
||||
"protocol_version": PROTOCOL_VERSION,
|
||||
"phase": phase,
|
||||
"split_seeds": list(split_seeds),
|
||||
"swarm_seeds": list(swarm_seeds),
|
||||
"official_test_data_loaded": False,
|
||||
"official_test_evaluations": 0,
|
||||
"candidate_config": {
|
||||
"ratio": ratio,
|
||||
"geometry_policy": geometry_policy,
|
||||
"projection_scope": projection_scope,
|
||||
"projection_seed_mode": projection_seed_mode,
|
||||
"projection_seed": projection_seed,
|
||||
"geometry_multiplier": float(geometry_multiplier),
|
||||
"particles": particles,
|
||||
"epochs": epochs,
|
||||
"subset_size": subset_size,
|
||||
},
|
||||
"workloads": {
|
||||
wl_id: {
|
||||
"workload_id": wl_id,
|
||||
"dataset_name": wl_cfg.dataset_name,
|
||||
"model_name": wl_cfg.model_name,
|
||||
"baseline_method": WORKLOAD_BASELINE_METHODS[wl_id],
|
||||
"projection_scope": get_effective_projection_scope(projection_scope, wl_id),
|
||||
"effective_projection_seed": (
|
||||
projection_seed[wl_id]
|
||||
if isinstance(projection_seed, dict)
|
||||
else projection_seed
|
||||
),
|
||||
}
|
||||
for wl_id, wl_cfg in WORKLOADS.items()
|
||||
},
|
||||
"splits": splits_payload,
|
||||
"resource_totals": {
|
||||
"total_runs": total_runs,
|
||||
"total_queries": total_queries,
|
||||
"total_samples_evaluated": total_samples_evaluated,
|
||||
"official_test_evaluations": 0,
|
||||
"wall_time_sec": round(time.time() - start_time, 4),
|
||||
},
|
||||
"provenance": {
|
||||
"hardware": hardware_info,
|
||||
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"pso_version": pso_version,
|
||||
},
|
||||
}
|
||||
|
||||
if output_path is not None:
|
||||
save_json_atomic(payload, output_path)
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Heavy PSO Cross-Split Experiment Runner (Development / Confirmation)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--phase",
|
||||
type=str,
|
||||
default="development",
|
||||
choices=list(PHASE_CONFIGS.keys()),
|
||||
help="Experiment phase ('development' or 'confirmation')",
|
||||
)
|
||||
parser.add_argument("--device", type=str, default=None, help="Device (cpu, mps, cuda)")
|
||||
parser.add_argument("--cache-dir", type=str, default=None, help="Dataset cache directory")
|
||||
parser.add_argument("--output", type=str, default=None, help="Output artifact JSON path")
|
||||
parser.add_argument(
|
||||
"--ratio",
|
||||
type=float,
|
||||
default=0.5,
|
||||
help="Subspace ratio for candidate PSO (default: 0.5)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--geometry-policy",
|
||||
type=str,
|
||||
default="baseline_aligned",
|
||||
help="Geometry policy (default: 'baseline_aligned')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--projection-scope",
|
||||
type=parse_projection_scope_arg,
|
||||
default="global",
|
||||
help="Projection scope ('global', 'tensor_local', 'balanced_global', 'two_hash_global', 'largest_tensor_hash', 'largest_tensor_row_hash', 'adjacent_pair', 'adjacent_difference', or workload dict)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--projection-seed-mode",
|
||||
type=str,
|
||||
default="explicit",
|
||||
help="Projection seed mode (default: 'explicit')",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--projection-seed",
|
||||
type=parse_projection_seed_arg,
|
||||
default=None,
|
||||
help="Exact projection seed (int or dict) for explicit mode",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--geometry-multiplier",
|
||||
type=float,
|
||||
default=DEFAULT_GEOMETRY_MULTIPLIER,
|
||||
help="Geometry multiplier (default: 1.0)",
|
||||
)
|
||||
parser.add_argument("--particles", type=int, default=12, help="Swarm size (default: 12)")
|
||||
parser.add_argument("--epochs", type=int, default=80, help="PSO epochs (default: 80)")
|
||||
parser.add_argument(
|
||||
"--subset-size",
|
||||
type=int,
|
||||
default=10000,
|
||||
help="Subset size (default: 10000)",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main():
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
cache_path = Path(args.cache_dir) if args.cache_dir else None
|
||||
out_path = Path(args.output) if args.output else None
|
||||
|
||||
run_heavy_pso_cross_split(
|
||||
phase=args.phase,
|
||||
ratio=args.ratio,
|
||||
geometry_policy=args.geometry_policy,
|
||||
projection_scope=args.projection_scope,
|
||||
projection_seed_mode=args.projection_seed_mode,
|
||||
projection_seed=args.projection_seed,
|
||||
geometry_multiplier=args.geometry_multiplier,
|
||||
particles=args.particles,
|
||||
epochs=args.epochs,
|
||||
subset_size=args.subset_size,
|
||||
device_str=args.device,
|
||||
cache_dir=cache_path,
|
||||
output_path=out_path,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
+94
-56
@@ -1,73 +1,111 @@
|
||||
import gc
|
||||
import os
|
||||
import sys
|
||||
|
||||
import argparse
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from sklearn.datasets import load_iris
|
||||
from sklearn.model_selection import train_test_split
|
||||
from tensorflow import keras
|
||||
from tensorflow.keras import layers
|
||||
from tensorflow.keras.models import Sequential
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
|
||||
from pso import optimizer
|
||||
|
||||
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
|
||||
from pso import Optimizer
|
||||
from cli import add_pso_args, build_optimizer_kwargs
|
||||
|
||||
|
||||
def make_model():
|
||||
model = Sequential()
|
||||
model.add(layers.Dense(10, activation="relu", input_shape=(4,)))
|
||||
model.add(layers.Dense(10, activation="relu"))
|
||||
model.add(layers.Dense(3, activation="softmax"))
|
||||
|
||||
return model
|
||||
|
||||
|
||||
def load_data():
|
||||
iris = load_iris()
|
||||
x = iris.data
|
||||
y = iris.target
|
||||
|
||||
y = keras.utils.to_categorical(y, 3)
|
||||
|
||||
x_train, x_test, y_train, y_test = train_test_split(
|
||||
x, y, test_size=0.2, shuffle=True, stratify=y
|
||||
def make_model(seed: int = 42):
|
||||
torch.manual_seed(seed)
|
||||
return nn.Sequential(
|
||||
nn.Linear(4, 10),
|
||||
nn.ReLU(),
|
||||
nn.Linear(10, 10),
|
||||
nn.ReLU(),
|
||||
nn.Linear(10, 3),
|
||||
)
|
||||
|
||||
return x_train, x_test, y_train, y_test
|
||||
|
||||
def load_data(seed: int = 42):
|
||||
iris = load_iris()
|
||||
x = iris.data.astype("float32")
|
||||
y = iris.target.astype("int64")
|
||||
|
||||
x_train, x_test, y_train, y_test = train_test_split(
|
||||
x, y, test_size=0.2, shuffle=True, stratify=y, random_state=seed
|
||||
)
|
||||
scaler = StandardScaler()
|
||||
x_train = scaler.fit_transform(x_train)
|
||||
x_test = scaler.transform(x_test)
|
||||
|
||||
return (
|
||||
torch.tensor(x_train, dtype=torch.float32),
|
||||
torch.tensor(x_test, dtype=torch.float32),
|
||||
torch.tensor(y_train, dtype=torch.int64),
|
||||
torch.tensor(y_test, dtype=torch.int64),
|
||||
)
|
||||
|
||||
|
||||
model = make_model()
|
||||
x_train, x_test, y_train, y_test = load_data()
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="PSO Iris Benchmark Script")
|
||||
add_pso_args(
|
||||
parser,
|
||||
defaults={
|
||||
"method": "original",
|
||||
"initialization": "model_noise",
|
||||
"evaluation": "full",
|
||||
"convergence": "particle_reset",
|
||||
"refinement": "adam",
|
||||
"n_particles": 24,
|
||||
"c0": None,
|
||||
"c1": None,
|
||||
"w_min": None,
|
||||
"w_max": None,
|
||||
"negative_swarm": 0.0,
|
||||
"mutation_swarm": 0.1,
|
||||
"particle_min": -3.0,
|
||||
"particle_max": 3.0,
|
||||
"velocity_limit_ratio": 0.1,
|
||||
"boundary_strategy": "reflect",
|
||||
"seed": 42,
|
||||
"epochs": 70,
|
||||
"renewal": "loss",
|
||||
"output_dir": "output/iris",
|
||||
"checkpoint_interval": 25,
|
||||
"refinement_epochs": 10,
|
||||
"refinement_lr": 0.001,
|
||||
},
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
model = make_model(seed=args.seed)
|
||||
x_train, x_test, y_train, y_test = load_data(seed=args.seed)
|
||||
|
||||
pso_iris = optimizer(
|
||||
fitness_size = args.fitness_size if args.evaluation == "fixed_subset" else None
|
||||
refinement_epochs = args.refinement_epochs if args.refinement == "adam" else 0
|
||||
|
||||
kwargs = build_optimizer_kwargs(
|
||||
args,
|
||||
model=model,
|
||||
loss="categorical_crossentropy",
|
||||
n_particles=100,
|
||||
c0=0.5,
|
||||
c1=0.3,
|
||||
w_min=0.1,
|
||||
w_max=0.9,
|
||||
negative_swarm=0,
|
||||
mutation_swarm=0.1,
|
||||
convergence_reset=True,
|
||||
convergence_reset_patience=10,
|
||||
convergence_reset_monitor="loss",
|
||||
convergence_reset_min_delta=0.001,
|
||||
)
|
||||
loss=nn.CrossEntropyLoss(),
|
||||
task="multiclass",
|
||||
inertia_profile={"c0": 0.5, "c1": 0.3, "w_min": 0.1, "w_max": 0.9},
|
||||
)
|
||||
pso_iris = Optimizer(**kwargs)
|
||||
|
||||
best_score = pso_iris.fit(
|
||||
print(f"Optimizer device: {pso_iris.device}")
|
||||
|
||||
best_score = pso_iris.fit(
|
||||
x_train,
|
||||
y_train,
|
||||
epochs=500,
|
||||
epochs=args.epochs,
|
||||
batch_size=args.batch_size,
|
||||
fitness_size=fitness_size,
|
||||
renewal=args.renewal,
|
||||
validation_data=(x_test, y_test),
|
||||
output_dir=args.output_dir,
|
||||
checkpoint_interval=25,
|
||||
save_info=True,
|
||||
log=2,
|
||||
log_name="iris",
|
||||
renewal="loss",
|
||||
check_point=25,
|
||||
validate_data=(x_test, y_test),
|
||||
)
|
||||
refinement_epochs=refinement_epochs,
|
||||
refinement_lr=args.refinement_lr,
|
||||
)
|
||||
|
||||
gc.collect()
|
||||
print("Done!")
|
||||
sys.exit(0)
|
||||
print(f"Done! Best score: {best_score}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
import os
|
||||
|
||||
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
|
||||
|
||||
import tensorflow as tf
|
||||
|
||||
gpus = tf.config.experimental.list_physical_devices("GPU")
|
||||
if gpus:
|
||||
try:
|
||||
# tf.config.experimental.set_visible_devices(gpus[0], "GPU")
|
||||
tf.config.experimental.set_memory_growth(gpus[0], True)
|
||||
except RuntimeError as e:
|
||||
print(e)
|
||||
|
||||
from sklearn.datasets import load_iris
|
||||
from sklearn.model_selection import train_test_split
|
||||
from tensorflow import keras
|
||||
from tensorflow.keras import layers
|
||||
from tensorflow.keras.models import Sequential
|
||||
|
||||
|
||||
def make_model():
|
||||
model = Sequential()
|
||||
model.add(layers.Dense(10, activation="relu", input_shape=(4,)))
|
||||
model.add(layers.Dense(10, activation="relu"))
|
||||
model.add(layers.Dense(3, activation="softmax"))
|
||||
|
||||
return model
|
||||
|
||||
|
||||
def load_data():
|
||||
iris = load_iris()
|
||||
x = iris.data
|
||||
y = iris.target
|
||||
|
||||
y = keras.utils.to_categorical(y, 3)
|
||||
|
||||
x_train, x_test, y_train, y_test = train_test_split(
|
||||
x, y, test_size=0.2, shuffle=True, stratify=y
|
||||
)
|
||||
|
||||
return x_train, x_test, y_train, y_test
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
model = make_model()
|
||||
x_train, x_test, y_train, y_test = load_data()
|
||||
print(x_train.shape, y_train.shape)
|
||||
|
||||
loss = ["categorical_crossentropy", "accuracy", "mse"]
|
||||
metrics = ["accuracy"]
|
||||
|
||||
model.compile(optimizer="sgd", loss=loss[0], metrics=metrics[0])
|
||||
model.fit(x_train, y_train, epochs=200, batch_size=32, validation_split=0.2)
|
||||
model.evaluate(x_test, y_test, batch_size=32)
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Iris dataset gradient baseline (PyTorch standard backprop optimizer, non-PSO)."""
|
||||
|
||||
import copy
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.utils.data import DataLoader, TensorDataset
|
||||
from sklearn.datasets import load_iris
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
|
||||
class IrisModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.net = nn.Sequential(
|
||||
nn.Linear(4, 10),
|
||||
nn.ReLU(),
|
||||
nn.Linear(10, 10),
|
||||
nn.ReLU(),
|
||||
nn.Linear(10, 3),
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
return self.net(x)
|
||||
|
||||
|
||||
def load_data(seed: int = 42):
|
||||
iris = load_iris()
|
||||
X = iris.data.astype("float32")
|
||||
y = iris.target.astype("int64")
|
||||
|
||||
x_train, x_test, y_train, y_test = train_test_split(
|
||||
X, y, test_size=0.2, shuffle=True, stratify=y, random_state=seed
|
||||
)
|
||||
return (
|
||||
torch.tensor(x_train, dtype=torch.float32),
|
||||
torch.tensor(x_test, dtype=torch.float32),
|
||||
torch.tensor(y_train, dtype=torch.int64),
|
||||
torch.tensor(y_test, dtype=torch.int64),
|
||||
)
|
||||
|
||||
|
||||
def get_device() -> torch.device:
|
||||
if (
|
||||
hasattr(torch.backends, "mps")
|
||||
and torch.backends.mps.is_built()
|
||||
and torch.backends.mps.is_available()
|
||||
):
|
||||
return torch.device("mps")
|
||||
elif torch.cuda.is_available():
|
||||
return torch.device("cuda")
|
||||
return torch.device("cpu")
|
||||
|
||||
|
||||
def main():
|
||||
torch.manual_seed(42)
|
||||
np.random.seed(42)
|
||||
|
||||
device = get_device()
|
||||
print(f"Selected device: {device}")
|
||||
|
||||
x_train, x_test, y_train, y_test = load_data(seed=42)
|
||||
train_loader = DataLoader(
|
||||
TensorDataset(x_train, y_train), batch_size=32, shuffle=True
|
||||
)
|
||||
val_loader = DataLoader(
|
||||
TensorDataset(x_test, y_test), batch_size=32, shuffle=False
|
||||
)
|
||||
|
||||
model = IrisModel().to(device)
|
||||
optimizer = torch.optim.SGD(model.parameters(), lr=0.01)
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
|
||||
best_val_loss = float("inf")
|
||||
best_state = None
|
||||
|
||||
for epoch in range(200):
|
||||
model.train()
|
||||
for bx, by in train_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
optimizer.zero_grad()
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
model.eval()
|
||||
val_loss = 0.0
|
||||
total = 0
|
||||
with torch.no_grad():
|
||||
for bx, by in val_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
val_loss += loss.item() * bx.size(0)
|
||||
total += bx.size(0)
|
||||
|
||||
val_loss /= total
|
||||
|
||||
if val_loss < best_val_loss:
|
||||
best_val_loss = val_loss
|
||||
best_state = copy.deepcopy(model.state_dict())
|
||||
|
||||
if best_state is not None:
|
||||
model.load_state_dict(best_state)
|
||||
|
||||
model.eval()
|
||||
test_loss = 0.0
|
||||
correct = 0
|
||||
total = 0
|
||||
with torch.no_grad():
|
||||
for bx, by in val_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
test_loss += loss.item() * bx.size(0)
|
||||
preds = out.argmax(dim=1)
|
||||
correct += (preds == by).sum().item()
|
||||
total += bx.size(0)
|
||||
|
||||
test_loss /= total
|
||||
test_acc = correct / total
|
||||
print(f"Final test loss: {test_loss:.4f}, accuracy: {test_acc:.4f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+87
-64
@@ -1,88 +1,111 @@
|
||||
# %%
|
||||
import os
|
||||
import sys
|
||||
import argparse
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from pso import optimizer
|
||||
|
||||
import tensorflow as tf
|
||||
from keras.datasets import mnist
|
||||
from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
|
||||
from keras.models import Sequential
|
||||
from pso import Optimizer
|
||||
from cli import add_pso_args, build_optimizer_kwargs
|
||||
|
||||
|
||||
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
|
||||
def get_data(seed: int = 42):
|
||||
from sklearn.decomposition import PCA
|
||||
from torchvision.datasets import MNIST
|
||||
|
||||
train_dataset = MNIST(root="./data", train=True, download=True)
|
||||
test_dataset = MNIST(root="./data", train=False, download=True)
|
||||
|
||||
def get_data():
|
||||
(x_train, y_train), (x_test, y_test) = mnist.load_data()
|
||||
x_train_raw = (train_dataset.data[:3000].float() / 255.0).reshape(3000, -1).numpy()
|
||||
y_train = train_dataset.targets[:3000].long()
|
||||
|
||||
x_train, x_test = x_train / 255.0, x_test / 255.0
|
||||
x_train = x_train.reshape((60000, 28, 28, 1))
|
||||
x_test = x_test.reshape((10000, 28, 28, 1))
|
||||
x_test_raw = (test_dataset.data[:1000].float() / 255.0).reshape(1000, -1).numpy()
|
||||
y_test = test_dataset.targets[:1000].long()
|
||||
|
||||
y_train, y_test = tf.one_hot(y_train, 10), tf.one_hot(y_test, 10)
|
||||
pca = PCA(n_components=32, whiten=True, random_state=seed)
|
||||
x_train_pca = pca.fit_transform(x_train_raw)
|
||||
x_test_pca = pca.transform(x_test_raw)
|
||||
|
||||
x_train, x_test = tf.convert_to_tensor(x_train), tf.convert_to_tensor(x_test)
|
||||
y_train, y_test = tf.convert_to_tensor(y_train), tf.convert_to_tensor(y_test)
|
||||
x_train = torch.tensor(x_train_pca, dtype=torch.float32)
|
||||
x_test = torch.tensor(x_test_pca, dtype=torch.float32)
|
||||
|
||||
print(f"x_train : {x_train[0].shape} | y_train : {y_train[0].shape}")
|
||||
print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}")
|
||||
print(f"x_train : {x_train.shape} | y_train : {y_train.shape}")
|
||||
print(f"x_test : {x_test.shape} | y_test : {y_test.shape}")
|
||||
|
||||
return x_train, y_train, x_test, y_test
|
||||
|
||||
|
||||
def make_model():
|
||||
model = Sequential()
|
||||
model.add(
|
||||
Conv2D(32, kernel_size=(5, 5), activation="relu", input_shape=(28, 28, 1))
|
||||
def make_model(seed: int = 42):
|
||||
torch.manual_seed(seed)
|
||||
return nn.Linear(32, 10)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="PSO MNIST Benchmark Script")
|
||||
add_pso_args(
|
||||
parser,
|
||||
defaults={
|
||||
"method": "inertia",
|
||||
"initialization": "model_noise",
|
||||
"evaluation": "fixed_subset",
|
||||
"convergence": "none",
|
||||
"refinement": "adam",
|
||||
"n_particles": 30,
|
||||
"c0": None,
|
||||
"c1": None,
|
||||
"w_min": None,
|
||||
"w_max": None,
|
||||
"negative_swarm": 0.0,
|
||||
"mutation_swarm": 0.02,
|
||||
"particle_min": -3.0,
|
||||
"particle_max": 3.0,
|
||||
"velocity_limit_ratio": 0.025,
|
||||
"boundary_strategy": "reflect",
|
||||
"initial_position_noise": 0.05,
|
||||
"seed": 42,
|
||||
"epochs": 80,
|
||||
"batch_size": 1000,
|
||||
"fitness_size": 2000,
|
||||
"renewal": "loss",
|
||||
"output_dir": "output/mnist",
|
||||
"checkpoint_interval": 25,
|
||||
"refinement_epochs": 100,
|
||||
"refinement_lr": 0.01,
|
||||
},
|
||||
)
|
||||
model.add(MaxPooling2D(pool_size=(2, 2)))
|
||||
model.add(Dropout(0.5))
|
||||
model.add(Conv2D(64, kernel_size=(3, 3), activation="relu"))
|
||||
model.add(MaxPooling2D(pool_size=(2, 2)))
|
||||
model.add(Flatten())
|
||||
model.add(Dropout(0.5))
|
||||
model.add(Dense(256, activation="relu"))
|
||||
model.add(Dense(128, activation="relu"))
|
||||
model.add(Dense(10, activation="softmax"))
|
||||
args = parser.parse_args()
|
||||
|
||||
return model
|
||||
model = make_model(seed=args.seed)
|
||||
x_train, y_train, x_test, y_test = get_data(seed=args.seed)
|
||||
|
||||
fitness_size = args.fitness_size if args.evaluation == "fixed_subset" else None
|
||||
refinement_epochs = args.refinement_epochs if args.refinement == "adam" else 0
|
||||
|
||||
# %%
|
||||
model = make_model()
|
||||
x_train, y_train, x_test, y_test = get_data()
|
||||
kwargs = build_optimizer_kwargs(
|
||||
args,
|
||||
model=model,
|
||||
loss=nn.CrossEntropyLoss(),
|
||||
task="multiclass",
|
||||
inertia_profile={"c0": 1.49618, "c1": 1.49618, "w_min": 0.7298, "w_max": 0.7298},
|
||||
)
|
||||
pso_mnist = Optimizer(**kwargs)
|
||||
|
||||
print(f"Optimizer device: {pso_mnist.device}")
|
||||
|
||||
pso_mnist = optimizer(
|
||||
model,
|
||||
loss="categorical_crossentropy",
|
||||
n_particles=200,
|
||||
c0=0.7,
|
||||
c1=0.4,
|
||||
w_min=0.1,
|
||||
w_max=0.9,
|
||||
negative_swarm=0.0,
|
||||
mutation_swarm=0.05,
|
||||
convergence_reset=True,
|
||||
convergence_reset_patience=10,
|
||||
convergence_reset_monitor="loss",
|
||||
convergence_reset_min_delta=0.005,
|
||||
)
|
||||
|
||||
best_score = pso_mnist.fit(
|
||||
best_score = pso_mnist.fit(
|
||||
x_train,
|
||||
y_train,
|
||||
epochs=1000,
|
||||
epochs=args.epochs,
|
||||
batch_size=args.batch_size,
|
||||
fitness_size=fitness_size,
|
||||
renewal=args.renewal,
|
||||
validation_data=(x_test, y_test),
|
||||
output_dir=args.output_dir,
|
||||
checkpoint_interval=25,
|
||||
save_info=True,
|
||||
log=2,
|
||||
log_name="mnist",
|
||||
renewal="loss",
|
||||
check_point=25,
|
||||
batch_size=5000,
|
||||
validate_data=(x_test, y_test),
|
||||
)
|
||||
refinement_epochs=refinement_epochs,
|
||||
refinement_lr=args.refinement_lr,
|
||||
)
|
||||
|
||||
print("Done!")
|
||||
print(f"Done! Best score: {best_score}")
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
from keras.models import Sequential
|
||||
from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
|
||||
from keras.datasets import mnist
|
||||
from keras.utils import to_categorical
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
# from tensorflow.data.Dataset import from_tensor_slices
|
||||
import tensorflow as tf
|
||||
import os
|
||||
|
||||
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
|
||||
|
||||
|
||||
gpus = tf.config.experimental.list_physical_devices("GPU")
|
||||
if gpus:
|
||||
try:
|
||||
tf.config.experimental.set_memory_growth(gpus[0], True)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
finally:
|
||||
del gpus
|
||||
|
||||
|
||||
def get_data():
|
||||
(x_train, y_train), (x_test, y_test) = mnist.load_data()
|
||||
|
||||
x_train, x_test = x_train / 255.0, x_test / 255.0
|
||||
x_train = x_train.reshape((60000, 28, 28, 1))
|
||||
x_test = x_test.reshape((10000, 28, 28, 1))
|
||||
|
||||
print(f"x_train : {x_train[0].shape} | y_train : {y_train[0].shape}")
|
||||
print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}")
|
||||
|
||||
return x_train, y_train, x_test, y_test
|
||||
|
||||
class _batch_generator_:
|
||||
def __init__(self, x, y, batch_size: int = None):
|
||||
self.index = 0
|
||||
self.x = x
|
||||
self.y = y
|
||||
self.setBatchSize(batch_size)
|
||||
|
||||
def next(self):
|
||||
self.index += 1
|
||||
if self.index >= self.max_index:
|
||||
self.index = 0
|
||||
self.__getBatchSlice(self.batch_size)
|
||||
return self.dataset[self.index][0], self.dataset[self.index][1]
|
||||
|
||||
def getMaxIndex(self):
|
||||
return self.max_index
|
||||
|
||||
def getIndex(self):
|
||||
return self.index
|
||||
|
||||
def setIndex(self, index):
|
||||
self.index = index
|
||||
|
||||
def getBatchSize(self):
|
||||
return self.batch_size
|
||||
|
||||
def setBatchSize(self, batch_size: int = None):
|
||||
if batch_size is None:
|
||||
batch_size = len(self.x) // 10
|
||||
elif batch_size > len(self.x):
|
||||
batch_size = len(self.x)
|
||||
self.batch_size = batch_size
|
||||
print(f"batch size : {self.batch_size}")
|
||||
self.dataset = self.__getBatchSlice(self.batch_size)
|
||||
self.max_index = len(self.dataset)
|
||||
|
||||
def __getBatchSlice(self, batch_size):
|
||||
return list(
|
||||
tf.data.Dataset.from_tensor_slices((self.x, self.y))
|
||||
.shuffle(len(self.x))
|
||||
.batch(batch_size)
|
||||
)
|
||||
|
||||
def getDataset(self):
|
||||
return self.dataset
|
||||
|
||||
|
||||
def make_model():
|
||||
model = Sequential()
|
||||
model.add(
|
||||
Conv2D(64, kernel_size=(5, 5), activation="relu", input_shape=(28, 28, 1))
|
||||
)
|
||||
model.add(MaxPooling2D(pool_size=(2, 2)))
|
||||
model.add(Dropout(0.5))
|
||||
model.add(Conv2D(128, kernel_size=(3, 3), activation="relu"))
|
||||
model.add(MaxPooling2D(pool_size=(2, 2)))
|
||||
model.add(Flatten())
|
||||
model.add(Dropout(0.5))
|
||||
model.add(Dense(2048, activation="relu"))
|
||||
model.add(Dropout(0.8))
|
||||
model.add(Dense(1024, activation="relu"))
|
||||
model.add(Dropout(0.8))
|
||||
model.add(Dense(10, activation="softmax"))
|
||||
|
||||
return model
|
||||
|
||||
|
||||
model = make_model()
|
||||
x_train, y_train, x_test, y_test = get_data()
|
||||
y_train = tf.one_hot(y_train, 10)
|
||||
y_test = tf.one_hot(y_test, 10)
|
||||
|
||||
batch = 64
|
||||
dataset = _batch_generator_(x_train, y_train, batch)
|
||||
|
||||
model.compile(
|
||||
optimizer="adam",
|
||||
loss="categorical_crossentropy",
|
||||
metrics=["accuracy", "mse"],
|
||||
)
|
||||
|
||||
count = 0
|
||||
print(f"batch size : {batch}")
|
||||
print("iter " + str(dataset.getMaxIndex()))
|
||||
print("Training model...")
|
||||
# while count < dataset.getMaxIndex():
|
||||
# x_batch, y_batch = dataset.next()
|
||||
# count += 1
|
||||
# print(f"iter {count}/{dataset.getMaxIndex()}")
|
||||
model.fit(x_train, y_train, epochs=1000, batch_size=batch, verbose=1)
|
||||
|
||||
print(count)
|
||||
|
||||
print("Evaluating model...")
|
||||
model.evaluate(x_test, y_test, verbose=1)
|
||||
|
||||
weights = model.get_weights()
|
||||
@@ -0,0 +1,136 @@
|
||||
"""MNIST dataset gradient baseline (PyTorch standard backprop optimizer, non-PSO)."""
|
||||
|
||||
import copy
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
|
||||
class MNISTModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.conv1 = nn.Conv2d(1, 64, kernel_size=5)
|
||||
self.relu1 = nn.ReLU()
|
||||
self.pool1 = nn.MaxPool2d(2, 2)
|
||||
self.drop1 = nn.Dropout(0.5)
|
||||
|
||||
self.conv2 = nn.Conv2d(64, 128, kernel_size=3)
|
||||
self.relu2 = nn.ReLU()
|
||||
self.pool2 = nn.MaxPool2d(2, 2)
|
||||
|
||||
self.drop2 = nn.Dropout(0.5)
|
||||
self.fc1 = nn.Linear(128 * 5 * 5, 2048)
|
||||
self.relu3 = nn.ReLU()
|
||||
self.drop3 = nn.Dropout(0.8)
|
||||
|
||||
self.fc2 = nn.Linear(2048, 1024)
|
||||
self.relu4 = nn.ReLU()
|
||||
self.drop4 = nn.Dropout(0.8)
|
||||
|
||||
self.fc3 = nn.Linear(1024, 10)
|
||||
|
||||
def forward(self, x):
|
||||
x = self.drop1(self.pool1(self.relu1(self.conv1(x))))
|
||||
x = self.pool2(self.relu2(self.conv2(x)))
|
||||
x = torch.flatten(x, 1)
|
||||
x = self.drop3(self.relu3(self.fc1(self.drop2(x))))
|
||||
x = self.drop4(self.relu4(self.fc2(x)))
|
||||
x = self.fc3(x)
|
||||
return x
|
||||
|
||||
|
||||
def get_data(download: bool = True):
|
||||
from torchvision import datasets, transforms
|
||||
|
||||
transform = transforms.ToTensor()
|
||||
train_dataset = datasets.MNIST(
|
||||
root="./data", train=True, transform=transform, download=download
|
||||
)
|
||||
test_dataset = datasets.MNIST(
|
||||
root="./data", train=False, transform=transform, download=download
|
||||
)
|
||||
return train_dataset, test_dataset
|
||||
|
||||
|
||||
def get_device() -> torch.device:
|
||||
if (
|
||||
hasattr(torch.backends, "mps")
|
||||
and torch.backends.mps.is_built()
|
||||
and torch.backends.mps.is_available()
|
||||
):
|
||||
return torch.device("mps")
|
||||
elif torch.cuda.is_available():
|
||||
return torch.device("cuda")
|
||||
return torch.device("cpu")
|
||||
|
||||
|
||||
def main():
|
||||
torch.manual_seed(42)
|
||||
np.random.seed(42)
|
||||
|
||||
device = get_device()
|
||||
print(f"Selected device: {device}")
|
||||
|
||||
train_dataset, test_dataset = get_data(download=True)
|
||||
train_loader = DataLoader(train_dataset, batch_size=64, shuffle=True)
|
||||
test_loader = DataLoader(test_dataset, batch_size=64, shuffle=False)
|
||||
|
||||
model = MNISTModel().to(device)
|
||||
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
|
||||
criterion = nn.CrossEntropyLoss()
|
||||
|
||||
best_val_loss = float("inf")
|
||||
best_state = None
|
||||
|
||||
for epoch in range(10):
|
||||
model.train()
|
||||
for bx, by in train_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
optimizer.zero_grad()
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
|
||||
model.eval()
|
||||
val_loss = 0.0
|
||||
total = 0
|
||||
with torch.no_grad():
|
||||
for bx, by in test_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
val_loss += loss.item() * bx.size(0)
|
||||
total += bx.size(0)
|
||||
|
||||
val_loss /= total
|
||||
|
||||
if val_loss < best_val_loss:
|
||||
best_val_loss = val_loss
|
||||
best_state = copy.deepcopy(model.state_dict())
|
||||
|
||||
if best_state is not None:
|
||||
model.load_state_dict(best_state)
|
||||
|
||||
model.eval()
|
||||
test_loss = 0.0
|
||||
correct = 0
|
||||
total = 0
|
||||
with torch.no_grad():
|
||||
for bx, by in test_loader:
|
||||
bx, by = bx.to(device), by.to(device)
|
||||
out = model(bx)
|
||||
loss = criterion(out, by)
|
||||
test_loss += loss.item() * bx.size(0)
|
||||
preds = out.argmax(dim=1)
|
||||
correct += (preds == by).sum().item()
|
||||
total += bx.size(0)
|
||||
|
||||
test_loss /= total
|
||||
test_acc = correct / total
|
||||
print(f"Final test loss: {test_loss:.4f}, accuracy: {test_acc:.4f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,269 @@
|
||||
"""Emit live, read-only TensorBoard progress for a convergence-study run."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from torch.utils.tensorboard import SummaryWriter
|
||||
|
||||
BASE_SEEDS = (501, 502, 503)
|
||||
SWARM_SEEDS = (601, 602, 603)
|
||||
CIFAR_WORKLOADS = ("cifar10_resnet18", "cifar10_resnet50")
|
||||
YOLO_WORKLOAD = "voc_yolo11n"
|
||||
|
||||
|
||||
def _cifar_seed(root: Path, seed: int) -> dict[str, Any]:
|
||||
baseline = root / f"baseline-{seed}.pt"
|
||||
searches = sum(
|
||||
(root / f"feature-{method}-{seed}-{swarm}.json").is_file()
|
||||
for method in ("pso", "random")
|
||||
for swarm in SWARM_SEEDS
|
||||
)
|
||||
controls = sum(
|
||||
(root / f"{method}-{seed}.pt").is_file()
|
||||
for method in ("feature-adam", "head-adam")
|
||||
)
|
||||
complete = baseline.is_file() and searches == 6 and controls == 2
|
||||
if complete:
|
||||
stage = "complete"
|
||||
elif not baseline.is_file():
|
||||
stage = "baseline_training"
|
||||
elif searches < 6:
|
||||
stage = f"pso_random_search_{searches}_of_6"
|
||||
else:
|
||||
stage = f"adam_controls_{controls}_of_2"
|
||||
return {
|
||||
"seed": seed,
|
||||
"stage": stage,
|
||||
"complete": complete,
|
||||
"searches": searches,
|
||||
"controls": controls,
|
||||
}
|
||||
|
||||
|
||||
def _read_yolo_metrics(path: Path) -> list[dict[str, float]]:
|
||||
if not path.is_file():
|
||||
return []
|
||||
try:
|
||||
with path.open(newline="", encoding="utf-8") as stream:
|
||||
rows = []
|
||||
for source in csv.DictReader(stream):
|
||||
rows.append(
|
||||
{
|
||||
key.strip(): float(value)
|
||||
for key, value in source.items()
|
||||
if key is not None
|
||||
and value is not None
|
||||
and value.strip()
|
||||
}
|
||||
)
|
||||
return rows
|
||||
except (OSError, ValueError):
|
||||
return []
|
||||
|
||||
|
||||
def _yolo_seed(
|
||||
root: Path,
|
||||
run_root: Path,
|
||||
seed: int,
|
||||
) -> dict[str, Any]:
|
||||
baseline = root / "baselines" / str(seed) / "ema_fp32.pt"
|
||||
training_root = (
|
||||
run_root / "ultralytics" / f"base-{seed}-100e"
|
||||
)
|
||||
metrics = _read_yolo_metrics(training_root / "results.csv")
|
||||
baseline_complete = (
|
||||
baseline.is_file()
|
||||
and len(metrics) >= 100
|
||||
and (training_root / "weights" / "last.pt").is_file()
|
||||
)
|
||||
arm_root = root / "arms" / str(seed)
|
||||
searches = sum(
|
||||
(arm_root / f"feature_{method}-{swarm}.pt").is_file()
|
||||
for method in ("pso", "random")
|
||||
for swarm in SWARM_SEEDS
|
||||
)
|
||||
controls = sum(
|
||||
(arm_root / f"{method}.pt").is_file()
|
||||
for method in ("feature_adam", "head_adam")
|
||||
)
|
||||
complete = (arm_root / "record.json").is_file()
|
||||
if complete:
|
||||
stage = "complete"
|
||||
elif not baseline_complete:
|
||||
stage = f"baseline_training_epoch_{len(metrics)}_of_100"
|
||||
elif searches < 6:
|
||||
stage = f"pso_random_search_{searches}_of_6"
|
||||
elif controls < 2:
|
||||
stage = f"adam_controls_{controls}_of_2"
|
||||
else:
|
||||
stage = "selection"
|
||||
return {
|
||||
"seed": seed,
|
||||
"stage": stage,
|
||||
"complete": complete,
|
||||
"searches": searches,
|
||||
"controls": controls,
|
||||
"training_metrics": metrics,
|
||||
}
|
||||
|
||||
|
||||
def snapshot(run_root: Path) -> dict[str, Any]:
|
||||
workloads = run_root / "workloads"
|
||||
status: dict[str, Any] = {}
|
||||
completed = 0
|
||||
for workload in CIFAR_WORKLOADS:
|
||||
seeds = [
|
||||
_cifar_seed(workloads / workload, seed)
|
||||
for seed in BASE_SEEDS
|
||||
]
|
||||
completed += sum(item["complete"] for item in seeds)
|
||||
status[workload] = seeds
|
||||
yolo = [
|
||||
_yolo_seed(
|
||||
workloads / YOLO_WORKLOAD,
|
||||
run_root,
|
||||
seed,
|
||||
)
|
||||
for seed in BASE_SEEDS
|
||||
]
|
||||
completed += sum(item["complete"] for item in yolo)
|
||||
status[YOLO_WORKLOAD] = yolo
|
||||
state_path = run_root / "state.json"
|
||||
state = "missing"
|
||||
if state_path.is_file():
|
||||
try:
|
||||
state = str(
|
||||
json.loads(
|
||||
state_path.read_text(encoding="utf-8")
|
||||
).get("state", "unknown")
|
||||
)
|
||||
except (OSError, ValueError):
|
||||
state = "unreadable"
|
||||
active = next(
|
||||
(
|
||||
f"{workload}/seed-{item['seed']}/{item['stage']}"
|
||||
for workload, items in status.items()
|
||||
for item in items
|
||||
if not item["complete"]
|
||||
),
|
||||
"",
|
||||
)
|
||||
if not active:
|
||||
for workload in CIFAR_WORKLOADS:
|
||||
selected = sum(
|
||||
(
|
||||
workloads
|
||||
/ workload
|
||||
/ f"selected-feature_{method}-{seed}.pt"
|
||||
).is_file()
|
||||
for method in ("pso", "random")
|
||||
for seed in BASE_SEEDS
|
||||
)
|
||||
if selected < 6:
|
||||
active = (
|
||||
f"{workload}/development_selection_"
|
||||
f"{selected}_of_6"
|
||||
)
|
||||
break
|
||||
if not active:
|
||||
active = "development_complete"
|
||||
return {
|
||||
"state": state,
|
||||
"active": active,
|
||||
"completed_base_seeds": int(completed),
|
||||
"total_base_seeds": 9,
|
||||
"workloads": status,
|
||||
"observed_at": time.time(),
|
||||
}
|
||||
|
||||
|
||||
def emit(
|
||||
writer: SummaryWriter,
|
||||
value: dict[str, Any],
|
||||
step: int,
|
||||
) -> None:
|
||||
writer.add_scalar(
|
||||
"progress/completed_base_seeds",
|
||||
value["completed_base_seeds"],
|
||||
step,
|
||||
)
|
||||
writer.add_scalar(
|
||||
"progress/completion_fraction",
|
||||
value["completed_base_seeds"] / value["total_base_seeds"],
|
||||
step,
|
||||
)
|
||||
for workload, seeds in value["workloads"].items():
|
||||
writer.add_scalar(
|
||||
f"progress/{workload}/completed_base_seeds",
|
||||
sum(item["complete"] for item in seeds),
|
||||
step,
|
||||
)
|
||||
for item in value["workloads"][YOLO_WORKLOAD]:
|
||||
seed = item["seed"]
|
||||
for row in item["training_metrics"]:
|
||||
epoch = int(row["epoch"])
|
||||
for metric, metric_value in row.items():
|
||||
if metric in {"epoch", "time"}:
|
||||
continue
|
||||
writer.add_scalar(
|
||||
f"training/{YOLO_WORKLOAD}/seed_{seed}/{metric}",
|
||||
metric_value,
|
||||
epoch,
|
||||
)
|
||||
writer.add_text(
|
||||
"progress/current",
|
||||
f"`{value['active']}`",
|
||||
step,
|
||||
)
|
||||
writer.add_text(
|
||||
"progress/snapshot",
|
||||
f"```json\n{json.dumps(value, indent=2)}\n```",
|
||||
step,
|
||||
)
|
||||
writer.flush()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--run-root", required=True, type=Path)
|
||||
parser.add_argument("--interval", type=float, default=10.0)
|
||||
parser.add_argument("--once", action="store_true")
|
||||
args = parser.parse_args()
|
||||
if args.interval <= 0:
|
||||
raise SystemExit("--interval must be positive")
|
||||
log_dir = args.run_root / "tensorboard"
|
||||
writer = SummaryWriter(log_dir=str(log_dir))
|
||||
print(f"monitoring {args.run_root} -> {log_dir}", flush=True)
|
||||
step = 0
|
||||
try:
|
||||
while True:
|
||||
value = snapshot(args.run_root)
|
||||
emit(writer, value, step)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"active": value["active"],
|
||||
"completed_base_seeds": value[
|
||||
"completed_base_seeds"
|
||||
],
|
||||
"state": value["state"],
|
||||
},
|
||||
sort_keys=True,
|
||||
),
|
||||
flush=True,
|
||||
)
|
||||
step += 1
|
||||
if args.once:
|
||||
break
|
||||
time.sleep(args.interval)
|
||||
finally:
|
||||
writer.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
Executable
+1751
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,655 @@
|
||||
"""
|
||||
Heavy PSO Cross-Split Experiment Results Publisher.
|
||||
|
||||
Protocol Version: HEAVY-PSO-CROSS-SPLIT-PUBLISH 1.0.0
|
||||
|
||||
Reads raw candidate and evaluation artifacts from a completed cross-split mission run,
|
||||
validates integrity, accounting, test seals, and candidate/evaluation alignment across
|
||||
all development variants, and exports deterministic public benchmark artifacts:
|
||||
1. benchmark_results/pso_v7_heavy_cross_split.json
|
||||
2. benchmark_results/pso_v7_heavy_cross_split.csv
|
||||
3. history_plt/pso_v7_heavy_cross_split.png
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple, Union
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
|
||||
# Ensure test directory and repo root are in sys.path
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent if Path(__file__).resolve().parent.name != "PSO" else Path(__file__).resolve().parent
|
||||
TEST_DIR = REPO_ROOT / "test"
|
||||
if str(TEST_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(TEST_DIR))
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from benchmark_suite import save_json_atomic
|
||||
from pso import __version__ as pso_version
|
||||
|
||||
PUBLISH_PROTOCOL_VERSION = "HEAVY-PSO-CROSS-SPLIT-PUBLISH 1.0.0"
|
||||
DEFAULT_SOURCE_DIR = Path(".omc/autoresearch/heavy-pso-cross-split-robustness/runs/20260903T162504Z")
|
||||
DEFAULT_JSON_OUTPUT = Path("benchmark_results/pso_v7_heavy_cross_split.json")
|
||||
DEFAULT_CSV_OUTPUT = Path("benchmark_results/pso_v7_heavy_cross_split.csv")
|
||||
DEFAULT_PLOT_OUTPUT = Path("history_plt/pso_v7_heavy_cross_split.png")
|
||||
|
||||
WORKLOADS = ["mnist_compact", "mnist_wide", "fashion_compact", "fashion_wide"]
|
||||
BASELINE_METHODS = {
|
||||
"mnist_compact": "G8",
|
||||
"mnist_wide": "G5",
|
||||
"fashion_compact": "G8",
|
||||
"fashion_wide": "G5",
|
||||
}
|
||||
EXPECTED_DEV_SPLITS = [20260905, 20260906]
|
||||
EXPECTED_DEV_SWARM_SEEDS = [101, 102, 103]
|
||||
EXPECTED_VARIANTS_COUNT = 9
|
||||
EXPECTED_VARIANT_IDS = (
|
||||
"iteration-0001-development",
|
||||
"iteration-0002-development",
|
||||
"iteration-0003-replica1-development",
|
||||
"iteration-0003-replica2-development",
|
||||
"iteration-0004-development",
|
||||
"iteration-0005-development",
|
||||
"iteration-0006-development",
|
||||
"iteration-0007-development",
|
||||
"iteration-0008-development",
|
||||
)
|
||||
EXPECTED_CELLS_PER_VARIANT = 8
|
||||
EXPECTED_RUNS_PER_VARIANT = 48
|
||||
EXPECTED_QUERIES_PER_VARIANT = 46080
|
||||
EXPECTED_SAMPLES_PER_VARIANT = 460800000
|
||||
|
||||
EXPECTED_TOTAL_RUNS = EXPECTED_VARIANTS_COUNT * EXPECTED_RUNS_PER_VARIANT # 432
|
||||
EXPECTED_TOTAL_QUERIES = EXPECTED_VARIANTS_COUNT * EXPECTED_QUERIES_PER_VARIANT # 414720
|
||||
EXPECTED_TOTAL_SAMPLES = EXPECTED_VARIANTS_COUNT * EXPECTED_SAMPLES_PER_VARIANT # 4147200000
|
||||
|
||||
|
||||
def discover_and_load_variants(
|
||||
source_dir: Path,
|
||||
) -> List[Tuple[Path, Path, Dict[str, Any], Dict[str, Any]]]:
|
||||
"""
|
||||
Discovers candidate and evaluation JSON file pairs in source_dir.
|
||||
Supports both subdirectories (candidates/ & evaluations/) and direct directory structure.
|
||||
"""
|
||||
if not source_dir.exists():
|
||||
raise FileNotFoundError(f"Source directory does not exist: {source_dir}")
|
||||
|
||||
cand_dir = source_dir / "candidates"
|
||||
eval_dir = source_dir / "evaluations"
|
||||
|
||||
if cand_dir.is_dir() and eval_dir.is_dir():
|
||||
candidate_files = sorted(cand_dir.glob("*.json"))
|
||||
else:
|
||||
candidate_files = sorted(source_dir.glob("*candidate*.json"))
|
||||
if not candidate_files:
|
||||
candidate_files = sorted(source_dir.glob("*.json"))
|
||||
|
||||
if not candidate_files:
|
||||
raise ValueError(f"No candidate JSON files found in {source_dir}")
|
||||
|
||||
observed_ids = tuple(path.stem for path in candidate_files)
|
||||
if observed_ids != EXPECTED_VARIANT_IDS:
|
||||
raise ValueError(
|
||||
f"Expected exact development variants {list(EXPECTED_VARIANT_IDS)}, "
|
||||
f"got {list(observed_ids)}"
|
||||
)
|
||||
|
||||
pairs = []
|
||||
for cf in candidate_files:
|
||||
if cand_dir.is_dir() and eval_dir.is_dir():
|
||||
ef = eval_dir / cf.name
|
||||
else:
|
||||
ef_name = cf.name.replace("candidate", "evaluation")
|
||||
ef = source_dir / ef_name
|
||||
if not ef.exists():
|
||||
ef = cf
|
||||
|
||||
if not ef.exists():
|
||||
raise FileNotFoundError(f"Missing corresponding evaluation file for candidate {cf.name}: {ef}")
|
||||
|
||||
with cf.open("r", encoding="utf-8") as f:
|
||||
cdata = json.load(f)
|
||||
with ef.open("r", encoding="utf-8") as f:
|
||||
edata = json.load(f)
|
||||
|
||||
pairs.append((cf, ef, cdata, edata))
|
||||
|
||||
return pairs
|
||||
|
||||
|
||||
def validate_variant_pair(
|
||||
cf_path: Path,
|
||||
ef_path: Path,
|
||||
cdata: Dict[str, Any],
|
||||
edata: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Validates each source candidate/evaluation artifact pair for expected IDs,
|
||||
phase, test seals, no development pass, cell count 8, and resource accounting.
|
||||
"""
|
||||
variant_id = cf_path.stem
|
||||
|
||||
# Phase check
|
||||
c_phase = cdata.get("phase")
|
||||
e_phase = edata.get("phase")
|
||||
if c_phase != "development":
|
||||
raise ValueError(f"[{variant_id}] Candidate phase must be 'development', got: {c_phase}")
|
||||
if e_phase not in (None, "development"):
|
||||
raise ValueError(f"[{variant_id}] Evaluation phase must be 'development', got: {e_phase}")
|
||||
|
||||
# Official test sealed check
|
||||
if cdata.get("official_test_data_loaded") is not False:
|
||||
raise ValueError(f"[{variant_id}] candidate.official_test_data_loaded must be False")
|
||||
if cdata.get("official_test_evaluations") != 0:
|
||||
raise ValueError(f"[{variant_id}] candidate.official_test_evaluations must be 0")
|
||||
|
||||
gates = edata.get("gates", {})
|
||||
if gates and "official_test_sealed" in gates:
|
||||
if not gates["official_test_sealed"].get("pass", False):
|
||||
raise ValueError(f"[{variant_id}] evaluation gate 'official_test_sealed' must pass")
|
||||
|
||||
# No development pass check
|
||||
if edata.get("development_pass") is not False:
|
||||
raise ValueError(f"[{variant_id}] development_pass must be False for all variants")
|
||||
if edata.get("pass") is not False:
|
||||
raise ValueError(f"[{variant_id}] pass must be False for all variants")
|
||||
if edata.get("eligible_for_confirmation") is not False:
|
||||
raise ValueError(f"[{variant_id}] eligible_for_confirmation must be False for all variants")
|
||||
|
||||
# Matching cell count 8
|
||||
summary_metrics = edata.get("summary_metrics", {})
|
||||
dev_cells = summary_metrics.get("development_cells")
|
||||
cell_metrics = edata.get("cell_metrics", [])
|
||||
if dev_cells != EXPECTED_CELLS_PER_VARIANT or len(cell_metrics) != EXPECTED_CELLS_PER_VARIANT:
|
||||
raise ValueError(
|
||||
f"[{variant_id}] Expected {EXPECTED_CELLS_PER_VARIANT} development cells, got "
|
||||
f"summary_metrics.development_cells={dev_cells}, len(cell_metrics)={len(cell_metrics)}"
|
||||
)
|
||||
|
||||
# Resource accounting check across per-seed runs
|
||||
splits = cdata.get("splits", {})
|
||||
c_runs = 0
|
||||
c_queries = 0
|
||||
c_samples = 0
|
||||
c_test_evals = 0
|
||||
|
||||
for split_key, split_data in splits.items():
|
||||
for role in ("baselines", "candidates"):
|
||||
for wl_key, wl_data in split_data.get(role, {}).items():
|
||||
for run in wl_data.get("per_seed_runs", []):
|
||||
c_runs += 1
|
||||
c_queries += run.get("total_queries", 0)
|
||||
c_samples += run.get("total_sample_evaluations", 0)
|
||||
c_test_evals += run.get("official_test_evaluations", 0)
|
||||
|
||||
if c_runs != EXPECTED_RUNS_PER_VARIANT:
|
||||
raise ValueError(f"[{variant_id}] Expected {EXPECTED_RUNS_PER_VARIANT} runs, got {c_runs}")
|
||||
if c_queries != EXPECTED_QUERIES_PER_VARIANT:
|
||||
raise ValueError(f"[{variant_id}] Expected {EXPECTED_QUERIES_PER_VARIANT} total queries, got {c_queries}")
|
||||
if c_samples != EXPECTED_SAMPLES_PER_VARIANT:
|
||||
raise ValueError(f"[{variant_id}] Expected {EXPECTED_SAMPLES_PER_VARIANT} total sample evaluations, got {c_samples}")
|
||||
if c_test_evals != 0:
|
||||
raise ValueError(f"[{variant_id}] Official test evaluations must be 0, got {c_test_evals}")
|
||||
resource_totals = cdata.get("resource_totals")
|
||||
if not isinstance(resource_totals, dict):
|
||||
raise ValueError(f"[{variant_id}] candidate.resource_totals must be present")
|
||||
expected_resource_totals = {
|
||||
"total_runs": c_runs,
|
||||
"total_queries": c_queries,
|
||||
"total_samples_evaluated": c_samples,
|
||||
"official_test_evaluations": c_test_evals,
|
||||
}
|
||||
for field, expected in expected_resource_totals.items():
|
||||
if resource_totals.get(field) != expected:
|
||||
raise ValueError(
|
||||
f"[{variant_id}] candidate.resource_totals.{field} must be {expected}, "
|
||||
f"got {resource_totals.get(field)!r}"
|
||||
)
|
||||
wall_time = resource_totals.get("wall_time_sec")
|
||||
if (
|
||||
not isinstance(wall_time, (int, float))
|
||||
or isinstance(wall_time, bool)
|
||||
or not math.isfinite(float(wall_time))
|
||||
or wall_time < 0
|
||||
):
|
||||
raise ValueError(
|
||||
f"[{variant_id}] candidate.resource_totals.wall_time_sec must be finite and non-negative"
|
||||
)
|
||||
|
||||
|
||||
return {
|
||||
"variant_id": variant_id,
|
||||
"candidate_file": cf_path.name,
|
||||
"evaluation_file": ef_path.name,
|
||||
"candidate_path": _repository_relative_path(cf_path),
|
||||
"evaluation_path": _repository_relative_path(ef_path),
|
||||
"phase": "development",
|
||||
"candidate_config": cdata.get("candidate_config", {}),
|
||||
"score": float(edata.get("score", 0.0)),
|
||||
"pass": False,
|
||||
"development_pass": False,
|
||||
"eligible_for_confirmation": False,
|
||||
"failed_hard_gate_count": edata.get("failed_hard_gate_count", 0),
|
||||
"failed_gates": edata.get("failed_gates", []),
|
||||
"summary_metrics": summary_metrics,
|
||||
"state_ratios": edata.get("state_ratios", {}),
|
||||
"cell_metrics": cell_metrics,
|
||||
"cdata": cdata,
|
||||
"edata": edata,
|
||||
"resources": {
|
||||
"runs": c_runs,
|
||||
"queries": c_queries,
|
||||
"sample_evaluations": c_samples,
|
||||
"wall_time_sec": float(wall_time),
|
||||
"official_test_evaluations": 0,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def validate_all_variants(
|
||||
variant_summaries: List[Dict[str, Any]],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Validates cumulative resources across all variants, identifies the best-observed variant,
|
||||
and constructs cumulative summary dictionary.
|
||||
"""
|
||||
observed_ids = tuple(v["variant_id"] for v in variant_summaries)
|
||||
if observed_ids != EXPECTED_VARIANT_IDS:
|
||||
raise ValueError(
|
||||
f"Expected exact development variants {list(EXPECTED_VARIANT_IDS)}, got {list(observed_ids)}"
|
||||
)
|
||||
|
||||
total_runs = sum(v["resources"]["runs"] for v in variant_summaries)
|
||||
total_queries = sum(v["resources"]["queries"] for v in variant_summaries)
|
||||
total_samples = sum(v["resources"]["sample_evaluations"] for v in variant_summaries)
|
||||
official_test_evals = sum(v["resources"]["official_test_evaluations"] for v in variant_summaries)
|
||||
|
||||
wall_time_sec = round(
|
||||
math.fsum(v["resources"]["wall_time_sec"] for v in variant_summaries),
|
||||
4,
|
||||
)
|
||||
if total_runs != EXPECTED_TOTAL_RUNS:
|
||||
raise ValueError(f"Cumulative total runs must be {EXPECTED_TOTAL_RUNS}, got {total_runs}")
|
||||
if total_queries != EXPECTED_TOTAL_QUERIES:
|
||||
raise ValueError(f"Cumulative total queries must be {EXPECTED_TOTAL_QUERIES}, got {total_queries}")
|
||||
if total_samples != EXPECTED_TOTAL_SAMPLES:
|
||||
raise ValueError(f"Cumulative total sample evaluations must be {EXPECTED_TOTAL_SAMPLES}, got {total_samples}")
|
||||
if official_test_evals != 0:
|
||||
raise ValueError(f"Cumulative official test evaluations must be 0, got {official_test_evals}")
|
||||
|
||||
# Mark best-observed-but-rejected variant (highest evaluation score)
|
||||
best_variant = max(variant_summaries, key=lambda v: v["score"])
|
||||
for v in variant_summaries:
|
||||
v["is_best_observed"] = (v["variant_id"] == best_variant["variant_id"])
|
||||
|
||||
return {
|
||||
"n_variants": len(variant_summaries),
|
||||
"total_runs": total_runs,
|
||||
"total_queries": total_queries,
|
||||
"total_sample_evaluations": total_samples,
|
||||
"official_test_evaluations": 0,
|
||||
"best_observed_variant_id": best_variant["variant_id"],
|
||||
"wall_time_sec": wall_time_sec,
|
||||
"best_observed_score": best_variant["score"],
|
||||
}
|
||||
|
||||
|
||||
def build_publish_json(
|
||||
source_dir: Path,
|
||||
variant_summaries: List[Dict[str, Any]],
|
||||
cum_resources: Dict[str, Any],
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Constructs the compact JSON dictionary matching all publication criteria.
|
||||
"""
|
||||
now_iso = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
|
||||
clean_variants = []
|
||||
for v in sorted(variant_summaries, key=lambda item: item["variant_id"]):
|
||||
clean_variants.append({
|
||||
"variant_id": v["variant_id"],
|
||||
"candidate_file": v["candidate_file"],
|
||||
"evaluation_file": v["evaluation_file"],
|
||||
"phase": v["phase"],
|
||||
"candidate_config": v["candidate_config"],
|
||||
"score": v["score"],
|
||||
"pass": v["pass"],
|
||||
"development_pass": v["development_pass"],
|
||||
"eligible_for_confirmation": v["eligible_for_confirmation"],
|
||||
"failed_hard_gate_count": v["failed_hard_gate_count"],
|
||||
"failed_gates": v["failed_gates"],
|
||||
"is_best_observed": v["is_best_observed"],
|
||||
"summary_metrics": v["summary_metrics"],
|
||||
"state_ratios": v["state_ratios"],
|
||||
"cell_metrics": v["cell_metrics"],
|
||||
"resources": v["resources"],
|
||||
})
|
||||
|
||||
payload = {
|
||||
"protocol_version": PUBLISH_PROTOCOL_VERSION,
|
||||
"pso_version": pso_version,
|
||||
"timestamp": now_iso,
|
||||
"mission_contract": {
|
||||
"phase": "development",
|
||||
"official_test_data_loaded": False,
|
||||
"official_test_evaluations": 0,
|
||||
"confirmation_executed": False,
|
||||
"retained_policy": None,
|
||||
},
|
||||
"official_test_data_loaded": False,
|
||||
"official_test_evaluations": 0,
|
||||
"confirmation_executed": False,
|
||||
"retained_policy": None,
|
||||
"verdict": {
|
||||
"status": "NO_RETAINED_POLICY_NO_CONFIRMATION",
|
||||
"retained_policy": None,
|
||||
"confirmation_executed": False,
|
||||
"official_test_data_loaded": False,
|
||||
"official_test_evaluations": 0,
|
||||
"best_observed_variant_id": cum_resources["best_observed_variant_id"],
|
||||
"best_observed_score": cum_resources["best_observed_score"],
|
||||
"description": (
|
||||
"All 9 development candidates failed the frozen evaluator gates (specifically maximum accuracy "
|
||||
"regression and/or development wide CNN improvement). No candidate qualified for confirmation. "
|
||||
"Confirmation split 20260907 and official test data remained completely unexecuted and sealed."
|
||||
),
|
||||
},
|
||||
"source_provenance": {
|
||||
"source_dir": _repository_relative_path(source_dir),
|
||||
"n_variants": cum_resources["n_variants"],
|
||||
"candidate_files": [v["candidate_file"] for v in clean_variants],
|
||||
"evaluation_files": [v["evaluation_file"] for v in clean_variants],
|
||||
},
|
||||
"cumulative_resources": cum_resources,
|
||||
"total_runs": cum_resources["total_runs"],
|
||||
"total_wall_time_sec": cum_resources["wall_time_sec"],
|
||||
"total_queries": cum_resources["total_queries"],
|
||||
"total_sample_evaluations": cum_resources["total_sample_evaluations"],
|
||||
"variants": clean_variants,
|
||||
}
|
||||
return payload
|
||||
|
||||
|
||||
def build_publish_csv(
|
||||
variant_summaries: List[Dict[str, Any]],
|
||||
csv_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
Writes CSV summary with header + exactly 72 data rows (9 variants x 2 splits x 4 workloads).
|
||||
Uses deterministic ordering and atomic writing.
|
||||
"""
|
||||
csv_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fieldnames = [
|
||||
"variant_id",
|
||||
"phase",
|
||||
"split_seed",
|
||||
"workload_id",
|
||||
"baseline_method",
|
||||
"baseline_acc",
|
||||
"candidate_acc",
|
||||
"acc_gain_pp",
|
||||
"baseline_nll",
|
||||
"candidate_nll",
|
||||
"nll_reduction_fraction",
|
||||
"state_ratio",
|
||||
"score",
|
||||
"pass",
|
||||
"is_best_observed",
|
||||
"candidate_path",
|
||||
"evaluation_path",
|
||||
]
|
||||
|
||||
rows = []
|
||||
# Sort variants deterministically
|
||||
sorted_variants = sorted(variant_summaries, key=lambda v: v["variant_id"])
|
||||
|
||||
for v in sorted_variants:
|
||||
dev_ratios = v.get("state_ratios", {}).get("development", {})
|
||||
cell_metrics = v.get("cell_metrics", [])
|
||||
|
||||
# Sort cells deterministically by split_seed then workload_id order
|
||||
def cell_sort_key(cm):
|
||||
wl_idx = WORKLOADS.index(cm["workload_id"]) if cm["workload_id"] in WORKLOADS else 99
|
||||
return (cm["split_seed"], wl_idx)
|
||||
|
||||
sorted_cells = sorted(cell_metrics, key=cell_sort_key)
|
||||
|
||||
for cm in sorted_cells:
|
||||
workload_id = cm["workload_id"]
|
||||
baseline_method = BASELINE_METHODS.get(workload_id, "G8" if "compact" in workload_id else "G5")
|
||||
st_ratio = dev_ratios.get(workload_id, 0.5) if isinstance(dev_ratios, dict) else 0.5
|
||||
|
||||
row = {
|
||||
"variant_id": v["variant_id"],
|
||||
"phase": cm["phase"],
|
||||
"split_seed": cm["split_seed"],
|
||||
"workload_id": workload_id,
|
||||
"baseline_method": baseline_method,
|
||||
"baseline_acc": cm["baseline_acc"],
|
||||
"candidate_acc": cm["candidate_acc"],
|
||||
"acc_gain_pp": cm["acc_gain_pp"],
|
||||
"baseline_nll": cm["baseline_nll"],
|
||||
"candidate_nll": cm["candidate_nll"],
|
||||
"nll_reduction_fraction": cm["nll_reduction_fraction"],
|
||||
"state_ratio": st_ratio,
|
||||
"score": v["score"],
|
||||
"pass": False,
|
||||
"is_best_observed": v["is_best_observed"],
|
||||
"candidate_path": v["candidate_path"],
|
||||
"evaluation_path": v["evaluation_path"],
|
||||
}
|
||||
rows.append(row)
|
||||
|
||||
tmp_csv = csv_path.with_suffix(".csv.tmp")
|
||||
with tmp_csv.open("w", encoding="utf-8", newline="") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
tmp_csv.replace(csv_path)
|
||||
|
||||
def _repository_relative_path(path: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(REPO_ROOT.resolve()))
|
||||
except ValueError:
|
||||
return str(path)
|
||||
|
||||
|
||||
def render_publish_plot(
|
||||
variant_summaries: List[Dict[str, Any]],
|
||||
plot_path: Path,
|
||||
) -> None:
|
||||
"""
|
||||
Renders readable 2-panel figure comparing mean gains and worst-cell regressions with frozen thresholds.
|
||||
Clearly marks all variants failed and confirmation withheld.
|
||||
"""
|
||||
plot_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
sorted_variants = sorted(variant_summaries, key=lambda v: v["variant_id"])
|
||||
|
||||
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(16, 6.5))
|
||||
|
||||
variant_labels = [
|
||||
v["variant_id"].replace("-development", "").replace("iteration-", "iter-")
|
||||
for v in sorted_variants
|
||||
]
|
||||
x = np.arange(len(variant_labels))
|
||||
width = 0.35
|
||||
|
||||
# Panel 1: Development Grand Mean Performance Gains
|
||||
acc_gains = [v["summary_metrics"].get("development_grand_mean_accuracy_gain_pp", 0.0) for v in sorted_variants]
|
||||
nll_reductions = [v["summary_metrics"].get("development_grand_mean_nll_reduction_fraction", 0.0) * 100.0 for v in sorted_variants]
|
||||
|
||||
ax1.bar(x - width/2, acc_gains, width, label="Grand Mean Acc Gain (pp)", color="#1f77b4")
|
||||
ax1.bar(x + width/2, nll_reductions, width, label="Grand Mean NLL Red. (%)", color="#2ca02c")
|
||||
|
||||
ax1.axhline(0.0, color="black", linestyle="--", linewidth=1.0, alpha=0.7)
|
||||
ax1.set_xticks(x)
|
||||
ax1.set_xticklabels(variant_labels, rotation=35, ha="right", fontsize=9)
|
||||
ax1.set_ylabel("Percentage Points (pp) / Percentage (%)")
|
||||
ax1.set_title("Panel A: Development Grand Mean Performance Gains")
|
||||
ax1.legend(loc="upper left")
|
||||
ax1.grid(True, linestyle="--", alpha=0.4)
|
||||
|
||||
# Panel 2: Worst-Cell Regressions & Wide CNN Improvement vs Gate Thresholds
|
||||
worst_acc_regs = []
|
||||
worst_nll_regs = []
|
||||
mnist_wide_accs = []
|
||||
|
||||
for v in sorted_variants:
|
||||
cell_acc_gains = [cm["acc_gain_pp"] for cm in v["cell_metrics"]]
|
||||
cell_nll_reds = [cm["nll_reduction_fraction"] * 100.0 for cm in v["cell_metrics"]]
|
||||
worst_acc_regs.append(min(cell_acc_gains))
|
||||
worst_nll_regs.append(min(cell_nll_reds))
|
||||
mnist_wide_accs.append(v["summary_metrics"].get("development_mnist_wide_accuracy_gain_pp", 0.0))
|
||||
|
||||
ax2.plot(x, worst_acc_regs, "o-", color="#d62728", linewidth=2, label="Worst-Cell Acc Delta (pp)")
|
||||
ax2.plot(x, worst_nll_regs, "s--", color="#ff7f0e", linewidth=2, label="Worst-Cell NLL Red. (%)")
|
||||
ax2.plot(x, mnist_wide_accs, "^-.", color="#9467bd", linewidth=2, label="MNIST Wide Acc Gain (pp)")
|
||||
|
||||
# Gate threshold lines
|
||||
ax2.axhline(-1.0, color="#d62728", linestyle=":", linewidth=1.5, label="Gate: Max Acc Reg. (-1.0 pp)")
|
||||
ax2.axhline(-5.0, color="#ff7f0e", linestyle=":", linewidth=1.5, label="Gate: Max NLL Reg. (-5.0%)")
|
||||
ax2.axhline(2.0, color="#2ca02c", linestyle="--", linewidth=1.5, label="Gate: MNIST Wide Gain (>= +2 pp)")
|
||||
panel_two_values = worst_acc_regs + worst_nll_regs + mnist_wide_accs + [-5.0, 2.0]
|
||||
panel_two_span = max(panel_two_values) - min(panel_two_values)
|
||||
panel_two_margin = max(1.0, panel_two_span * 0.08)
|
||||
ax2.set_ylim(
|
||||
min(panel_two_values) - panel_two_margin,
|
||||
max(panel_two_values) + panel_two_margin,
|
||||
)
|
||||
|
||||
ax2.set_xticks(x)
|
||||
ax2.set_xticklabels(variant_labels, rotation=35, ha="right", fontsize=9)
|
||||
ax2.set_ylabel("Metrics vs Gate Thresholds")
|
||||
ax2.set_title("Panel B: Worst-Cell Regressions & Wide CNN Improvement vs Gates")
|
||||
ax2.legend(loc="lower left", fontsize=8)
|
||||
ax2.grid(True, linestyle="--", alpha=0.4)
|
||||
|
||||
# Highlight best observed candidate
|
||||
best_idx = next(i for i, v in enumerate(sorted_variants) if v["is_best_observed"])
|
||||
ax1.annotate(
|
||||
f"Best observed, still rejected\nScore: {sorted_variants[best_idx]['score']:.2f}",
|
||||
xy=(best_idx, acc_gains[best_idx]),
|
||||
xycoords="data",
|
||||
xytext=(0.62, 0.94),
|
||||
textcoords="axes fraction",
|
||||
arrowprops=dict(facecolor="black", shrink=0.05, width=1, headwidth=5),
|
||||
bbox=dict(boxstyle="round,pad=0.3", facecolor="white", edgecolor="black", alpha=0.95),
|
||||
fontsize=8,
|
||||
ha="center",
|
||||
va="top",
|
||||
weight="bold",
|
||||
)
|
||||
|
||||
# Mission Outcome Text Banner
|
||||
fig.suptitle(
|
||||
"HEAVY PSO CROSS-SPLIT ROBUSTNESS MISSION REPORT\n"
|
||||
"STATUS: ALL 9 VARIANTS FAILED FROZEN EVALUATOR GATES | CONFIRMATION WITHHELD & SEALED OFFICIAL TEST UNEXECUTED | RETAINED POLICY: NONE",
|
||||
fontsize=11,
|
||||
weight="bold",
|
||||
color="#8b0000",
|
||||
y=0.99,
|
||||
)
|
||||
|
||||
plt.tight_layout(rect=[0, 0, 1, 0.93])
|
||||
tmp_plot = plot_path.with_name(plot_path.stem + "_tmp.png")
|
||||
fig.savefig(tmp_plot, dpi=200, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
tmp_plot.replace(plot_path)
|
||||
|
||||
|
||||
def publish_heavy_cross_split(
|
||||
source_dir: Union[str, Path] = DEFAULT_SOURCE_DIR,
|
||||
output_json: Union[str, Path] = DEFAULT_JSON_OUTPUT,
|
||||
output_csv: Union[str, Path] = DEFAULT_CSV_OUTPUT,
|
||||
output_plot: Union[str, Path] = DEFAULT_PLOT_OUTPUT,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Main programmatic interface for Heavy PSO Cross-Split results publication.
|
||||
Parses artifacts, validates all contracts and resource totals, and writes
|
||||
the compact JSON, CSV summary, and PNG plot atomically.
|
||||
"""
|
||||
source_dir = Path(source_dir).resolve()
|
||||
output_json = Path(output_json).resolve()
|
||||
output_csv = Path(output_csv).resolve()
|
||||
output_plot = Path(output_plot).resolve()
|
||||
|
||||
pairs = discover_and_load_variants(source_dir)
|
||||
|
||||
variant_summaries = []
|
||||
for cf, ef, cdata, edata in pairs:
|
||||
summary = validate_variant_pair(cf, ef, cdata, edata)
|
||||
variant_summaries.append(summary)
|
||||
|
||||
cum_resources = validate_all_variants(variant_summaries)
|
||||
|
||||
json_payload = build_publish_json(source_dir, variant_summaries, cum_resources)
|
||||
save_json_atomic(json_payload, output_json)
|
||||
|
||||
build_publish_csv(variant_summaries, output_csv)
|
||||
|
||||
render_publish_plot(variant_summaries, output_plot)
|
||||
|
||||
return json_payload
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Publish Heavy PSO Cross-Split Robustness Mission Results"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--source-dir",
|
||||
type=Path,
|
||||
default=DEFAULT_SOURCE_DIR,
|
||||
help=f"Raw experiment runs directory (default: {DEFAULT_SOURCE_DIR})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-json",
|
||||
type=Path,
|
||||
default=DEFAULT_JSON_OUTPUT,
|
||||
help=f"Output compact JSON path (default: {DEFAULT_JSON_OUTPUT})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-csv",
|
||||
type=Path,
|
||||
default=DEFAULT_CSV_OUTPUT,
|
||||
help=f"Output CSV summary path (default: {DEFAULT_CSV_OUTPUT})",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-plot",
|
||||
type=Path,
|
||||
default=DEFAULT_PLOT_OUTPUT,
|
||||
help=f"Output PNG plot path (default: {DEFAULT_PLOT_OUTPUT})",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main():
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
publish_heavy_cross_split(
|
||||
source_dir=args.source_dir,
|
||||
output_json=args.output_json,
|
||||
output_csv=args.output_csv,
|
||||
output_plot=args.output_plot,
|
||||
)
|
||||
print(f"[{PUBLISH_PROTOCOL_VERSION}] Successfully published cross-split results!")
|
||||
print(f" JSON: {args.output_json}")
|
||||
print(f" CSV: {args.output_csv}")
|
||||
print(f" Plot: {args.output_plot}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,464 @@
|
||||
"""
|
||||
Adaptive Moment 120-Particle x 80-Epoch MNIST Scaling Replication Check
|
||||
|
||||
Validates the published 120-particle x 80-epoch fixed-epoch Adaptive Moment MNIST scaling result.
|
||||
Performs exact replay on seeds 71-75 and fresh independent cohort evaluation on seeds 81-85.
|
||||
|
||||
Predeclared Acceptance Criteria:
|
||||
1. Exact Replay (seeds 71-75): Max per-seed absolute test accuracy delta <= 0.005 (0.5%p).
|
||||
2. Independent Cohort (seeds 81-85): Mean test accuracy absolute difference <= 0.03 (3%p)
|
||||
AND 95% t-confidence intervals overlap between baseline and independent cohorts.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import datetime
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Tuple
|
||||
|
||||
import torch
|
||||
from sklearn.decomposition import PCA
|
||||
|
||||
# Path setup for imports from test/ directory
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from benchmark_suite import (
|
||||
calc_stats,
|
||||
compute_data_fingerprint,
|
||||
get_hardware_provenance,
|
||||
resolve_execution_device,
|
||||
save_json_atomic,
|
||||
)
|
||||
from pso import __version__ as pso_version
|
||||
from tuning_suite import (
|
||||
TUNING_PROTOCOL_VERSION,
|
||||
CandidateConfig,
|
||||
get_mnist_raw_data,
|
||||
get_search_candidates,
|
||||
run_single_experiment,
|
||||
)
|
||||
|
||||
REPLAY_TOLERANCE = 0.005
|
||||
INDEPENDENT_MEAN_MARGIN = 0.03
|
||||
REPLICATION_PROTOCOL_VERSION = "1.0.0"
|
||||
REPLAY_SEEDS = [71, 72, 73, 74, 75]
|
||||
INDEPENDENT_SEEDS = [81, 82, 83, 84, 85]
|
||||
|
||||
|
||||
def validate_and_load_baseline(
|
||||
baseline_path: Path,
|
||||
) -> Tuple[Dict[str, Any], List[Dict[str, Any]], CandidateConfig, str]:
|
||||
if not baseline_path.exists():
|
||||
raise FileNotFoundError(f"Baseline JSON file not found at: {baseline_path}")
|
||||
|
||||
with open(baseline_path, "r", encoding="utf-8") as f:
|
||||
baseline_data = json.load(f)
|
||||
|
||||
if baseline_data.get("tuning_protocol_version") != TUNING_PROTOCOL_VERSION:
|
||||
raise ValueError(
|
||||
"Baseline tuning protocol mismatch: "
|
||||
f"expected {TUNING_PROTOCOL_VERSION}, "
|
||||
f"got {baseline_data.get('tuning_protocol_version')}"
|
||||
)
|
||||
if baseline_data.get("quick") is not False:
|
||||
raise ValueError("Replication requires the full, non-quick tuning baseline.")
|
||||
|
||||
winners = baseline_data.get("winners", {})
|
||||
if "adaptive_moment" not in winners:
|
||||
raise ValueError(f"Baseline JSON {baseline_path} missing 'adaptive_moment' winner entry.")
|
||||
|
||||
am_winner_info = winners["adaptive_moment"]
|
||||
winner_label = am_winner_info.get("candidate_label")
|
||||
|
||||
all_candidates = get_search_candidates()
|
||||
am_candidates = all_candidates.get("adaptive_moment", [])
|
||||
winner_cfg = None
|
||||
for cfg in am_candidates:
|
||||
if cfg.candidate_label == winner_label:
|
||||
winner_cfg = cfg
|
||||
break
|
||||
|
||||
if winner_cfg is None:
|
||||
raise ValueError(
|
||||
f"Could not find CandidateConfig matching label '{winner_label}' in search candidates."
|
||||
)
|
||||
expected_optimizer_config = winner_cfg.to_optimizer_kwargs(quick=False)
|
||||
if am_winner_info.get("config") != expected_optimizer_config:
|
||||
raise ValueError(
|
||||
"Adaptive Moment winner configuration in the baseline no longer matches "
|
||||
f"CandidateConfig '{winner_label}'."
|
||||
)
|
||||
|
||||
|
||||
scaling_runs = baseline_data.get("scaling_runs", [])
|
||||
baseline_records = []
|
||||
for r in scaling_runs:
|
||||
if (
|
||||
r.get("completed")
|
||||
and r.get("method") == "adaptive_moment"
|
||||
and r.get("candidate_label") == winner_label
|
||||
and r.get("n_particles") == 120
|
||||
and r.get("epochs") == 80
|
||||
and r.get("regimen") == "fixed_epoch"
|
||||
and r.get("seed") in REPLAY_SEEDS
|
||||
):
|
||||
baseline_records.append(r)
|
||||
|
||||
baseline_records.sort(key=lambda x: x["seed"])
|
||||
|
||||
if len(baseline_records) != 5:
|
||||
raise ValueError(
|
||||
f"Expected exactly 5 baseline records for seeds {REPLAY_SEEDS}, "
|
||||
f"found {len(baseline_records)} in {baseline_path}."
|
||||
)
|
||||
|
||||
expected_seeds = sorted(REPLAY_SEEDS)
|
||||
actual_seeds = [r["seed"] for r in baseline_records]
|
||||
if actual_seeds != expected_seeds:
|
||||
raise ValueError(f"Baseline seeds mismatch: expected {expected_seeds}, got {actual_seeds}")
|
||||
|
||||
expected_fp = baseline_data.get("split_fingerprints", {}).get("full")
|
||||
if not isinstance(expected_fp, str) or not expected_fp:
|
||||
raise ValueError("Baseline JSON is missing split_fingerprints.full.")
|
||||
for r in baseline_records:
|
||||
if r.get("data_fingerprint") != expected_fp:
|
||||
raise ValueError(
|
||||
f"Baseline run seed {r['seed']} data_fingerprint {r.get('data_fingerprint')} "
|
||||
f"does not match split_fingerprints.full {expected_fp}"
|
||||
)
|
||||
run_config = r.get("config", {})
|
||||
for key, value in expected_optimizer_config.items():
|
||||
if run_config.get(key) != value:
|
||||
raise ValueError(
|
||||
f"Baseline run seed {r['seed']} config[{key!r}]={run_config.get(key)!r} "
|
||||
f"does not match selected winner value {value!r}."
|
||||
)
|
||||
expected_run_config = {
|
||||
"n_particles": 120,
|
||||
"epochs": 80,
|
||||
"batch_size": 1000,
|
||||
"renewal": "loss",
|
||||
}
|
||||
for key, value in expected_run_config.items():
|
||||
if run_config.get(key) != value:
|
||||
raise ValueError(
|
||||
f"Baseline run seed {r['seed']} config[{key!r}]={run_config.get(key)!r}; "
|
||||
f"expected {value!r}."
|
||||
)
|
||||
|
||||
return baseline_data, baseline_records, winner_cfg, expected_fp
|
||||
|
||||
|
||||
def prepare_full_pca_data() -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, str]:
|
||||
x_train_raw, x_test_raw, y_train_3000, y_test_1000 = get_mnist_raw_data()
|
||||
pca_full = PCA(n_components=32, whiten=True, random_state=42)
|
||||
x_full_tr = torch.tensor(pca_full.fit_transform(x_train_raw), dtype=torch.float32)
|
||||
x_full_test = torch.tensor(pca_full.transform(x_test_raw), dtype=torch.float32)
|
||||
data_fp = compute_data_fingerprint(x_full_tr, x_full_test, y_train_3000, y_test_1000)
|
||||
return x_full_tr, y_train_3000, x_full_test, y_test_1000, data_fp
|
||||
|
||||
|
||||
def run_replication_cohort(
|
||||
cfg: CandidateConfig,
|
||||
seeds: List[int],
|
||||
x_train: torch.Tensor,
|
||||
y_train: torch.Tensor,
|
||||
x_eval: torch.Tensor,
|
||||
y_eval: torch.Tensor,
|
||||
device: torch.device,
|
||||
data_fp: str,
|
||||
run_type: str,
|
||||
) -> List[Dict[str, Any]]:
|
||||
runs = []
|
||||
for seed in seeds:
|
||||
res = run_single_experiment(
|
||||
cfg=cfg,
|
||||
seed=seed,
|
||||
x_train=x_train,
|
||||
y_train=y_train,
|
||||
x_eval=x_eval,
|
||||
y_eval=y_eval,
|
||||
n_particles=120,
|
||||
epochs=80,
|
||||
batch_size=1000,
|
||||
device=device,
|
||||
quick=False,
|
||||
eval_metric_name="test",
|
||||
data_fp=data_fp,
|
||||
run_type=run_type,
|
||||
extra_meta={"regimen": "fixed_epoch"},
|
||||
)
|
||||
runs.append(res)
|
||||
return runs
|
||||
|
||||
|
||||
def evaluate_replication(
|
||||
baseline_records: List[Dict[str, Any]],
|
||||
replay_runs: List[Dict[str, Any]],
|
||||
independent_runs: List[Dict[str, Any]],
|
||||
) -> Tuple[Dict[str, Any], Dict[str, float], Dict[str, float], Dict[str, float]]:
|
||||
base_acc_by_seed = {r["seed"]: float(r["test_acc"]) for r in baseline_records}
|
||||
replay_acc_by_seed = {r["seed"]: float(r["test_acc"]) for r in replay_runs}
|
||||
expected_seeds = set(REPLAY_SEEDS)
|
||||
if set(base_acc_by_seed) != expected_seeds or set(replay_acc_by_seed) != expected_seeds:
|
||||
raise ValueError("Baseline and replay cohorts must each contain exactly seeds 71-75.")
|
||||
if len(independent_runs) != len(INDEPENDENT_SEEDS) or {
|
||||
r["seed"] for r in independent_runs
|
||||
} != set(INDEPENDENT_SEEDS):
|
||||
raise ValueError("Independent cohort must contain exactly seeds 81-85.")
|
||||
|
||||
baseline_model_fp = {r["seed"]: r.get("model_fingerprint") for r in baseline_records}
|
||||
replay_model_fp = {r["seed"]: r.get("model_fingerprint") for r in replay_runs}
|
||||
replay_model_fingerprint_match = baseline_model_fp == replay_model_fp
|
||||
replay_deltas = {}
|
||||
max_replay_delta = 0.0
|
||||
for seed in sorted(base_acc_by_seed.keys()):
|
||||
b_acc = base_acc_by_seed[seed]
|
||||
r_acc = replay_acc_by_seed[seed]
|
||||
delta = abs(r_acc - b_acc)
|
||||
replay_deltas[str(seed)] = round(delta, 6)
|
||||
if delta > max_replay_delta:
|
||||
max_replay_delta = delta
|
||||
|
||||
replay_pass = bool(max_replay_delta <= REPLAY_TOLERANCE)
|
||||
|
||||
baseline_accs = [base_acc_by_seed[s] for s in sorted(base_acc_by_seed.keys())]
|
||||
replay_accs = [replay_acc_by_seed[s] for s in sorted(replay_acc_by_seed.keys())]
|
||||
indep_accs = [float(r["test_acc"]) for r in independent_runs]
|
||||
|
||||
baseline_stats = calc_stats(baseline_accs)
|
||||
replay_stats = calc_stats(replay_accs)
|
||||
independent_stats = calc_stats(indep_accs)
|
||||
|
||||
indep_mean_diff = abs(independent_stats["mean"] - baseline_stats["mean"])
|
||||
independent_mean_pass = bool(indep_mean_diff <= INDEPENDENT_MEAN_MARGIN)
|
||||
|
||||
baseline_ci_low = round(baseline_stats["mean"] - baseline_stats["ci95_t"], 6)
|
||||
baseline_ci_high = round(baseline_stats["mean"] + baseline_stats["ci95_t"], 6)
|
||||
|
||||
indep_ci_low = round(independent_stats["mean"] - independent_stats["ci95_t"], 6)
|
||||
indep_ci_high = round(independent_stats["mean"] + independent_stats["ci95_t"], 6)
|
||||
|
||||
ci_overlap_pass = bool(max(baseline_ci_low, indep_ci_low) <= min(baseline_ci_high, indep_ci_high))
|
||||
independent_pass = bool(independent_mean_pass and ci_overlap_pass)
|
||||
|
||||
comparison = {
|
||||
"replay_per_seed_deltas": replay_deltas,
|
||||
"replay_max_abs_delta": round(max_replay_delta, 6),
|
||||
"replay_model_fingerprint_match": replay_model_fingerprint_match,
|
||||
"replay_pass": bool(replay_pass and replay_model_fingerprint_match),
|
||||
"independent_mean_abs_diff": round(indep_mean_diff, 6),
|
||||
"independent_mean_pass": independent_mean_pass,
|
||||
"baseline_ci95_t_interval": [baseline_ci_low, baseline_ci_high],
|
||||
"independent_ci95_t_interval": [indep_ci_low, indep_ci_high],
|
||||
"ci_overlap_pass": ci_overlap_pass,
|
||||
"independent_pass": independent_pass,
|
||||
"overall_pass": bool(
|
||||
replay_pass and replay_model_fingerprint_match and independent_pass
|
||||
),
|
||||
}
|
||||
|
||||
return comparison, baseline_stats, replay_stats, independent_stats
|
||||
|
||||
|
||||
def write_replication_csv(
|
||||
baseline_records: List[Dict[str, Any]],
|
||||
replay_runs: List[Dict[str, Any]],
|
||||
independent_runs: List[Dict[str, Any]],
|
||||
output_csv: Path,
|
||||
):
|
||||
output_csv.parent.mkdir(parents=True, exist_ok=True)
|
||||
fields = [
|
||||
"cohort",
|
||||
"method",
|
||||
"candidate_label",
|
||||
"regimen",
|
||||
"seed",
|
||||
"n_particles",
|
||||
"epochs",
|
||||
"particle_epochs",
|
||||
"train_loss",
|
||||
"train_acc",
|
||||
"test_loss",
|
||||
"test_acc",
|
||||
"test_mse",
|
||||
"fit_time_sec",
|
||||
"data_fingerprint",
|
||||
"model_fingerprint",
|
||||
"device",
|
||||
"completed",
|
||||
"error",
|
||||
]
|
||||
all_rows = []
|
||||
for r in baseline_records:
|
||||
row = dict(r)
|
||||
row["cohort"] = "baseline"
|
||||
all_rows.append(row)
|
||||
for r in replay_runs:
|
||||
row = dict(r)
|
||||
row["cohort"] = "replay"
|
||||
all_rows.append(row)
|
||||
for r in independent_runs:
|
||||
row = dict(r)
|
||||
row["cohort"] = "independent"
|
||||
all_rows.append(row)
|
||||
|
||||
with open(output_csv, "w", newline="", encoding="utf-8") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore")
|
||||
writer.writeheader()
|
||||
for r in all_rows:
|
||||
writer.writerow(r)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Replicate and verify published Adaptive Moment 120p x 80e MNIST scaling result"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--baseline-json",
|
||||
type=Path,
|
||||
default=Path("benchmark_results/pso_v4_tuning.json"),
|
||||
help="Path to baseline tuning JSON",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-json",
|
||||
type=Path,
|
||||
default=Path("benchmark_results/pso_v4_120p80_replication.json"),
|
||||
help="Path for replication output JSON",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-csv",
|
||||
type=Path,
|
||||
default=Path("benchmark_results/pso_v4_120p80_replication.csv"),
|
||||
help="Path for replication output CSV",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--device",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Execution device (cpu, cuda, mps)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
device = resolve_execution_device(args.device)
|
||||
|
||||
print("=== Adaptive Moment 120p x 80e Replication Check ===")
|
||||
print(f"Device: {device}")
|
||||
print(f"Baseline JSON: {args.baseline_json}")
|
||||
print(f"Output JSON: {args.output_json}")
|
||||
print(f"Output CSV: {args.output_csv}")
|
||||
|
||||
baseline_data, baseline_records, winner_cfg, expected_fp = validate_and_load_baseline(
|
||||
args.baseline_json
|
||||
)
|
||||
print(f"Validated baseline winner '{winner_cfg.candidate_label}' across 5 records.")
|
||||
if baseline_data.get("device") != str(device):
|
||||
raise ValueError(
|
||||
f"Exact replay requires baseline device {baseline_data.get('device')!r}; "
|
||||
f"got {str(device)!r}."
|
||||
)
|
||||
if baseline_data.get("pso_version") != pso_version:
|
||||
raise ValueError(
|
||||
f"Exact replay requires pso version {baseline_data.get('pso_version')!r}; "
|
||||
f"got {pso_version!r}."
|
||||
)
|
||||
if baseline_data.get("torch_version") != torch.__version__:
|
||||
raise ValueError(
|
||||
f"Exact replay requires torch version {baseline_data.get('torch_version')!r}; "
|
||||
f"got {torch.__version__!r}."
|
||||
)
|
||||
|
||||
x_full_tr, y_train_3000, x_full_test, y_test_1000, data_fp = prepare_full_pca_data()
|
||||
if expected_fp and data_fp != expected_fp:
|
||||
raise ValueError(
|
||||
f"Reconstructed data fingerprint {data_fp} does not match baseline {expected_fp}"
|
||||
)
|
||||
print(f"Reconstructed PCA32 data (fingerprint: {data_fp})")
|
||||
|
||||
print("\n--- Running Cohort 1: Exact Replay (Seeds 71-75) ---")
|
||||
replay_runs = run_replication_cohort(
|
||||
cfg=winner_cfg,
|
||||
seeds=REPLAY_SEEDS,
|
||||
x_train=x_full_tr,
|
||||
y_train=y_train_3000,
|
||||
x_eval=x_full_test,
|
||||
y_eval=y_test_1000,
|
||||
device=device,
|
||||
data_fp=data_fp,
|
||||
run_type="replication_replay",
|
||||
)
|
||||
|
||||
print("\n--- Running Cohort 2: Independent Fresh Seeds (Seeds 81-85) ---")
|
||||
independent_runs = run_replication_cohort(
|
||||
cfg=winner_cfg,
|
||||
seeds=INDEPENDENT_SEEDS,
|
||||
x_train=x_full_tr,
|
||||
y_train=y_train_3000,
|
||||
x_eval=x_full_test,
|
||||
y_eval=y_test_1000,
|
||||
device=device,
|
||||
data_fp=data_fp,
|
||||
run_type="replication_independent",
|
||||
)
|
||||
|
||||
comparison, baseline_stats, replay_stats, independent_stats = evaluate_replication(
|
||||
baseline_records, replay_runs, independent_runs
|
||||
)
|
||||
|
||||
payload = {
|
||||
"replication_protocol_version": REPLICATION_PROTOCOL_VERSION,
|
||||
"source_tuning_protocol_version": baseline_data["tuning_protocol_version"],
|
||||
"pso_version": pso_version,
|
||||
"torch_version": torch.__version__,
|
||||
"timestamp": datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"device": str(device),
|
||||
"hardware": get_hardware_provenance(device),
|
||||
"baseline_json": str(args.baseline_json),
|
||||
"source_tuning_timestamp": baseline_data.get("timestamp"),
|
||||
"candidate_label": winner_cfg.candidate_label,
|
||||
"config": winner_cfg.to_optimizer_kwargs(),
|
||||
"data_fingerprint": data_fp,
|
||||
"criteria": {
|
||||
"replay_seeds": REPLAY_SEEDS,
|
||||
"replay_max_abs_delta_tolerance": REPLAY_TOLERANCE,
|
||||
"require_replay_model_fingerprint_match": True,
|
||||
"independent_seeds": INDEPENDENT_SEEDS,
|
||||
"independent_mean_abs_diff_margin": INDEPENDENT_MEAN_MARGIN,
|
||||
"require_ci_overlap": True,
|
||||
},
|
||||
"summaries": {
|
||||
"baseline": baseline_stats,
|
||||
"replay": replay_stats,
|
||||
"independent": independent_stats,
|
||||
},
|
||||
"comparison": comparison,
|
||||
"baseline_runs": baseline_records,
|
||||
"replay_runs": replay_runs,
|
||||
"independent_runs": independent_runs,
|
||||
"completed": True,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
save_json_atomic(payload, args.output_json)
|
||||
write_replication_csv(baseline_records, replay_runs, independent_runs, args.output_csv)
|
||||
|
||||
print("\n=== Replication Results Summary ===")
|
||||
print(f"Baseline Mean Test Acc: {baseline_stats['mean']:.4f} ± {baseline_stats['std']:.4f}")
|
||||
print(f"Replay Mean Test Acc: {replay_stats['mean']:.4f} ± {replay_stats['std']:.4f}")
|
||||
print(f"Independent Mean Test Acc: {independent_stats['mean']:.4f} ± {independent_stats['std']:.4f}")
|
||||
print(f"Max Replay Delta: {comparison['replay_max_abs_delta']:.6f} (Limit: {REPLAY_TOLERANCE}) -> Pass: {comparison['replay_pass']}")
|
||||
print(f"Indep Mean Diff: {comparison['independent_mean_abs_diff']:.6f} (Limit: {INDEPENDENT_MEAN_MARGIN}) -> Pass: {comparison['independent_mean_pass']}")
|
||||
print(f"CI Overlap Pass: {comparison['ci_overlap_pass']} (Baseline CI: {comparison['baseline_ci95_t_interval']}, Indep CI: {comparison['independent_ci95_t_interval']})")
|
||||
print(f"OVERALL PASS: {comparison['overall_pass']}")
|
||||
|
||||
if not comparison["overall_pass"]:
|
||||
print("\nREPLICATION CHECK FAILED!")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("\nREPLICATION CHECK PASSED!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+90
-74
@@ -1,23 +1,16 @@
|
||||
# %%
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import tensorflow as tf
|
||||
from keras.layers import Dense
|
||||
from keras.models import Sequential
|
||||
from keras.utils import to_categorical
|
||||
import argparse
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from sklearn.model_selection import train_test_split
|
||||
from tensorflow import keras
|
||||
from sklearn.preprocessing import StandardScaler
|
||||
|
||||
from pso import optimizer
|
||||
|
||||
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
|
||||
from pso import Optimizer
|
||||
from cli import add_pso_args, build_optimizer_kwargs
|
||||
|
||||
|
||||
def get_data():
|
||||
def get_data(seed: int = 42):
|
||||
with open("data/seeds/seeds_dataset.txt", "r", encoding="utf-8") as f:
|
||||
data = f.readlines()
|
||||
df = pd.DataFrame([d.split() for d in data])
|
||||
@@ -33,80 +26,103 @@ def get_data():
|
||||
]
|
||||
|
||||
df = df.astype(float)
|
||||
df["target"] = df["target"].astype(int)
|
||||
df["target"] = df["target"].astype(int) - 1
|
||||
|
||||
x = df.iloc[:, :-1].values.round(0).astype(int)
|
||||
y = df.iloc[:, -1].values
|
||||
|
||||
y_class = to_categorical(y)
|
||||
x = df.iloc[:, :-1].values.astype(np.float32)
|
||||
y = df.iloc[:, -1].values.astype(np.int64)
|
||||
|
||||
x_train, x_test, y_train, y_test = train_test_split(
|
||||
x, y_class, test_size=0.2, shuffle=True
|
||||
x, y, test_size=0.2, shuffle=True, random_state=seed
|
||||
)
|
||||
scaler = StandardScaler()
|
||||
x_train = scaler.fit_transform(x_train)
|
||||
x_test = scaler.transform(x_test)
|
||||
|
||||
return (
|
||||
torch.tensor(x_train, dtype=torch.float32),
|
||||
torch.tensor(y_train, dtype=torch.int64),
|
||||
torch.tensor(x_test, dtype=torch.float32),
|
||||
torch.tensor(y_test, dtype=torch.int64),
|
||||
)
|
||||
|
||||
return x_train, y_train, x_test, y_test
|
||||
|
||||
def make_model(seed: int = 42):
|
||||
torch.manual_seed(seed)
|
||||
return nn.Sequential(
|
||||
nn.Linear(7, 16),
|
||||
nn.ReLU(),
|
||||
nn.Linear(16, 32),
|
||||
nn.ReLU(),
|
||||
nn.Linear(32, 3),
|
||||
)
|
||||
|
||||
|
||||
def make_model():
|
||||
model = Sequential()
|
||||
model.add(Dense(16, activation="relu", input_shape=(7,)))
|
||||
model.add(Dense(32, activation="relu"))
|
||||
model.add(Dense(4, activation="softmax"))
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="PSO Seeds Benchmark Script")
|
||||
add_pso_args(
|
||||
parser,
|
||||
defaults={
|
||||
"method": "original",
|
||||
"initialization": "model_noise",
|
||||
"evaluation": "full",
|
||||
"convergence": "particle_reset",
|
||||
"refinement": "adam",
|
||||
"n_particles": 24,
|
||||
"c0": None,
|
||||
"c1": None,
|
||||
"w_min": None,
|
||||
"w_max": None,
|
||||
"negative_swarm": 0.0,
|
||||
"mutation_swarm": 0.3,
|
||||
"particle_min": -3.0,
|
||||
"particle_max": 3.0,
|
||||
"velocity_limit_ratio": 0.1,
|
||||
"boundary_strategy": "reflect",
|
||||
"seed": 42,
|
||||
"epochs": 80,
|
||||
"renewal": "acc",
|
||||
"output_dir": "output/seeds",
|
||||
"checkpoint_interval": 25,
|
||||
"refinement_epochs": 10,
|
||||
"refinement_lr": 0.001,
|
||||
},
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
return model
|
||||
model = make_model(seed=args.seed)
|
||||
x_train, y_train, x_test, y_test = get_data(seed=args.seed)
|
||||
|
||||
fitness_size = args.fitness_size if args.evaluation == "fixed_subset" else None
|
||||
refinement_epochs = args.refinement_epochs if args.refinement == "adam" else 0
|
||||
|
||||
# %%
|
||||
model = make_model()
|
||||
x_train, y_train, x_test, y_test = get_data()
|
||||
kwargs = build_optimizer_kwargs(
|
||||
args,
|
||||
model=model,
|
||||
loss=nn.CrossEntropyLoss(),
|
||||
task="multiclass",
|
||||
inertia_profile={"c0": 0.5, "c1": 1.0, "w_min": 0.7, "w_max": 1.2},
|
||||
)
|
||||
pso_seeds = Optimizer(**kwargs)
|
||||
|
||||
loss = [
|
||||
"mean_squared_error",
|
||||
"categorical_crossentropy",
|
||||
"sparse_categorical_crossentropy",
|
||||
"binary_crossentropy",
|
||||
"kullback_leibler_divergence",
|
||||
"poisson",
|
||||
"cosine_similarity",
|
||||
"log_cosh",
|
||||
"huber_loss",
|
||||
"mean_absolute_error",
|
||||
"mean_absolute_percentage_error",
|
||||
]
|
||||
print(f"Optimizer device: {pso_seeds.device}")
|
||||
|
||||
# rs = random_state()
|
||||
|
||||
pso_mnist = optimizer(
|
||||
model,
|
||||
loss="categorical_crossentropy",
|
||||
n_particles=100,
|
||||
c0=0.5,
|
||||
c1=1.0,
|
||||
w_min=0.7,
|
||||
w_max=1.2,
|
||||
negative_swarm=0.0,
|
||||
mutation_swarm=0.3,
|
||||
convergence_reset=True,
|
||||
convergence_reset_patience=10,
|
||||
convergence_reset_monitor="mse",
|
||||
convergence_reset_min_delta=0.0005,
|
||||
)
|
||||
|
||||
best_score = pso_mnist.fit(
|
||||
best_score = pso_seeds.fit(
|
||||
x_train,
|
||||
y_train,
|
||||
epochs=500,
|
||||
epochs=args.epochs,
|
||||
batch_size=args.batch_size,
|
||||
fitness_size=fitness_size,
|
||||
renewal=args.renewal,
|
||||
validation_data=(x_test, y_test),
|
||||
output_dir=args.output_dir,
|
||||
checkpoint_interval=25,
|
||||
save_info=True,
|
||||
log=2,
|
||||
log_name="seeds",
|
||||
renewal="acc",
|
||||
check_point=25,
|
||||
empirical_balance=False,
|
||||
dispersion=False,
|
||||
back_propagation=False,
|
||||
validate_data=(x_test, y_test),
|
||||
)
|
||||
refinement_epochs=refinement_epochs,
|
||||
refinement_lr=args.refinement_lr,
|
||||
)
|
||||
|
||||
print("Done!")
|
||||
print(f"Done! Best score: {best_score}")
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user