jung-geun/PSO
1
0
Fork 0
mirror of https://github.com/jung-geun/PSO.git synced 2025-12-20 04:50:45 +09:00
Code Issues Packages Projects Releases Wiki Activity

23-05-31

Browse Source 
전체 파티클 중 일부를 현재 속도의 음수 방향으로 진행하도록 하여 지역해에 갇혀 조기수렴하는 문제의 방안으로 사용
This commit is contained in:
jung-geun
2023-05-31 02:52:32 +09:00
parent c5731c6870
commit 8012cf3557
7 changed files with 171 additions and 121 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
iris.py
View File

@@ -39,7 +39,7 @@ x_train, x_test, y_train, y_test = load_data()
loss = 'categorical_crossentropy'
pso_iris = Optimizer(model, loss=loss, n_particles=50, c0=0.5, c1=0.8, w_min=0.7, w_max=1.3)
pso_iris = Optimizer(model, loss=loss, n_particles=50, c0=0.5, c1=0.8, w_min=0.75, w_max=1.3)
weight, score = pso_iris.fit(
x_train, y_train, epochs=500, save=True, save_path="./result/iris", renewal="acc", empirical_balance=False, Dispersion=False, check_point=50)

BIN
iris_50_99.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 177 KiB

BIN
iris_99.16.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 143 KiB

10
mnist.py
View File

@@ -23,9 +23,9 @@ from tqdm import tqdm
import gc
print(tf.__version__)
print(tf.config.list_physical_devices())
print(f"Num GPUs Available: {len(tf.config.list_physical_devices('GPU'))}")
# print(tf.__version__)
# print(tf.config.list_physical_devices())
# print(f"Num GPUs Available: {len(tf.config.list_physical_devices('GPU'))}")
def get_data():
@@ -79,9 +79,9 @@ loss = 'huber_loss'
# loss = 'mean_squared_error'
pso_mnist = Optimizer(model, loss=loss, n_particles=50, c0=0.5, c1=0.8, w_min=0.75, w_max=1.3)
pso_mnist = Optimizer(model, loss=loss, n_particles=50, c0=0.4, c1=0.8, w_min=0.7, w_max=1.2, random=0.3)
weight, score = pso_mnist.fit(
x_test, y_test, epochs=1000, save=True, save_path="./result/mnist", renewal="acc", empirical_balance=False, Dispersion=True)
x_test, y_test, epochs=200, save=True, save_path="./result/mnist", renewal="acc", empirical_balance=False, Dispersion=False, check_point=10)
pso_mnist.model_save("./result/mnist")
pso_mnist.save_info("./result/mnist")

23
plt.ipynb
View File

File diff suppressed because one or more lines are too long

111
pso/optimizer.py
View File

@@ -1,4 +1,5 @@
import os
import sys
import tensorflow as tf
from tensorflow import keras
@@ -15,7 +16,19 @@ import gc
from pso.particle import Particle
class Optimizer:
"""
Args:
model (keras.models): 모델 구조
loss (str): 손실함수
n_particles (int): 파티클 개수
c0 (float): local rate - 지역 최적값 관성 수치
c1 (float): global rate - 전역 최적값 관성 수치
w_min (float): 최소 관성 수치
w_max (float): 최대 관성 수치
random (float): 랜덤 파티클 비율 - 0 ~ 1 사이의 값
"""
def __init__(
self,
model: keras.models,
@@ -25,6 +38,7 @@ class Optimizer:
c1=1.5,
w_min=0.5,
w_max=1.5,
random:float = 0,
):
self.model = model # 모델 구조
self.loss = loss # 손실함수
@@ -38,23 +52,28 @@ class Optimizer:
self.g_best_score = 0 # 최고 점수 - 시작은 0으로 초기화
self.g_best = None # 최고 점수를 받은 가중치
self.g_best_ = None # 최고 점수를 받은 가중치 - 값의 분산을 위한 변수
self.avg_score = 0 # 평균 점수
for i in tqdm(range(self.n_particles), desc="Initializing Particles"):
m = keras.models.model_from_json(model.to_json())
init_weights = m.get_weights()
w_, sh_, len_ = self._encode(init_weights)
w_ = np.random.uniform(-0.1, 0.1, len(w_))
w_ = np.random.uniform(-3, 3, len(w_))
m.set_weights(self._decode(w_, sh_, len_))
m.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"])
self.particles[i] = Particle(m, loss)
if i < random * self.n_particles:
self.particles[i] = Particle(m, loss, random=True)
else:
self.particles[i] = Particle(m, loss, random=False)
"""
Args:
weights (list) : keras model의 가중치
Returns:
(cupy array) : 가중치 - 1차원으로 풀어서 반환
(numpy array) : 가중치 - 1차원으로 풀어서 반환
(list) : 가중치의 원본 shape
(list) : 가중치의 원본 shape의 길이
"""
def _encode(self, weights):
# w_gpu = cp.array([])
w_gpu = np.array([])
@@ -70,6 +89,10 @@ class Optimizer:
return w_gpu, shape, lenght
"""
Args:
weight (numpy array) : 가중치 - 1차원으로 풀어진 상태
shape (list) : 가중치의 원본 shape
lenght (list) : 가중치의 원본 shape의 길이
Returns:
(list) : 가중치 원본 shape으로 복원
"""
@@ -102,27 +125,16 @@ class Optimizer:
return 1 + np.abs(score)
"""
parameters
----------
x : numpy.ndarray
y : numpy.ndarray
epochs : int
save : bool
save_path : str ex) "./result"
renewal : str ex) "acc" or "loss"
"""
"""
parameters
fit(
Args:
x_test : numpy.ndarray,
y_test : numpy.ndarray,
epochs : int,
save : bool - True : save, False : not save
save_path : str ex) "./result",
renewal : str ex) "acc" or "loss",
empirical_balance : bool - True : empirical balance, False : no balance
Dispersion : bool - True : random search, False : PSO
empirical_balance : bool - True :
Dispersion : bool - True : g_best 의 값을 분산시켜 전역해를 찾음, False : g_best 의 값만 사용
check_point : int - 저장할 위치 - None : 저장 안함
"""
def fit(
self,
@@ -136,6 +148,8 @@ class Optimizer:
Dispersion: bool = False,
check_point: int = None,
):
self.save_path = save_path
self.renewal = renewal
if renewal == "acc":
self.g_best_score = 0
@@ -150,7 +164,9 @@ class Optimizer:
os.makedirs(save_path, exist_ok=True)
self.day = datetime.now().strftime("%m-%d-%H-%M")
for i, p in enumerate(self.particles):
# for i, p in enumerate(self.particles):
for i in tqdm(range(self.n_particles), desc="Initializing Particles"):
p = self.particles[i]
local_score = p.get_score(x, y, renewal=renewal)
if renewal == "acc":
@@ -166,7 +182,9 @@ class Optimizer:
print(f"initial g_best_score : {self.g_best_score}")
try:
for _ in range(epochs):
print(f"epoch {_ + 1}/{epochs}")
acc = 0
loss = 0
min_score = np.inf
@@ -176,7 +194,7 @@ class Optimizer:
# for i in tqdm(range(len(self.particles)), desc=f"epoch {_ + 1}/{epochs}", ascii=True):
for i in range(len(self.particles)):
w = self.w_min + (self.w_max - self.w_min) * _ / epochs
w = self.w_max - (self.w_max - self.w_min) * _ / epochs
if Dispersion:
g_best = self.g_best_
@@ -189,12 +207,18 @@ class Optimizer:
w_g_ = self.f(x, y, self.g_best)
w_p = w_p_ / (w_p_ + w_g_)
w_g = w_p_ / (w_p_ + w_g_)
del w_p_
del w_g_
else:
p = 1 / (self.n_particles * np.linalg.norm(self.c1 - self.c0))
p = np.exp(-p)
w_p = p
w_g = 1 - p
p_b = self.particles[i].get_best_score()
g_a = self.avg_score
l_b = p_b - g_a
l_b = np.sqrt(np.power(l_b, 2))
p_ = 1 / (self.n_particles * np.linalg.norm(self.c1 - self.c0)) * l_b
p_ = np.exp(-1 * p_)
w_p = p_
w_g = 1 - p_
score = self.particles[i].step_w(
x, y, self.c0, self.c1, w, g_best, w_p, w_g, renewal=renewal
@@ -226,6 +250,8 @@ class Optimizer:
if score[1] > max_score:
max_score = score[1]
gc.collect()
if save:
with open(
f"./{save_path}/{self.day}_{self.n_particles}_{epochs}_{self.c0}_{self.c1}_{self.w_min}_{renewal}.csv",
@@ -235,6 +261,7 @@ class Optimizer:
if i != self.n_particles - 1:
f.write(", ")
TS = self.c0 + np.random.rand() * (self.c1 - self.c0)
g_, g_sh, g_len = self._encode(self.g_best)
decrement = (epochs - (_) + 1) / epochs
@@ -248,11 +275,10 @@ class Optimizer:
) as f:
f.write("\n")
print(f"epoch {_ + 1}/{epochs} finished")
# print(f"loss min : {min_loss} | loss max : {max_loss} | acc min : {min_score} | acc max : {max_score}")
# print(f"loss avg : {loss/self.n_particles} | acc avg : {acc/self.n_particles} | Best {renewal} : {self.g_best_score}")
print(
f"loss min : {min_loss} | acc avg : {max_score} | Best {renewal} : {self.g_best_score}"
f"loss min : {min_loss} | acc max : {max_score} | Best {renewal} : {self.g_best_score}"
)
gc.collect()
@@ -260,10 +286,28 @@ class Optimizer:
if check_point is not None:
if _ % check_point == 0:
os.makedirs(f"./{save_path}/{self.day}", exist_ok=True)
self._check_point_save(f"./{save_path}/{self.day}/check_point_{_}.h5")
self._check_point_save(f"./{save_path}/{self.day}/ckpt-{_}")
self.avg_score = acc/self.n_particles
except KeyboardInterrupt:
print("Keyboard Interrupt")
self.model_save(save_path)
print("model saved")
self.save_info(save_path)
print("info saved")
sys.exit(0)
except MemoryError:
print("Memory Error")
self.model_save(save_path)
print("model save")
self.save_info(save_path)
print("save info")
sys.exit(1)
except Exception as e:
print(e)
finally:
return self.g_best, self.g_best_score
def get_best_model(self):
model = keras.models.model_from_json(self.model.to_json())
model.set_weights(self.g_best)
@@ -290,16 +334,17 @@ class Optimizer:
}
with open(
f"./{path}/{self.day}_{self.loss}_{self.n_particles}_{self.g_best_score}.json",
"w",
f"./{path}/{self.day}/{self.loss}_{self.n_particles}.json",
"a",
) as f:
json.dump(json_save, f, indent=4)
f.write(",\n")
def _check_point_save(self, save_path: str = f"./result/check_point"):
model = self.get_best_model()
model.save(save_path)
model.save_weights(save_path)
def model_save(self, save_path: str = "./result/model"):
def model_save(self, save_path: str = "./result"):
model = self.get_best_model()
model.save(
f"./{save_path}/{self.day}/{self.n_particles}_{self.c0}_{self.c1}_{self.w_min}.h5"

6
pso/particle.py
View File

@@ -6,14 +6,14 @@ from tensorflow import keras
import numpy as np
class Particle:
def __init__(self, model:keras.models, loss):
def __init__(self, model:keras.models, loss, random:bool = False):
self.model = model
self.loss = loss
self.init_weights = self.model.get_weights()
i_w_,s_,l_ = self._encode(self.init_weights)
i_w_ = np.random.rand(len(i_w_)) / 5 - 0.10
self.velocities = self._decode(i_w_,s_,l_)
self.random = random
self.best_score = 0
self.best_weights = self.init_weights
@@ -94,6 +94,8 @@ class Particle:
def _update_weights(self):
encode_w, w_sh, w_len = self._encode(weights = self.model.get_weights())
encode_v, _, _ = self._encode(weights = self.velocities)
if self.random:
encode_v = -1 * encode_v
new_w = encode_w + encode_v
self.model.set_weights(self._decode(new_w, w_sh, w_len))