23-06-05

env 파일 이름 변경
돌연변이 설정 수정

.gitignore

@@ -10,4 +10,7 @@ result/
 
 # 논문 관련 파일
 *.pdf
-*.pptx
+*.pptx
+*.png
+관련 논문/
+발표 자료/

iris.py

@@ -46,8 +46,8 @@ pso_iris = Optimizer(
     c1=0.8, 
     w_min=0.7,
     w_max=1.0, 
-    negative_swarm=0,
+    negative_swarm=0.1,
-    mutation_swarm=0,
+    mutation_swarm=0.2,
     )
 
 best_score = pso_iris.fit(

mnist.py

@@ -10,15 +10,10 @@ from keras import backend as K
 from keras.datasets import mnist
 from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
 from keras.models import Sequential
-# from pso_tf import PSO
 from pso import Optimizer
 from tensorflow import keras
 from tqdm import tqdm
 
-# print(tf.__version__)
-# print(tf.config.list_physical_devices())
-# print(f"Num GPUs Available: {len(tf.config.list_physical_devices('GPU'))}")
-
 
 def get_data():
     (x_train, y_train), (x_test, y_test) = mnist.load_data()
@@ -62,13 +57,13 @@ if __name__ == "__main__":
         pso_mnist = Optimizer(
             model,
             loss=loss[0], 
-            n_particles=75,
+            n_particles=100,
-            c0=0.3, 
+            c0=0.35, 
-            c1=0.7, 
+            c1=0.8, 
-            w_min=0.6,
+            w_min=0.7,
-            w_max=0.9,
+            w_max=1.0,
-            negative_swarm=0.25,
+            negative_swarm=0.2,
-            mutation_swarm=0,
+            mutation_swarm=0.2,
             )
 
         best_score = pso_mnist.fit(

pso/__init__.py

@@ -1,5 +1,6 @@
 from .optimizer import Optimizer
-
+from .particle import Particle
 __all__ = [
-    'Optimizer'
+    'Optimizer',
+    'Particle'
 ]

pso/optimizer.py

@@ -6,11 +6,10 @@ from datetime import datetime
 
 import numpy as np
 import tensorflow as tf
-from pso.particle import Particle
 from tensorflow import keras
 from tqdm import tqdm
 
-# import cupy as cp
+from .particle import Particle
 
 gpus = tf.config.experimental.list_physical_devices("GPU")
 if gpus:
@@ -21,6 +20,10 @@ if gpus:
         print(e)
 
 class Optimizer:
+    """
+    particle swarm optimization
+    PSO 실행을 위한 클래스
+    """
 
     def __init__(
         self,
@@ -47,11 +50,12 @@ class Optimizer:
             c1 (float): global rate - 전역 최적값 관성 수치
             w_min (float): 최소 관성 수치
             w_max (float): 최대 관성 수치
-            nefative_swarm (float): 최적해와 반대로 이동할 파티클 비율 - 0 ~ 1 사이의 값
+            negative_swarm (float): 최적해와 반대로 이동할 파티클 비율 - 0 ~ 1 사이의 값
-            momentun_swarm (float): 관성을 추가로 사용할 파티클 비율 - 0 ~ 1 사이의 값
+            mutation_swarm (float): 돌연변이가 일어날 확률
         """
         np.random.seed(np_seed)
         tf.random.set_seed(tf_seed)
+        
         self.model = model  # 모델 구조
         self.loss = loss  # 손실함수
         self.n_particles = n_particles  # 파티클 개수
@@ -66,22 +70,30 @@ class Optimizer:
         self.g_best = None  # 최고 점수를 받은 가중치
         self.g_best_ = None  # 최고 점수를 받은 가중치 - 값의 분산을 위한 변수
         self.avg_score = 0  # 평균 점수
+        
+        negative_count = 0
 
         for i in tqdm(range(self.n_particles), desc="Initializing Particles"):
             m = keras.models.model_from_json(model.to_json())
             init_weights = m.get_weights()
             w_, sh_, len_ = self._encode(init_weights)
-            w_ = np.random.uniform(-0.5, 0.5, len(w_))
+            w_ = np.random.uniform(-1, 2, len(w_))
             m.set_weights(self._decode(w_, sh_, len_))
             m.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"])
             self.particles[i] = Particle(
-                m, loss, 
+                m, 
+                loss, 
                 negative=True if i < negative_swarm * self.n_particles else False, 
-                mutation=True if i > self.n_particles * (1 - self.mutation_swarm) else False
+                mutation=mutation_swarm,
                 )
+            if i < negative_swarm * self.n_particles:
+                negative_count += 1
                 
+        print(f"negative swarm : {negative_count} / {self.n_particles}")
+        print(f"mutation swarm : {mutation_swarm*100/self.n_particles} / {self.n_particles}")
+
         gc.collect()
-        
+
     def __del__(self):
         del self.model
         del self.loss
@@ -110,46 +122,44 @@ class Optimizer:
             (list) : 가중치의 원본 shape
             (list) : 가중치의 원본 shape의 길이
         """
-        # w_gpu = cp.array([])
         w_gpu = np.array([])
-        lenght = []
+        length = []
         shape = []
         for layer in weights:
             shape.append(layer.shape)
             w_ = layer.reshape(-1)
-            lenght.append(len(w_))
+            length.append(len(w_))
-            # w_gpu = cp.append(w_gpu, w_)
             w_gpu = np.append(w_gpu, w_)
 
         del weights
-        return w_gpu, shape, lenght
+        
+        return w_gpu, shape, length
 
 
-    def _decode(self, weight, shape, lenght):
+    def _decode(self, weight, shape, length):
         """
         _encode 로 인코딩된 가중치를 원본 shape으로 복원
         파라미터는 encode의 리턴값을 그대로 사용을 권장
         
         Args:
-            weight (numpy|cupy array): 가중치 - 1차원으로 풀어서 반환
+            weight (numpy array): 가중치 - 1차원으로 풀어서 반환
             shape (list): 가중치의 원본 shape
-            lenght (list): 가중치의 원본 shape의 길이
+            length (list): 가중치의 원본 shape의 길이
         Returns:
             (list) : 가중치 원본 shape으로 복원
         """
         weights = []
         start = 0
         for i in range(len(shape)):
-            end = start + lenght[i]
+            end = start + length[i]
             w_ = weight[start:end]
-            # w_ = weight[start:end].get()
             w_ = np.reshape(w_, shape[i])
-            # w_ = w_.reshape(shape[i])
             weights.append(w_)
             start = end
+            
         del weight
         del shape
-        del lenght
+        del length
 
         return weights
 
@@ -188,8 +198,8 @@ class Optimizer:
     ):
         """
         Args:
-            x_test : numpy.ndarray,
+            x_test : numpy array,
-            y_test : numpy.ndarray,
+            y_test : numpy array,
             epochs : int,
             save : bool - True : save, False : not save
             save_path : str ex) "./result",
@@ -248,6 +258,8 @@ class Optimizer:
                         f.write(", ")
                     else:
                         f.write("\n")
+                        
+                    f.close()
             del local_score
             gc.collect()
 
@@ -255,7 +267,7 @@ class Optimizer:
 
         try:
             epochs_pbar = tqdm(range(epochs), desc=f"best {self.g_best_score[0]:.4f}|{self.g_best_score[1]:.4f}", ascii=True, leave=True)
-            for _ in epochs_pbar:
+            for epoch in epochs_pbar:
                 acc = 0
                 loss = 0
                 min_score = np.inf
@@ -264,15 +276,16 @@ class Optimizer:
                 max_loss = 0
 
                 ts = self.c0 + np.random.rand() * (self.c1 - self.c0)
-                g_, g_sh, g_len = self._encode(self.g_best)
-                decrement = (epochs - (_) + 1) / epochs
-                g_ = (1 - decrement) * g_ + decrement * ts
-                self.g_best_ = self._decode(g_, g_sh, g_len)
 
                 part_pbar = tqdm(range(len(self.particles)), desc=f"acc : {max_score:.4f} loss : {min_loss:.4f}", ascii=True, leave=False)
                 for i in part_pbar:
                     part_pbar.set_description(f"acc : {max_score:.4f} loss : {min_loss:.4f}")
-                    w = self.w_max - (self.w_max - self.w_min) * _ / epochs
+                    w = self.w_max - (self.w_max - self.w_min) * epoch / epochs
+
+                    g_, g_sh, g_len = self._encode(self.g_best)
+                    decrement = (epochs - (epoch) + 1) / epochs
+                    g_ = (1 - decrement) * g_ + decrement * ts
+                    self.g_best_ = self._decode(g_, g_sh, g_len)
 
                     if Dispersion:
                         g_best = self.g_best_
@@ -280,11 +293,12 @@ class Optimizer:
                         g_best = self.g_best
 
                     if empirical_balance:
-                        if np.random.rand() < np.exp(-(_) / epochs):
+                        if np.random.rand() < np.exp(-(epoch) / epochs):
                             w_p_ = self.f(x, y, self.particles[i].get_best_weights())
                             w_g_ = self.f(x, y, self.g_best)
                             w_p = w_p_ / (w_p_ + w_g_)
                             w_g = w_p_ / (w_p_ + w_g_)
+                            
                             del w_p_
                             del w_g_
 
@@ -301,6 +315,7 @@ class Optimizer:
                             p_ = np.exp(-1 * p_)
                             w_p = p_
                             w_g = 1 - p_
+                            
                             del p_b
                             del g_a
                             del l_b
@@ -362,11 +377,12 @@ class Optimizer:
                                 f.write(", ")
                             else:
                                 f.write("\n")
+                        f.close()
 
                 if check_point is not None:
-                    if _ % check_point == 0:
+                    if epoch % check_point == 0:
                         os.makedirs(f"./{save_path}/{self.day}", exist_ok=True)
-                        self._check_point_save(f"./{save_path}/{self.day}/ckpt-{_}")
+                        self._check_point_save(f"./{save_path}/{self.day}/ckpt-{epoch}")
                 self.avg_score = acc / self.n_particles
 
                 gc.collect()
@@ -439,10 +455,12 @@ class Optimizer:
         }
 
         with open(
-            f"./{path}/{self.day}/{self.loss}_{self.n_particles}.json",
+            f"./{path}/{self.day}/{self.loss}_{self.g_best_score}.json",
             "a",
         ) as f:
             json.dump(json_save, f, indent=4)
+            
+        f.close()
 
     def _check_point_save(self, save_path: str = f"./result/check_point"):
         """

pso/particle.py

@@ -1,6 +1,5 @@
 import gc
 
-# import cupy as cp
 import numpy as np
 import tensorflow as tf
 from tensorflow import keras
@@ -9,8 +8,14 @@ from tensorflow import keras
 class Particle:
     """
     Particle Swarm Optimization의 Particle을 구현한 클래스
+    한 파티클의 life cycle은 다음과 같다.
+    1. 초기화
+    2. 손실 함수 계산
+    3. 속도 업데이트
+    4. 가중치 업데이트
+    5. 2번으로 돌아가서 반복
     """
-    def __init__(self, model: keras.models, loss, negative: bool = False, mutation: bool = False):
+    def __init__(self, model: keras.models, loss, negative: bool = False, mutation: float = 0):
         """
         Args:
             model (keras.models): 학습 및 검증을 위한 모델
@@ -21,7 +26,7 @@ class Particle:
         self.loss = loss
         init_weights = self.model.get_weights()
         i_w_, s_, l_ = self._encode(init_weights)
-        i_w_ = np.random.rand(len(i_w_)) / 2 - 0.25
+        i_w_ = np.random.uniform(-0.5, 0.5, len(i_w_))
         self.velocities = self._decode(i_w_, s_, l_)
         self.negative = negative
         self.mutation = mutation
@@ -52,44 +57,40 @@ class Particle:
             (list) : 가중치의 원본 shape
             (list) : 가중치의 원본 shape의 길이
         """
-        # w_gpu = cp.array([])
         w_gpu = np.array([])
-        lenght = []
+        length = []
         shape = []
         for layer in weights:
             shape.append(layer.shape)
             w_ = layer.reshape(-1)
-            lenght.append(len(w_))
+            length.append(len(w_))
-            # w_gpu = cp.append(w_gpu, w_)
             w_gpu = np.append(w_gpu, w_)
 
-        return w_gpu, shape, lenght
+        return w_gpu, shape, length
 
 
-    def _decode(self, weight: list, shape, lenght):
+    def _decode(self, weight: list, shape, length):
         """
         _encode 로 인코딩된 가중치를 원본 shape으로 복원
         파라미터는 encode의 리턴값을 그대로 사용을 권장
         
         Args:
-            weight (numpy|cupy array): 가중치 - 1차원으로 풀어서 반환
+            weight (numpy array): 가중치 - 1차원으로 풀어서 반환
             shape (list): 가중치의 원본 shape
-            lenght (list): 가중치의 원본 shape의 길이
+            length (list): 가중치의 원본 shape의 길이
         Returns:
             (list) : 가중치 원본 shape으로 복원
         """
         weights = []
         start = 0
         for i in range(len(shape)):
-            end = start + lenght[i]
+            end = start + length[i]
             w_ = weight[start:end]
-            # w_ = weight[start:end].get()
             w_ = np.reshape(w_, shape[i])
-            # w_ = w_.reshape(shape[i])
             weights.append(w_)
             start = end
         del start, end, w_
-        del shape, lenght
+        del shape, length
         del weight
 
         return weights
@@ -104,11 +105,10 @@ class Particle:
             renewal (str, optional): 점수 갱신 방식. Defaults to "acc" | "acc" or "loss".
 
         Returns:
-            _type_: _description_
+            (float): 점수
         """
         self.model.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"])
         score = self.model.evaluate(x, y, verbose=0)
-        # print(score)
         if renewal == "acc":
             if score[1] > self.best_score:
                 self.best_score = score[1]
@@ -127,7 +127,7 @@ class Particle:
         현재 속도 업데이트
 
         Args:
-            local_rate (flost): 지역 최적해의 영향력
+            local_rate (float): 지역 최적해의 영향력
             global_rate (float): 전역 최적해의 영향력
             w (float): 현재 속도의 영향력 - 관성 | 0.9 ~ 0.4 이 적당
             g_best (list): 전역 최적해
@@ -150,9 +150,11 @@ class Particle:
                 + local_rate * r0 * (encode_p - encode_w)
                 + global_rate * r1 * (encode_g - encode_w)
             )
-        if self.mutation:
+        if np.random.rand() < self.mutation:
+            
             new_v += 0.5 * encode_v
         self.velocities = self._decode(new_v, w_sh, w_len)
+        
         del encode_w, w_sh, w_len
         del encode_v, v_sh, v_len
         del encode_p, p_sh, p_len
@@ -178,6 +180,7 @@ class Particle:
         encode_g, g_sh, g_len = self._encode(weights=g_best)
         r0 = np.random.rand()
         r1 = np.random.rand()
+        
         if self.negative:
             new_v = (
                 w * encode_v
@@ -193,6 +196,7 @@ class Particle:
         if self.mutation:
             new_v += 0.5 * encode_v
         self.velocities = self._decode(new_v, w_sh, w_len)
+        
         del encode_w, w_sh, w_len
         del encode_v, v_sh, v_len
         del encode_p, p_sh, p_len
@@ -207,6 +211,7 @@ class Particle:
         encode_v, v_sh, v_len = self._encode(weights=self.velocities)
         new_w = encode_w + encode_v
         self.model.set_weights(self._decode(new_w, w_sh, w_len))
+        
         del encode_w, w_sh, w_len
         del encode_v, v_sh, v_len
 
@@ -220,10 +225,11 @@ class Particle:
             weights (list): 가중치
 
         Returns:
-            flost: 목적함수 값
+            float: 목적함수 값
         """
         self.model.set_weights(weights)
         score = self.model.evaluate(x, y, verbose=0)[1]
+        
         if score > 0:
             return 1 / (1 + score)
         else:
@@ -247,6 +253,7 @@ class Particle:
         """
         self._update_velocity(local_rate, global_rate, w, g_best)
         self._update_weights()
+        
         return self.get_score(x, y, renewal)
 
     def step_w(
@@ -272,6 +279,7 @@ class Particle:
         """
         self._update_velocity_w(local_rate, global_rate, w, w_p, w_g, g_best)
         self._update_weights()
+        
         return self.get_score(x, y, renewal)
 
     def get_best_score(self):