23-06-01

함수 실행마다 사용안하는 변수 delete 및 gc.collect() 를 실행하여 메모리 문제 해결을 위해 변경

metacode/pso_meta.py

@@ -16,7 +16,7 @@ class PSO(object):
         """
         self.func = func
         self.n_particles = n_particles
-        self.init_pos = init_pos           # 곰샥헐 차원
+        self.init_pos = init_pos           # 검색할 차원
         self.particle_dim = len(init_pos)  # 검색할 차원의 크기
         self.particles_pos = np.random.uniform(size=(n_particles, self.particle_dim)) \
             * self.init_pos

mnist.py

@@ -79,9 +79,9 @@ loss = 'huber_loss'
 # loss = 'mean_squared_error'
 
 
-pso_mnist = Optimizer(model, loss=loss, n_particles=75, c0=0.4, c1=0.8, w_min=0.6, w_max=0.95, random=0.3)
+pso_mnist = Optimizer(model, loss=loss, n_particles=50, c0=0.4, c1=0.8, w_min=0.4, w_max=0.95, negative_swarm=0.3)
 weight, score = pso_mnist.fit(
-    x_test, y_test, epochs=500, save=True, save_path="./result/mnist", renewal="acc", empirical_balance=False, Dispersion=False, check_point=10)
+    x_test, y_test, epochs=500, save=True, save_path="./result/mnist", renewal="acc", empirical_balance=True, Dispersion=False, check_point=10)
 # pso_mnist.model_save("./result/mnist")
 # pso_mnist.save_info("./result/mnist")
 

pso/optimizer.py

@@ -12,11 +12,11 @@ from tqdm import tqdm
 from datetime import datetime
 import json
 import gc
+from copy import copy, deepcopy
 
 from pso.particle import Particle
 
 
-
 class Optimizer:
     """
     Args:
@@ -27,8 +27,9 @@ class Optimizer:
         c1 (float): global rate - 전역 최적값 관성 수치
         w_min (float): 최소 관성 수치
         w_max (float): 최대 관성 수치
-        random (float): 랜덤 파티클 비율 - 0 ~ 1 사이의 값
+        nefative_swarm (float): 최적해와 반대로 이동할 파티클 비율 - 0 ~ 1 사이의 값
     """
+
     def __init__(
         self,
         model: keras.models,
@@ -38,7 +39,7 @@ class Optimizer:
         c1=1.5,
         w_min=0.5,
         w_max=1.5,
-        random:float = 0,
+        negative_swarm: float = 0,
     ):
         self.model = model  # 모델 구조
         self.loss = loss  # 손실함수
@@ -61,10 +62,11 @@ class Optimizer:
             w_ = np.random.uniform(-1.5, 1.5, len(w_))
             m.set_weights(self._decode(w_, sh_, len_))
             m.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"])
-            if i < random * self.n_particles:
+            if i < negative_swarm * self.n_particles:
-                self.particles[i] = Particle(m, loss, random=True)
+                self.particles[i] = Particle(m, loss, negative=True)
             else:
-                self.particles[i] = Particle(m, loss, random=False)
+                self.particles[i] = Particle(m, loss, negative=False)
+        gc.collect()
 
     """
     Args:
@@ -74,6 +76,7 @@ class Optimizer:
         (list) : 가중치의 원본 shape
         (list) : 가중치의 원본 shape의 길이
     """
+
     def _encode(self, weights):
         # w_gpu = cp.array([])
         w_gpu = np.array([])
@@ -86,6 +89,8 @@ class Optimizer:
             # w_gpu = cp.append(w_gpu, w_)
             w_gpu = np.append(w_gpu, w_)
 
+        del weights
+        gc.collect()
         return w_gpu, shape, lenght
 
     """
@@ -119,6 +124,8 @@ class Optimizer:
         self.model.set_weights(weights)
         self.model.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"])
         score = self.model.evaluate(x, y, verbose=0)[1]
+
+        gc.collect()
         if score > 0:
             return 1 / (1 + score)
         else:
@@ -136,6 +143,7 @@ class Optimizer:
         Dispersion : bool - True : g_best 의 값을 분산시켜 전역해를 찾음, False : g_best 의 값만 사용
         check_point : int - 저장할 위치 - None : 저장 안함
     """
+
     def fit(
         self,
         x,
@@ -155,18 +163,21 @@ class Optimizer:
             self.g_best_score = 0
         elif renewal == "loss":
             self.g_best_score = np.inf
-
+        try:
             if save:
                 if save_path is None:
                     raise ValueError("save_path is None")
                 else:
                     self.save_path = save_path
+                    if not os.path.exists(save_path):
                         os.makedirs(save_path, exist_ok=True)
                     self.day = datetime.now().strftime("%m-%d-%H-%M")
-
+        except ValueError as e:
+            print(e)
+            sys.exit(1)
         # for i, p in enumerate(self.particles):
         for i in tqdm(range(self.n_particles), desc="Initializing Particles"):
-            p = self.particles[i]
+            p = copy(self.particles[i])
             local_score = p.get_score(x, y, renewal=renewal)
 
             if renewal == "acc":
@@ -179,6 +190,9 @@ class Optimizer:
                     self.g_best_score = local_score[0]
                     self.g_best = p.get_best_weights()
                     self.g_best_ = p.get_best_weights()
+            del local_score
+            del p
+            gc.collect()
 
         print(f"initial g_best_score : {self.g_best_score}")
 
@@ -192,6 +206,12 @@ class Optimizer:
                 min_loss = np.inf
                 max_loss = 0
 
+                ts = self.c0 + np.random.rand() * (self.c1 - self.c0)
+                g_, g_sh, g_len = self._encode(self.g_best)
+                decrement = (epochs - (_) + 1) / epochs
+                g_ = (1 - decrement) * g_ + decrement * ts
+                self.g_best_ = self._decode(g_, g_sh, g_len)
+
                 # for i in tqdm(range(len(self.particles)), desc=f"epoch {_ + 1}/{epochs}", ascii=True):
                 for i in range(len(self.particles)):
                     w = self.w_max - (self.w_max - self.w_min) * _ / epochs
@@ -215,10 +235,18 @@ class Optimizer:
                             g_a = self.avg_score
                             l_b = p_b - g_a
                             l_b = np.sqrt(np.power(l_b, 2))
-                            p_ = 1 / (self.n_particles * np.linalg.norm(self.c1 - self.c0)) * l_b
+                            p_ = (
+                                1
+                                / (self.n_particles * np.linalg.norm(self.c1 - self.c0))
+                                * l_b
+                            )
                             p_ = np.exp(-1 * p_)
                             w_p = p_
                             w_g = 1 - p_
+                            del p_b
+                            del g_a
+                            del l_b
+                            del p_
 
                         score = self.particles[i].step_w(
                             x, y, self.c0, self.c1, w, g_best, w_p, w_g, renewal=renewal
@@ -238,8 +266,8 @@ class Optimizer:
                             self.g_best_score = score[0]
                             self.g_best = self.particles[i].get_best_weights()
 
-                    loss += score[0]
+                    loss = loss + score[0]
-                    acc += score[1]
+                    acc = acc + score[1]
                     if score[0] < min_loss:
                         min_loss = score[0]
                     if score[0] > max_loss:
@@ -258,18 +286,7 @@ class Optimizer:
                             f.write(f"{score[0]}, {score[1]}")
                             if i != self.n_particles - 1:
                                 f.write(", ")
-                                
+                            else:
-                TS = self.c0 + np.random.rand() * (self.c1 - self.c0)
-                g_, g_sh, g_len = self._encode(self.g_best)
-                decrement = (epochs - (_) + 1) / epochs
-                g_ = (1 - decrement) * g_ + decrement * TS
-                self.g_best_ = self._decode(g_, g_sh, g_len)
-
-                if save:
-                    with open(
-                        f"./{save_path}/{self.day}_{self.n_particles}_{epochs}_{self.c0}_{self.c1}_{self.w_min}_{renewal}.csv",
-                        "a",
-                    ) as f:
                                 f.write("\n")
 
                 # print(f"loss min : {min_loss} | loss max : {max_loss} | acc min : {min_score} | acc max : {max_score}")
@@ -285,6 +302,7 @@ class Optimizer:
                         os.makedirs(f"./{save_path}/{self.day}", exist_ok=True)
                         self._check_point_save(f"./{save_path}/{self.day}/ckpt-{_}")
                 self.avg_score = acc / self.n_particles
+
         except KeyboardInterrupt:
             print("Ctrl + C : Stop Training")
         except MemoryError:
@@ -296,8 +314,8 @@ class Optimizer:
             print("model save")
             self.save_info(save_path)
             print("save info")
-            return self.g_best, self.g_best_score
 
+            return self.g_best, self.g_best_score
 
     def get_best_model(self):
         model = keras.models.model_from_json(self.model.to_json())
@@ -329,7 +347,6 @@ class Optimizer:
             "a",
         ) as f:
             json.dump(json_save, f, indent=4)
-            f.write(",\n")
 
     def _check_point_save(self, save_path: str = f"./result/check_point"):
         model = self.get_best_model()

pso/particle.py

@@ -1,21 +1,26 @@
-
 import tensorflow as tf
 from tensorflow import keras
 
 # import cupy as cp
 import numpy as np
+import gc
+
 
 class Particle:
-    def __init__(self, model:keras.models, loss, random:bool = False):
+    def __init__(self, model: keras.models, loss, negative: bool = False):
         self.model = model
         self.loss = loss
-        self.init_weights = self.model.get_weights()
+        init_weights = self.model.get_weights()
-        i_w_,s_,l_ = self._encode(self.init_weights)
+        i_w_, s_, l_ = self._encode(init_weights)
         i_w_ = np.random.rand(len(i_w_)) / 5 - 0.10
         self.velocities = self._decode(i_w_, s_, l_)
-        self.random = random
+        self.negative = negative
         self.best_score = 0
-        self.best_weights = self.init_weights
+        self.best_weights = init_weights
+
+        del i_w_, s_, l_
+        del init_weights
+        gc.collect()
 
     """
     Returns:
@@ -23,6 +28,7 @@ class Particle:
         (list) : 가중치의 원본 shape
         (list) : 가중치의 원본 shape의 길이
     """
+
     def _encode(self, weights: list):
         # w_gpu = cp.array([])
         w_gpu = np.array([])
@@ -34,6 +40,7 @@ class Particle:
             lenght.append(len(w_))
             # w_gpu = cp.append(w_gpu, w_)
             w_gpu = np.append(w_gpu, w_)
+        gc.collect()
         return w_gpu, shape, lenght
 
     """
@@ -52,7 +59,10 @@ class Particle:
             # w_ = w_.reshape(shape[i])
             weights.append(w_)
             start = end
-
+        del start, end, w_
+        del shape, lenght
+        del weight
+        gc.collect()
         return weights
 
     def get_score(self, x, y, renewal: str = "acc"):
@@ -67,39 +77,76 @@ class Particle:
             if score[0] < self.best_score:
                 self.best_score = score[0]
                 self.best_weights = self.model.get_weights()
-            
+        gc.collect()
         return score
+
     def _update_velocity(self, local_rate, global_rate, w, g_best):
         encode_w, w_sh, w_len = self._encode(weights=self.model.get_weights())
-        encode_v, _, _ = self._encode(weights = self.velocities)
+        encode_v, v_sh, v_len = self._encode(weights=self.velocities)
-        encode_p, _, _ = self._encode(weights = self.best_weights)
+        encode_p, p_sh, p_len = self._encode(weights=self.best_weights)
-        encode_g, _, _ = self._encode(weights = g_best)
+        encode_g, g_sh, g_len = self._encode(weights=g_best)
         r0 = np.random.rand()
         r1 = np.random.rand()
-        new_v = w * encode_v + local_rate * r0 * (encode_p - encode_w) + global_rate * r1 * (encode_g - encode_w)
+        if self.negative:
+            new_v = (
+                w * encode_v
+                + -1 * local_rate * r0 * (encode_p - encode_w)
+                + -1 * global_rate * r1 * (encode_g - encode_w)
+            )
+        else:
+            new_v = (
+                w * encode_v
+                + local_rate * r0 * (encode_p - encode_w)
+                + global_rate * r1 * (encode_g - encode_w)
+            )
         self.velocities = self._decode(new_v, w_sh, w_len)
+        del encode_w, w_sh, w_len
+        del encode_v, v_sh, v_len
+        del encode_p, p_sh, p_len
+        del encode_g, g_sh, g_len
+        del r0, r1
+        gc.collect()
 
     def _update_velocity_w(self, local_rate, global_rate, w, w_p, w_g, g_best):
         encode_w, w_sh, w_len = self._encode(weights=self.model.get_weights())
-        encode_v, _, _ = self._encode(weights = self.velocities)
+        encode_v, v_sh, v_len = self._encode(weights=self.velocities)
-        encode_p, _, _ = self._encode(weights = self.best_weights)
+        encode_p, p_sh, p_len = self._encode(weights=self.best_weights)
-        encode_g, _, _ = self._encode(weights = g_best)
+        encode_g, g_sh, g_len = self._encode(weights=g_best)
         r0 = np.random.rand()
         r1 = np.random.rand()
-        new_v = w * encode_v + local_rate * r0 * (w_p * encode_p - encode_w) + global_rate * r1 * (w_g * encode_g - encode_w)
+        if self.negative:
+            new_v = (
+                w * encode_v
+                + -1 * local_rate * r0 * (w_p * encode_p - encode_w)
+                + -1 * global_rate * r1 * (w_g * encode_g - encode_w)
+            )
+        else:
+            new_v = (
+                w * encode_v
+                + local_rate * r0 * (w_p * encode_p - encode_w)
+                + global_rate * r1 * (w_g * encode_g - encode_w)
+            )
         self.velocities = self._decode(new_v, w_sh, w_len)
+        del encode_w, w_sh, w_len
+        del encode_v, v_sh, v_len
+        del encode_p, p_sh, p_len
+        del encode_g, g_sh, g_len
+        del r0, r1
+        gc.collect()
 
     def _update_weights(self):
         encode_w, w_sh, w_len = self._encode(weights=self.model.get_weights())
-        encode_v, _, _ = self._encode(weights = self.velocities)
+        encode_v, v_sh, v_len = self._encode(weights=self.velocities)
-        if self.random:
-            encode_v = -0.5 * encode_v
         new_w = encode_w + encode_v
         self.model.set_weights(self._decode(new_w, w_sh, w_len))
+        del encode_w, w_sh, w_len
+        del encode_v, v_sh, v_len
+        gc.collect()
 
     def f(self, x, y, weights):
         self.model.set_weights(weights)
         score = self.model.evaluate(x, y, verbose=0)[1]
+        gc.collect()
         if score > 0:
             return 1 / (1 + score)
         else:
@@ -108,11 +155,15 @@ class Particle:
     def step(self, x, y, local_rate, global_rate, w, g_best, renewal: str = "acc"):
         self._update_velocity(local_rate, global_rate, w, g_best)
         self._update_weights()
+        gc.collect()
         return self.get_score(x, y, renewal)
 
-    def step_w(self, x, y, local_rate, global_rate, w, g_best, w_p, w_g, renewal:str = "acc"):     
+    def step_w(
+        self, x, y, local_rate, global_rate, w, g_best, w_p, w_g, renewal: str = "acc"
+    ):
         self._update_velocity_w(local_rate, global_rate, w, w_p, w_g, g_best)
         self._update_weights()
+        gc.collect()
         return self.get_score(x, y, renewal)
 
     def get_best_score(self):

readme.md

@@ -72,7 +72,7 @@ pso 알고리즘을 이용하여 오차역전파 함수를 최적화 하는 방
 
 <br>
 위의 아이디어는 원래의 목표와 다른 방향으로 가고 있습니다. 따라서 다른 방법을 모색해야할 것 같습니다
-<br><br>
+<br>
 
 ### Trouble Shooting
 
@@ -89,5 +89,11 @@ pso 알고리즘을 이용하여 오차역전파 함수를 최적화 하는 방
 >
 > > pso 와 random forest 방식이 매우 유사하다고 생각하여 학습할 때 뿐만 아니라 예측 할 때도 이러한 방식으로 사용할 수 있을 것 같습니다
 
-이곳의 코드를 참고하여 좀더 효율적인 코드로 수정하였습니다
+# 참고 자료
-> <https://github.com/mike-holcomb/PSOkeras>
+
+> A partilce swarm optimization algorithm with empirical balance stategy - <https://www.sciencedirect.com/science/article/pii/S2590054422000185#bib0005> <br>
+> psokeras - <https://github.com/mike-holcomb/PSOkeras> <br>
+> PSO의 다양한 영역 탐색과
+지역적 미니멈 인식을 위한 전략 - <https://koreascience.kr/article/JAKO200925836515680.pdf> <br>
+> PC 클러스터 기반의 Multi-HPSO를 이용한 안전도 제약의 경제 급전 - <https://koreascience.kr/article/JAKO200932056732373.pdf> <br>
+> Particle 2-Swarm Optimization for Robust Search - <https://s-space.snu.ac.kr/bitstream/10371/29949/3/management_information_v18_01_p01.pdf> <br>