23-06-09

자동 튜닝을 위한 스크립트 추가
메모리 관리를 위해 소멸자 추가

auto_tunning.py

@@ -0,0 +1,126 @@
+# %%
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+
+import tensorflow as tf
+tf.random.set_seed(777)  # for reproducibility
+
+from tensorflow import keras
+from keras.datasets import mnist
+from keras.models import Sequential
+from keras.layers import Dense, Dropout, Flatten
+from keras.layers import Conv2D, MaxPooling2D
+from keras import backend as K
+
+# from pso_tf import PSO
+from pso import Optimizer
+# from optimizer import Optimizer
+
+import numpy as np
+
+from datetime import date
+from tqdm import tqdm
+
+import gc
+
+# print(tf.__version__)
+# print(tf.config.list_physical_devices())
+# print(f"Num GPUs Available: {len(tf.config.list_physical_devices('GPU'))}")
+
+
+def get_data():
+    (x_train, y_train), (x_test, y_test) = mnist.load_data()
+
+    x_train, x_test = x_train / 255.0, x_test / 255.0
+    x_train = x_train.reshape((60000, 28, 28, 1))
+    x_test = x_test.reshape((10000, 28, 28, 1))
+
+    print(f"x_train : {x_train[0].shape} | y_train : {y_train[0].shape}")
+    print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}")
+    return x_train, y_train, x_test, y_test
+
+def get_data_test():
+    (x_train, y_train), (x_test, y_test) = mnist.load_data()
+    x_test = x_test.reshape((10000, 28, 28, 1))
+    
+    return x_test, y_test
+
+def make_model():
+    model = Sequential()
+    model.add(Conv2D(32, kernel_size=(5, 5),
+              activation='relu', input_shape=(28, 28, 1)))
+    model.add(MaxPooling2D(pool_size=(3, 3)))
+    model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
+    model.add(MaxPooling2D(pool_size=(2, 2)))
+    model.add(Dropout(0.25))
+    model.add(Flatten())
+    model.add(Dense(128, activation='relu'))
+    model.add(Dense(10, activation='softmax'))
+
+    return model
+
+# %%
+model = make_model()
+x_test, y_test = get_data_test()
+# loss = 'binary_crossentropy'
+# loss = 'categorical_crossentropy'
+# loss = 'sparse_categorical_crossentropy'
+# loss = 'kullback_leibler_divergence'
+# loss = 'poisson'
+# loss = 'cosine_similarity'
+# loss = 'log_cosh'
+# loss = 'huber_loss' 
+# loss = 'mean_absolute_error'
+# loss = 'mean_absolute_percentage_error'
+# loss = 'mean_squared_error'
+
+loss = ['mse', 'categorical_crossentropy', 'binary_crossentropy', 'kullback_leibler_divergence', 'poisson', 'cosine_similarity', 'log_cosh', 'huber_loss', 'mean_absolute_error', 'mean_absolute_percentage_error']
+n_particles = [50, 75, 100]
+c0 = [0.25, 0.35, 0.45, 0.55]
+c1 = [0.5, 0.6, 0.7, 0.8, 0.9]
+w_min = [0.5, 0.6, 0.7]
+w_max = [1.1, 1.2, 1.3]
+negative_swarm = [0.25, 0.3, 0.5]
+eb = [True, False]
+dispersion = [True, False]
+
+if __name__ == "__main__":
+    try:
+        for loss_ in loss:
+            for n in n_particles:
+                for c_0 in c0:
+                    for c_1 in c1:
+                        for w_m in w_min:
+                            for w_M in w_max:
+                                for n_s in negative_swarm:
+                                    pso_mnist = Optimizer(
+                                        model,
+                                        loss=loss_, 
+                                        n_particles=n,
+                                        c0=c_0, 
+                                        c1=c_1, 
+                                        w_min=w_m,
+                                        w_max=w_M,
+                                        negative_swarm=n_s
+                                        )
+
+                                    best_score = pso_mnist.fit(
+                                        x_test,
+                                        y_test,
+                                        epochs=200,
+                                        save=True,
+                                        save_path="./result/mnist", 
+                                        renewal="acc", 
+                                        empirical_balance=False,
+                                        Dispersion=False, 
+                                        check_point=25
+                                        )
+                                    
+                                    del pso_mnist
+                                    gc.collect() 
+                                    tf.keras.backend.clear_session()
+                                    
+    except KeyboardInterrupt:
+        print("KeyboardInterrupt")
+    finally:
+        print("Finish")

mnist.py

@@ -62,30 +62,23 @@ def make_model():
 # %%
 model = make_model()
 x_test, y_test = get_data_test()
-# loss = 'binary_crossentropy'
-# loss = 'categorical_crossentropy'
-# loss = 'sparse_categorical_crossentropy'
-# loss = 'kullback_leibler_divergence'
-# loss = 'poisson'
-# loss = 'cosine_similarity'
-# loss = 'log_cosh'
-# loss = 'huber_loss' 
-# loss = 'mean_absolute_error'
-# loss = 'mean_absolute_percentage_error'
-loss = 'mean_squared_error'
 
-pso_mnist = Optimizer(
+loss = ['mse', 'categorical_crossentropy', 'binary_crossentropy', 'kullback_leibler_divergence', 'poisson', 'cosine_similarity', 'log_cosh', 'huber_loss', 'mean_absolute_error', 'mean_absolute_percentage_error']
+
+if __name__ == "__main__":
+    try:
+        pso_mnist = Optimizer(
             model,
-    loss=loss, 
+            loss=loss[0], 
-    n_particles=50,
+            n_particles=200,
             c0=0.35, 
             c1=0.8, 
             w_min=0.7,
-    w_max=1.0,
+            w_max=1.15,
             negative_swarm=0.25
             )
 
-best_score = pso_mnist.fit(
+        best_score = pso_mnist.fit(
             x_test,
             y_test,
             epochs=200,
@@ -96,5 +89,8 @@ best_score = pso_mnist.fit(
             Dispersion=False, 
             check_point=25
             )
+    except Exception as e:
+        print(e)
+
 # pso_mnist.model_save("./result/mnist")
 # pso_mnist.save_info("./result/mnist")

pso/optimizer.py

@@ -56,7 +56,7 @@ class Optimizer:
         self.c1 = c1  # global rate - 전역 최적값 관성 수치
         self.w_min = w_min  # 최소 관성 수치
         self.w_max = w_max  # 최대 관성 수치
-
+        self.negative_swarm = negative_swarm  # 최적해와 반대로 이동할 파티클 비율 - 0 ~ 1 사이의 값
         self.g_best_score = 0  # 최고 점수 - 시작은 0으로 초기화
         self.g_best = None  # 최고 점수를 받은 가중치
         self.g_best_ = None  # 최고 점수를 받은 가중치 - 값의 분산을 위한 변수
@@ -75,6 +75,22 @@ class Optimizer:
                 self.particles[i] = Particle(m, loss, negative=False)
         gc.collect()
         
+    def __del__(self):
+        del self.model
+        del self.loss
+        del self.n_particles
+        del self.particles
+        del self.c0
+        del self.c1
+        del self.w_min
+        del self.w_max
+        del self.negative_swarm
+        del self.g_best_score
+        del self.g_best
+        del self.g_best_
+        del self.avg_score
+        gc.collect()
+
     """
     Args:
         weights (list) : keras model의 가중치
@@ -160,6 +176,8 @@ class Optimizer:
         check_point: int = None,
     ):
         self.save_path = save_path
+        self.empirical_balance = empirical_balance
+        self.Dispersion = Dispersion
 
         self.renewal = renewal
         if renewal == "acc":
@@ -180,7 +198,7 @@ class Optimizer:
             print(e)
             sys.exit(1)
             
-        for i in tqdm(range(self.n_particles), desc="Initializing Particles"):
+        for i in tqdm(range(self.n_particles), desc="Initializing velocity"):
             p = self.particles[i]
             local_score = p.get_score(x, y, renewal=renewal)
 
@@ -364,6 +382,9 @@ class Optimizer:
             "w_min": self.w_min,
             "w_max": self.w_max,
             "loss_method": self.loss,
+            "empirical_balance": self.empirical_balance,
+            "Dispersion": self.Dispersion,
+            "negative_swarm": self.negative_swarm,
             "renewal": self.renewal,
         }
 

pso/particle.py

@@ -22,6 +22,15 @@ class Particle:
         del init_weights
         gc.collect()
         
+    def __del__(self):
+        del self.model
+        del self.loss
+        del self.velocities
+        del self.negative
+        del self.best_score
+        del self.best_weights
+        gc.collect()
+
     """
     Returns:
         (cupy array) : 가중치 - 1차원으로 풀어서 반환