23-11-05

Update PSO and neural network parameters
best score 초기화 를 무작위 값에서 계산 후 설정으로 변경

digits.py

@@ -19,7 +19,7 @@ os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
 def make_model():
     model = Sequential()
     model.add(Dense(12, input_dim=64, activation="relu"))
-    model.add(Dense(8, activation="relu"))
+    model.add(Dense(10, activation="relu"))
     model.add(Dense(10, activation="softmax"))
 
     return model
@@ -55,7 +55,7 @@ digits_pso = optimizer(
     mutation_swarm=0.1,
     convergence_reset=True,
     convergence_reset_patience=10,
-    convergence_reset_monitor="acc",
+    convergence_reset_monitor="loss",
     convergence_reset_min_delta=0.001,
 )
 
@@ -66,7 +66,7 @@ digits_pso.fit(
     validate_data=(x_test, y_test),
     log=2,
     save_info=True,
-    renewal="acc",
+    renewal="loss",
     log_name="digits",
 )
 

digits_tf.py

@@ -0,0 +1,81 @@
+import os
+import sys
+
+import pandas as pd
+import tensorflow as tf
+from sklearn.datasets import load_digits
+from sklearn.model_selection import train_test_split
+from tensorflow import keras
+from tensorflow.keras import layers
+from tensorflow.keras.layers import Dense
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.utils import to_categorical
+
+
+gpus = tf.config.experimental.list_physical_devices("GPU")
+if gpus:
+    try:
+        tf.config.experimental.set_memory_growth(gpus[0], True)
+    except RuntimeError as r:
+        print(r)
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
+
+
+def make_model():
+    model = Sequential()
+    model.add(Dense(12, input_dim=64, activation="relu"))
+    model.add(Dense(12, activation="relu"))
+    model.add(Dense(10, activation="softmax"))
+
+    return model
+
+
+def get_data():
+    digits = load_digits()
+    X = digits.data
+    y = digits.target
+
+    x = X.astype("float32")
+
+    y_class = to_categorical(y)
+
+    x_train, x_test, y_train, y_test = train_test_split(
+        x, y_class, test_size=0.2, random_state=42, shuffle=True
+    )
+    return x_train, x_test, y_train, y_test
+
+
+if __name__ == "__main__":
+    model = make_model()
+    x_train, x_test, y_train, y_test = get_data()
+
+    callbacks = [
+        tf.keras.callbacks.EarlyStopping(
+            monitor="val_loss", patience=10, restore_best_weights=True
+        )
+    ]
+
+    print(x_train.shape, y_train.shape)
+
+    model.compile(
+        optimizer="adam",
+        loss="categorical_crossentropy",
+        metrics=["accuracy", "mse"],
+    )
+
+    print(model.summary())
+
+    history = model.fit(
+        x_train,
+        y_train,
+        epochs=500,
+        batch_size=32,
+        verbose=1,
+        validation_data=(x_test, y_test),
+        callbacks=callbacks,
+    )
+
+    print("Done!")
+
+    sys.exit(0)

iris.py

@@ -44,13 +44,13 @@ pso_iris = optimizer(
     n_particles=100,
     c0=0.5,
     c1=0.3,
-    w_min=0.2,
+    w_min=0.1,
     w_max=0.9,
     negative_swarm=0,
     mutation_swarm=0.1,
     convergence_reset=True,
     convergence_reset_patience=10,
-    convergence_reset_monitor="mse",
+    convergence_reset_monitor="loss",
     convergence_reset_min_delta=0.001,
 )
 
@@ -61,7 +61,7 @@ best_score = pso_iris.fit(
     save_info=True,
     log=2,
     log_name="iris",
-    renewal="mse",
+    renewal="loss",
     check_point=25,
     validate_data=(x_test, y_test),
 )

mnist.py

@@ -65,6 +65,7 @@ loss = [
     "huber_loss",
     "mean_absolute_error",
     "mean_absolute_percentage_error",
+
 ]
 
 # rs = random_state()
@@ -75,13 +76,13 @@ pso_mnist = optimizer(
     n_particles=500,
     c0=0.5,
     c1=0.3,
-    w_min=0.2,
+    w_min=0.1,
     w_max=0.9,
     negative_swarm=0.0,
     mutation_swarm=0.1,
     convergence_reset=True,
     convergence_reset_patience=10,
-    convergence_reset_monitor="mse",
+    convergence_reset_monitor="loss",
     convergence_reset_min_delta=0.005,
 )
 
@@ -92,7 +93,7 @@ best_score = pso_mnist.fit(
     save_info=True,
     log=2,
     log_name="mnist",
-    renewal="mse",
+    renewal="loss",
     check_point=25,
     empirical_balance=False,
     dispersion=False,

pso/__init__.py

@@ -1,7 +1,7 @@
 from .optimizer import Optimizer as optimizer
 from .particle import Particle as particle
 
-__version__ = "1.0.4"
+__version__ = "1.0.5"
 
 print("pso2keras version : " + __version__)
 

pso/optimizer.py

@@ -152,7 +152,6 @@ class Optimizer:
                 tf.keras.backend.reset_uids()
                 tf.keras.backend.clear_session()
 
-            self.particles[0].update_global_best()
             print(f"negative swarm : {negative_count} / {n_particles}")
             print(f"mutation swarm : {mutation_swarm * 100}%")
 
@@ -449,6 +448,15 @@ class Optimizer:
 
         dataset = self.batch_generator(x, y, batch_size=batch_size)
 
+        for i in tqdm(
+            range(len(self.particles)),
+            desc="best score init",
+            ascii=True,
+            leave=True,
+        ):
+            score = self.particles[i].get_score(x, y, self.renewal)
+            self.particles[i].check_global_best(self.renewal)
+
         try:
             epoch_sum = 0
             epochs_pbar = tqdm(
@@ -477,7 +485,8 @@ class Optimizer:
                     position=1,
                 )
 
-                w = self.w_max - (self.w_max - self.w_min) * epoch / epochs
+                # w = self.w_max - (self.w_max - self.w_min) * epoch / epochs
+                w = self.w_max - (self.w_max - self.w_min) * (epoch % 100) / 100
                 for i in part_pbar:
                     part_pbar.set_description(
                         f"loss: {min_loss:.4f} acc: {max_acc:.4f} mse: {min_mse:.4f}"

pso/particle.py

@@ -12,6 +12,7 @@ class Particle:
     4. 가중치 업데이트
     5. 2번으로 돌아가서 반복
     """
+
     g_best_score = [np.inf, 0, np.inf]
     g_best_weights = None
     count = 0
@@ -40,7 +41,12 @@ class Particle:
         self.loss = loss
 
         try:
-            if converge_reset and converge_reset_monitor not in ["acc", "accuracy", "loss", "mse"]:
+            if converge_reset and converge_reset_monitor not in [
+                "acc",
+                "accuracy",
+                "loss",
+                "mse",
+            ]:
                 raise ValueError(
                     "converge_reset_monitor must be 'acc' or 'accuracy' or 'loss'"
                 )
@@ -154,7 +160,13 @@ class Particle:
 
         return score
 
-    def __check_converge_reset(self, score, monitor: str = None, patience: int = 10, min_delta: float = 0.0001):
+    def __check_converge_reset(
+        self,
+        score,
+        monitor: str = None,
+        patience: int = 10,
+        min_delta: float = 0.0001,
+    ):
         """
         early stop을 구현한 함수
 
@@ -173,8 +185,7 @@ class Particle:
         elif monitor in ["mse"]:
             self.score_history.append(score[2])
         else:
-            raise ValueError(
+            raise ValueError("monitor must be 'acc' or 'accuracy' or 'loss' or 'mse'")
-                "monitor must be 'acc' or 'accuracy' or 'loss' or 'mse'")
 
         if len(self.score_history) > patience:
             last_scores = self.score_history[-patience:]
@@ -187,10 +198,10 @@ class Particle:
         self.model.compile(
             optimizer="adam",
             loss=self.loss,
-            metrics=["accuracy", "mse"]
+            metrics=["accuracy", "mse"],
         )
         i_w_, i_s, i_l = self._encode(self.model.get_weights())
-        i_w_ = np.random.uniform(-0.05, 0.1, len(i_w_))
+        i_w_ = np.random.uniform(-0.1, 0.1, len(i_w_))
         self.velocities = self._decode(i_w_, i_s, i_l)
 
         del i_w_, i_s, i_l
@@ -228,10 +239,13 @@ class Particle:
             # 지역 최적해와 전역 최적해를 음수로 사용하여 전역 탐색을 유도
             new_v = (
                 w * encode_v
-                - local_rate * r_0 * (encode_p - encode_w)
+                + local_rate * r_0 * (encode_p - encode_w)
                 - global_rate * r_1 * (encode_g - encode_w)
             )
-            if len(self.score_history) > 10 and max(self.score_history[-10:]) - min(self.score_history[-10:]) < 0.01:
+            if (
+                len(self.score_history) > 10
+                and max(self.score_history[-10:]) - min(self.score_history[-10:]) < 0.01
+            ):
                 self.__reset_particle()
 
         else:
@@ -285,7 +299,7 @@ class Particle:
         if self.negative:
             new_v = (
                 w * encode_v
-                - local_rate * r_0 * (w_p * encode_p - encode_w)
+                + local_rate * r_0 * (w_p * encode_p - encode_w)
                 - global_rate * r_1 * (w_g * encode_g - encode_w)
             )
         else:
@@ -296,7 +310,7 @@ class Particle:
             )
 
         if np.random.rand() < self.mutation:
-            m_v = np.random.uniform(-0.05, 0.05, len(encode_v))
+            m_v = np.random.uniform(-0.1, 0.1, len(encode_v))
             new_v = m_v
 
         self.velocities = self._decode(new_v, w_sh, w_len)
@@ -341,11 +355,28 @@ class Particle:
         score = self.get_score(x, y, renewal)
 
         if self.converge_reset and self.__check_converge_reset(
-                score, self.converge_reset_monitor, self.converge_reset_patience, self.converge_reset_min_delta):
+            score,
+            self.converge_reset_monitor,
+            self.converge_reset_patience,
+            self.converge_reset_min_delta,
+        ):
             self.__reset_particle()
             score = self.get_score(x, y, renewal)
 
-        while np.isnan(score[0]) or np.isnan(score[1]) or np.isnan(score[2]) or score[0] == 0 or score[1] == 0 or score[2] == 0 or np.isinf(score[0]) or np.isinf(score[1]) or np.isinf(score[2]) or score[0] > 1000 or score[1] > 1 or score[2] > 1000:
+        while (
+            np.isnan(score[0])
+            or np.isnan(score[1])
+            or np.isnan(score[2])
+            or score[0] == 0
+            or score[1] == 0
+            or score[2] == 0
+            or np.isinf(score[0])
+            or np.isinf(score[1])
+            or np.isinf(score[2])
+            or score[0] > 1000
+            or score[1] > 1
+            or score[2] > 1000
+        ):
             self.__reset_particle()
             score = self.get_score(x, y, renewal)
 
@@ -362,9 +393,7 @@ class Particle:
 
         return score
 
-    def step_w(
+    def step_w(self, x, y, local_rate, global_rate, w, w_p, w_g, renewal: str = "acc"):
-        self, x, y, local_rate, global_rate, w, w_p, w_g, renewal: str = "acc"
-    ):
         """
         파티클의 한 스텝을 진행합니다.
         기본 스텝의 변형으로, 지역최적해와 전역최적해의 분산 정도를 조정할 수 있습니다
@@ -389,11 +418,28 @@ class Particle:
         score = self.get_score(x, y, renewal)
 
         if self.converge_reset and self.__check_converge_reset(
-                score, self.converge_reset_monitor, self.converge_reset_patience, self.converge_reset_min_delta):
+            score,
+            self.converge_reset_monitor,
+            self.converge_reset_patience,
+            self.converge_reset_min_delta,
+        ):
             self.__reset_particle()
             score = self.get_score(x, y, renewal)
 
-        while np.isnan(score[0]) or np.isnan(score[1]) or np.isnan(score[2]) or score[0] == 0 or score[1] == 0 or score[2] == 0 or np.isinf(score[0]) or np.isinf(score[1]) or np.isinf(score[2]) or score[0] > 1000 or score[1] > 1 or score[2] > 1000:
+        while (
+            np.isnan(score[0])
+            or np.isnan(score[1])
+            or np.isnan(score[2])
+            or score[0] == 0
+            or score[1] == 0
+            or score[2] == 0
+            or np.isinf(score[0])
+            or np.isinf(score[1])
+            or np.isinf(score[2])
+            or score[0] > 1000
+            or score[1] > 1
+            or score[2] > 1000
+        ):
             self.__reset_particle()
             score = self.get_score(x, y, renewal)
 
@@ -429,17 +475,25 @@ class Particle:
         return self.best_weights
 
     def set_global_score(self):
-        """전역 최고점수를 현재 파티클의 최고점수로 설정합니다
+        """전역 최고점수를 현재 파티클의 최고점수로 설정합니다"""
-        """
         Particle.g_best_score = self.best_score
 
     def set_global_weights(self):
-        """전역 최고점수를 받은 가중치를 현재 파티클의 최고점수를 받은 가중치로 설정합니다
+        """전역 최고점수를 받은 가중치를 현재 파티클의 최고점수를 받은 가중치로 설정합니다"""
-        """
         Particle.g_best_weights = self.best_weights
 
     def update_global_best(self):
-        """현재 파티클의 점수와 가중치를 전역 최고점수와 가중치로 설정합니다
+        """현재 파티클의 점수와 가중치를 전역 최고점수와 가중치로 설정합니다"""
-        """
         self.set_global_score()
         self.set_global_weights()
+
+    def check_global_best(self, renewal: str = "loss"):
+        if renewal == "loss":
+            if self.best_score[0] < Particle.g_best_score[0]:
+                self.update_global_best()
+        elif renewal == "acc":
+            if self.best_score[1] > Particle.g_best_score[1]:
+                self.update_global_best()
+        elif renewal == "mse":
+            if self.best_score[2] < Particle.g_best_score[2]:
+                self.update_global_best()