코드 변경 내용: digits.py, iris.py, mnist.py, bean.py

Keras 모듈을 사용하여 코드를 업데이트했습니다.

bean.py

@@ -1,24 +1,21 @@
 import os
 
-import pandas as pd
+from keras.layers import Dense
-import tensorflow as tf
+from keras.models import Sequential
+from keras.utils import to_categorical
 from sklearn.model_selection import train_test_split
 from tensorflow import keras
-from tensorflow.keras import layers
-from tensorflow.keras.layers import Dense
-from tensorflow.keras.models import Sequential
-from tensorflow.keras.utils import to_categorical
 from ucimlrepo import fetch_ucirepo
 
-os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
-os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
+os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "true"
 
 
 def make_model():
     model = Sequential()
-    model.add(Dense(12, input_dim=16, activation='relu'))
+    model.add(Dense(12, input_dim=16, activation="relu"))
-    model.add(Dense(8, activation='relu'))
+    model.add(Dense(8, activation="relu"))
-    model.add(Dense(7, activation='softmax'))
+    model.add(Dense(7, activation="softmax"))
 
     return model
 
@@ -34,7 +31,7 @@ def get_data():
     x = X.to_numpy()
     # object to categorical
 
-    x = x.astype('float32')
+    x = x.astype("float32")
 
     y_class = to_categorical(y)
 
@@ -49,21 +46,27 @@ def get_data():
     # y_class = to_categorical(y)
 
     x_train, x_test, y_train, y_test = train_test_split(
-        x, y_class, test_size=0.2, random_state=42, shuffle=True)
+        x, y_class, test_size=0.2, random_state=42, shuffle=True
+    )
     return x_train, x_test, y_train, y_test
 
 
 x_train, x_test, y_train, y_test = get_data()
 model = make_model()
 early_stopping = keras.callbacks.EarlyStopping(
-    patience=10, min_delta=0.001, restore_best_weights=True)
+    patience=10, min_delta=0.001, restore_best_weights=True
+)
 
 
-model.compile(loss='sparse_categorical_crossentropy',
+model.compile(
-              optimizer='adam', metrics=['accuracy', "mse"])
+    loss="sparse_categorical_crossentropy",
+    optimizer="adam",
+    metrics=["accuracy", "mse"],
+)
 
 model.summary()
 
-history = model.fit(x_train, y_train, epochs=150,
+history = model.fit(
-                    batch_size=10, callbacks=[early_stopping])
+    x_train, y_train, epochs=150, batch_size=10, callbacks=[early_stopping]
+)
 score = model.evaluate(x_test, y_test, verbose=2)

digits.py

@@ -1,15 +1,11 @@
 import os
 import sys
 
-import pandas as pd
+from keras.layers import Dense
-import tensorflow as tf
+from keras.models import Sequential
+from keras.utils import to_categorical
 from sklearn.datasets import load_digits
 from sklearn.model_selection import train_test_split
-from tensorflow import keras
-from tensorflow.keras import layers
-from tensorflow.keras.layers import Dense
-from tensorflow.keras.models import Sequential
-from tensorflow.keras.utils import to_categorical
 
 from pso import optimizer
 

fashion_mnist.py

@@ -1,15 +1,16 @@
 # %%
-from pso import optimizer
-from tensorflow import keras
-from keras.models import Sequential
-from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
-from keras.datasets import mnist, fashion_mnist
-import tensorflow as tf
-import numpy as np
 import json
 import os
 import sys
 
+import numpy as np
+import tensorflow as tf
+from keras.datasets import fashion_mnist
+from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
+from keras.models import Sequential
+
+from pso import optimizer
+
 os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
 
 
@@ -22,10 +23,8 @@ def get_data():
 
     y_train, y_test = tf.one_hot(y_train, 10), tf.one_hot(y_test, 10)
 
-    x_train, x_test = tf.convert_to_tensor(
+    x_train, x_test = tf.convert_to_tensor(x_train), tf.convert_to_tensor(x_test)
-        x_train), tf.convert_to_tensor(x_test)
+    y_train, y_test = tf.convert_to_tensor(y_train), tf.convert_to_tensor(y_test)
-    y_train, y_test = tf.convert_to_tensor(
-        y_train), tf.convert_to_tensor(y_test)
 
     print(f"x_train : {x_train[0].shape} | y_train : {y_train[0].shape}")
     print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}")
@@ -36,8 +35,7 @@ def get_data():
 def make_model():
     model = Sequential()
     model.add(
-        Conv2D(32, kernel_size=(5, 5), activation="relu",
+        Conv2D(32, kernel_size=(5, 5), activation="relu", input_shape=(28, 28, 1))
-               input_shape=(28, 28, 1))
     )
     model.add(MaxPooling2D(pool_size=(2, 2)))
     model.add(Conv2D(64, kernel_size=(3, 3), activation="relu"))
@@ -51,72 +49,39 @@ def make_model():
     return model
 
 
-def random_state():
-    with open(
-        "result/mnist/20230723-061626/mean_squared_error_[0.6384999752044678, 0.0723000094294548].json",
-        "r",
-    ) as f:
-        json_ = json.load(f)
-        rs = (
-            json_["random_state_0"],
-            np.array(json_["random_state_1"]),
-            json_["random_state_2"],
-            json_["random_state_3"],
-            json_["random_state_4"],
-        )
-
-    return rs
-
-
 # %%
 model = make_model()
 x_train, y_train, x_test, y_test = get_data()
 
-loss = [
-    "mean_squared_error",
-    "categorical_crossentropy",
-    "sparse_categorical_crossentropy",
-    "binary_crossentropy",
-    "kullback_leibler_divergence",
-    "poisson",
-    "cosine_similarity",
-    "log_cosh",
-    "huber_loss",
-    "mean_absolute_error",
-    "mean_absolute_percentage_error",
-]
-
-# rs = random_state()
 
 pso_mnist = optimizer(
     model,
     loss="categorical_crossentropy",
-    n_particles=500,
+    n_particles=200,
-    c0=0.5,
+    c0=0.7,
-    c1=1.0,
+    c1=0.5,
-    w_min=0.7,
+    w_min=0.1,
-    w_max=1.2,
+    w_max=0.8,
-    negative_swarm=0.05,
+    negative_swarm=0.0,
-    mutation_swarm=0.3,
+    mutation_swarm=0.05,
     convergence_reset=True,
     convergence_reset_patience=10,
+    convergence_reset_monitor="loss",
 )
 
 best_score = pso_mnist.fit(
     x_train,
     y_train,
-    epochs=200,
+    epochs=1000,
     save_info=True,
     log=2,
     log_name="fashion_mnist",
-    renewal="acc",
+    renewal="loss",
     check_point=25,
-    empirical_balance=False,
-    dispersion=False,
     batch_size=5000,
-    back_propagation=True,
 )
 
 print("Done!")
 
 sys.exit(0)
+

iris.py

@@ -1,13 +1,15 @@
-from pso import optimizer
-from tensorflow.keras.models import Sequential
-from tensorflow.keras import layers
-from tensorflow import keras
-from sklearn.model_selection import train_test_split
-from sklearn.datasets import load_iris
 import gc
 import os
 import sys
 
+from sklearn.datasets import load_iris
+from sklearn.model_selection import train_test_split
+from tensorflow import keras
+from tensorflow.keras import layers
+from tensorflow.keras.models import Sequential
+
+from pso import optimizer
+
 os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
 
 

mnist.py

@@ -1,15 +1,15 @@
 # %%
-from pso import optimizer
-from tensorflow import keras
-from keras.models import Sequential
-from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
-from keras.datasets import mnist
-import tensorflow as tf
-import numpy as np
-import json
 import os
 import sys
 
+from pso import optimizer
+
+import tensorflow as tf
+from keras.datasets import mnist
+from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
+from keras.models import Sequential
+
+
 os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
 
 
@@ -53,33 +53,17 @@ def make_model():
 model = make_model()
 x_train, y_train, x_test, y_test = get_data()
 
-loss = [
-    "mean_squared_error",
-    "categorical_crossentropy",
-    "sparse_categorical_crossentropy",
-    "binary_crossentropy",
-    "kullback_leibler_divergence",
-    "poisson",
-    "cosine_similarity",
-    "log_cosh",
-    "huber_loss",
-    "mean_absolute_error",
-    "mean_absolute_percentage_error",
-
-]
-
-# rs = random_state()
 
 pso_mnist = optimizer(
     model,
     loss="categorical_crossentropy",
-    n_particles=500,
+    n_particles=200,
-    c0=0.5,
+    c0=0.7,
-    c1=0.3,
+    c1=0.4,
     w_min=0.1,
     w_max=0.9,
     negative_swarm=0.0,
-    mutation_swarm=0.1,
+    mutation_swarm=0.05,
     convergence_reset=True,
     convergence_reset_patience=10,
     convergence_reset_monitor="loss",
@@ -89,16 +73,13 @@ pso_mnist = optimizer(
 best_score = pso_mnist.fit(
     x_train,
     y_train,
-    epochs=500,
+    epochs=1000,
     save_info=True,
     log=2,
     log_name="mnist",
     renewal="loss",
     check_point=25,
-    empirical_balance=False,
+    batch_size=5000,
-    dispersion=False,
-    batch_size=10000,
-    back_propagation=False,
     validate_data=(x_test, y_test),
 )
 

mnist_tf.py

@@ -2,6 +2,8 @@ from keras.models import Sequential
 from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
 from keras.datasets import mnist
 from keras.utils import to_categorical
+from sklearn.model_selection import train_test_split
+
 # from tensorflow.data.Dataset import from_tensor_slices
 import tensorflow as tf
 import os
@@ -31,14 +33,6 @@ def get_data():
 
     return x_train, y_train, x_test, y_test
 
-
-def get_data_test():
-    (x_train, y_train), (x_test, y_test) = mnist.load_data()
-    x_test = x_test.reshape((10000, 28, 28, 1))
-
-    return x_test, y_test
-
-
 class _batch_generator_:
     def __init__(self, x, y, batch_size: int = None):
         self.index = 0
@@ -77,8 +71,9 @@ class _batch_generator_:
 
     def __getBatchSlice(self, batch_size):
         return list(
-            tf.data.Dataset.from_tensor_slices(
+            tf.data.Dataset.from_tensor_slices((self.x, self.y))
-                (self.x, self.y)).shuffle(len(self.x)).batch(batch_size)
+            .shuffle(len(self.x))
+            .batch(batch_size)
         )
 
     def getDataset(self):
@@ -88,17 +83,18 @@ class _batch_generator_:
 def make_model():
     model = Sequential()
     model.add(
-        Conv2D(32, kernel_size=(5, 5), activation="relu",
+        Conv2D(64, kernel_size=(5, 5), activation="relu", input_shape=(28, 28, 1))
-               input_shape=(28, 28, 1))
     )
     model.add(MaxPooling2D(pool_size=(2, 2)))
     model.add(Dropout(0.5))
-    model.add(Conv2D(64, kernel_size=(3, 3), activation="relu"))
+    model.add(Conv2D(128, kernel_size=(3, 3), activation="relu"))
     model.add(MaxPooling2D(pool_size=(2, 2)))
     model.add(Flatten())
     model.add(Dropout(0.5))
-    model.add(Dense(256, activation="relu"))
+    model.add(Dense(2048, activation="relu"))
-    model.add(Dense(128, activation="relu"))
+    model.add(Dropout(0.8))
+    model.add(Dense(1024, activation="relu"))
+    model.add(Dropout(0.8))
     model.add(Dense(10, activation="softmax"))
 
     return model
@@ -112,18 +108,21 @@ y_test = tf.one_hot(y_test, 10)
 batch = 64
 dataset = _batch_generator_(x_train, y_train, batch)
 
-model.compile(optimizer="adam", loss="categorical_crossentropy",
+model.compile(
-              metrics=["accuracy", "mse", "mae"])
+    optimizer="adam",
+    loss="categorical_crossentropy",
+    metrics=["accuracy", "mse"],
+)
 
 count = 0
 print(f"batch size : {batch}")
 print("iter " + str(dataset.getMaxIndex()))
 print("Training model...")
-while count < dataset.getMaxIndex():
+# while count < dataset.getMaxIndex():
-    x_batch, y_batch = dataset.next()
+# x_batch, y_batch = dataset.next()
-    count += 1
+# count += 1
-    print(f"iter {count}/{dataset.getMaxIndex()}")
+# print(f"iter {count}/{dataset.getMaxIndex()}")
-    model.fit(x_batch, y_batch, epochs=1, batch_size=batch, verbose=1)
+model.fit(x_train, y_train, epochs=1000, batch_size=batch, verbose=1)
 
 print(count)
 

pso/optimizer.py

@@ -1,3 +1,4 @@
+import atexit
 import gc
 import json
 import os
@@ -8,10 +9,10 @@ from datetime import datetime
 
 import numpy as np
 import tensorflow as tf
+from sklearn.model_selection import train_test_split
 from tensorboard.plugins.hparams import api as hp
 from tensorflow import keras
 from tqdm.auto import tqdm
-import atexit
 
 from .particle import Particle
 
@@ -46,7 +47,7 @@ class Optimizer:
         n_particles: int = None,
         c0: float = 0.5,
         c1: float = 0.3,
-        w_min: float = 0.2,
+        w_min: float = 0.1,
         w_max: float = 0.9,
         negative_swarm: float = 0,
         mutation_swarm: float = 0,
@@ -56,7 +57,7 @@ class Optimizer:
         convergence_reset: bool = False,
         convergence_reset_patience: int = 10,
         convergence_reset_min_delta: float = 0.0001,
-        convergence_reset_monitor: str = "mse",
+        convergence_reset_monitor: str = "loss",
     ):
         """
         particle swarm optimization
@@ -284,10 +285,10 @@ class Optimizer:
 
         def next(self):
             self.index += 1
-            if self.index >= self.max_index:
+            if self.index > self.max_index:
                 self.index = 0
                 self.__getBatchSlice(self.batch_size)
-            return self.dataset[self.index][0], self.dataset[self.index][1]
+            return self.dataset[self.index - 1][0], self.dataset[self.index - 1][1]
 
         def getMaxIndex(self):
             return self.max_index
@@ -306,13 +307,12 @@ class Optimizer:
                 batch_size = len(self.x) // 10
             elif batch_size > len(self.x):
                 batch_size = len(self.x)
+
             self.batch_size = batch_size
 
             print(f"batch size : {self.batch_size}")
             self.dataset = self.__getBatchSlice(self.batch_size)
             self.max_index = len(self.dataset)
-            if batch_size % len(self.x) != 0:
-                self.max_index -= 1
 
         def __getBatchSlice(self, batch_size):
             return list(
@@ -331,14 +331,16 @@ class Optimizer:
         epochs: int = 1,
         log: int = 0,
         log_name: str = None,
-        save_info: bool = False,
+        save_info: bool = None,
-        renewal: str = "mse",
+        renewal: str = None,
-        empirical_balance: bool = False,
+        empirical_balance: bool = None,
-        dispersion: bool = False,
+        dispersion: bool = None,
         check_point: int = None,
         batch_size: int = None,
         validate_data: tuple = None,
+        validation_split: float = None,
         back_propagation: bool = False,
+        weight_reduction: int = None,
     ):
         """
         # Args:
@@ -355,12 +357,16 @@ class Optimizer:
             batch_size : int - batch size default : None => len(x) // 10
                 batch_size > len(x) : auto max batch size
             validate_data : tuple - (x, y) default : None => (x, y)
-            back_propagation : bool - True : back propagation, False : not back propagation
+            back_propagation : bool - True : back propagation, False : not back propagation default : False
+            weight_reduction : int - 가중치 감소 초기화 주기 default : None => epochs
         """
         try:
             if x.shape[0] != y.shape[0]:
                 raise ValueError("x, y shape error")
 
+            if save_info is None:
+                save_info = False
+
             if log not in [0, 1, 2]:
                 raise ValueError(
                     """log not in [0, 1, 2]
@@ -370,9 +376,18 @@ class Optimizer:
                     """
                 )
 
+            if renewal is None:
+                renewal = "loss"
+
             if renewal not in ["acc", "loss", "mse"]:
                 raise ValueError("renewal not in ['acc', 'loss', 'mse']")
 
+            if empirical_balance is None:
+                empirical_balance = False
+
+            if dispersion is None:
+                dispersion = False
+
             if validate_data is not None:
                 if validate_data[0].shape[0] != validate_data[1].shape[0]:
                     raise ValueError("validate_data shape error")
@@ -380,12 +395,23 @@ class Optimizer:
             if validate_data is None:
                 validate_data = (x, y)
 
+            if validation_split is not None:
+                if validation_split < 0 or validation_split > 1:
+                    raise ValueError("validation_split not in [0, 1]")
+
+                x, validate_data[0], y, validate_data[1] = train_test_split(
+                    x, y, test_size=validation_split, shuffle=True
+                )
+
             if batch_size is not None and batch_size < 1:
                 raise ValueError("batch_size < 1")
 
             if batch_size is None or batch_size > len(x):
                 batch_size = len(x)
 
+            if weight_reduction == None:
+                weight_reduction = epochs
+
         except ValueError as ve:
             sys.exit(ve)
         except Exception as e:
@@ -430,34 +456,39 @@ class Optimizer:
         except Exception as e:
             sys.exit(e)
 
-        if back_propagation:
-            model_ = keras.models.model_from_json(self.model.to_json())
-            model_.compile(
-                loss=self.loss,
-                optimizer="adam",
-                metrics=["accuracy", "mse"],
-            )
-            model_.fit(x, y, epochs=1, verbose=0)
-            score = model_.evaluate(x, y, verbose=1)
-
-            Particle.g_best_score = score
-
-            Particle.g_best_weights = model_.get_weights()
-
-            del model_
-
-        dataset = self.batch_generator(x, y, batch_size=batch_size)
-
-        for i in tqdm(
-            range(len(self.particles)),
-            desc="best score init",
-            ascii=True,
-            leave=True,
-        ):
-            score = self.particles[i].get_score(x, y, self.renewal)
-            self.particles[i].check_global_best(self.renewal)
-
         try:
+            dataset = self.batch_generator(x, y, batch_size=batch_size)
+
+            if back_propagation:
+                model_ = keras.models.model_from_json(self.model.to_json())
+                model_.compile(
+                    loss=self.loss,
+                    optimizer="adam",
+                    metrics=["accuracy", "mse"],
+                )
+                model_.fit(x, y, epochs=1, verbose=0)
+                score = model_.evaluate(x, y, verbose=1)
+
+                Particle.g_best_score = score
+
+                Particle.g_best_weights = model_.get_weights()
+
+                del model_
+
+            else:
+                for i in tqdm(
+                    range(len(self.particles)),
+                    desc="best score init",
+                    ascii=True,
+                    leave=False,
+                ):
+                    score = self.particles[i].get_score(
+                        validate_data[0], validate_data[1], self.renewal
+                    )
+                    self.particles[i].check_global_best(self.renewal)
+
+            print("best score init complete" + str(Particle.g_best_score))
+
             epoch_sum = 0
             epochs_pbar = tqdm(
                 range(epochs),
@@ -486,7 +517,12 @@ class Optimizer:
                 )
 
                 # w = self.w_max - (self.w_max - self.w_min) * epoch / epochs
-                w = self.w_max - (self.w_max - self.w_min) * (epoch % 100) / 100
+                w = (
+                    self.w_max
+                    - (self.w_max - self.w_min)
+                    * (epoch % weight_reduction)
+                    / weight_reduction
+                )
                 for i in part_pbar:
                     part_pbar.set_description(
                         f"loss: {min_loss:.4f} acc: {max_acc:.4f} mse: {min_mse:.4f}"