23-10-18

batch size 적용 -> 속도 개선 역전파 1회 적용 -> 조기 수렴을 일부 방지
2025-12-19 20:44:39 +09:00 · 2023-10-18 14:42:59 +09:00
parent f32433d935
commit dfc0df7d48
10 changed files with 157 additions and 63 deletions
--- a/.github/workflows/pypi.yml
+++ b/.github/workflows/pypi.yml
@@ -16,7 +16,7 @@ jobs:
    strategy:
      max-parallel: 5
      matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9"]

    steps:
      - uses: actions/checkout@v3
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -1,5 +1,6 @@
 {
    "[python]": {
-        "editor.defaultFormatter": "ms-python.black-formatter"
+        "editor.defaultFormatter": "ms-python.autopep8"
    },
+    "python.formatting.provider": "none"
 }
--- a/README.md
+++ b/README.md
@@ -1,4 +1,5 @@
 [![Python Package Index publish](https://github.com/jung-geun/PSO/actions/workflows/pypi.yml/badge.svg?event=push)](https://github.com/jung-geun/PSO/actions/workflows/pypi.yml)
+[![PyPI - Version](https://img.shields.io/pypi/v/pso2keras)](https://pypi.org/project/pso2keras/)

 ### 목차

@@ -14,6 +15,8 @@

 # PSO 알고리즘 구현 및 새로운 시도

+Particle Swarm Optimization on tensorflow package
+
 pso 알고리즘을 사용하여 새로운 학습 방법을 찾는중 입니다</br>
 병렬처리로 사용하는 논문을 찾아보았지만 이보다 더 좋은 방법이 있을 것 같아서 찾아보고 있습니다 - [[1]](#참고-자료)</br>

--- a/mnist.py
+++ b/mnist.py
@@ -1,19 +1,17 @@
 # %%
+from pso import optimizer
+from tensorflow import keras
+from keras.models import Sequential
+from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
+from keras.datasets import mnist
+import tensorflow as tf
+import numpy as np
 import json
 import os
 import sys

 os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"

-import numpy as np
-import tensorflow as tf
-from keras.datasets import mnist
-from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
-from keras.models import Sequential
-from tensorflow import keras
-
-from pso import optimizer
-

 def get_data():
    (x_train, y_train), (x_test, y_test) = mnist.load_data()
@@ -24,8 +22,10 @@ def get_data():

    y_train, y_test = tf.one_hot(y_train, 10), tf.one_hot(y_test, 10)

-    x_train, x_test = tf.convert_to_tensor(x_train), tf.convert_to_tensor(x_test)
-    y_train, y_test = tf.convert_to_tensor(y_train), tf.convert_to_tensor(y_test)
+    x_train, x_test = tf.convert_to_tensor(
+        x_train), tf.convert_to_tensor(x_test)
+    y_train, y_test = tf.convert_to_tensor(
+        y_train), tf.convert_to_tensor(y_test)

    print(f"x_train : {x_train[0].shape} | y_train : {y_train[0].shape}")
    print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}")
@@ -40,8 +40,10 @@ def get_data_test():

    y_train, y_test = tf.one_hot(y_train, 10), tf.one_hot(y_test, 10)

-    x_train, x_test = tf.convert_to_tensor(x_train), tf.convert_to_tensor(x_test)
-    y_train, y_test = tf.convert_to_tensor(y_train), tf.convert_to_tensor(y_test)
+    x_train, x_test = tf.convert_to_tensor(
+        x_train), tf.convert_to_tensor(x_test)
+    y_train, y_test = tf.convert_to_tensor(
+        y_train), tf.convert_to_tensor(y_test)

    print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}")

@@ -51,7 +53,8 @@ def get_data_test():
 def make_model():
    model = Sequential()
    model.add(
-        Conv2D(32, kernel_size=(5, 5), activation="sigmoid", input_shape=(28, 28, 1))
+        Conv2D(32, kernel_size=(5, 5), activation="sigmoid",
+               input_shape=(28, 28, 1))
    )
    model.add(MaxPooling2D(pool_size=(3, 3)))
    model.add(Conv2D(64, kernel_size=(3, 3), activation="sigmoid"))
@@ -83,7 +86,7 @@ def random_state():

 # %%
 model = make_model()
-x_train, y_train = get_data_test()
+x_train, y_train, x_test, y_test = get_data()

 loss = [
    "mean_squared_error",
@@ -104,12 +107,12 @@ loss = [
 pso_mnist = optimizer(
    model,
    loss="mean_squared_error",
-    n_particles=2000,
+    n_particles=600,
    c0=0.2,
    c1=0.4,
    w_min=0.3,
-    w_max=0.7,
-    negative_swarm=0.1,
+    w_max=0.5,
+    negative_swarm=0.05,
    mutation_swarm=0.3,
    particle_min=-4,
    particle_max=4,
@@ -118,16 +121,16 @@ pso_mnist = optimizer(
 best_score = pso_mnist.fit(
    x_train,
    y_train,
-    epochs=300,
+    epochs=200,
    save_info=True,
-    log=1,
+    log=2,
    log_name="mnist",
    save_path="./logs/mnist",
    renewal="acc",
    check_point=25,
    empirical_balance=False,
    dispersion=False,
-
+    batch_size=32,
 )

 print("Done!")
--- a/mnist_tf.py
+++ b/mnist_tf.py
@@ -1,8 +1,13 @@
+from keras.models import Sequential
+from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
+from keras.datasets import mnist
+from keras.utils import to_categorical
+# from tensorflow.data.Dataset import from_tensor_slices
+import tensorflow as tf
 import os

 os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"

-import tensorflow as tf

 gpus = tf.config.experimental.list_physical_devices("GPU")
 if gpus:
@@ -13,10 +18,6 @@ if gpus:
    finally:
        del gpus

-from keras.datasets import mnist
-from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
-from keras.models import Sequential
-

 def get_data():
    (x_train, y_train), (x_test, y_test) = mnist.load_data()
@@ -38,10 +39,26 @@ def get_data_test():
    return x_test, y_test


+class _batch_generator:
+    def __init__(self, x, y, batch_size: int = 32):
+        self.batch_size = batch_size
+        self.index = 0
+        dataset = tf.data.Dataset.from_tensor_slices((x, y))
+        self.dataset = list(dataset.batch(batch_size))
+        self.max_index = len(dataset) // batch_size
+
+    def next(self):
+        self.index += 1
+        if self.index >= self.max_index:
+            self.index = 0
+        return self.dataset[self.index][0], self.dataset[self.index][1]
+
+
 def make_model():
    model = Sequential()
    model.add(
-        Conv2D(32, kernel_size=(5, 5), activation="relu", input_shape=(28, 28, 1))
+        Conv2D(32, kernel_size=(5, 5), activation="relu",
+               input_shape=(28, 28, 1))
    )
    model.add(MaxPooling2D(pool_size=(3, 3)))
    model.add(Conv2D(64, kernel_size=(3, 3), activation="relu"))
@@ -56,17 +73,24 @@ def make_model():

 model = make_model()
 x_train, y_train, x_test, y_test = get_data()
-
 y_train = tf.one_hot(y_train, 10)
 y_test = tf.one_hot(y_test, 10)

+dataset = _batch_generator(x_train, y_train, 64)
+
 model.compile(optimizer="adam", loss="mse", metrics=["accuracy"])

-print("Training model...")
-model.fit(x_train, y_train, epochs=100, batch_size=128, verbose=1)
+count = 0
+
+while count < 20:
+    x_batch, y_batch = dataset.next()
+    count += 1
+    print("Training model...")
+    model.fit(x_batch, y_batch, epochs=1, batch_size=1, verbose=1)
+
+print(count)

 print("Evaluating model...")
-model.evaluate(x_test, y_test, verbose=1)
+model.evaluate(x_test, y_test, verbose=2)

 weights = model.get_weights()
-
--- a/pso/init.py
+++ b/pso/init.py
@@ -1,7 +1,7 @@
 from .optimizer import Optimizer as optimizer
 from .particle import Particle as particle

-__version__ = "0.1.7"
+__version__ = "0.1.8"

 __all__ = [
    "optimizer",
--- a/pso/optimizer.py
+++ b/pso/optimizer.py
@@ -106,7 +106,11 @@ class Optimizer:
                w_ = np.random.uniform(particle_min, particle_max, len(w_))
                model_.set_weights(self._decode(w_, sh_, len_))

-                model_.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"])
+                model_.compile(
+                    loss=self.loss,
+                    optimizer="sgd",
+                    metrics=["accuracy"]
+                )
                self.particles[i] = Particle(
                    model_,
                    loss,
@@ -224,6 +228,20 @@ class Optimizer:
        else:
            return 1 + np.abs(score_)

+    class _batch_generator:
+        def __init__(self, x, y, batch_size: int = 32):
+            self.batch_size = batch_size
+            self.index = 0
+            dataset = tf.data.Dataset.from_tensor_slices((x, y))
+            self.dataset = list(dataset.batch(batch_size))
+            self.max_index = len(dataset) // batch_size
+
+        def next(self):
+            self.index += 1
+            if self.index >= self.max_index:
+                self.index = 0
+            return self.dataset[self.index][0], self.dataset[self.index][1]
+
    def fit(
        self,
        x,
@@ -237,6 +255,7 @@ class Optimizer:
        empirical_balance: bool = False,
        dispersion: bool = False,
        check_point: int = None,
+        batch_size: int = 128,
    ):
        """
        # Args:
@@ -250,6 +269,7 @@ class Optimizer:
            empirical_balance : bool - True : EBPSO, False : PSO,
            dispersion : bool - True : g_best 의 값을 분산시켜 전역해를 찾음, False : g_best 의 값만 사용
            check_point : int - 저장할 위치 - None : 저장 안함
+            batch_size : int - batch size default : 128
        """
        self.save_path = save_path
        self.empirical_balance = empirical_balance
@@ -278,10 +298,33 @@ class Optimizer:
        except ValueError as ve:
            sys.exit(ve)

+        model_ = keras.models.model_from_json(self.model.to_json())
+        model_.compile(loss=self.loss, optimizer="adam", metrics=["accuracy"])
+        model_.fit(x, y, epochs=1, batch_size=64, verbose=0)
+        score = model_.evaluate(x, y, verbose=1)
+
+        if renewal == "acc":
+            self.g_best_score[0] = score[1]
+            self.g_best_score[1] = score[0]
+        else:
+            self.g_best_score[0] = score[0]
+            self.g_best_score[1] = score[1]
+
+        self.g_best = model_.get_weights()
+        self.g_best_ = model_.get_weights()
+
+        del model_
+
+        dataset = self._batch_generator(x, y, batch_size=batch_size)
+
        for i in tqdm(range(self.n_particles), desc="Initializing velocity"):
            p = self.particles[i]
-            local_score = p.get_score(x, y, renewal=renewal)
+
+            x_batch, y_batch = dataset.next()
+            local_score = p.get_score(x_batch, y_batch, renewal=renewal)
+
            particle_sum += local_score[1]
+
            if renewal == "acc":
                if local_score[1] > self.g_best_score[0]:
                    self.g_best_score[0] = local_score[1]
@@ -329,7 +372,7 @@ class Optimizer:
            epoch_sum = 0
            epochs_pbar = tqdm(
                range(epochs),
-                desc=f"best {self.g_best_score[0]:.4f}|{self.g_best_score[1]:.4f}",
+                desc=f"best {self.g_best_score[0]:.4f} | {self.g_best_score[1]:.4f}",
                ascii=True,
                leave=True,
                position=0,
@@ -355,6 +398,8 @@ class Optimizer:

                    g_best = self.g_best

+                    x_batch, y_batch = dataset.next()
+
                    if dispersion:
                        ts = self.particle_min + np.random.rand() * (
                            self.particle_max - self.particle_min
@@ -367,7 +412,9 @@ class Optimizer:

                    if empirical_balance:
                        if np.random.rand() < np.exp(-(epoch) / epochs):
-                            w_p_ = self._f(x, y, self.particles[i].get_best_weights())
+                            w_p_ = self._f(
+                                x, y, self.particles[i].get_best_weights()
+                            )
                            w_g_ = self._f(x, y, self.g_best)
                            w_p = w_p_ / (w_p_ + w_g_)
                            w_g = w_p_ / (w_p_ + w_g_)
@@ -417,8 +464,8 @@ class Optimizer:
                            del p_

                        score = self.particles[i].step_w(
-                            x,
-                            y,
+                            x_batch,
+                            y_batch,
                            self.c0,
                            self.c1,
                            w,
@@ -431,13 +478,15 @@ class Optimizer:

                    else:
                        score = self.particles[i].step(
-                            x, y, self.c0, self.c1, w, g_best, renewal=renewal
+                            x_batch, y_batch, self.c0, self.c1, w, g_best, renewal=renewal
                        )

                    if log == 2:
                        with self.train_summary_writer[i].as_default():
                            tf.summary.scalar("loss", score[0], step=epoch + 1)
-                            tf.summary.scalar("accuracy", score[1], step=epoch + 1)
+                            tf.summary.scalar(
+                                "accuracy", score[1], step=epoch + 1
+                            )

                    if renewal == "acc":
                        if score[1] >= max_score:
@@ -447,11 +496,13 @@ class Optimizer:
                        if score[1] >= self.g_best_score[0]:
                            if score[1] > self.g_best_score[0]:
                                self.g_best_score[0] = score[1]
-                                self.g_best = self.particles[i].get_best_weights()
+                                self.g_best = self.particles[i].get_best_weights(
+                                )
                            else:
                                if score[0] < self.g_best_score[1]:
                                    self.g_best_score[1] = score[0]
-                                    self.g_best = self.particles[i].get_best_weights()
+                                    self.g_best = self.particles[i].get_best_weights(
+                                    )
                            epochs_pbar.set_description(
                                f"best {self.g_best_score[0]:.4f} | {self.g_best_score[1]:.4f}"
                            )
@@ -463,11 +514,13 @@ class Optimizer:
                        if score[0] <= self.g_best_score[1]:
                            if score[0] < self.g_best_score[1]:
                                self.g_best_score[1] = score[0]
-                                self.g_best = self.particles[i].get_best_weights()
+                                self.g_best = self.particles[i].get_best_weights(
+                                )
                            else:
                                if score[1] > self.g_best_score[0]:
                                    self.g_best_score[0] = score[1]
-                                    self.g_best = self.particles[i].get_best_weights()
+                                    self.g_best = self.particles[i].get_best_weights(
+                                    )
                            epochs_pbar.set_description(
                                f"best {self.g_best_score[0]:.4f} | {self.g_best_score[1]:.4f}"
                            )
@@ -505,7 +558,8 @@ class Optimizer:
                if check_point is not None:
                    if epoch % check_point == 0:
                        os.makedirs(f"./{save_path}/{self.day}", exist_ok=True)
-                        self._check_point_save(f"./{save_path}/{self.day}/ckpt-{epoch}")
+                        self._check_point_save(
+                            f"./{save_path}/{self.day}/ckpt-{epoch}")

                gc.collect()
                tf.keras.backend.reset_uids()
@@ -513,10 +567,13 @@ class Optimizer:

        except KeyboardInterrupt:
            print("Ctrl + C : Stop Training")
+
        except MemoryError:
            print("Memory Error : Stop Training")
+
        except Exception as e:
            print(e)
+
        finally:
            self.model_save(save_path)
            print("model save")
--- a/pso/particle.py
+++ b/pso/particle.py
@@ -133,7 +133,9 @@ class Particle:
        encode_v, v_sh, v_len = self._encode(weights=self.velocities)
        encode_p, p_sh, p_len = self._encode(weights=self.best_weights)
        encode_g, g_sh, g_len = self._encode(weights=g_best)
-        encode_before, before_sh, before_len = self._encode(weights=self.before_best)
+        encode_before, before_sh, before_len = self._encode(
+            weights=self.before_best
+        )
        r_0 = np.random.rand()
        r_1 = np.random.rand()

@@ -187,7 +189,9 @@ class Particle:
        encode_v, v_sh, v_len = self._encode(weights=self.velocities)
        encode_p, p_sh, p_len = self._encode(weights=self.best_weights)
        encode_g, g_sh, g_len = self._encode(weights=g_best)
-        encode_before, before_sh, before_len = self._encode(weights=self.before_best)
+        encode_before, before_sh, before_len = self._encode(
+            weights=self.before_best
+        )
        r_0 = np.random.rand()
        r_1 = np.random.rand()

--- a/setup.py
+++ b/setup.py
@@ -7,7 +7,7 @@ VERSION = pso.__version__
 setup(
    name="pso2keras",
    version=VERSION,
-    description="Particle Swarm Optimization to tensorflow package",
+    description="Particle Swarm Optimization on tensorflow package",
    author="pieroot",
    author_email="jgbong0306@gmail.com",
    url="https://github.com/jung-geun/PSO",
@@ -16,8 +16,8 @@ setup(
        "numpy",
        "pandas",
        "ipython",
-        "tensorflow<=2.11.1",
-        "keras"
+        "tensorflow<=2.11.1,>=2.8.0",
+        "keras<=2.11.1,>=2.8.0",
    ],
    packages=find_packages(exclude=[]),
    keywords=["pso", "tensorflow", "keras"],
@@ -26,4 +26,13 @@ setup(
    zip_safe=False,
    long_description=open("README.md", encoding="UTF8").read(),
    long_description_content_type="text/markdown",
+    license="MIT",
+    classifiers=[
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python :: 3:: Only"
+        "Programming Language :: Python :: 3.7"
+        "Programming Language :: Python :: 3.8",
+        "programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+    ],
 )
--- a/test01.py
+++ b/test01.py
@@ -1,11 +1,4 @@
-# 반복문을 사용해서 자동 생성하는 python 코드
+it =iter(range(10))

-
-def pibonachi(n):
-    if n <= 1:
-        return n
-    else:
-        return pibonachi(n - 1) + pibonachi(n - 2)
-
-
-print(pibonachi(10))
+for i in range(100):
+    print(next(it))