23-10-18

batch size 적용 -> 속도 개선
역전파 1회 적용 -> 조기 수렴을 일부 방지

.github/workflows/pypi.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       max-parallel: 5
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.9"]
 
     steps:
       - uses: actions/checkout@v3

.vscode/settings.json

@@ -1,5 +1,6 @@
 {
     "[python]": {
-        "editor.defaultFormatter": "ms-python.black-formatter"
+        "editor.defaultFormatter": "ms-python.autopep8"
     },
+    "python.formatting.provider": "none"
 }

README.md

@@ -1,4 +1,5 @@
 [![Python Package Index publish](https://github.com/jung-geun/PSO/actions/workflows/pypi.yml/badge.svg?event=push)](https://github.com/jung-geun/PSO/actions/workflows/pypi.yml)
+[![PyPI - Version](https://img.shields.io/pypi/v/pso2keras)](https://pypi.org/project/pso2keras/)
 
 ### 목차
 
@@ -14,6 +15,8 @@
 
 # PSO 알고리즘 구현 및 새로운 시도
 
+Particle Swarm Optimization on tensorflow package
+
 pso 알고리즘을 사용하여 새로운 학습 방법을 찾는중 입니다</br>
 병렬처리로 사용하는 논문을 찾아보았지만 이보다 더 좋은 방법이 있을 것 같아서 찾아보고 있습니다 - [[1]](#참고-자료)</br>
 

mnist.py

@@ -1,19 +1,17 @@
 # %%
+from pso import optimizer
+from tensorflow import keras
+from keras.models import Sequential
+from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
+from keras.datasets import mnist
+import tensorflow as tf
+import numpy as np
 import json
 import os
 import sys
 
 os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
 
-import numpy as np
-import tensorflow as tf
-from keras.datasets import mnist
-from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
-from keras.models import Sequential
-from tensorflow import keras
-
-from pso import optimizer
-
 
 def get_data():
     (x_train, y_train), (x_test, y_test) = mnist.load_data()
@@ -24,8 +22,10 @@ def get_data():
 
     y_train, y_test = tf.one_hot(y_train, 10), tf.one_hot(y_test, 10)
 
-    x_train, x_test = tf.convert_to_tensor(x_train), tf.convert_to_tensor(x_test)
+    x_train, x_test = tf.convert_to_tensor(
-    y_train, y_test = tf.convert_to_tensor(y_train), tf.convert_to_tensor(y_test)
+        x_train), tf.convert_to_tensor(x_test)
+    y_train, y_test = tf.convert_to_tensor(
+        y_train), tf.convert_to_tensor(y_test)
 
     print(f"x_train : {x_train[0].shape} | y_train : {y_train[0].shape}")
     print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}")
@@ -40,8 +40,10 @@ def get_data_test():
 
     y_train, y_test = tf.one_hot(y_train, 10), tf.one_hot(y_test, 10)
 
-    x_train, x_test = tf.convert_to_tensor(x_train), tf.convert_to_tensor(x_test)
+    x_train, x_test = tf.convert_to_tensor(
-    y_train, y_test = tf.convert_to_tensor(y_train), tf.convert_to_tensor(y_test)
+        x_train), tf.convert_to_tensor(x_test)
+    y_train, y_test = tf.convert_to_tensor(
+        y_train), tf.convert_to_tensor(y_test)
 
     print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}")
 
@@ -51,7 +53,8 @@ def get_data_test():
 def make_model():
     model = Sequential()
     model.add(
-        Conv2D(32, kernel_size=(5, 5), activation="sigmoid", input_shape=(28, 28, 1))
+        Conv2D(32, kernel_size=(5, 5), activation="sigmoid",
+               input_shape=(28, 28, 1))
     )
     model.add(MaxPooling2D(pool_size=(3, 3)))
     model.add(Conv2D(64, kernel_size=(3, 3), activation="sigmoid"))
@@ -83,7 +86,7 @@ def random_state():
 
 # %%
 model = make_model()
-x_train, y_train = get_data_test()
+x_train, y_train, x_test, y_test = get_data()
 
 loss = [
     "mean_squared_error",
@@ -104,12 +107,12 @@ loss = [
 pso_mnist = optimizer(
     model,
     loss="mean_squared_error",
-    n_particles=2000,
+    n_particles=600,
     c0=0.2,
     c1=0.4,
     w_min=0.3,
-    w_max=0.7,
+    w_max=0.5,
-    negative_swarm=0.1,
+    negative_swarm=0.05,
     mutation_swarm=0.3,
     particle_min=-4,
     particle_max=4,
@@ -118,16 +121,16 @@ pso_mnist = optimizer(
 best_score = pso_mnist.fit(
     x_train,
     y_train,
-    epochs=300,
+    epochs=200,
     save_info=True,
-    log=1,
+    log=2,
     log_name="mnist",
     save_path="./logs/mnist",
     renewal="acc",
     check_point=25,
     empirical_balance=False,
     dispersion=False,
-
+    batch_size=32,
 )
 
 print("Done!")

mnist_tf.py

@@ -1,8 +1,13 @@
+from keras.models import Sequential
+from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
+from keras.datasets import mnist
+from keras.utils import to_categorical
+# from tensorflow.data.Dataset import from_tensor_slices
+import tensorflow as tf
 import os
 
 os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
 
-import tensorflow as tf
 
 gpus = tf.config.experimental.list_physical_devices("GPU")
 if gpus:
@@ -13,10 +18,6 @@ if gpus:
     finally:
         del gpus
 
-from keras.datasets import mnist
-from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D
-from keras.models import Sequential
-
 
 def get_data():
     (x_train, y_train), (x_test, y_test) = mnist.load_data()
@@ -38,10 +39,26 @@ def get_data_test():
     return x_test, y_test
 
 
+class _batch_generator:
+    def __init__(self, x, y, batch_size: int = 32):
+        self.batch_size = batch_size
+        self.index = 0
+        dataset = tf.data.Dataset.from_tensor_slices((x, y))
+        self.dataset = list(dataset.batch(batch_size))
+        self.max_index = len(dataset) // batch_size
+
+    def next(self):
+        self.index += 1
+        if self.index >= self.max_index:
+            self.index = 0
+        return self.dataset[self.index][0], self.dataset[self.index][1]
+
+
 def make_model():
     model = Sequential()
     model.add(
-        Conv2D(32, kernel_size=(5, 5), activation="relu", input_shape=(28, 28, 1))
+        Conv2D(32, kernel_size=(5, 5), activation="relu",
+               input_shape=(28, 28, 1))
     )
     model.add(MaxPooling2D(pool_size=(3, 3)))
     model.add(Conv2D(64, kernel_size=(3, 3), activation="relu"))
@@ -56,17 +73,24 @@ def make_model():
 
 model = make_model()
 x_train, y_train, x_test, y_test = get_data()
-
 y_train = tf.one_hot(y_train, 10)
 y_test = tf.one_hot(y_test, 10)
 
+dataset = _batch_generator(x_train, y_train, 64)
+
 model.compile(optimizer="adam", loss="mse", metrics=["accuracy"])
 
-print("Training model...")
+count = 0
-model.fit(x_train, y_train, epochs=100, batch_size=128, verbose=1)
+
+while count < 20:
+    x_batch, y_batch = dataset.next()
+    count += 1
+    print("Training model...")
+    model.fit(x_batch, y_batch, epochs=1, batch_size=1, verbose=1)
+
+print(count)
 
 print("Evaluating model...")
-model.evaluate(x_test, y_test, verbose=1)
+model.evaluate(x_test, y_test, verbose=2)
 
 weights = model.get_weights()
-

pso/__init__.py

@@ -1,7 +1,7 @@
 from .optimizer import Optimizer as optimizer
 from .particle import Particle as particle
 
-__version__ = "0.1.7"
+__version__ = "0.1.8"
 
 __all__ = [
     "optimizer",

pso/optimizer.py

@@ -106,7 +106,11 @@ class Optimizer:
                 w_ = np.random.uniform(particle_min, particle_max, len(w_))
                 model_.set_weights(self._decode(w_, sh_, len_))
 
-                model_.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"])
+                model_.compile(
+                    loss=self.loss,
+                    optimizer="sgd",
+                    metrics=["accuracy"]
+                )
                 self.particles[i] = Particle(
                     model_,
                     loss,
@@ -224,6 +228,20 @@ class Optimizer:
         else:
             return 1 + np.abs(score_)
 
+    class _batch_generator:
+        def __init__(self, x, y, batch_size: int = 32):
+            self.batch_size = batch_size
+            self.index = 0
+            dataset = tf.data.Dataset.from_tensor_slices((x, y))
+            self.dataset = list(dataset.batch(batch_size))
+            self.max_index = len(dataset) // batch_size
+
+        def next(self):
+            self.index += 1
+            if self.index >= self.max_index:
+                self.index = 0
+            return self.dataset[self.index][0], self.dataset[self.index][1]
+
     def fit(
         self,
         x,
@@ -237,6 +255,7 @@ class Optimizer:
         empirical_balance: bool = False,
         dispersion: bool = False,
         check_point: int = None,
+        batch_size: int = 128,
     ):
         """
         # Args:
@@ -250,6 +269,7 @@ class Optimizer:
             empirical_balance : bool - True : EBPSO, False : PSO,
             dispersion : bool - True : g_best 의 값을 분산시켜 전역해를 찾음, False : g_best 의 값만 사용
             check_point : int - 저장할 위치 - None : 저장 안함
+            batch_size : int - batch size default : 128
         """
         self.save_path = save_path
         self.empirical_balance = empirical_balance
@@ -278,10 +298,33 @@ class Optimizer:
         except ValueError as ve:
             sys.exit(ve)
 
+        model_ = keras.models.model_from_json(self.model.to_json())
+        model_.compile(loss=self.loss, optimizer="adam", metrics=["accuracy"])
+        model_.fit(x, y, epochs=1, batch_size=64, verbose=0)
+        score = model_.evaluate(x, y, verbose=1)
+
+        if renewal == "acc":
+            self.g_best_score[0] = score[1]
+            self.g_best_score[1] = score[0]
+        else:
+            self.g_best_score[0] = score[0]
+            self.g_best_score[1] = score[1]
+
+        self.g_best = model_.get_weights()
+        self.g_best_ = model_.get_weights()
+
+        del model_
+
+        dataset = self._batch_generator(x, y, batch_size=batch_size)
+
         for i in tqdm(range(self.n_particles), desc="Initializing velocity"):
             p = self.particles[i]
-            local_score = p.get_score(x, y, renewal=renewal)
+
+            x_batch, y_batch = dataset.next()
+            local_score = p.get_score(x_batch, y_batch, renewal=renewal)
+
             particle_sum += local_score[1]
+
             if renewal == "acc":
                 if local_score[1] > self.g_best_score[0]:
                     self.g_best_score[0] = local_score[1]
@@ -329,7 +372,7 @@ class Optimizer:
             epoch_sum = 0
             epochs_pbar = tqdm(
                 range(epochs),
-                desc=f"best {self.g_best_score[0]:.4f}|{self.g_best_score[1]:.4f}",
+                desc=f"best {self.g_best_score[0]:.4f} | {self.g_best_score[1]:.4f}",
                 ascii=True,
                 leave=True,
                 position=0,
@@ -355,6 +398,8 @@ class Optimizer:
 
                     g_best = self.g_best
 
+                    x_batch, y_batch = dataset.next()
+
                     if dispersion:
                         ts = self.particle_min + np.random.rand() * (
                             self.particle_max - self.particle_min
@@ -367,7 +412,9 @@ class Optimizer:
 
                     if empirical_balance:
                         if np.random.rand() < np.exp(-(epoch) / epochs):
-                            w_p_ = self._f(x, y, self.particles[i].get_best_weights())
+                            w_p_ = self._f(
+                                x, y, self.particles[i].get_best_weights()
+                            )
                             w_g_ = self._f(x, y, self.g_best)
                             w_p = w_p_ / (w_p_ + w_g_)
                             w_g = w_p_ / (w_p_ + w_g_)
@@ -417,8 +464,8 @@ class Optimizer:
                             del p_
 
                         score = self.particles[i].step_w(
-                            x,
+                            x_batch,
-                            y,
+                            y_batch,
                             self.c0,
                             self.c1,
                             w,
@@ -431,13 +478,15 @@ class Optimizer:
 
                     else:
                         score = self.particles[i].step(
-                            x, y, self.c0, self.c1, w, g_best, renewal=renewal
+                            x_batch, y_batch, self.c0, self.c1, w, g_best, renewal=renewal
                         )
 
                     if log == 2:
                         with self.train_summary_writer[i].as_default():
                             tf.summary.scalar("loss", score[0], step=epoch + 1)
-                            tf.summary.scalar("accuracy", score[1], step=epoch + 1)
+                            tf.summary.scalar(
+                                "accuracy", score[1], step=epoch + 1
+                            )
 
                     if renewal == "acc":
                         if score[1] >= max_score:
@@ -447,11 +496,13 @@ class Optimizer:
                         if score[1] >= self.g_best_score[0]:
                             if score[1] > self.g_best_score[0]:
                                 self.g_best_score[0] = score[1]
-                                self.g_best = self.particles[i].get_best_weights()
+                                self.g_best = self.particles[i].get_best_weights(
+                                )
                             else:
                                 if score[0] < self.g_best_score[1]:
                                     self.g_best_score[1] = score[0]
-                                    self.g_best = self.particles[i].get_best_weights()
+                                    self.g_best = self.particles[i].get_best_weights(
+                                    )
                             epochs_pbar.set_description(
                                 f"best {self.g_best_score[0]:.4f} | {self.g_best_score[1]:.4f}"
                             )
@@ -463,11 +514,13 @@ class Optimizer:
                         if score[0] <= self.g_best_score[1]:
                             if score[0] < self.g_best_score[1]:
                                 self.g_best_score[1] = score[0]
-                                self.g_best = self.particles[i].get_best_weights()
+                                self.g_best = self.particles[i].get_best_weights(
+                                )
                             else:
                                 if score[1] > self.g_best_score[0]:
                                     self.g_best_score[0] = score[1]
-                                    self.g_best = self.particles[i].get_best_weights()
+                                    self.g_best = self.particles[i].get_best_weights(
+                                    )
                             epochs_pbar.set_description(
                                 f"best {self.g_best_score[0]:.4f} | {self.g_best_score[1]:.4f}"
                             )
@@ -505,7 +558,8 @@ class Optimizer:
                 if check_point is not None:
                     if epoch % check_point == 0:
                         os.makedirs(f"./{save_path}/{self.day}", exist_ok=True)
-                        self._check_point_save(f"./{save_path}/{self.day}/ckpt-{epoch}")
+                        self._check_point_save(
+                            f"./{save_path}/{self.day}/ckpt-{epoch}")
 
                 gc.collect()
                 tf.keras.backend.reset_uids()
@@ -513,10 +567,13 @@ class Optimizer:
 
         except KeyboardInterrupt:
             print("Ctrl + C : Stop Training")
+
         except MemoryError:
             print("Memory Error : Stop Training")
+
         except Exception as e:
             print(e)
+
         finally:
             self.model_save(save_path)
             print("model save")

pso/particle.py

@@ -133,7 +133,9 @@ class Particle:
         encode_v, v_sh, v_len = self._encode(weights=self.velocities)
         encode_p, p_sh, p_len = self._encode(weights=self.best_weights)
         encode_g, g_sh, g_len = self._encode(weights=g_best)
-        encode_before, before_sh, before_len = self._encode(weights=self.before_best)
+        encode_before, before_sh, before_len = self._encode(
+            weights=self.before_best
+        )
         r_0 = np.random.rand()
         r_1 = np.random.rand()
 
@@ -187,7 +189,9 @@ class Particle:
         encode_v, v_sh, v_len = self._encode(weights=self.velocities)
         encode_p, p_sh, p_len = self._encode(weights=self.best_weights)
         encode_g, g_sh, g_len = self._encode(weights=g_best)
-        encode_before, before_sh, before_len = self._encode(weights=self.before_best)
+        encode_before, before_sh, before_len = self._encode(
+            weights=self.before_best
+        )
         r_0 = np.random.rand()
         r_1 = np.random.rand()
 

setup.py

@@ -7,7 +7,7 @@ VERSION = pso.__version__
 setup(
     name="pso2keras",
     version=VERSION,
-    description="Particle Swarm Optimization to tensorflow package",
+    description="Particle Swarm Optimization on tensorflow package",
     author="pieroot",
     author_email="jgbong0306@gmail.com",
     url="https://github.com/jung-geun/PSO",
@@ -16,8 +16,8 @@ setup(
         "numpy",
         "pandas",
         "ipython",
-        "tensorflow<=2.11.1",
+        "tensorflow<=2.11.1,>=2.8.0",
-        "keras"
+        "keras<=2.11.1,>=2.8.0",
     ],
     packages=find_packages(exclude=[]),
     keywords=["pso", "tensorflow", "keras"],
@@ -26,4 +26,13 @@ setup(
     zip_safe=False,
     long_description=open("README.md", encoding="UTF8").read(),
     long_description_content_type="text/markdown",
+    license="MIT",
+    classifiers=[
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python :: 3:: Only"
+        "Programming Language :: Python :: 3.7"
+        "Programming Language :: Python :: 3.8",
+        "programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+    ],
 )

test01.py

@@ -1,11 +1,4 @@
-# 반복문을 사용해서 자동 생성하는 python 코드
+it =iter(range(10))
 
-
+for i in range(100):
-def pibonachi(n):
+    print(next(it))
-    if n <= 1:
-        return n
-    else:
-        return pibonachi(n - 1) + pibonachi(n - 2)
-
-
-print(pibonachi(10))