From dfc0df7d481b7dc0d94fda384edbf74b38dcfa3d Mon Sep 17 00:00:00 2001 From: jung-geun Date: Wed, 18 Oct 2023 14:42:59 +0900 Subject: [PATCH] =?UTF-8?q?23-10-18=20batch=20size=20=EC=A0=81=EC=9A=A9=20?= =?UTF-8?q?->=20=EC=86=8D=EB=8F=84=20=EA=B0=9C=EC=84=A0=20=EC=97=AD?= =?UTF-8?q?=EC=A0=84=ED=8C=8C=201=ED=9A=8C=20=EC=A0=81=EC=9A=A9=20->=20?= =?UTF-8?q?=EC=A1=B0=EA=B8=B0=20=EC=88=98=EB=A0=B4=EC=9D=84=20=EC=9D=BC?= =?UTF-8?q?=EB=B6=80=20=EB=B0=A9=EC=A7=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/pypi.yml | 2 +- .vscode/settings.json | 3 +- README.md | 3 ++ mnist.py | 45 +++++++++++---------- mnist_tf.py | 46 ++++++++++++++++----- pso/__init__.py | 2 +- pso/optimizer.py | 83 ++++++++++++++++++++++++++++++++------ pso/particle.py | 8 +++- setup.py | 15 +++++-- test01.py | 13 ++---- 10 files changed, 157 insertions(+), 63 deletions(-) diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index 29a844d..c0da476 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -16,7 +16,7 @@ jobs: strategy: max-parallel: 5 matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 diff --git a/.vscode/settings.json b/.vscode/settings.json index cb8f564..9ee86e7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "[python]": { - "editor.defaultFormatter": "ms-python.black-formatter" + "editor.defaultFormatter": "ms-python.autopep8" }, + "python.formatting.provider": "none" } \ No newline at end of file diff --git a/README.md b/README.md index b0a2f03..6b25ba4 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ [![Python Package Index publish](https://github.com/jung-geun/PSO/actions/workflows/pypi.yml/badge.svg?event=push)](https://github.com/jung-geun/PSO/actions/workflows/pypi.yml) +[![PyPI - Version](https://img.shields.io/pypi/v/pso2keras)](https://pypi.org/project/pso2keras/) ### 목차 @@ -14,6 +15,8 @@ # PSO 알고리즘 구현 및 새로운 시도 +Particle Swarm Optimization on tensorflow package + pso 알고리즘을 사용하여 새로운 학습 방법을 찾는중 입니다
병렬처리로 사용하는 논문을 찾아보았지만 이보다 더 좋은 방법이 있을 것 같아서 찾아보고 있습니다 - [[1]](#참고-자료)
diff --git a/mnist.py b/mnist.py index 1a20724..60317a8 100644 --- a/mnist.py +++ b/mnist.py @@ -1,19 +1,17 @@ # %% +from pso import optimizer +from tensorflow import keras +from keras.models import Sequential +from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D +from keras.datasets import mnist +import tensorflow as tf +import numpy as np import json import os import sys os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" -import numpy as np -import tensorflow as tf -from keras.datasets import mnist -from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D -from keras.models import Sequential -from tensorflow import keras - -from pso import optimizer - def get_data(): (x_train, y_train), (x_test, y_test) = mnist.load_data() @@ -24,8 +22,10 @@ def get_data(): y_train, y_test = tf.one_hot(y_train, 10), tf.one_hot(y_test, 10) - x_train, x_test = tf.convert_to_tensor(x_train), tf.convert_to_tensor(x_test) - y_train, y_test = tf.convert_to_tensor(y_train), tf.convert_to_tensor(y_test) + x_train, x_test = tf.convert_to_tensor( + x_train), tf.convert_to_tensor(x_test) + y_train, y_test = tf.convert_to_tensor( + y_train), tf.convert_to_tensor(y_test) print(f"x_train : {x_train[0].shape} | y_train : {y_train[0].shape}") print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}") @@ -40,8 +40,10 @@ def get_data_test(): y_train, y_test = tf.one_hot(y_train, 10), tf.one_hot(y_test, 10) - x_train, x_test = tf.convert_to_tensor(x_train), tf.convert_to_tensor(x_test) - y_train, y_test = tf.convert_to_tensor(y_train), tf.convert_to_tensor(y_test) + x_train, x_test = tf.convert_to_tensor( + x_train), tf.convert_to_tensor(x_test) + y_train, y_test = tf.convert_to_tensor( + y_train), tf.convert_to_tensor(y_test) print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}") @@ -51,7 +53,8 @@ def get_data_test(): def make_model(): model = Sequential() model.add( - Conv2D(32, kernel_size=(5, 5), activation="sigmoid", input_shape=(28, 28, 1)) + Conv2D(32, kernel_size=(5, 5), activation="sigmoid", + input_shape=(28, 28, 1)) ) model.add(MaxPooling2D(pool_size=(3, 3))) model.add(Conv2D(64, kernel_size=(3, 3), activation="sigmoid")) @@ -83,7 +86,7 @@ def random_state(): # %% model = make_model() -x_train, y_train = get_data_test() +x_train, y_train, x_test, y_test = get_data() loss = [ "mean_squared_error", @@ -104,12 +107,12 @@ loss = [ pso_mnist = optimizer( model, loss="mean_squared_error", - n_particles=2000, + n_particles=600, c0=0.2, c1=0.4, w_min=0.3, - w_max=0.7, - negative_swarm=0.1, + w_max=0.5, + negative_swarm=0.05, mutation_swarm=0.3, particle_min=-4, particle_max=4, @@ -118,16 +121,16 @@ pso_mnist = optimizer( best_score = pso_mnist.fit( x_train, y_train, - epochs=300, + epochs=200, save_info=True, - log=1, + log=2, log_name="mnist", save_path="./logs/mnist", renewal="acc", check_point=25, empirical_balance=False, dispersion=False, - + batch_size=32, ) print("Done!") diff --git a/mnist_tf.py b/mnist_tf.py index 1aa78c7..c8e3023 100644 --- a/mnist_tf.py +++ b/mnist_tf.py @@ -1,8 +1,13 @@ +from keras.models import Sequential +from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D +from keras.datasets import mnist +from keras.utils import to_categorical +# from tensorflow.data.Dataset import from_tensor_slices +import tensorflow as tf import os os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" -import tensorflow as tf gpus = tf.config.experimental.list_physical_devices("GPU") if gpus: @@ -13,10 +18,6 @@ if gpus: finally: del gpus -from keras.datasets import mnist -from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D -from keras.models import Sequential - def get_data(): (x_train, y_train), (x_test, y_test) = mnist.load_data() @@ -38,10 +39,26 @@ def get_data_test(): return x_test, y_test +class _batch_generator: + def __init__(self, x, y, batch_size: int = 32): + self.batch_size = batch_size + self.index = 0 + dataset = tf.data.Dataset.from_tensor_slices((x, y)) + self.dataset = list(dataset.batch(batch_size)) + self.max_index = len(dataset) // batch_size + + def next(self): + self.index += 1 + if self.index >= self.max_index: + self.index = 0 + return self.dataset[self.index][0], self.dataset[self.index][1] + + def make_model(): model = Sequential() model.add( - Conv2D(32, kernel_size=(5, 5), activation="relu", input_shape=(28, 28, 1)) + Conv2D(32, kernel_size=(5, 5), activation="relu", + input_shape=(28, 28, 1)) ) model.add(MaxPooling2D(pool_size=(3, 3))) model.add(Conv2D(64, kernel_size=(3, 3), activation="relu")) @@ -56,17 +73,24 @@ def make_model(): model = make_model() x_train, y_train, x_test, y_test = get_data() - y_train = tf.one_hot(y_train, 10) y_test = tf.one_hot(y_test, 10) +dataset = _batch_generator(x_train, y_train, 64) + model.compile(optimizer="adam", loss="mse", metrics=["accuracy"]) -print("Training model...") -model.fit(x_train, y_train, epochs=100, batch_size=128, verbose=1) +count = 0 + +while count < 20: + x_batch, y_batch = dataset.next() + count += 1 + print("Training model...") + model.fit(x_batch, y_batch, epochs=1, batch_size=1, verbose=1) + +print(count) print("Evaluating model...") -model.evaluate(x_test, y_test, verbose=1) +model.evaluate(x_test, y_test, verbose=2) weights = model.get_weights() - diff --git a/pso/__init__.py b/pso/__init__.py index d973856..d87eb54 100644 --- a/pso/__init__.py +++ b/pso/__init__.py @@ -1,7 +1,7 @@ from .optimizer import Optimizer as optimizer from .particle import Particle as particle -__version__ = "0.1.7" +__version__ = "0.1.8" __all__ = [ "optimizer", diff --git a/pso/optimizer.py b/pso/optimizer.py index 4602299..94500b3 100644 --- a/pso/optimizer.py +++ b/pso/optimizer.py @@ -106,7 +106,11 @@ class Optimizer: w_ = np.random.uniform(particle_min, particle_max, len(w_)) model_.set_weights(self._decode(w_, sh_, len_)) - model_.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"]) + model_.compile( + loss=self.loss, + optimizer="sgd", + metrics=["accuracy"] + ) self.particles[i] = Particle( model_, loss, @@ -224,6 +228,20 @@ class Optimizer: else: return 1 + np.abs(score_) + class _batch_generator: + def __init__(self, x, y, batch_size: int = 32): + self.batch_size = batch_size + self.index = 0 + dataset = tf.data.Dataset.from_tensor_slices((x, y)) + self.dataset = list(dataset.batch(batch_size)) + self.max_index = len(dataset) // batch_size + + def next(self): + self.index += 1 + if self.index >= self.max_index: + self.index = 0 + return self.dataset[self.index][0], self.dataset[self.index][1] + def fit( self, x, @@ -237,6 +255,7 @@ class Optimizer: empirical_balance: bool = False, dispersion: bool = False, check_point: int = None, + batch_size: int = 128, ): """ # Args: @@ -250,6 +269,7 @@ class Optimizer: empirical_balance : bool - True : EBPSO, False : PSO, dispersion : bool - True : g_best 의 값을 분산시켜 전역해를 찾음, False : g_best 의 값만 사용 check_point : int - 저장할 위치 - None : 저장 안함 + batch_size : int - batch size default : 128 """ self.save_path = save_path self.empirical_balance = empirical_balance @@ -278,10 +298,33 @@ class Optimizer: except ValueError as ve: sys.exit(ve) + model_ = keras.models.model_from_json(self.model.to_json()) + model_.compile(loss=self.loss, optimizer="adam", metrics=["accuracy"]) + model_.fit(x, y, epochs=1, batch_size=64, verbose=0) + score = model_.evaluate(x, y, verbose=1) + + if renewal == "acc": + self.g_best_score[0] = score[1] + self.g_best_score[1] = score[0] + else: + self.g_best_score[0] = score[0] + self.g_best_score[1] = score[1] + + self.g_best = model_.get_weights() + self.g_best_ = model_.get_weights() + + del model_ + + dataset = self._batch_generator(x, y, batch_size=batch_size) + for i in tqdm(range(self.n_particles), desc="Initializing velocity"): p = self.particles[i] - local_score = p.get_score(x, y, renewal=renewal) + + x_batch, y_batch = dataset.next() + local_score = p.get_score(x_batch, y_batch, renewal=renewal) + particle_sum += local_score[1] + if renewal == "acc": if local_score[1] > self.g_best_score[0]: self.g_best_score[0] = local_score[1] @@ -329,7 +372,7 @@ class Optimizer: epoch_sum = 0 epochs_pbar = tqdm( range(epochs), - desc=f"best {self.g_best_score[0]:.4f}|{self.g_best_score[1]:.4f}", + desc=f"best {self.g_best_score[0]:.4f} | {self.g_best_score[1]:.4f}", ascii=True, leave=True, position=0, @@ -355,6 +398,8 @@ class Optimizer: g_best = self.g_best + x_batch, y_batch = dataset.next() + if dispersion: ts = self.particle_min + np.random.rand() * ( self.particle_max - self.particle_min @@ -367,7 +412,9 @@ class Optimizer: if empirical_balance: if np.random.rand() < np.exp(-(epoch) / epochs): - w_p_ = self._f(x, y, self.particles[i].get_best_weights()) + w_p_ = self._f( + x, y, self.particles[i].get_best_weights() + ) w_g_ = self._f(x, y, self.g_best) w_p = w_p_ / (w_p_ + w_g_) w_g = w_p_ / (w_p_ + w_g_) @@ -417,8 +464,8 @@ class Optimizer: del p_ score = self.particles[i].step_w( - x, - y, + x_batch, + y_batch, self.c0, self.c1, w, @@ -431,13 +478,15 @@ class Optimizer: else: score = self.particles[i].step( - x, y, self.c0, self.c1, w, g_best, renewal=renewal + x_batch, y_batch, self.c0, self.c1, w, g_best, renewal=renewal ) if log == 2: with self.train_summary_writer[i].as_default(): tf.summary.scalar("loss", score[0], step=epoch + 1) - tf.summary.scalar("accuracy", score[1], step=epoch + 1) + tf.summary.scalar( + "accuracy", score[1], step=epoch + 1 + ) if renewal == "acc": if score[1] >= max_score: @@ -447,11 +496,13 @@ class Optimizer: if score[1] >= self.g_best_score[0]: if score[1] > self.g_best_score[0]: self.g_best_score[0] = score[1] - self.g_best = self.particles[i].get_best_weights() + self.g_best = self.particles[i].get_best_weights( + ) else: if score[0] < self.g_best_score[1]: self.g_best_score[1] = score[0] - self.g_best = self.particles[i].get_best_weights() + self.g_best = self.particles[i].get_best_weights( + ) epochs_pbar.set_description( f"best {self.g_best_score[0]:.4f} | {self.g_best_score[1]:.4f}" ) @@ -463,11 +514,13 @@ class Optimizer: if score[0] <= self.g_best_score[1]: if score[0] < self.g_best_score[1]: self.g_best_score[1] = score[0] - self.g_best = self.particles[i].get_best_weights() + self.g_best = self.particles[i].get_best_weights( + ) else: if score[1] > self.g_best_score[0]: self.g_best_score[0] = score[1] - self.g_best = self.particles[i].get_best_weights() + self.g_best = self.particles[i].get_best_weights( + ) epochs_pbar.set_description( f"best {self.g_best_score[0]:.4f} | {self.g_best_score[1]:.4f}" ) @@ -505,7 +558,8 @@ class Optimizer: if check_point is not None: if epoch % check_point == 0: os.makedirs(f"./{save_path}/{self.day}", exist_ok=True) - self._check_point_save(f"./{save_path}/{self.day}/ckpt-{epoch}") + self._check_point_save( + f"./{save_path}/{self.day}/ckpt-{epoch}") gc.collect() tf.keras.backend.reset_uids() @@ -513,10 +567,13 @@ class Optimizer: except KeyboardInterrupt: print("Ctrl + C : Stop Training") + except MemoryError: print("Memory Error : Stop Training") + except Exception as e: print(e) + finally: self.model_save(save_path) print("model save") diff --git a/pso/particle.py b/pso/particle.py index afef9fd..23f53e0 100644 --- a/pso/particle.py +++ b/pso/particle.py @@ -133,7 +133,9 @@ class Particle: encode_v, v_sh, v_len = self._encode(weights=self.velocities) encode_p, p_sh, p_len = self._encode(weights=self.best_weights) encode_g, g_sh, g_len = self._encode(weights=g_best) - encode_before, before_sh, before_len = self._encode(weights=self.before_best) + encode_before, before_sh, before_len = self._encode( + weights=self.before_best + ) r_0 = np.random.rand() r_1 = np.random.rand() @@ -187,7 +189,9 @@ class Particle: encode_v, v_sh, v_len = self._encode(weights=self.velocities) encode_p, p_sh, p_len = self._encode(weights=self.best_weights) encode_g, g_sh, g_len = self._encode(weights=g_best) - encode_before, before_sh, before_len = self._encode(weights=self.before_best) + encode_before, before_sh, before_len = self._encode( + weights=self.before_best + ) r_0 = np.random.rand() r_1 = np.random.rand() diff --git a/setup.py b/setup.py index b70ff3f..e4d8ba9 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ VERSION = pso.__version__ setup( name="pso2keras", version=VERSION, - description="Particle Swarm Optimization to tensorflow package", + description="Particle Swarm Optimization on tensorflow package", author="pieroot", author_email="jgbong0306@gmail.com", url="https://github.com/jung-geun/PSO", @@ -16,8 +16,8 @@ setup( "numpy", "pandas", "ipython", - "tensorflow<=2.11.1", - "keras" + "tensorflow<=2.11.1,>=2.8.0", + "keras<=2.11.1,>=2.8.0", ], packages=find_packages(exclude=[]), keywords=["pso", "tensorflow", "keras"], @@ -26,4 +26,13 @@ setup( zip_safe=False, long_description=open("README.md", encoding="UTF8").read(), long_description_content_type="text/markdown", + license="MIT", + classifiers=[ + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3:: Only" + "Programming Language :: Python :: 3.7" + "Programming Language :: Python :: 3.8", + "programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + ], ) diff --git a/test01.py b/test01.py index 6c10d3e..7602de8 100644 --- a/test01.py +++ b/test01.py @@ -1,11 +1,4 @@ -# 반복문을 사용해서 자동 생성하는 python 코드 +it =iter(range(10)) - -def pibonachi(n): - if n <= 1: - return n - else: - return pibonachi(n - 1) + pibonachi(n - 2) - - -print(pibonachi(10)) +for i in range(100): + print(next(it)) \ No newline at end of file