diff --git a/bean.py b/bean.py index f78be37..89c74b2 100644 --- a/bean.py +++ b/bean.py @@ -1,24 +1,21 @@ import os -import pandas as pd -import tensorflow as tf +from keras.layers import Dense +from keras.models import Sequential +from keras.utils import to_categorical from sklearn.model_selection import train_test_split from tensorflow import keras -from tensorflow.keras import layers -from tensorflow.keras.layers import Dense -from tensorflow.keras.models import Sequential -from tensorflow.keras.utils import to_categorical from ucimlrepo import fetch_ucirepo -os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' -os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true' +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" +os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "true" def make_model(): model = Sequential() - model.add(Dense(12, input_dim=16, activation='relu')) - model.add(Dense(8, activation='relu')) - model.add(Dense(7, activation='softmax')) + model.add(Dense(12, input_dim=16, activation="relu")) + model.add(Dense(8, activation="relu")) + model.add(Dense(7, activation="softmax")) return model @@ -34,7 +31,7 @@ def get_data(): x = X.to_numpy() # object to categorical - x = x.astype('float32') + x = x.astype("float32") y_class = to_categorical(y) @@ -49,21 +46,27 @@ def get_data(): # y_class = to_categorical(y) x_train, x_test, y_train, y_test = train_test_split( - x, y_class, test_size=0.2, random_state=42, shuffle=True) + x, y_class, test_size=0.2, random_state=42, shuffle=True + ) return x_train, x_test, y_train, y_test x_train, x_test, y_train, y_test = get_data() model = make_model() early_stopping = keras.callbacks.EarlyStopping( - patience=10, min_delta=0.001, restore_best_weights=True) + patience=10, min_delta=0.001, restore_best_weights=True +) -model.compile(loss='sparse_categorical_crossentropy', - optimizer='adam', metrics=['accuracy', "mse"]) +model.compile( + loss="sparse_categorical_crossentropy", + optimizer="adam", + metrics=["accuracy", "mse"], +) model.summary() -history = model.fit(x_train, y_train, epochs=150, - batch_size=10, callbacks=[early_stopping]) +history = model.fit( + x_train, y_train, epochs=150, batch_size=10, callbacks=[early_stopping] +) score = model.evaluate(x_test, y_test, verbose=2) diff --git a/digits.py b/digits.py index 256f55d..e9fb16a 100644 --- a/digits.py +++ b/digits.py @@ -1,15 +1,11 @@ import os import sys -import pandas as pd -import tensorflow as tf +from keras.layers import Dense +from keras.models import Sequential +from keras.utils import to_categorical from sklearn.datasets import load_digits from sklearn.model_selection import train_test_split -from tensorflow import keras -from tensorflow.keras import layers -from tensorflow.keras.layers import Dense -from tensorflow.keras.models import Sequential -from tensorflow.keras.utils import to_categorical from pso import optimizer diff --git a/fashion_mnist.py b/fashion_mnist.py index 5a77184..e0af751 100644 --- a/fashion_mnist.py +++ b/fashion_mnist.py @@ -1,15 +1,16 @@ # %% -from pso import optimizer -from tensorflow import keras -from keras.models import Sequential -from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D -from keras.datasets import mnist, fashion_mnist -import tensorflow as tf -import numpy as np import json import os import sys +import numpy as np +import tensorflow as tf +from keras.datasets import fashion_mnist +from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D +from keras.models import Sequential + +from pso import optimizer + os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" @@ -22,10 +23,8 @@ def get_data(): y_train, y_test = tf.one_hot(y_train, 10), tf.one_hot(y_test, 10) - x_train, x_test = tf.convert_to_tensor( - x_train), tf.convert_to_tensor(x_test) - y_train, y_test = tf.convert_to_tensor( - y_train), tf.convert_to_tensor(y_test) + x_train, x_test = tf.convert_to_tensor(x_train), tf.convert_to_tensor(x_test) + y_train, y_test = tf.convert_to_tensor(y_train), tf.convert_to_tensor(y_test) print(f"x_train : {x_train[0].shape} | y_train : {y_train[0].shape}") print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}") @@ -36,8 +35,7 @@ def get_data(): def make_model(): model = Sequential() model.add( - Conv2D(32, kernel_size=(5, 5), activation="relu", - input_shape=(28, 28, 1)) + Conv2D(32, kernel_size=(5, 5), activation="relu", input_shape=(28, 28, 1)) ) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(64, kernel_size=(3, 3), activation="relu")) @@ -51,72 +49,39 @@ def make_model(): return model -def random_state(): - with open( - "result/mnist/20230723-061626/mean_squared_error_[0.6384999752044678, 0.0723000094294548].json", - "r", - ) as f: - json_ = json.load(f) - rs = ( - json_["random_state_0"], - np.array(json_["random_state_1"]), - json_["random_state_2"], - json_["random_state_3"], - json_["random_state_4"], - ) - - return rs - - # %% model = make_model() x_train, y_train, x_test, y_test = get_data() -loss = [ - "mean_squared_error", - "categorical_crossentropy", - "sparse_categorical_crossentropy", - "binary_crossentropy", - "kullback_leibler_divergence", - "poisson", - "cosine_similarity", - "log_cosh", - "huber_loss", - "mean_absolute_error", - "mean_absolute_percentage_error", -] - -# rs = random_state() pso_mnist = optimizer( model, loss="categorical_crossentropy", - n_particles=500, - c0=0.5, - c1=1.0, - w_min=0.7, - w_max=1.2, - negative_swarm=0.05, - mutation_swarm=0.3, + n_particles=200, + c0=0.7, + c1=0.5, + w_min=0.1, + w_max=0.8, + negative_swarm=0.0, + mutation_swarm=0.05, convergence_reset=True, convergence_reset_patience=10, + convergence_reset_monitor="loss", ) best_score = pso_mnist.fit( x_train, y_train, - epochs=200, + epochs=1000, save_info=True, log=2, log_name="fashion_mnist", - renewal="acc", + renewal="loss", check_point=25, - empirical_balance=False, - dispersion=False, batch_size=5000, - back_propagation=True, ) print("Done!") sys.exit(0) + diff --git a/iris.py b/iris.py index bd96218..6a356a4 100644 --- a/iris.py +++ b/iris.py @@ -1,13 +1,15 @@ -from pso import optimizer -from tensorflow.keras.models import Sequential -from tensorflow.keras import layers -from tensorflow import keras -from sklearn.model_selection import train_test_split -from sklearn.datasets import load_iris import gc import os import sys +from sklearn.datasets import load_iris +from sklearn.model_selection import train_test_split +from tensorflow import keras +from tensorflow.keras import layers +from tensorflow.keras.models import Sequential + +from pso import optimizer + os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" diff --git a/mnist.py b/mnist.py index 4fc45c9..415e2a0 100644 --- a/mnist.py +++ b/mnist.py @@ -1,15 +1,15 @@ # %% -from pso import optimizer -from tensorflow import keras -from keras.models import Sequential -from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D -from keras.datasets import mnist -import tensorflow as tf -import numpy as np -import json import os import sys +from pso import optimizer + +import tensorflow as tf +from keras.datasets import mnist +from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D +from keras.models import Sequential + + os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2" @@ -53,33 +53,17 @@ def make_model(): model = make_model() x_train, y_train, x_test, y_test = get_data() -loss = [ - "mean_squared_error", - "categorical_crossentropy", - "sparse_categorical_crossentropy", - "binary_crossentropy", - "kullback_leibler_divergence", - "poisson", - "cosine_similarity", - "log_cosh", - "huber_loss", - "mean_absolute_error", - "mean_absolute_percentage_error", - -] - -# rs = random_state() pso_mnist = optimizer( model, loss="categorical_crossentropy", - n_particles=500, - c0=0.5, - c1=0.3, + n_particles=200, + c0=0.7, + c1=0.4, w_min=0.1, w_max=0.9, negative_swarm=0.0, - mutation_swarm=0.1, + mutation_swarm=0.05, convergence_reset=True, convergence_reset_patience=10, convergence_reset_monitor="loss", @@ -89,16 +73,13 @@ pso_mnist = optimizer( best_score = pso_mnist.fit( x_train, y_train, - epochs=500, + epochs=1000, save_info=True, log=2, log_name="mnist", renewal="loss", check_point=25, - empirical_balance=False, - dispersion=False, - batch_size=10000, - back_propagation=False, + batch_size=5000, validate_data=(x_test, y_test), ) diff --git a/mnist_tf.py b/mnist_tf.py index 0d12fbc..e153d46 100644 --- a/mnist_tf.py +++ b/mnist_tf.py @@ -2,6 +2,8 @@ from keras.models import Sequential from keras.layers import Conv2D, Dense, Dropout, Flatten, MaxPooling2D from keras.datasets import mnist from keras.utils import to_categorical +from sklearn.model_selection import train_test_split + # from tensorflow.data.Dataset import from_tensor_slices import tensorflow as tf import os @@ -31,14 +33,6 @@ def get_data(): return x_train, y_train, x_test, y_test - -def get_data_test(): - (x_train, y_train), (x_test, y_test) = mnist.load_data() - x_test = x_test.reshape((10000, 28, 28, 1)) - - return x_test, y_test - - class _batch_generator_: def __init__(self, x, y, batch_size: int = None): self.index = 0 @@ -77,8 +71,9 @@ class _batch_generator_: def __getBatchSlice(self, batch_size): return list( - tf.data.Dataset.from_tensor_slices( - (self.x, self.y)).shuffle(len(self.x)).batch(batch_size) + tf.data.Dataset.from_tensor_slices((self.x, self.y)) + .shuffle(len(self.x)) + .batch(batch_size) ) def getDataset(self): @@ -88,17 +83,18 @@ class _batch_generator_: def make_model(): model = Sequential() model.add( - Conv2D(32, kernel_size=(5, 5), activation="relu", - input_shape=(28, 28, 1)) + Conv2D(64, kernel_size=(5, 5), activation="relu", input_shape=(28, 28, 1)) ) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Dropout(0.5)) - model.add(Conv2D(64, kernel_size=(3, 3), activation="relu")) + model.add(Conv2D(128, kernel_size=(3, 3), activation="relu")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dropout(0.5)) - model.add(Dense(256, activation="relu")) - model.add(Dense(128, activation="relu")) + model.add(Dense(2048, activation="relu")) + model.add(Dropout(0.8)) + model.add(Dense(1024, activation="relu")) + model.add(Dropout(0.8)) model.add(Dense(10, activation="softmax")) return model @@ -112,18 +108,21 @@ y_test = tf.one_hot(y_test, 10) batch = 64 dataset = _batch_generator_(x_train, y_train, batch) -model.compile(optimizer="adam", loss="categorical_crossentropy", - metrics=["accuracy", "mse", "mae"]) +model.compile( + optimizer="adam", + loss="categorical_crossentropy", + metrics=["accuracy", "mse"], +) count = 0 print(f"batch size : {batch}") print("iter " + str(dataset.getMaxIndex())) print("Training model...") -while count < dataset.getMaxIndex(): - x_batch, y_batch = dataset.next() - count += 1 - print(f"iter {count}/{dataset.getMaxIndex()}") - model.fit(x_batch, y_batch, epochs=1, batch_size=batch, verbose=1) +# while count < dataset.getMaxIndex(): +# x_batch, y_batch = dataset.next() +# count += 1 +# print(f"iter {count}/{dataset.getMaxIndex()}") +model.fit(x_train, y_train, epochs=1000, batch_size=batch, verbose=1) print(count) diff --git a/pso/optimizer.py b/pso/optimizer.py index 2540ee6..ac912bf 100644 --- a/pso/optimizer.py +++ b/pso/optimizer.py @@ -1,3 +1,4 @@ +import atexit import gc import json import os @@ -8,10 +9,10 @@ from datetime import datetime import numpy as np import tensorflow as tf +from sklearn.model_selection import train_test_split from tensorboard.plugins.hparams import api as hp from tensorflow import keras from tqdm.auto import tqdm -import atexit from .particle import Particle @@ -46,7 +47,7 @@ class Optimizer: n_particles: int = None, c0: float = 0.5, c1: float = 0.3, - w_min: float = 0.2, + w_min: float = 0.1, w_max: float = 0.9, negative_swarm: float = 0, mutation_swarm: float = 0, @@ -56,7 +57,7 @@ class Optimizer: convergence_reset: bool = False, convergence_reset_patience: int = 10, convergence_reset_min_delta: float = 0.0001, - convergence_reset_monitor: str = "mse", + convergence_reset_monitor: str = "loss", ): """ particle swarm optimization @@ -284,10 +285,10 @@ class Optimizer: def next(self): self.index += 1 - if self.index >= self.max_index: + if self.index > self.max_index: self.index = 0 self.__getBatchSlice(self.batch_size) - return self.dataset[self.index][0], self.dataset[self.index][1] + return self.dataset[self.index - 1][0], self.dataset[self.index - 1][1] def getMaxIndex(self): return self.max_index @@ -306,13 +307,12 @@ class Optimizer: batch_size = len(self.x) // 10 elif batch_size > len(self.x): batch_size = len(self.x) + self.batch_size = batch_size print(f"batch size : {self.batch_size}") self.dataset = self.__getBatchSlice(self.batch_size) self.max_index = len(self.dataset) - if batch_size % len(self.x) != 0: - self.max_index -= 1 def __getBatchSlice(self, batch_size): return list( @@ -331,14 +331,16 @@ class Optimizer: epochs: int = 1, log: int = 0, log_name: str = None, - save_info: bool = False, - renewal: str = "mse", - empirical_balance: bool = False, - dispersion: bool = False, + save_info: bool = None, + renewal: str = None, + empirical_balance: bool = None, + dispersion: bool = None, check_point: int = None, batch_size: int = None, validate_data: tuple = None, + validation_split: float = None, back_propagation: bool = False, + weight_reduction: int = None, ): """ # Args: @@ -355,12 +357,16 @@ class Optimizer: batch_size : int - batch size default : None => len(x) // 10 batch_size > len(x) : auto max batch size validate_data : tuple - (x, y) default : None => (x, y) - back_propagation : bool - True : back propagation, False : not back propagation + back_propagation : bool - True : back propagation, False : not back propagation default : False + weight_reduction : int - 가중치 감소 초기화 주기 default : None => epochs """ try: if x.shape[0] != y.shape[0]: raise ValueError("x, y shape error") + if save_info is None: + save_info = False + if log not in [0, 1, 2]: raise ValueError( """log not in [0, 1, 2] @@ -370,9 +376,18 @@ class Optimizer: """ ) + if renewal is None: + renewal = "loss" + if renewal not in ["acc", "loss", "mse"]: raise ValueError("renewal not in ['acc', 'loss', 'mse']") + if empirical_balance is None: + empirical_balance = False + + if dispersion is None: + dispersion = False + if validate_data is not None: if validate_data[0].shape[0] != validate_data[1].shape[0]: raise ValueError("validate_data shape error") @@ -380,12 +395,23 @@ class Optimizer: if validate_data is None: validate_data = (x, y) + if validation_split is not None: + if validation_split < 0 or validation_split > 1: + raise ValueError("validation_split not in [0, 1]") + + x, validate_data[0], y, validate_data[1] = train_test_split( + x, y, test_size=validation_split, shuffle=True + ) + if batch_size is not None and batch_size < 1: raise ValueError("batch_size < 1") if batch_size is None or batch_size > len(x): batch_size = len(x) + if weight_reduction == None: + weight_reduction = epochs + except ValueError as ve: sys.exit(ve) except Exception as e: @@ -430,34 +456,39 @@ class Optimizer: except Exception as e: sys.exit(e) - if back_propagation: - model_ = keras.models.model_from_json(self.model.to_json()) - model_.compile( - loss=self.loss, - optimizer="adam", - metrics=["accuracy", "mse"], - ) - model_.fit(x, y, epochs=1, verbose=0) - score = model_.evaluate(x, y, verbose=1) - - Particle.g_best_score = score - - Particle.g_best_weights = model_.get_weights() - - del model_ - - dataset = self.batch_generator(x, y, batch_size=batch_size) - - for i in tqdm( - range(len(self.particles)), - desc="best score init", - ascii=True, - leave=True, - ): - score = self.particles[i].get_score(x, y, self.renewal) - self.particles[i].check_global_best(self.renewal) - try: + dataset = self.batch_generator(x, y, batch_size=batch_size) + + if back_propagation: + model_ = keras.models.model_from_json(self.model.to_json()) + model_.compile( + loss=self.loss, + optimizer="adam", + metrics=["accuracy", "mse"], + ) + model_.fit(x, y, epochs=1, verbose=0) + score = model_.evaluate(x, y, verbose=1) + + Particle.g_best_score = score + + Particle.g_best_weights = model_.get_weights() + + del model_ + + else: + for i in tqdm( + range(len(self.particles)), + desc="best score init", + ascii=True, + leave=False, + ): + score = self.particles[i].get_score( + validate_data[0], validate_data[1], self.renewal + ) + self.particles[i].check_global_best(self.renewal) + + print("best score init complete" + str(Particle.g_best_score)) + epoch_sum = 0 epochs_pbar = tqdm( range(epochs), @@ -486,7 +517,12 @@ class Optimizer: ) # w = self.w_max - (self.w_max - self.w_min) * epoch / epochs - w = self.w_max - (self.w_max - self.w_min) * (epoch % 100) / 100 + w = ( + self.w_max + - (self.w_max - self.w_min) + * (epoch % weight_reduction) + / weight_reduction + ) for i in part_pbar: part_pbar.set_description( f"loss: {min_loss:.4f} acc: {max_acc:.4f} mse: {min_mse:.4f}"