23-05-29

EBPSO 알고리즘 구현 - 선택지로 추가
random 으로 분산시키는 방법 구현 - 선택지로 추가
iris 기준 98퍼센트로 나오나 정확한 결과를 지켜봐야 할것으로 보임

.gitignore

@@ -2,4 +2,4 @@
 __pycache__/
 .ipynb_checkpoints/
 *.pdf
-model/
+result/

example.py

@@ -0,0 +1,103 @@
+"""
+example.py
+
+Demonstrates usage of PSOkeras module by training dense Keras model for classifying Iris data set.  Also compares
+results with a number of independent runs of standard Backpropagation algorithm (Adam) equal to the particle count.
+
+@author Mike Holcomb (mjh170630@utdallas.edu)
+"""
+
+from sklearn.datasets import load_iris
+from sklearn.model_selection import train_test_split
+import tensorflow as tf
+from tensorflow import keras
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import Dense
+
+from psokeras import Optimizer
+
+N = 50 # number of particles
+STEPS = 500 # number of steps
+LOSS = 'mse' # Loss function
+BATCH_SIZE = 32 # Size of batches to train on
+
+
+def build_model(loss):
+    """
+    Builds test Keras model for predicting Iris classifications
+
+    :param loss (str): Type of loss - must be one of Keras accepted keras losses
+    :return: Keras dense model of predefined structure
+    """
+    model = Sequential()
+    model.add(Dense(4, activation='sigmoid', input_dim=4, use_bias=True))
+    model.add(Dense(4, activation='sigmoid', use_bias=True))
+    model.add(Dense(3, activation='softmax', use_bias=True))
+
+    model.compile(loss=loss,
+                  optimizer='adam')
+
+    return model
+
+
+def vanilla_backpropagation(x_train, y_train):
+    """
+    Runs N number of backpropagation model training simulations
+    :param x_train: x values to train on
+    :param y_train: target labels to train with
+    :return: best model run as measured by LOSS
+    """
+    best_model = None
+    best_score = 100.0
+
+    for i in range(N):
+        model_s = build_model(LOSS)
+        model_s.fit(x_train, y_train,
+                    epochs=STEPS,
+                    batch_size=BATCH_SIZE,
+                    verbose=0)
+        train_score = model_s.evaluate(x_train, y_train, batch_size=BATCH_SIZE, verbose=0)
+        if train_score < best_score:
+            best_model = model_s
+            best_score = train_score
+    return best_model
+
+
+if __name__ == "__main__":
+    # Section I: Build the data set
+    iris = load_iris()
+    x_train, x_test, y_train, y_test = train_test_split(iris.data,
+                                                        keras.utils.to_categorical(iris.target, num_classes=None),
+                                                        test_size=0.5,
+                                                        random_state=0,
+                                                        stratify=iris.target)
+
+    # Section II: First run the backpropagation simulation
+    model_s = vanilla_backpropagation(x_train=x_train, y_train=y_train)
+
+    b_train_score = model_s.evaluate(x_train, y_train, batch_size=BATCH_SIZE, verbose=0)
+    b_test_score = model_s.evaluate(x_test, y_test, batch_size=BATCH_SIZE, verbose=0)
+    print("Backprop -- train: {:.4f}  test: {:.4f}".format(b_train_score, b_test_score))
+
+    # Section III: Then run the particle swarm optimization
+    # First build model to train on (primarily used for structure, also included in swarm)
+    model_p = build_model(LOSS)
+
+    # Instantiate optimizer with model, loss function, and hyperparameters
+    pso = Optimizer(model=model_p,
+                    loss=LOSS,
+                    n=N,  # Number of particles
+                    acceleration=1.0,  # Contribution of recursive particle velocity (acceleration)
+                    local_rate=0.6,    # Contribution of locally best weights to new velocity
+                    global_rate=0.4    # Contribution of globally best weights to new velocity
+                    )
+
+    # Train model on provided data
+    pso.fit(x_train, y_train, steps=STEPS, batch_size=BATCH_SIZE)
+
+    # Get a copy of the model with the globally best weights
+    model_p = pso.get_best_model()
+
+    p_train_score = model_p.evaluate(x_train, y_train, batch_size=BATCH_SIZE, verbose=0)
+    p_test_score = model_p.evaluate(x_test, y_test, batch_size=BATCH_SIZE, verbose=0)
+    print("PSO -- train: {:.4f}  test: {:.4f}".format(p_train_score, p_test_score))

iris.py

@@ -0,0 +1,48 @@
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+
+import tensorflow as tf
+tf.random.set_seed(777)  # for reproducibility
+
+from sklearn.datasets import load_iris
+from sklearn.model_selection import train_test_split
+
+from tensorflow import keras
+from tensorflow.keras.models import Sequential
+from tensorflow.keras import layers
+
+from pso import Optimizer
+
+import gc
+
+def make_model():
+    model = Sequential()
+    model.add(layers.Dense(10, activation='relu', input_shape=(4,)))
+    model.add(layers.Dense(10, activation='relu'))
+    model.add(layers.Dense(3, activation='softmax'))
+
+    return model
+
+def load_data():
+    iris = load_iris()
+    x = iris.data
+    y = iris.target
+
+    y = keras.utils.to_categorical(y, 3)
+
+    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, shuffle=True, stratify=y)
+
+    return x_train, x_test, y_train, y_test
+
+model = make_model()
+x_train, x_test, y_train, y_test = load_data()
+
+loss = 'categorical_crossentropy'
+
+pso_iris = Optimizer(model, loss=loss, n_particles=50, c0=0.4, c1=0.8, w_min=0.7, w_max=1.3)
+weight, score = pso_iris.fit(
+    x_train, y_train, epochs=500, save=True, save_path="./result/iris", renewal="acc", empirical_balance=True, Dispersion=True, check_point=50)
+pso_iris.model_save("./result/iris")
+pso_iris.save_info("./result/iris/")
+
+gc.collect()

metacode/pso_tf.py

@@ -0,0 +1,268 @@
+import os
+import numpy as np
+from tqdm import tqdm
+from matplotlib import pyplot as plt
+import pandas as pd
+
+import tensorflow as tf
+from tensorflow import keras
+
+import datetime
+import gc
+import cupy as cp
+
+
+
+class PSO(object):
+    """
+    Class implementing PSO algorithm
+    """
+
+    def __init__(self, model: keras.models, loss_method=keras.losses.MeanSquaredError(), n_particles: int = 5):
+        """
+        Initialize the key variables.
+
+        Args:
+            model : 학습할 모델 객체 (Sequential)
+            loss_method : 손실 함수
+            n_particles(int) : 파티클의 개수
+        """
+        self.model = model                                       # 모델
+        self.n_particles = n_particles                           # 파티클의 개수
+        self.loss_method = loss_method                           # 손실 함수
+        model_structure = self.model.to_json()              # 모델의 구조 정보
+        self.init_weights = self.model.get_weights()             # 검색할 차원
+        self.particle_depth = len(self.model.get_weights())      # 검색할 차원의 깊이
+        self.particles_weights = [None] * n_particles            # 파티클의 위치
+        for _ in tqdm(range(self.n_particles), desc="init particles position"):
+            m = keras.models.model_from_json(model_structure)
+            m.compile(loss=self.loss_method,
+                      optimizer="adam", metrics=["accuracy"])
+            self.particles_weights[_] = m.get_weights()     
+
+        # 입력받은 파티클의 개수 * 검색할 차원의 크기 만큼의 균등한 위치를 생성
+        self.velocities = [
+            [0 for i in range(self.particle_depth)] for n in range(n_particles)]
+        for i in tqdm(range(n_particles), desc="init velocities"):
+            
+            self.init_weights = self.model.get_weights()
+            w_,s_,l_ = self._encode(self.init_weights)
+            w_ = np.random.rand(len(w_)) / 5 - 0.10
+            self.velocities[i] = self._decode(w_,s_,l_)
+            # for index, layer in enumerate(self.init_weights):
+            #     self.velocities[i][index] = np.random.rand(
+            #         *layer.shape) / 5 - 0.10
+        
+
+        # 입력받은 파티클의 개수 * 검색할 차원의 크기 만큼의 속도를 무작위로 초기화
+        # 최대 사이즈로 전역 최적갑 저장 - global best
+        self.p_best = self.particles_weights            # 각 파티클의 최적값(최적의 가중치)
+        self.g_best=self.model.get_weights()            # 전역 최적값(최적의 가중치) | 초기값은 모델의 가중치
+
+        # 각 파티클의 최적값의 점수
+        self.p_best_score = [0 for i in range(n_particles)]
+
+        # 전역 최적값의 점수(초기화 - 0)
+        self.g_best_score = 0
+        
+    def __del__(self):
+        del self.model
+        del self.n_particles
+        del self.loss_method
+        del self.init_weights
+        del self.particles_weights
+        del self.velocities
+        del self.p_best
+        del self.g_best
+        del self.p_best_score
+        del self.g_best_score
+
+    def _encode(self,weights: list):
+        # w_gpu = cp.array([])
+        w_gpu = np.array([])
+        lenght = []
+        shape = []
+        for layer in weights:
+            shape.append(layer.shape)
+            w_ = layer.reshape(-1)
+            lenght.append(len(w_))
+            w_gpu = np.append(w_gpu, w_)
+            # w_gpu = cp.append(w_gpu, w_)
+        
+        return w_gpu, shape, lenght
+
+    def _decode(self,weight, shape, lenght):
+        weights = []
+        start = 0
+        for i in range(len(shape)):
+            end = start + lenght[i]
+            # print(f"{start} ~ {end}")
+            # print(f"{shape[i]}")
+            w_ = weight[start:end]
+            w_ = np.reshape(w_, shape[i])
+            # w_ = w_.reshape(shape[i])
+            weights.append(w_)
+            start = end
+
+        return weights
+    
+    def _update_weights(self, weights, v):
+        """
+        Update particle position
+
+        Args:
+            weights (array-like) : 파티클의 현재 가중치
+            v (array-like) : 가중치의 속도
+
+        Returns:
+            (array-like) : 파티클의 새로운 가중치(위치)
+        """
+        # w = np.array(w)  # 각 파티클의 위치
+        # v = np.array(v)  # 각 파티클의 속도(방향과 속력을 가짐)
+        # new_weights = [0 for i in range(len(weights))]
+        # print(f"weights : {weights}")
+        encode_w, w_sh, w_len = self._encode(weights = weights)
+        encode_v, _, _ = self._encode(weights = v)
+        new_w = encode_w + encode_v
+        new_weights = self._decode(new_w, w_sh, w_len)
+        
+        # for i in range(len(weights)):
+            # new_weights[i] = tf.add(weights[i], v[i])
+        # new_w = tf.add(w, v)   # 각 파티클을 랜덤한 속도만큼 진행
+        return new_weights    # 진행한 파티클들의 위치를 반환
+
+    def _update_velocity(self, weights, v, p_best, c0=0.5, c1=1.5, w=0.75):
+        """
+            Update particle velocity
+
+            Args:
+                weights (array-like) : 파티클의 현재 가중치
+                v (array-like) : 속도
+                p_best(array-like) : 각 파티클의 최적의 위치 (최적의 가중치)
+                c0 (float) : 인지 스케일링 상수 (가중치의 중요도 - 지역) - 지역 관성
+                c1 (float) : 사회 스케일링 상수 (가중치의 중요도 - 전역) - 전역 관성
+                w (float) : 관성 상수 (현재 속도의 중요도)
+
+            Returns:
+                (array-like) : 각 파티클의 새로운 속도
+        """
+        # x = np.array(x)
+        # v = np.array(v)
+        # assert np.shape(weights) == np.shape(v), "Position and velocity must have same shape."
+        # 두 데이터의 shape 이 같지 않으면 오류 출력
+        # 0에서 1사이의 숫자를 랜덤 생성
+        r0 = np.random.rand()
+        r1 = np.random.rand()
+        # p_best = np.array(p_best)
+        # g_best = np.array(g_best)
+
+        # 가중치(상수)*속도 + \
+        # 스케일링 상수*랜덤 가중치*(나의 최적값 - 처음 위치) + \
+        # 전역 스케일링 상수*랜덤 가중치*(전체 최적값 - 처음 위치)
+        
+        encode_w, w_sh, w_len = self._encode(weights = weights)
+        encode_v, _, _ = self._encode(weights = v)
+        encode_p, _, _ = self._encode(weights = p_best)
+        encode_g, _, _ = self._encode(weights = self.g_best)
+        
+        new_v = encode_w * encode_v + c0*r0*(encode_p - encode_w) + c1*r1*(encode_g - encode_w)
+        new_velocity = self._decode(new_v, w_sh, w_len)
+        # new_velocity = [None] * len(weights)
+        # for i, layer in enumerate(weights):
+
+            # new_v = w*v[i]
+            # new_v = new_v + c0*r0*(p_best[i] - layer)
+            # new_v = new_v + c1*r1*(self.g_best[i] - layer)
+            # new_velocity[i] = new_v
+
+        # new_v = w*v + c0*r0*(p_best - weights) + c1*r1*(g_best - weights)
+        return new_velocity
+
+    def _get_score(self, x, y):
+        """
+        Compute the score of the current position of the particles.
+
+        Args:
+            x (array-like): The current position of the particles
+            y (array-like): The current position of the particles
+        Returns:
+            (array-like) : 추론에 대한 점수
+        """
+        score = self.model.evaluate(x, y, verbose=0)
+
+        return score
+
+    def optimize(self, x_, y_, maxiter=10, c0=0.5, c1=1.5, w=0.75, save=False, save_path="./result/history"):
+        """
+        Run the PSO optimization process utill the stoping critera is met.
+        Cas for minization. The aim is to minimize the cost function
+
+        Args:
+            maxiter (int): the maximum number of iterations before stopping the optimization
+            파티클의 최종 위치를 위한 반복 횟수
+        Returns:
+            The best solution found (array-like)
+        """
+        if save:
+            os.makedirs(save_path, exist_ok=True)
+            day = datetime.datetime.now().strftime('%m-%d-%H-%M')
+        
+        for _ in range(maxiter):
+
+            for i in tqdm(range(self.n_particles), desc=f"Iter {_}/{maxiter} ", ascii=True):
+                weights = self.particles_weights[i]  # 각 파티클 추출
+                v = self.velocities[i]    # 각 파티클의 다음 속도 추출
+                p_best = self.p_best[i]   # 결과치 저장할 변수 지정
+                # 2. 속도 계산
+                self.velocities[i] = self._update_velocity(
+                    weights, v, p_best, c0, c1, w)
+                # 다음에 움직일 속도 = 최초 위치, 현재 속도, 현재 위치, 최종 위치
+                # 3. 위치 업데이트
+                self.particles_weights[i] = self._update_weights(weights, v)
+                # 현재 위치 = 이전 위치 + 현재 속도
+                # 내 현재 위치가 내 위치의 최소치보다 작으면 갱신
+                self.model.set_weights(self.particles_weights[i])
+                # self.particles_weights[i] = self.model.get_weights()
+                # 4. 평가
+                self.model.compile(loss=self.loss_method,
+                                   optimizer='sgd', metrics=['accuracy'])
+                score = self._get_score(x_, y_)
+
+                if score[1] > self.p_best_score[i]:
+                    self.p_best_score[i] = score[1]
+                    self.p_best[i] = self.particles_weights[i]
+                    if score[1] > self.g_best_score:
+                        self.g_best_score = score[1]
+                        self.g_best = self.particles_weights[i]
+                
+                if save:
+                    with open(f"{save_path}/{day}_{self.n_particles}_{maxiter}_{c0}_{c1}_{w}.csv",'a')as f:
+                        f.write(f"{score[0]}, {score[1]}")
+                        if i != self.n_particles - 1:
+                            f.write(",")
+                            
+            if save:   
+                with open(f"{save_path}/{day}_{self.n_particles}_{maxiter}_{c0}_{c1}_{w}.csv",'a')as f:
+                    f.write("\n")
+            print(
+                f"loss avg : {score[0]/self.n_particles} | acc avg : {score[1]/self.n_particles} | best score : {self.g_best_score}")
+            gc.collect()
+
+        # 전체 최소 위치, 전체 최소 벡터
+        return self.g_best, self._get_score(x_, y_)
+
+    """
+    Returns:
+        최종 가중치
+    """
+
+    def best_weights(self):
+        return self.g_best
+
+    """
+    Returns:
+        최종 가중치의 스코어
+    """
+
+    def best_score(self):
+        return self.g_best_score

metacode/pso_tf_bak.py

@@ -1,7 +1,10 @@
+import os
 import numpy as np
+from tqdm import tqdm
+from matplotlib import pyplot as plt
+
 import tensorflow as tf
 from tensorflow import keras
-from tqdm import tqdm
 
 
 class PSO(object):
@@ -9,77 +12,49 @@ class PSO(object):
     Class implementing PSO algorithm
     """
 
-    def __init__(self, model: keras.models, loss_method=keras.losses.MeanSquaredError(), n_particles=5):
+    def __init__(self, model: keras.models, loss_method=keras.losses.MeanSquaredError(), n_particles: int = 5):
         """
         Initialize the key variables.
 
         Args:
             model : 학습할 모델 객체 (Sequential)
             loss_method : 손실 함수
-            optimizer : 최적화 함수
             n_particles(int) : 파티클의 개수
         """
         self.model = model                                       # 모델
         self.n_particles = n_particles                           # 파티클의 개수
         self.loss_method = loss_method                           # 손실 함수
-        self.model_structure = self.model.to_json()  # 모델의 구조
+        self.model_structure = self.model.to_json()              # 모델의 구조 정보
         self.init_weights = self.model.get_weights()             # 검색할 차원
         self.particle_depth = len(self.model.get_weights())      # 검색할 차원의 깊이
         self.particles_weights = [None] * n_particles            # 파티클의 위치
         for _ in tqdm(range(self.n_particles), desc="init particles position"):
-            # particle_node = []
             m = keras.models.model_from_json(self.model_structure)
             m.compile(loss=self.loss_method,
                       optimizer="adam", metrics=["accuracy"])
             self.particles_weights[_] = m.get_weights()
-            # print(f"shape > {self.particles_weights[_][0]}")
 
-            # self.particles_weights.append(particle_node)
-
-        # print(f"particles_weights > {self.particles_weights}")
-        # self.particles_weights = np.random.uniform(size=(n_particles, self.particle_depth)) \
-            # * self.init_pos
         # 입력받은 파티클의 개수 * 검색할 차원의 크기 만큼의 균등한 위치를 생성
-        # self.velocities = [None] * self.n_particles
         self.velocities = [
             [0 for i in range(self.particle_depth)] for n in range(n_particles)]
         for i in tqdm(range(n_particles), desc="init velocities"):
-            # print(i)
             for index, layer in enumerate(self.init_weights):
-                # print(f"index > {index}")
-                # print(f"layer > {layer.shape}")
                 self.velocities[i][index] = np.random.rand(
                     *layer.shape) / 5 - 0.10
-                # if layer.ndim == 1:
-                #     self.velocities[i][index] = np.random.uniform(
-                #         size=(layer.shape[0],))
-                # elif layer.ndim == 2:
-                #     self.velocities[i][index] = np.random.uniform(
-                #         size=(layer.shape[0], layer.shape[1]))
-                # elif layer.ndim == 3:
-                #     self.velocities[i][index] = np.random.uniform(
-                #         size=(layer.shape[0], layer.shape[1], layer.shape[2]))
-        # print(f"type > {type(self.velocities)}")
-        # print(f"velocities > {self.velocities}")
 
-        # print(f"velocities > {self.velocities}")
-        # for i, layer in enumerate(self.init_weights):
-        #     self.velocities[i] = np.random.rand(*layer.shape) / 5 - 0.10
-
-        # self.velocities = np.random.uniform(
-        #     size=(n_particles, self.particle_depth))
         # 입력받은 파티클의 개수 * 검색할 차원의 크기 만큼의 속도를 무작위로 초기화
         # 최대 사이즈로 전역 최적갑 저장 - global best
-        self.g_best = self.model.get_weights()      # 전역 최적값(최적의 가중치)
         self.p_best = self.particles_weights            # 각 파티클의 최적값(최적의 가중치)
-        self.p_best_score = [0 for i in range(
+        self.g_best = self.model.get_weights()          # 전역 최적값(최적의 가중치) | 초기값은 모델의 가중치
-            n_particles)]  # 각 파티클의 최적값의 점수
+
-        self.g_best_score = 0                  # 전역 최적값의 점수(초기화 - 무한대)
+        # 각 파티클의 최적값의 점수
-        self.g_history = []
+        self.p_best_score = [0 for i in range(n_particles)]
-        self.loss_history = [[] for i in range(n_particles)]
+
-        self.acc_history = [[] for i in range(n_particles)]
+        # 전역 최적값의 점수(초기화 - 0)
-        self.g_best_score_history = []
+        self.g_best_score = 0
-        self.history = []
+        self.loss_history = [[] for i in range(n_particles)]    # 각 파티클의 손실값 변화
+        self.acc_history = [[] for i in range(n_particles)]     # 각 파티클의 정확도 변화
+        self.g_best_score_history = []                          # 전역 최적값의 점수 변화
 
     def _update_weights(self, weights, v):
         """
@@ -94,11 +69,8 @@ class PSO(object):
         """
         # w = np.array(w)  # 각 파티클의 위치
         # v = np.array(v)  # 각 파티클의 속도(방향과 속력을 가짐)
-        # print(f"len(w) > {len(w)}")
-        # print(f"len(v) > {len(v)}")
         new_weights = [0 for i in range(len(weights))]
         for i in range(len(weights)):
-            # print(f"shape > w : {np.shape(w[i])}, v : {np.shape(v[i])}")
             new_weights[i] = tf.add(weights[i], v[i])
         # new_w = tf.add(w, v)   # 각 파티클을 랜덤한 속도만큼 진행
         return new_weights    # 진행한 파티클들의 위치를 반환
@@ -125,18 +97,12 @@ class PSO(object):
         # 0에서 1사이의 숫자를 랜덤 생성
         r0 = np.random.rand()
         r1 = np.random.rand()
-        # print(f"type > weights : {type(weights)}")
-        # print(f"type > v : {type(v)}")
-        # print(
-        # f"shape > weights : {np.shape(weights[0])}, v : {np.shape(v[0])}")
-        # print(f"len > weights : {len(weights)}, v : {len(v)}")
         # p_best = np.array(p_best)
         # g_best = np.array(g_best)
 
         # 가중치(상수)*속도 + \
         # 스케일링 상수*랜덤 가중치*(나의 최적값 - 처음 위치) + \
         # 전역 스케일링 상수*랜덤 가중치*(전체 최적값 - 처음 위치)
-        # for i, layer in enumerate(weights):
         new_velocity = [None] * len(weights)
         for i, layer in enumerate(weights):
 
@@ -145,18 +111,6 @@ class PSO(object):
             new_v = new_v + c1*r1*(self.g_best[i] - layer)
             new_velocity[i] = new_v
 
-            # m2 = tf.multiply(tf.multiply(c0, r0),
-            #  tf.subtract(p_best[i], layer))
-            # m3 = tf.multiply(tf.multiply(c1, r1),
-            #  tf.subtract(g_best[i], layer))
-            # new_v[i] = tf.add(m1, tf.add(m2, m3))
-            # new_v[i] = tf.add_n([m1, m2, m3])
-            # new_v[i] = tf.add_n(
-            # tf.multiply(w, v[i]),
-            # tf.multiply(tf.multiply(c0, r0),
-            # tf.subtract(p_best[i], layer)),
-            # tf.multiply(tf.multiply(c1, r1),
-            # tf.subtract(g_best[i], layer)))
         # new_v = w*v + c0*r0*(p_best - weights) + c1*r1*(g_best - weights)
         return new_velocity
 
@@ -170,13 +124,11 @@ class PSO(object):
         Returns:
             (array-like) : 추론에 대한 점수
         """
-        #  = self.model
-        # model.set_weights(weights)
         score = self.model.evaluate(x, y, verbose=0)
 
         return score
 
-    def optimize(self, x_train, y_train, x_test, y_test, maxiter=10, c0=0.5, c1=1.5, w=0.75):
+    def optimize(self, x_, y_, maxiter=10, c0=0.5, c1=1.5, w=0.75):
         """
         Run the PSO optimization process utill the stoping critera is met.
         Cas for minization. The aim is to minimize the cost function
@@ -188,8 +140,7 @@ class PSO(object):
             The best solution found (array-like)
         """
         for _ in range(maxiter):
-            loss = 0
+
-            acc = 0
             for i in tqdm(range(self.n_particles), desc=f"Iter {_}/{maxiter} ", ascii=True):
                 weights = self.particles_weights[i]  # 각 파티클 추출
                 v = self.velocities[i]    # 각 파티클의 다음 속도 추출
@@ -200,22 +151,14 @@ class PSO(object):
                 # 다음에 움직일 속도 = 최초 위치, 현재 속도, 현재 위치, 최종 위치
                 # 3. 위치 업데이트
                 self.particles_weights[i] = self._update_weights(weights, v)
-                # 현재 위치 = 최초 위치 현재 속도
+                # 현재 위치 = 이전 위치 + 현재 속도
-                # Update the besst position for particle i
                 # 내 현재 위치가 내 위치의 최소치보다 작으면 갱신
                 self.model.set_weights(self.particles_weights[i].copy())
-                # self.model.fit(x_train, y_train, epochs=epochs, batch_size=batch_size,
-                #    verbose=0, validation_data=(x_test, y_test))
                 # self.particles_weights[i] = self.model.get_weights()
                 # 4. 평가
                 self.model.compile(loss=self.loss_method,
                                    optimizer='adam', metrics=['accuracy'])
-                score = self._get_score(x_test, y_test)
+                score = self._get_score(x_, y_)
-                # print(score)
-
-                # print(f"score : {score}")
-                # print(f"loss : {loss}")
-                # print(f"p_best_score : {self.p_best_score[i]}")
 
                 if score[1] > self.p_best_score[i]:
                     self.p_best_score[i] = score[1]
@@ -223,54 +166,33 @@ class PSO(object):
                     if score[1] > self.g_best_score:
                         self.g_best_score = score[1]
                         self.g_best = self.particles_weights[i].copy()
-                        self.g_history.append(self.g_best.copy())
                         self.g_best_score_history.append(
                             self.g_best_score)
 
-                self.score = score
                 self.loss_history[i].append(score[0])
                 self.acc_history[i].append(score[1])
-                # if self.func(self.particles_weights[i]) < self.func(p_best):
-                # self.p_best[i] = self.particles_weights[i]
-                # if self.
-                # Update the best position overall
-                # 내 현재 위치가 전체 위치 최소치보다 작으면 갱신
-                # if self.func(self.particles_weights[i]) < self.func(self.g_best):
-                # self.g_best = self.particles_weights[i]
-                # self.g_history.append(self.g_best)
-                # print(f"{i} particle score : {score[0]}")
-            print(
-                f"loss avg : {self.score[0]/self.n_particles} | acc avg : {self.score[1]/self.n_particles} | best score : {self.g_best_score}")
 
-            # self.history.append(self.particles_weights.copy())
+            print(
+                f"loss avg : {score[0]/self.n_particles} | acc avg : {score[1]/self.n_particles} | best score : {self.g_best_score}")
 
         # 전체 최소 위치, 전체 최소 벡터
-        return self.g_best, self._get_score(x_test, y_test)
+        return self.g_best, self._get_score(x_, y_)
 
     """
     Returns:
-        현재 전체 위치
+        최종 가중치
     """
 
-    def position(self):
+    def best_weights(self):
-        return self.particles_weights.copy()
+        return self.g_best
 
     """
     Returns:
-        전체 위치 벡터 history
+        최종 가중치의 스코어
     """
 
-    def position_history(self):
+    def best_score(self):
-        return self.history.copy()
+        return self.g_best_score
-
-    """
-    Returns:
-        global best 의 갱신된 값의 변화를 반환
-    """
-
-    def global_history(self):
-        return self.g_history.copy()
-
     """
     Returns:
         global best score 의 갱신된 값의 변화를 반환
@@ -279,5 +201,10 @@ class PSO(object):
     def global_score_history(self):
         return self.g_best_score_history.copy()
 
+    """
+    Returns:
+        모든 파티클의 손실값과 정확도의 변화를 반환
+    """
+
     def all_history(self):
         return self.loss_history, self.acc_history.copy()

mnist.py

@@ -3,7 +3,7 @@ import os
 os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
 
 import tensorflow as tf
-# tf.random.set_seed(777)  # for reproducibility
+tf.random.set_seed(777)  # for reproducibility
 
 from tensorflow import keras
 from keras.datasets import mnist
@@ -12,17 +12,21 @@ from keras.layers import Dense, Dropout, Flatten
 from keras.layers import Conv2D, MaxPooling2D
 from keras import backend as K
 
-from pso_tf import PSO
+# from pso_tf import PSO
+from pso import Optimizer
+# from optimizer import Optimizer
 
 import numpy as np
-import matplotlib.pyplot as plt
 
 from datetime import date
 from tqdm import tqdm
-import json
+
+import gc
 
 print(tf.__version__)
 print(tf.config.list_physical_devices())
+print(f"Num GPUs Available: {len(tf.config.list_physical_devices('GPU'))}")
+
 
 def get_data():
     (x_train, y_train), (x_test, y_test) = mnist.load_data()
@@ -35,9 +39,16 @@ def get_data():
     print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}")
     return x_train, y_train, x_test, y_test
 
+def get_data_test():
+    (x_train, y_train), (x_test, y_test) = mnist.load_data()
+    x_test = x_test.reshape((10000, 28, 28, 1))
+    
+    return x_test, y_test
+
 def make_model():
     model = Sequential()
-    model.add(Conv2D(32, kernel_size=(5, 5), activation='relu', input_shape=(28,28,1)))
+    model.add(Conv2D(32, kernel_size=(5, 5),
+              activation='relu', input_shape=(28, 28, 1)))
     model.add(MaxPooling2D(pool_size=(3, 3)))
     model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
     model.add(MaxPooling2D(pool_size=(2, 2)))
@@ -50,87 +61,28 @@ def make_model():
 
     return model
 
+
 # %%
-'''
-optimizer parameter
-'''
-lr = 0.1
-momentun = 0.8
-decay = 1e-04
-nestrov = True
 
-'''
-pso parameter
-'''
-n_particles = 30
-maxiter = 50
-# epochs = 1
-w = 0.8
-c0 = 0.6
-c1 = 1.6
-
-def auto_tuning(n_particles=n_particles, maxiter=maxiter, c0=c0, c1=c1, w=w):
-    x_train, y_train, x_test, y_test = get_data()
 model = make_model()
-
+x_test, y_test = get_data_test()
-    loss = keras.losses.MeanSquaredError()
+# loss = 'binary_crossentropy'
-    optimizer = keras.optimizers.SGD(lr=lr, momentum=momentun, decay=decay, nesterov=nestrov)
+# loss = 'categorical_crossentropy'
-
+# loss = 'sparse_categorical_crossentropy'
-
+# loss = 'kullback_leibler_divergence'
-    pso_m = PSO(model=model, loss_method=loss, n_particles=n_particles)
+# loss = 'poisson'
-    # c0 : 지역 최적값 중요도
+# loss = 'cosine_similarity'
-    # c1 : 전역 최적값 중요도
+# loss = 'log_cosh'
-    # w : 관성 (현재 속도를 유지하는 정도)
+# loss = 'huber_loss'
-    best_weights, score = pso_m.optimize(x_train, y_train, x_test, y_test, maxiter=maxiter, c0=c0, c1=c1, w=w)
+# loss = 'mean_absolute_error'
-    model.set_weights(best_weights)
+# loss = 'mean_absolute_percentage_error'
-
+loss = 'mean_squared_error'
-    score_ = model.evaluate(x_test, y_test, verbose=2)
-    print(f" Test loss: {score_}")
-    score = round(score_[1]*100, 2)
-
-    day = date.today().strftime("%Y-%m-%d")
-
-    os.makedirs(f'./model', exist_ok=True)
-    model.save(f'./model/{day}_{score}_mnist.h5')
-    json_save = {
-        "name" : f"{day}_{score}_mnist.h5",
-        "score" : score_,
-        "maxiter" : maxiter,
-        "c0" : c0,
-        "c1" : c1,
-        "w" : w    
-    }
-    with open(f'./model/{day}_{score}_pso_mnist.json', 'a') as f:
-        json.dump(json_save, f)
-        f.write(',\n')
-                    
-    return model
-
-# auto_tuning(n_particles=30, maxiter=1000, c0=0.5, c1=1.5, w=0.75)
-
-
-# %%
-# print(f"정답 > {y_test}")
-def get_score(model):
-    x_train, y_train, x_test, y_test = get_data()
-    
-    predicted_result = model.predict(x_test)
-    predicted_labels = np.argmax(predicted_result, axis=1)
-    not_correct = []
-    for i in tqdm(range(len(y_test)), desc="진행도"):
-        if predicted_labels[i] != y_test[i]:
-            not_correct.append(i)
-            # print(f"추론 > {predicted_labels[i]} | 정답 > {y_test[i]}")
-            
-    print(f"틀린 갯수 > {len(not_correct)}/{len(y_test)}")
-
-    # for i in range(3):
-        # plt.imshow(x_test[not_correct[i]].reshape(28,28), cmap='Greys')
-    # plt.show() 
-    
-get_score(auto_tuning(n_particles=30, maxiter=50, c0=0.5, c1=1.5, w=0.75))
-
-# %%
 
 
+pso_mnist = Optimizer(model, loss=loss, n_particles=50, c0=0.4, c1=0.8, w_min=0.75, w_max=1.4)
+weight, score = pso_mnist.fit(
+    x_test, y_test, epochs=1000, save=True, save_path="./result/mnist", renewal="acc", empirical_balance=False, Dispersion=True)
+pso_mnist.model_save("./result/mnist")
+pso_mnist.save_info("./result/mnist")
 
+gc.collect()

pso/__init__.py

@@ -0,0 +1,5 @@
+from .optimizer import Optimizer
+
+__all__ = [
+    'Optimizer'
+]

pso/optimizer.py

@@ -0,0 +1,303 @@
+import os
+
+import tensorflow as tf
+from tensorflow import keras
+
+import numpy as np
+
+# import cupy as cp
+
+from tqdm import tqdm
+from datetime import datetime
+import json
+import gc
+
+from pso.particle import Particle
+
+
+class Optimizer:
+    def __init__(
+        self,
+        model: keras.models,
+        loss = "mse",
+        n_particles: int = 10,
+        c0=0.5,
+        c1=1.5,
+        w_min=0.5,
+        w_max=1.5,
+    ):
+        self.model = model  # 모델 구조
+        self.loss = loss  # 손실함수
+        self.n_particles = n_particles  # 파티클 개수
+        self.particles = [None] * n_particles  # 파티클 리스트
+        self.c0 = c0  # local rate - 지역 최적값 관성 수치
+        self.c1 = c1  # global rate - 전역 최적값 관성 수치
+        self.w_min = w_min  # 최소 관성 수치
+        self.w_max = w_max  # 최대 관성 수치
+
+        self.g_best_score = 0  # 최고 점수 - 시작은 0으로 초기화
+        self.g_best = None  # 최고 점수를 받은 가중치
+        self.g_best_ = None  # 최고 점수를 받은 가중치 - 값의 분산을 위한 변수
+
+        for i in tqdm(range(self.n_particles), desc="Initializing Particles"):
+            m = keras.models.model_from_json(model.to_json())
+            m.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"])
+
+            self.particles[i] = Particle(m, loss)
+
+    """
+    Returns:
+        (cupy array) : 가중치 - 1차원으로 풀어서 반환
+        (list) : 가중치의 원본 shape
+        (list) : 가중치의 원본 shape의 길이
+    """
+
+    def _encode(self, weights):
+        # w_gpu = cp.array([])
+        w_gpu = np.array([])
+        lenght = []
+        shape = []
+        for layer in weights:
+            shape.append(layer.shape)
+            w_ = layer.reshape(-1)
+            lenght.append(len(w_))
+            # w_gpu = cp.append(w_gpu, w_)
+            w_gpu = np.append(w_gpu, w_)
+
+        return w_gpu, shape, lenght
+
+    """
+    Returns:
+        (list) : 가중치 원본 shape으로 복원
+    """
+
+    def _decode(self, weight, shape, lenght):
+        weights = []
+        start = 0
+        for i in range(len(shape)):
+            end = start + lenght[i]
+            w_ = weight[start:end]
+            # w_ = weight[start:end].get()
+            w_ = np.reshape(w_, shape[i])
+            # w_ = w_.reshape(shape[i])
+            weights.append(w_)
+            start = end
+        del weight
+        del shape
+        del lenght
+        gc.collect()
+
+        return weights
+
+    def f(self, x, y, weights):
+        self.model.set_weights(weights)
+        self.model.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"])
+        score = self.model.evaluate(x, y, verbose=0)[1]
+        if score > 0:
+            return 1 / (1 + score)
+        else:
+            return 1 + np.abs(score)
+
+    """
+    parameters
+    ----------
+    x : numpy.ndarray
+    y : numpy.ndarray
+    epochs : int
+    save : bool
+    save_path : str ex) "./result"
+    renewal : str ex) "acc" or "loss"
+    """
+
+    """
+    parameters
+    fit(
+        x_test : numpy.ndarray,
+        y_test : numpy.ndarray,
+        epochs : int,
+        save : bool - True : save, False : not save
+        save_path : str ex) "./result",
+        renewal : str ex) "acc" or "loss",
+        empirical_balance : bool - True : empirical balance, False : no balance
+        Dispersion : bool - True : random search, False : PSO
+    """
+    def fit(
+        self,
+        x,
+        y,
+        epochs: int = 100,
+        save: bool = False,
+        save_path: str = "./result",
+        renewal: str = "acc",
+        empirical_balance: bool = False,
+        Dispersion: bool = False,
+        check_point: int = None,
+    ):
+        self.renewal = renewal
+        if renewal == "acc":
+            self.g_best_score = 0
+        elif renewal == "loss":
+            self.g_best_score = np.inf
+
+        if save:
+            if save_path is None:
+                raise ValueError("save_path is None")
+            else:
+                self.save_path = save_path
+                os.makedirs(save_path, exist_ok=True)
+                self.day = datetime.now().strftime("%m-%d-%H-%M")
+
+        for i, p in enumerate(self.particles):
+            local_score = p.get_score(x, y, renewal=renewal)
+
+            if renewal == "acc":
+                if local_score[1] > self.g_best_score:
+                    self.g_best_score = local_score[1]
+                    self.g_best = p.get_best_weights()
+                    self.g_best_ = p.get_best_weights()
+            elif renewal == "loss":
+                if local_score[0] < self.g_best_score:
+                    self.g_best_score = local_score[0]
+                    self.g_best = p.get_best_weights()
+                    self.g_best_ = p.get_best_weights()
+        
+        print(f"initial g_best_score : {self.g_best_score}")
+        
+        for _ in range(epochs):
+            acc = 0
+            loss = 0
+            min_score = np.inf
+            max_score = 0
+            min_loss = np.inf
+            max_loss = 0
+
+            # for i in tqdm(range(len(self.particles)), desc=f"epoch {_ + 1}/{epochs}", ascii=True):
+            for i in range(len(self.particles)):
+                w = self.w_min + (self.w_max - self.w_min) * _ / epochs
+
+                if Dispersion:
+                    g_best = self.g_best_
+                else:
+                    g_best = self.g_best
+
+                if empirical_balance:
+                    if np.random.rand() < np.exp(-(_) / epochs):
+                        w_p_ = self.f(x, y, self.particles[i].get_best_weights())
+                        w_g_ = self.f(x, y, self.g_best)
+                        w_p = w_p_ / (w_p_ + w_g_)
+                        w_g = w_p_ / (w_p_ + w_g_)
+
+                    else:
+                        p = 1 / (self.n_particles * np.linalg.norm(self.c1 - self.c0))
+                        p = np.exp(-p)
+                        w_p = p
+                        w_g = 1 - p
+
+                    score = self.particles[i].step_w(
+                        x, y, self.c0, self.c1, w, g_best, w_p, w_g, renewal=renewal
+                    )
+
+                else:
+                    score = self.particles[i].step(
+                        x, y, self.c0, self.c1, w, g_best, renewal=renewal
+                    )
+
+                if renewal == "acc":
+                    if score[1] >= self.g_best_score:
+                        self.g_best_score = score[1]
+                        self.g_best = self.particles[i].get_best_weights()
+                elif renewal == "loss":
+                    if score[0] <= self.g_best_score:
+                        self.g_best_score = score[0]
+                        self.g_best = self.particles[i].get_best_weights()
+
+                loss += score[0]
+                acc += score[1]
+                if score[0] < min_loss:
+                    min_loss = score[0]
+                if score[0] > max_loss:
+                    max_loss = score[0]
+
+                if score[1] < min_score:
+                    min_score = score[1]
+                if score[1] > max_score:
+                    max_score = score[1]
+
+                if save:
+                    with open(
+                        f"./{save_path}/{self.day}_{self.n_particles}_{epochs}_{self.c0}_{self.c1}_{self.w_min}_{renewal}.csv",
+                        "a",
+                    ) as f:
+                        f.write(f"{score[0]}, {score[1]}")
+                        if i != self.n_particles - 1:
+                            f.write(", ")
+
+            TS = self.c0 + np.random.rand() * (self.c1 - self.c0)
+            g_, g_sh, g_len = self._encode(self.g_best)
+            decrement = (epochs - (_) + 1) / epochs
+            g_ = (1 - decrement) * g_ + decrement * TS
+            self.g_best_ = self._decode(g_, g_sh, g_len)
+
+            if save:
+                with open(
+                    f"./{save_path}/{self.day}_{self.n_particles}_{epochs}_{self.c0}_{self.c1}_{self.w_min}_{renewal}.csv",
+                    "a",
+                ) as f:
+                    f.write("\n")
+
+            print(f"epoch {_ + 1}/{epochs} finished")
+            # print(f"loss min : {min_loss} | loss max : {max_loss} | acc min : {min_score} | acc max : {max_score}")
+            # print(f"loss avg : {loss/self.n_particles} | acc avg : {acc/self.n_particles} | Best {renewal} : {self.g_best_score}")
+            print(
+                f"loss min : {min_loss} | acc avg : {max_score} | Best {renewal} : {self.g_best_score}"
+            )
+
+            gc.collect()
+            
+            if check_point is not None:
+                if _ % check_point == 0:
+                    self._check_point_save(f"./{save_path}/{self.day}/check_point_{_}.h5")
+
+        return self.g_best, self.g_best_score
+
+    def get_best_model(self):
+        model = keras.models.model_from_json(self.model.to_json())
+        model.set_weights(self.g_best)
+        model.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"])
+        return model
+
+    def get_best_score(self):
+        return self.g_best_score
+
+    def get_best_weights(self):
+        return self.g_best
+
+    def save_info(self, path: str = "./result"):
+        json_save = {
+            "name": f"{self.day}_{self.n_particles}_{self.c0}_{self.c1}_{self.w_min}.h5",
+            "n_particles": self.n_particles,
+            "score": self.g_best_score,
+            "c0": self.c0,
+            "c1": self.c1,
+            "w_min": self.w_min,
+            "w_max": self.w_max,
+            "loss_method": self.loss,
+            "renewal": self.renewal,
+        }
+
+        with open(
+            f"./{path}/{self.day}_{self.loss}_{self.n_particles}_{self.g_best_score}.json",
+            "w",
+        ) as f:
+            json.dump(json_save, f, indent=4)
+            
+    def _check_point_save(self, save_path: str = f"./result/check_point"):
+        model = self.get_best_model()
+        model.save(save_path)
+            
+    def model_save(self, save_path: str = "./result/model"):
+        model = self.get_best_model()
+        model.save(
+            f"./{save_path}/{self.day}/{self.n_particles}_{self.c0}_{self.c1}_{self.w_min}.h5"
+        )
+        return model

pso/particle.py

@@ -0,0 +1,124 @@
+
+import tensorflow as tf
+from tensorflow import keras
+
+# import cupy as cp
+import numpy as np
+
+class Particle:
+    def __init__(self, model:keras.models, loss):
+        self.model = model
+        self.loss = loss
+        self.init_weights = self.model.get_weights()
+        i_w_,s_,l_ = self._encode(self.init_weights)
+        i_w_ = np.random.rand(len(i_w_)) / 5 - 0.10
+        self.velocities = self._decode(i_w_,s_,l_)
+        
+        self.best_score = 0
+        self.best_weights = self.init_weights
+
+
+    """
+    Returns:
+        (cupy array) : 가중치 - 1차원으로 풀어서 반환
+        (list) : 가중치의 원본 shape
+        (list) : 가중치의 원본 shape의 길이
+    """
+    def _encode(self, weights:list):
+        # w_gpu = cp.array([])
+        w_gpu = np.array([])
+        lenght = []
+        shape = []
+        for layer in weights:
+            shape.append(layer.shape)
+            w_ = layer.reshape(-1)
+            lenght.append(len(w_))
+            # w_gpu = cp.append(w_gpu, w_)
+            w_gpu = np.append(w_gpu, w_)
+        
+        return w_gpu, shape, lenght
+
+    """
+    Returns:
+        (list) : 가중치 원본 shape으로 복원
+    """
+
+    def _decode(self, weight:list, shape, lenght):
+        weights = []
+        start = 0
+        for i in range(len(shape)):
+            end = start + lenght[i]
+            w_ = weight[start:end]
+            # w_ = weight[start:end].get()
+            w_ = np.reshape(w_, shape[i])
+            # w_ = w_.reshape(shape[i])
+            weights.append(w_)
+            start = end
+
+        return weights
+
+    def get_score(self, x, y, renewal:str = "acc"):
+        self.model.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"])
+        score = self.model.evaluate(x, y, verbose=0)
+        # print(score)
+        if renewal == "acc":
+            if score[1] > self.best_score:
+                self.best_score = score[1]
+                self.best_weights = self.model.get_weights()
+        elif renewal == "loss":
+            if score[0] < self.best_score:
+                self.best_score = score[0]
+                self.best_weights = self.model.get_weights()
+            
+        return score
+    def _update_velocity(self, local_rate, global_rate, w, g_best):
+        encode_w, w_sh, w_len = self._encode(weights = self.model.get_weights())
+        encode_v, _, _ = self._encode(weights = self.velocities)
+        encode_p, _, _ = self._encode(weights = self.best_weights)
+        encode_g, _, _ = self._encode(weights = g_best)
+        r0 = np.random.rand()
+        r1 = np.random.rand()
+        new_v = w * encode_v + local_rate * r0 * (encode_p - encode_w) + global_rate * r1 * (encode_g - encode_w)
+        self.velocities = self._decode(new_v, w_sh, w_len)
+    
+    def _update_velocity_w(self, local_rate, global_rate, w, w_p, w_g, g_best):
+        encode_w, w_sh, w_len = self._encode(weights = self.model.get_weights())
+        encode_v, _, _ = self._encode(weights = self.velocities)
+        encode_p, _, _ = self._encode(weights = self.best_weights)
+        encode_g, _, _ = self._encode(weights = g_best)
+        r0 = np.random.rand()
+        r1 = np.random.rand()
+        new_v = w * encode_v + local_rate * r0 * (w_p * encode_p - encode_w) + global_rate * r1 * (w_g * encode_g - encode_w)
+        self.velocities = self._decode(new_v, w_sh, w_len)
+    
+    def _update_weights(self):
+        encode_w, w_sh, w_len = self._encode(weights = self.model.get_weights())
+        encode_v, _, _ = self._encode(weights = self.velocities)
+        new_w = encode_w + encode_v
+        self.model.set_weights(self._decode(new_w, w_sh, w_len))
+
+
+    def f(self, x, y, weights):
+        self.model.set_weights(weights)
+        score = self.model.evaluate(x, y, verbose = 0)[1]
+        if score > 0:
+            return 1 / (1 + score)
+        else:
+            return 1 + np.abs(score)
+
+    def step(self, x, y, local_rate, global_rate, w, g_best, renewal:str = "acc"):
+        self._update_velocity(local_rate, global_rate, w, g_best)
+        self._update_weights()
+        return self.get_score(x, y, renewal)
+    
+    def step_w(self, x, y, local_rate, global_rate, w, g_best, w_p, w_g, renewal:str = "acc"):     
+        self._update_velocity_w(local_rate, global_rate, w, w_p, w_g, g_best)
+        self._update_weights()
+        return self.get_score(x, y, renewal)
+    
+    
+    def get_best_score(self):
+        return self.best_score
+    
+    def get_best_weights(self):
+        return self.best_weights

pso_tuning.py

@@ -1,155 +0,0 @@
-# %%
-import json
-from tqdm import tqdm
-from datetime import date
-import matplotlib.pyplot as plt
-import numpy as np
-from PSO.pso_bp import PSO
-from keras import backend as K
-from keras.layers import Conv2D, MaxPooling2D
-from keras.layers import Dense, Dropout, Flatten
-from keras.models import Sequential
-from keras.datasets import mnist
-from tensorflow import keras
-import tensorflow as tf
-import os
-os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
-
-tf.random.set_seed(777)  # for reproducibility
-
-
-print(tf.__version__)
-print(tf.config.list_physical_devices())
-
-
-def get_data():
-    (x_train, y_train), (x_test, y_test) = mnist.load_data()
-
-    x_train, x_test = x_train / 255.0, x_test / 255.0
-    x_train = x_train.reshape((60000, 28, 28, 1))
-    x_test = x_test.reshape((10000, 28, 28, 1))
-
-    print(f"x_train : {x_train[0].shape} | y_train : {y_train[0].shape}")
-    print(f"x_test : {x_test[0].shape} | y_test : {y_test[0].shape}")
-    return x_train, y_train, x_test, y_test
-
-
-def make_model():
-    model = Sequential()
-    model.add(Conv2D(32, kernel_size=(5, 5),
-              activation='relu', input_shape=(28, 28, 1)))
-    model.add(MaxPooling2D(pool_size=(3, 3)))
-    model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
-    model.add(MaxPooling2D(pool_size=(2, 2)))
-    model.add(Dropout(0.25))
-    model.add(Flatten())
-    model.add(Dense(128, activation='relu'))
-    model.add(Dense(10, activation='softmax'))
-
-    model.compile(loss='sparse_categorical_crossentropy',
-                  optimizer='adam', metrics=['accuracy'])
-
-    # model.summary()
-
-    return model
-
-
-# %%
-'''
-optimizer parameter
-'''
-lr = 0.1
-momentun = 0.8
-decay = 1e-04
-nestrov = True
-
-'''
-pso parameter
-'''
-n_particles = 100
-maxiter = 500
-# epochs = 1
-w = 0.8
-c0 = 0.6
-c1 = 1.6
-
-def auto_tuning():
-    x_train, y_train, x_test, y_test = get_data()
-    model = make_model()
-
-    loss = keras.losses.MeanSquaredError()
-    optimizer = keras.optimizers.SGD(lr=lr, momentum=momentun, decay=decay, nesterov=nestrov)
-
-
-    pso_m = PSO(model=model, loss_method=loss, n_particles=n_particles)
-    # c0 : 지역 최적값 중요도
-    # c1 : 전역 최적값 중요도
-    # w : 관성 (현재 속도를 유지하는 정도)
-    best_weights, score = pso_m.optimize(x_train, y_train, x_test, y_test, maxiter=maxiter, c0=c0, c1=c1, w=w)
-    model.set_weights(best_weights)
-
-    score_ = model.evaluate(x_test, y_test, verbose=2)
-    print(f" Test loss: {score_}")
-    score = round(score_[0]*100, 2)
-
-    day = date.today().strftime("%Y-%m-%d")
-
-    model.save(f'./model/{day}_{score}_mnist.h5')
-    json_save = {
-        "name" : f"{day}_{score}_mnist.h5",
-        "score" : score_,
-        "maxiter" : maxiter,
-        "c0" : c0,
-        "c1" : c1,
-        "w" : w    
-    }
-    with open(f'./model/{day}_{score}_bp_mnist.json', 'a') as f:
-        json.dump(json_save, f)
-        f.write(',\n')
-                    
-    return model
-auto_tuning()
-
-
-# %%
-# print(f"정답 > {y_test}")
-def get_score(model):
-    x_train, y_train, x_test, y_test = get_data()
-
-    predicted_result = model.predict(x_test)
-    predicted_labels = np.argmax(predicted_result, axis=1)
-    not_correct = []
-    for i in tqdm(range(len(y_test)), desc="진행도"):
-        if predicted_labels[i] != y_test[i]:
-            not_correct.append(i)
-            # print(f"추론 > {predicted_labels[i]} | 정답 > {y_test[i]}")
-
-    print(f"틀린 갯수 > {len(not_correct)}/{len(y_test)}")
-
-    for i in range(3):
-        plt.imshow(x_test[not_correct[i]].reshape(28, 28), cmap='Greys')
-    plt.show()
-
-# %%
-
-
-def default_mnist(epochs=5):
-    x_train, y_train, x_test, y_test = get_data()
-    model = make_model()
-
-    hist = model.fit(x_train, y_train, epochs=epochs, batch_size=32, verbose=1)
-    print(hist.history['loss'][-1])
-    print(hist.history['accuracy'][-1])
-
-    predicted_result = model.predict(x_test)
-    predicted_labels = np.argmax(predicted_result, axis=1)
-    not_correct = []
-    for i in tqdm(range(len(y_test)), desc="진행도"):
-        if predicted_labels[i] != y_test[i]:
-            not_correct.append(i)
-            # print(f"추론 > {predicted_labels[i]} | 정답 > {y_test[i]}")
-
-    print(f"틀린 갯수 > {len(not_correct)}/{len(y_test)}")
-
-
-# %%

psokeras/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+"""PSOkeras - Particle Swarm Optimizer for Keras models
+
+This module implements a particle swarm optimizer for training the weights of Keras models.  The
+
+"""
+
+
+from .version import __version__
+from .optimizer import Optimizer
+
+__all__ = [
+    'Optimizer',
+]

psokeras/optimizer.py

@@ -0,0 +1,67 @@
+BIG_SCORE = 1.e6  # type: float
+
+import keras
+from psokeras.particle import Particle
+from .util import ProgressBar
+
+
+class Optimizer:
+    def __init__(self, model, loss,
+                 n=10,
+                 acceleration=0.1,
+                 local_rate=1.0,
+                 global_rate=1.0):
+
+        self.n_particles = n
+        self.structure = model.to_json()
+        self.particles = [None] * n
+        self.loss = loss
+        self.length = len(model.get_weights())
+
+        params = {'acc': acceleration, 'local_acc': local_rate, 'global_acc': global_rate}
+
+        for i in range(n-1):
+            m = keras.models.model_from_json(self.structure)
+            m.compile(loss=loss,optimizer='sgd')
+            self.particles[i] = Particle(m, params)
+
+        self.particles[n-1] = Particle(model, params)
+
+        self.global_best_weights = None
+        self.global_best_score = BIG_SCORE
+
+    def fit(self, x, y, steps=0, batch_size=32):
+        num_batches = x.shape[0] // batch_size
+
+        for i, p in enumerate(self.particles):
+            local_score = p.get_score(x, y)
+
+            if local_score < self.global_best_score:
+                self.global_best_score = local_score
+                self.global_best_weights = p.get_best_weights()
+
+        print("PSO -- Initial best score {:0.4f}".format(self.global_best_score))
+
+        bar = ProgressBar(steps, updates=20)
+
+        for i in range(steps):
+            for j in range(num_batches):
+                x_ = x[j*batch_size:(j+1)*batch_size,:]
+                y_ = y[j*batch_size:(j+1)*batch_size]
+
+                for p in self.particles:
+                    local_score = p.step(x_, y_, self.global_best_weights)
+
+                    if local_score < self.global_best_score:
+                        self.global_best_score = local_score
+                        self.global_best_weights = p.get_best_weights()
+
+            bar.update(i)
+
+        bar.done()
+
+    def get_best_model(self):
+        best_model = keras.models.model_from_json(self.structure)
+        best_model.set_weights(self.global_best_weights)
+        best_model.compile(loss=self.loss,optimizer='sgd')
+        return best_model

psokeras/particle.py

@@ -0,0 +1,66 @@
+import random
+
+import numpy as np
+
+from psokeras.optimizer import BIG_SCORE
+
+
+class Particle:
+    def __init__(self, model, params):
+        self.model = model
+        self.params = params
+        self.init_weights = model.get_weights()
+        self.velocities = [None] * len(self.init_weights)
+        self.length = len(self.init_weights)
+        for i, layer in enumerate(self.init_weights):
+            self.velocities[i] = np.random.rand(*layer.shape) / 5 - 0.10
+            # self.velocities[i] = np.zeros(layer.shape)
+
+        self.best_weights = None
+        self.best_score = BIG_SCORE
+
+    def get_score(self, x, y, update=True):
+        local_score = self.model.evaluate(x, y, verbose=0)
+        if local_score < self.best_score and update:
+            self.best_score = local_score
+            self.best_weights = self.model.get_weights()
+
+        return local_score
+
+    def _update_velocities(self, global_best_weights, depth):
+        new_velocities = [None] * len(self.init_weights)
+        weights = self.model.get_weights()
+        local_rand, global_rand = random.random(), random.random()
+
+        for i, layer in enumerate(weights):
+            if i >= depth:
+                new_velocities[i] = self.velocities[i]
+                continue
+            new_v = self.params['acc'] * self.velocities[i]
+            new_v = new_v + self.params['local_acc'] * local_rand * (self.best_weights[i] - layer)
+            new_v = new_v + self.params['global_acc'] * global_rand * (global_best_weights[i] - layer)
+            new_velocities[i] = new_v
+
+        self.velocities = new_velocities
+
+    def _update_weights(self, depth):
+        old_weights = self.model.get_weights()
+        new_weights = [None] * len(old_weights)
+        for i, layer in enumerate(old_weights):
+            if i>= depth:
+                new_weights[i] = layer
+                continue
+            new_w = layer + self.velocities[i]
+            new_weights[i] = new_w
+
+        self.model.set_weights(new_weights)
+
+    def step(self, x, y, global_best_weights,depth=None):
+        if depth is None:
+            depth = self.length
+        self._update_velocities(global_best_weights, depth)
+        self._update_weights(depth)
+        return self.get_score(x, y)
+
+    def get_best_weights(self):
+        return self.best_weights

psokeras/util.py

@@ -0,0 +1,31 @@
+class ProgressBar:
+    def __init__(self, steps, updates=10):
+        self.step = 0
+        self.step_size = (steps // updates)
+        self.total_steps = steps
+        self.updates = updates
+
+        bar = self._make_bar(0)
+        print(bar, end=' ')
+
+    def update(self, i):
+        if i % self.step_size > 0:
+            return
+
+        self.step = i // self.step_size
+        bar = self._make_bar(i)
+
+        print(bar, end=' ')
+
+    def done(self):
+        self.step = self.total_steps
+        bar = self._make_bar(self.updates)
+        print(bar)
+
+    def _make_bar(self, x):
+        bar = "["
+        for x in range(self.updates):
+            print("\r", end=' ')
+            bar += "=" if x < self.step else " "
+        bar += "]"
+        return bar

psokeras/version.py

@@ -0,0 +1 @@
+__version__ = '0.2.0'

pyswarms/pyswarm.py

@@ -0,0 +1,93 @@
+from sklearn.datasets import load_iris
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import OneHotEncoder, StandardScaler
+
+import tensorflow as tf
+from tensorflow import keras
+from tensorflow.keras.models import Sequential
+from tensorflow.keras.layers import Dense, Dropout, Flatten
+
+import time as time
+import numpy as np
+
+
+iris = load_iris()
+iris = load_iris()
+X = iris['data']
+y = iris['target']
+names = iris['target_names']
+feature_names = iris['feature_names']
+enc = OneHotEncoder()
+Y = enc.fit_transform(y[:, np.newaxis]).toarray()
+scaler = StandardScaler()
+X_scaled = scaler.fit_transform(X)
+X_train, X_test, Y_train, Y_test = train_test_split(
+    X_scaled, Y, test_size=0.5, random_state=2)
+n_features = X.shape[1]
+n_classes = Y.shape[1]
+
+def create_custom_model(input_dim, output_dim, nodes, n=1, name='model'):
+    model = Sequential(name=name)
+    for i in range(n):
+        model.add(Dense(nodes, input_dim=input_dim, activation='relu'))
+    model.add(Dense(output_dim, activation='softmax'))
+    model.compile(loss='categorical_crossentropy',
+                  optimizer='adam',
+                  metrics=['accuracy'])
+    return model
+
+
+# n_layers = 1
+# model = create_custom_model(n_features, n_classes,
+#                             4, n_layers)
+# model.summary()
+
+# start_time = time.time()
+# print('Model name:', model.name)
+# history_callback = model.fit(X_train, Y_train,
+#                              batch_size=5,
+#                              epochs=400,
+#                              verbose=0,
+#                              validation_data=(X_test, Y_test)
+#                              )
+# score = model.evaluate(X_test, Y_test)
+# print('Test loss:', score[0])
+# print('Test accuracy:', score[1])
+# print("--- %s seconds ---" % (time.time() - start_time))
+
+
+def get_shape(model):
+    weights_layer = model.get_weights()
+    shapes = []
+    for weights in weights_layer:
+        shapes.append(weights.shape)
+    return shapes
+
+def set_shape(weights,shapes):
+    new_weights = []
+    index=0
+    for shape in shapes:
+        if(len(shape)>1):
+            n_nodes = np.prod(shape)+index
+        else:
+            n_nodes=shape[0]+index
+        tmp = np.array(weights[index:n_nodes]).reshape(shape)
+        new_weights.append(tmp)
+        index=n_nodes
+    return new_weights
+
+def evaluate_nn(W, shape,X_train=X_train, Y_train=Y_train):
+    results = []
+    for weights in W:
+        model.set_weights(set_shape(weights,shape))
+        score = model.evaluate(X_train, Y_train, verbose=0)
+        results.append(1-score[1])
+    return results
+
+shape = get_shape(model)
+x_max = 1.0 * np.ones(83)
+x_min = -1.0 * x_max
+bounds = (x_min, x_max)
+options = {'c1': 0.4, 'c2': 0.8, 'w': 0.4}
+optimizer = GlobalBestPSO(n_particles=25, dimensions=83,
+                          options=options, bounds=bounds)

pyswarms/report.log

@@ -0,0 +1,11 @@
+2023-05-28 17:29:35,386 - pyswarms.single.global_best - INFO - Optimize for 20 iters with {'c1': 0.4, 'c2': 0.6, 'w': 0.4}
+2023-05-28 17:30:07,637 - pyswarms.single.global_best - INFO - Optimization finished | best cost: 0.013333320617675781, best pos: [ 0.17027965  0.17696722 -0.07395054  0.31544984  0.17052408 -0.37810479
+  0.24267479  0.16931148  0.65606942 -0.24207116 -0.66562722  0.02191478
+  0.5870387   0.78966943 -0.4457816   0.0907434  -0.1808341   0.29282655
+  0.61472003  0.90660508  0.16469465 -0.55057763  0.54702005 -0.22636745
+  0.01125538  0.62431828  0.02128613 -0.26723577 -0.43527016  0.51223244
+  0.76388399 -0.02073011  0.15949622  0.45878514  0.01787211]
+2023-05-28 17:30:08,140 - matplotlib.animation - WARNING - MovieWriter pillowwritter unavailable; using Pillow instead.
+2023-05-28 17:30:08,141 - matplotlib.animation - INFO - Animation.save using <class 'matplotlib.animation.PillowWriter'>
+2023-05-28 17:31:01,885 - tensorflow - WARNING - 5 out of the last 3010 calls to <function Model.make_test_function.<locals>.test_function at 0x7fee6622df70> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.
+2023-05-28 17:31:08,794 - tensorflow - WARNING - 6 out of the last 3011 calls to <function Model.make_test_function.<locals>.test_function at 0x7fee66257b80> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.

readme.md

@@ -30,6 +30,7 @@ pso_bp.py # 오차역전파 함수를 최적화하는 PSO 알고리즘 구현 -
 pso_tuning.py # pso 알고리즘의 하이퍼 파라미터를 자동으로 튜닝하는 파일
 
 xor.ipynb # xor 문제를 pso 알고리즘으로 풀이
+iris.ipynb # iris 문제를 pso 알고리즘으로 풀이
 mnist.ipynb # mnist 문제를 pso 알고리즘으로 풀이
 mnist.py # mnist 문제를 pso 알고리즘으로 풀이 - shell 실행용
 ```
@@ -61,3 +62,7 @@ pso 알고리즘을 이용하여 오차역전파 함수를 최적화 하는 방
 > <br>
 >
 > > pso 와 random forest 방식이 매우 유사하다고 생각하여 학습할 때 뿐만 아니라 예측 할 때도 이러한 방식으로 사용할 수 있을 것 같습니다
+
+
+이곳의 코드를 참고하여 좀더 효율적인 코드로 수정하였습니다
+> https://github.com/mike-holcomb/PSOkeras