diff --git a/fashion_mnist.py b/fashion_mnist.py index 1da433b..1d3574b 100644 --- a/fashion_mnist.py +++ b/fashion_mnist.py @@ -36,16 +36,16 @@ def get_data(): def make_model(): model = Sequential() model.add( - Conv2D(32, kernel_size=(5, 5), activation="sigmoid", + Conv2D(32, kernel_size=(5, 5), activation="relu", input_shape=(28, 28, 1)) ) model.add(MaxPooling2D(pool_size=(2, 2))) - model.add(Conv2D(64, kernel_size=(3, 3), activation="sigmoid")) + model.add(Conv2D(64, kernel_size=(3, 3), activation="relu")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dropout(0.25)) - model.add(Dense(256, activation="sigmoid")) - model.add(Dense(128, activation="sigmoid")) + model.add(Dense(256, activation="relu")) + model.add(Dense(128, activation="relu")) model.add(Dense(10, activation="softmax")) return model @@ -90,18 +90,16 @@ loss = [ pso_mnist = optimizer( model, - loss="mean_squared_error", + loss="categorical_crossentropy", n_particles=500, - c0=0.2, - c1=0.4, - w_min=0.3, - w_max=0.5, + c0=0.5, + c1=1.0, + w_min=0.7, + w_max=1.2, negative_swarm=0.05, mutation_swarm=0.3, - particle_min=-0.3, - particle_max=0.3, - early_stopping=True, - early_stopping_patience=10, + convergence_reset=True, + convergence_reset_patience=10, ) best_score = pso_mnist.fit( @@ -112,11 +110,11 @@ best_score = pso_mnist.fit( log=2, log_name="fashion_mnist", save_path="./logs/fashion_mnist", - renewal="acc", + renewal="mse", check_point=25, empirical_balance=False, dispersion=False, - batch_size=1024, + batch_size=5000, ) print("Done!") diff --git a/mnist.py b/mnist.py index cd5961e..97ef676 100644 --- a/mnist.py +++ b/mnist.py @@ -36,16 +36,16 @@ def get_data(): def make_model(): model = Sequential() model.add( - Conv2D(32, kernel_size=(5, 5), activation="sigmoid", + Conv2D(32, kernel_size=(5, 5), activation="relu", input_shape=(28, 28, 1)) ) model.add(MaxPooling2D(pool_size=(2, 2))) - model.add(Conv2D(64, kernel_size=(3, 3), activation="sigmoid")) + model.add(Conv2D(64, kernel_size=(3, 3), activation="relu")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dropout(0.25)) - model.add(Dense(256, activation="sigmoid")) - model.add(Dense(128, activation="sigmoid")) + model.add(Dense(256, activation="relu")) + model.add(Dense(128, activation="relu")) model.add(Dense(10, activation="softmax")) return model @@ -90,26 +90,24 @@ loss = [ pso_mnist = optimizer( model, - loss="mean_squared_error", - n_particles=900, - c0=0.2, - c1=0.4, - w_min=0.3, - w_max=0.5, + loss="categorical_crossentropy", + n_particles=500, + c0=0.5, + c1=1.0, + w_min=0.7, + w_max=0.9, negative_swarm=0.05, mutation_swarm=0.3, - particle_min=-0.3, - particle_max=0.3, - early_stopping=True, - early_stopping_patience=10, - early_stopping_monitor="loss", - early_stopping_min_delta=0.0005, + convergence_reset=True, + convergence_reset_patience=10, + convergence_reset_monitor="mse", + convergence_reset_min_delta=0.0005, ) best_score = pso_mnist.fit( x_train, y_train, - epochs=200, + epochs=300, save_info=True, log=2, log_name="mnist", @@ -118,7 +116,7 @@ best_score = pso_mnist.fit( check_point=25, empirical_balance=False, dispersion=False, - batch_size=1024, + batch_size=5000, ) print("Done!") diff --git a/pso/optimizer.py b/pso/optimizer.py index 4fde5f6..a0590bc 100644 --- a/pso/optimizer.py +++ b/pso/optimizer.py @@ -28,23 +28,21 @@ class Optimizer: def __init__( self, model: keras.models, - loss="mean_squared_error", - n_particles: int = 10, - c0=0.5, - c1=1.5, - w_min=0.5, - w_max=1.5, + loss: any = None, + n_particles: int = None, + c0: float = 0.5, + c1: float = 1.5, + w_min: float = 0.5, + w_max: float = 1.5, negative_swarm: float = 0, mutation_swarm: float = 0, np_seed: int = None, tf_seed: int = None, random_state: tuple = None, - particle_min: float = -0.3, - particle_max: float = 0.3, convergence_reset: bool = False, convergence_reset_patience: int = 10, convergence_reset_min_delta: float = 0.0001, - convergence_reset_monitor: str = "loss", + convergence_reset_monitor: str = "mse", ): """ particle swarm optimization @@ -61,54 +59,70 @@ class Optimizer: mutation_swarm (float): 돌연변이가 일어날 확률 np_seed (int, optional): numpy seed. Defaults to None. tf_seed (int, optional): tensorflow seed. Defaults to None. - particle_min (float, optional): 가중치 초기화 최소값. Defaults to -5. - particle_max (float, optional): 가중치 초기화 최대값. Defaults to 5. convergence_reset (bool, optional): early stopping 사용 여부. Defaults to False. convergence_reset_patience (int, optional): early stopping 사용시 얼마나 기다릴지. Defaults to 10. convergence_reset_min_delta (float, optional): early stopping 사용시 얼마나 기다릴지. Defaults to 0.0001. - convergence_reset_monitor (str, optional): early stopping 사용시 어떤 값을 기준으로 할지. Defaults to "loss". + convergence_reset_monitor (str, optional): early stopping 사용시 어떤 값을 기준으로 할지. Defaults to "loss". - "loss" or "acc" or "mse" """ - if np_seed is not None: - np.random.seed(np_seed) - if tf_seed is not None: - tf.random.set_seed(tf_seed) - - self.random_state = np.random.get_state() - - if random_state is not None: - np.random.set_state(random_state) - - model.compile(loss=loss, optimizer="sgd", metrics=["accuracy"]) - self.model = model # 모델 구조 - self.loss = loss # 손실함수 - self.n_particles = n_particles # 파티클 개수 - self.particles = [None] * n_particles # 파티클 리스트 - self.c0 = c0 # local rate - 지역 최적값 관성 수치 - self.c1 = c1 # global rate - 전역 최적값 관성 수치 - self.w_min = w_min # 최소 관성 수치 - self.w_max = w_max # 최대 관성 수치 - self.negative_swarm = negative_swarm # 최적해와 반대로 이동할 파티클 비율 - 0 ~ 1 사이의 값 - self.mutation_swarm = mutation_swarm # 관성을 추가로 사용할 파티클 비율 - 0 ~ 1 사이의 값 - self.particle_min = particle_min # 가중치 초기화 최소값 - self.particle_max = particle_max - self.g_best_score = [0, np.inf] # 최고 점수 - 시작은 0으로 초기화 - self.g_best = None # 최고 점수를 받은 가중치 - self.g_best_ = None # 최고 점수를 받은 가중치 - 값의 분산을 위한 변수 - self.avg_score = 0 # 평균 점수 - self.sigma = 1.0 - - self.save_path = None # 저장 위치 - self.renewal = "acc" - self.dispersion = False - self.day = datetime.now().strftime("%Y%m%d-%H%M%S") - - self.empirical_balance = False - - negative_count = 0 - - self.train_summary_writer = [None] * self.n_particles try: + if model is None: + raise ValueError("model is None") + if model is not None and not isinstance(model, keras.models.Model): + raise ValueError("model is not keras.models.Model") + + if loss is None: + raise ValueError("loss is None") + + if n_particles is None: + raise ValueError("n_particles is None") + if n_particles < 1: + raise ValueError("n_particles < 1") + + if c0 < 0 or c1 < 0: + raise ValueError("c0 or c1 < 0") + + if np_seed is not None: + np.random.seed(np_seed) + if tf_seed is not None: + tf.random.set_seed(tf_seed) + + self.random_state = np.random.get_state() + + if random_state is not None: + np.random.set_state(random_state) + + model.compile( + loss=loss, + optimizer="adam", + metrics=["accuracy", "mse"] + ) + self.model = model # 모델 구조 + self.loss = loss # 손실함수 + self.n_particles = n_particles # 파티클 개수 + self.particles = [None] * n_particles # 파티클 리스트 + self.c0 = c0 # local rate - 지역 최적값 관성 수치 + self.c1 = c1 # global rate - 전역 최적값 관성 수치 + self.w_min = w_min # 최소 관성 수치 + self.w_max = w_max # 최대 관성 수치 + self.negative_swarm = negative_swarm # 최적해와 반대로 이동할 파티클 비율 - 0 ~ 1 사이의 값 + self.mutation_swarm = mutation_swarm # 관성을 추가로 사용할 파티클 비율 - 0 ~ 1 사이의 값 + self.g_best_score = [np.inf, 0, np.inf] # 최고 점수 - 시작은 0으로 초기화 + self.g_best = model.get_weights() # 최고 점수를 받은 가중치 + self.avg_score = 0 # 평균 점수 + # self.sigma = 1.0 + + self.save_path = None # 저장 위치 + self.renewal = "acc" + self.dispersion = False + self.day = datetime.now().strftime("%Y%m%d-%H%M%S") + + self.empirical_balance = False + + negative_count = 0 + + self.train_summary_writer = [None] * self.n_particles + print(f"start running time : {self.day}") for i in tqdm(range(self.n_particles), desc="Initializing Particles"): @@ -142,6 +156,10 @@ class Optimizer: sys.exit("Ctrl + C : Stop Training") except MemoryError: sys.exit("Memory Error : Stop Training") + except ValueError as ve: + sys.exit(ve) + except Exception as e: + sys.exit(e) def __del__(self): del self.model @@ -155,14 +173,13 @@ class Optimizer: del self.negative_swarm del self.g_best_score del self.g_best - del self.g_best_ del self.avg_score gc.collect() tf.keras.backend.reset_uids() tf.keras.backend.clear_session() - def _encode(self, weights): + def _encode_(self, weights): """ 가중치를 1차원으로 풀어서 반환 @@ -186,9 +203,9 @@ class Optimizer: return w_gpu, shape, length - def _decode(self, weight, shape, length): + def _decode_(self, weight, shape, length): """ - _encode 로 인코딩된 가중치를 원본 shape으로 복원 + _encode_ 로 인코딩된 가중치를 원본 shape으로 복원 파라미터는 encode의 리턴값을 그대로 사용을 권장 Args: @@ -236,7 +253,23 @@ class Optimizer: else: return 1 + np.abs(score_) - class _batch_generator: + def __weight_range__(self): + """ + 가중치의 범위를 반환 + + Returns: + (float): 가중치의 최소값 + (float): 가중치의 최대값 + """ + w_, w_s, w_l = self._encode_(self.g_best) + weight_min = np.min(w_) + weight_max = np.max(w_) + + del w_, w_s, w_l + + return weight_min, weight_max + + class _batch_generator_: def __init__(self, x, y, batch_size: int = 32): self.batch_size = batch_size self.index = 0 @@ -284,12 +317,12 @@ class Optimizer: self, x, y, - epochs: int = 100, + epochs: int = 1, log: int = 0, log_name: str = None, save_info: bool = False, - save_path: str = "./result", - renewal: str = "acc", + save_path: str = "./logs", + renewal: str = "mse", empirical_balance: bool = False, dispersion: bool = False, check_point: int = None, @@ -304,7 +337,7 @@ class Optimizer: log : int - 0 : log 기록 안함, 1 : csv, 2 : tensorboard, save_info : bool - 종료시 학습 정보 저장 여부 default : False, save_path : str - ex) "./result", - renewal : str ex) "acc" or "loss" or "both", + renewal : str ex) "acc" or "loss" or "mse", empirical_balance : bool - True : EBPSO, False : PSO, dispersion : bool - True : g_best 의 값을 분산시켜 전역해를 찾음, False : g_best 의 값만 사용 check_point : int - 저장할 위치 - None : 저장 안함 @@ -321,8 +354,8 @@ class Optimizer: if save_info and save_path is None: raise ValueError("save_path is None") - if renewal not in ["acc", "loss", "both"]: - raise ValueError("renewal not in ['acc', 'loss', 'both']") + if renewal not in ["acc", "loss", "mse"]: + raise ValueError("renewal not in ['acc', 'loss', 'mse']") if check_point is not None and save_path is None: raise ValueError("save_path is None") @@ -361,93 +394,43 @@ class Optimizer: sys.exit(ve) model_ = keras.models.model_from_json(self.model.to_json()) - model_.compile(loss=self.loss, optimizer="adam", metrics=["accuracy"]) + model_.compile( + loss=self.loss, + optimizer="adam", + metrics=["accuracy", "mse"] + ) model_.fit(x, y, epochs=1, verbose=0) score = model_.evaluate(x, y, verbose=1) - if renewal == "acc": - self.g_best_score[0] = score[1] - self.g_best_score[1] = score[0] - else: - self.g_best_score[0] = score[0] - self.g_best_score[1] = score[1] + self.g_best_score = score self.g_best = model_.get_weights() - self.g_best_ = model_.get_weights() del model_ - dataset = self._batch_generator(x, y, batch_size=batch_size) - - for i in tqdm(range(self.n_particles), desc="Initializing velocity"): - p = self.particles[i] - - x_batch, y_batch = dataset.next() - local_score = p.get_score(x_batch, y_batch, renewal=renewal) - - particle_sum += local_score[1] - - if renewal == "acc": - if local_score[1] > self.g_best_score[0]: - self.g_best_score[0] = local_score[1] - self.g_best_score[1] = local_score[0] - self.g_best = p.get_best_weights() - self.g_best_ = p.get_best_weights() - elif renewal == "loss": - if local_score[0] < self.g_best_score[1]: - self.g_best_score[1] = local_score[0] - self.g_best_score[0] = local_score[1] - self.g_best = p.get_best_weights() - self.g_best_ = p.get_best_weights() - elif renewal == "both": - if local_score[1] > self.g_best_score[0]: - self.g_best_score[0] = local_score[1] - self.g_best_score[1] = local_score[0] - self.g_best = p.get_best_weights() - self.g_best_ = p.get_best_weights() - - if log == 1: - with open( - f"./{save_path}/{self.day}/{self.n_particles}_{epochs}_{self.c0}_{self.c1}_{self.w_min}_{renewal}.csv", - "a", - ) as f: - f.write(f"{local_score[0]}, {local_score[1]}") - if i != self.n_particles - 1: - f.write(", ") - else: - f.write("\n") - - elif log == 2: - with self.train_summary_writer[i].as_default(): - tf.summary.scalar("loss", local_score[0], step=0) - tf.summary.scalar("accuracy", local_score[1], step=0) - - del local_score - # gc.collect() - # tf.keras.backend.reset_uids() - # tf.keras.backend.clear_session() - print( - f"initial g_best_score : {self.g_best_score[0] if self.renewal == 'acc' else self.g_best_score[1]}" - ) + dataset = self._batch_generator_(x, y, batch_size=batch_size) try: epoch_sum = 0 epochs_pbar = tqdm( range(epochs), - desc=f"best {self.g_best_score[0]:.4f} | {self.g_best_score[1]:.4f}", + desc=f"best - loss: {self.g_best_score[0]:.4f} - acc: {self.g_best_score[1]:.4f} - mse: {self.g_best_score[2]:.4f}", ascii=True, leave=True, position=0, ) for epoch in epochs_pbar: + # 이번 epoch의 평균 점수 particle_avg = particle_sum / self.n_particles # x_j particle_sum = 0 - max_score = 0 + # 각 최고 점수, 최저 loss, 최저 mse + max_acc = 0 min_loss = np.inf + min_mse = np.inf # epoch_particle_sum = 0 part_pbar = tqdm( range(len(self.particles)), - desc=f"acc : {max_score:.4f} loss : {min_loss:.4f}", + desc=f"loss: {min_loss:.4f} acc: {max_acc:.4f} mse: {min_mse:.4f}", ascii=True, leave=False, position=1, @@ -455,29 +438,30 @@ class Optimizer: w = self.w_max - (self.w_max - self.w_min) * epoch / epochs for i in part_pbar: part_pbar.set_description( - f"acc : {max_score:.4f} loss : {min_loss:.4f}" + f"loss: {min_loss:.4f} acc: {max_acc:.4f} mse: {min_mse:.4f}" ) g_best = self.g_best x_batch, y_batch = dataset.next() + weight_min, weight_max = self.__weight_range__() + if dispersion: - ts = self.particle_min + np.random.rand() * ( - self.particle_max - self.particle_min - ) - g_, g_sh, g_len = self._encode(self.g_best) + ts = weight_min + np.random.rand() * (weight_max - weight_min) + + g_, g_sh, g_len = self._encode_(self.g_best) decrement = (epochs - epoch + 1) / epochs g_ = (1 - decrement) * g_ + decrement * ts - self.g_best_ = self._decode(g_, g_sh, g_len) - g_best = self.g_best_ + g_best = self._decode(g_, g_sh, g_len) if empirical_balance: if np.random.rand() < np.exp(-(epoch) / epochs): w_p_ = self._f( x, y, self.particles[i].get_best_weights() ) - w_g_ = self._f(x, y, self.g_best) + w_g_ = self._f(x, y, g_best) + w_p = w_p_ / (w_p_ + w_g_) w_g = w_p_ / (w_p_ + w_g_) @@ -494,7 +478,7 @@ class Optimizer: / ( self.n_particles * np.linalg.norm( - self.particle_max - self.particle_min + weight_min - weight_max ) ) * sigma_post @@ -549,60 +533,81 @@ class Optimizer: tf.summary.scalar( "accuracy", score[1], step=epoch + 1 ) + tf.summary.scalar("mse", score[2], step=epoch + 1) if renewal == "acc": - if score[1] >= max_score: - max_score = score[1] - min_loss = score[0] - - if score[1] >= self.g_best_score[0]: - if score[1] > self.g_best_score[0]: - self.g_best_score[0] = score[1] + # 최고 점수 보다 높거나 같을 경우 + if score[1] >= max_acc: + # 각 점수 갱신 + min_loss, max_acc, min_mse = score + # 최고 점수 보다 같거나 높을 경우 + if score[1] >= self.g_best_score[1]: + # 최고 점수 보다 높을 경우 + if score[1] > self.g_best_score[1]: + # 최고 점수 갱신 + self.g_best_score = score + # 최고 weight 갱신 self.g_best = self.particles[i].get_best_weights( ) + # 최고 점수 와 같을 경우 else: - if score[0] < self.g_best_score[1]: - self.g_best_score[1] = score[0] + # 최저 loss 보다 낮을 경우 + if score[0] < self.g_best_score[0]: + self.g_best_score[0] = score[0] self.g_best = self.particles[i].get_best_weights( ) epochs_pbar.set_description( - f"best {self.g_best_score[0]:.4f} | {self.g_best_score[1]:.4f}" + f"best - loss: {self.g_best_score[0]:.4f} - acc: {self.g_best_score[1]:.4f} - mse: {self.g_best_score[2]:.4f}" ) + elif renewal == "loss": + # 최저 loss 보다 작거나 같을 경우 if score[0] <= min_loss: - min_loss = score[0] - max_score = score[1] + # 각 점수 갱신 + min_loss, max_acc, min_mse = score - if score[0] <= self.g_best_score[1]: - if score[0] < self.g_best_score[1]: - self.g_best_score[1] = score[0] + # 최저 loss 와 같거나 작을 경우 + if score[0] <= self.g_best_score[0]: + # 최저 loss 보다 작을 경우 + if score[0] < self.g_best_score[0]: + # 최고 점수 갱신 + self.g_best_score = score + # 최고 weight 갱신 self.g_best = self.particles[i].get_best_weights( ) + # 최저 loss 와 같을 경우 else: - if score[1] > self.g_best_score[0]: - self.g_best_score[0] = score[1] + # 최고 acc 보다 높을 경우 + if score[1] > self.g_best_score[1]: + self.g_best_score[1] = score[1] self.g_best = self.particles[i].get_best_weights( ) epochs_pbar.set_description( - f"best {self.g_best_score[0]:.4f} | {self.g_best_score[1]:.4f}" + f"best - loss: {self.g_best_score[0]:.4f} - acc: {self.g_best_score[1]:.4f} - mse: {self.g_best_score[2]:.4f}" ) - elif renewal == "both": - if score[0] <= min_loss: - min_loss = score[0] - if score[1] >= self.g_best_score[0]: - self.g_best_score[0] = score[1] - self.g_best = self.particles[i].get_best_weights() + + elif renewal == "mse": + if score[2] <= min_mse: + min_loss, max_acc, min_mse = score + + if score[2] <= self.g_best_score[2]: + if score[2] < self.g_best_score[2]: + self.g_best_score[0] = score[0] + self.g_best_score[1] = score[1] + self.g_best_score[2] = score[2] + + self.g_best = self.particles[i].get_best_weights( + ) + + else: + if score[1] > self.g_best_score[1]: + self.g_best_score[1] = score[1] + self.g_best = self.particles[i].get_best_weights( + ) epochs_pbar.set_description( - f"best {self.g_best_score[0]:.4f} | {self.g_best_score[1]:.4f}" - ) - if score[1] >= max_score: - max_score = score[1] - if score[0] <= self.g_best_score[1]: - self.g_best_score[1] = score[0] - self.g_best = self.particles[i].get_best_weights() - epochs_pbar.set_description( - f"best {self.g_best_score[0]:.4f} | {self.g_best_score[1]:.4f}" + f"best - loss: {self.g_best_score[0]:.4f} - acc: {self.g_best_score[1]:.4f} - mse: {self.g_best_score[2]:.4f}" ) + particle_sum += score[1] if log == 1: @@ -623,9 +628,9 @@ class Optimizer: self._check_point_save( f"./{save_path}/{self.day}/ckpt-{epoch}") - gc.collect() tf.keras.backend.reset_uids() tf.keras.backend.clear_session() + gc.collect() except KeyboardInterrupt: print("Ctrl + C : Stop Training") @@ -654,7 +659,11 @@ class Optimizer: """ model = keras.models.model_from_json(self.model.to_json()) model.set_weights(self.g_best) - model.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"]) + model.compile( + loss=self.loss, + optimizer="adam", + metrics=["accuracy", "mse"] + ) return model @@ -732,8 +741,9 @@ class Optimizer: """ model = self.get_best_model() score = model.evaluate(x, y, verbose=1) - print(f"model acc : {score[1]}, loss : {score[0]}") + print( + f"model score - loss: {score[0]} - acc: {score[1]} - mse: {score[2]}") model.save( - f"./{save_path}/{self.day}/{self.n_particles}_{self.c0}_{self.c1}_{self.w_min}.h5" + f"./{save_path}/{self.day}/model_{score[0 if self.renewal == 'loss' else 1 if self.renewal == 'acc' else 2 ]}.h5" ) return model diff --git a/pso/particle.py b/pso/particle.py index 305a8fa..26eb626 100644 --- a/pso/particle.py +++ b/pso/particle.py @@ -38,7 +38,7 @@ class Particle: self.loss = loss try: - if converge_reset and converge_reset_monitor not in ["acc", "accuracy", "loss"]: + if converge_reset and converge_reset_monitor not in ["acc", "accuracy", "loss", "mse"]: raise ValueError( "converge_reset_monitor must be 'acc' or 'accuracy' or 'loss'" ) @@ -50,10 +50,12 @@ class Particle: print(e) exit(1) - self.reset_particle() + self.__reset_particle__() + self.best_weights = self.model.get_weights() + self.before_best = self.model.get_weights() self.negative = negative self.mutation = mutation - self.best_score = 0 + self.best_score = [np.inf, 0, np.inf] self.before_w = 0 self.score_history = [] self.converge_reset = converge_reset @@ -131,14 +133,20 @@ class Particle: (float): 점수 """ score = self.model.evaluate(x, y, verbose=0) - if renewal == "acc": - if score[1] > self.best_score: - self.best_score = score[1] + if renewal == "loss": + if score[0] < self.best_score[0]: + self.best_score[0] = score[0] self.best_weights = self.model.get_weights() - elif renewal == "loss": - if score[0] < self.best_score: - self.best_score = score[0] + elif renewal == "acc": + if score[1] > self.best_score[1]: + self.best_score[1] = score[1] self.best_weights = self.model.get_weights() + elif renewal == "mse": + if score[2] < self.best_score[2]: + self.best_score[2] = score[2] + self.best_weights = self.model.get_weights() + else: + raise ValueError("renewal must be 'acc' or 'loss' or 'mse'") return score @@ -156,6 +164,11 @@ class Particle: self.score_history.append(score[1]) elif monitor in ["loss"]: self.score_history.append(score[0]) + elif monitor in ["mse"]: + self.score_history.append(score[2]) + else: + raise ValueError( + "monitor must be 'acc' or 'accuracy' or 'loss' or 'mse'") if len(self.score_history) > patience: last_scores = self.score_history[-patience:] @@ -163,19 +176,18 @@ class Particle: return True return False - def reset_particle(self): + def __reset_particle__(self): self.model = keras.models.model_from_json(self.model.to_json()) - self.model.compile(optimizer="adam", loss=self.loss, - metrics=["accuracy"]) - init_weights = self.model.get_weights() - i_w_, i_s, i_l = self._encode(init_weights) + self.model.compile( + optimizer="adam", + loss=self.loss, + metrics=["accuracy", "mse"] + ) + i_w_, i_s, i_l = self._encode(self.model.get_weights()) i_w_ = np.random.uniform(-0.05, 0.05, len(i_w_)) self.velocities = self._decode(i_w_, i_s, i_l) - self.best_weights = init_weights - self.before_best = init_weights - - del init_weights, i_w_, i_s, i_l + del i_w_, i_s, i_l self.score_history = [] def _update_velocity(self, local_rate, global_rate, w, g_best): @@ -212,7 +224,7 @@ class Particle: + -1 * global_rate * r_1 * (encode_g - encode_w) ) if len(self.score_history) > 10 and max(self.score_history[-10:]) - min(self.score_history[-10:]) < 0.01: - self.reset_particle() + self.__reset_particle__() else: new_v = ( @@ -324,7 +336,7 @@ class Particle: if self.converge_reset and self.__check_converge_reset__( score, self.converge_reset_monitor, self.converge_reset_patience, self.converge_reset_min_delta): - self.reset_particle() + self.__reset_particle__() return score