jung-geun/PSO
1
0
Fork 0
mirror of https://github.com/jung-geun/PSO.git synced 2025-12-20 04:50:45 +09:00
Code Issues Packages Projects Releases Wiki Activity

23-06-03

Browse Source 
tensorflow gpu 의 메모리 용량 제한을 추가
readme에 분류 문제별 해결 현황 추가
This commit is contained in:
jung-geun
2023-06-03 17:25:30 +09:00
parent 4ffc6cc6e5
commit 0d99329a43
12 changed files with 357 additions and 1706 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
iris.py
View File

@@ -39,9 +39,27 @@ x_train, x_test, y_train, y_test = load_data()
loss = 'categorical_crossentropy'
pso_iris = Optimizer(model, loss=loss, n_particles=50, c0=0.4, c1=0.8, w_min=0.7, w_max=1.0, random=0.2)
pso_iris = Optimizer(
model,
loss=loss,
n_particles=75,
c0=0.4,
c1=0.8,
w_min=0.7,
w_max=1.0,
negative_swarm=0.25
)
weight, score = pso_iris.fit(
x_train, y_train, epochs=500, save=True, save_path="./result/iris", renewal="acc", empirical_balance=False, Dispersion=False, check_point=25)
best_score = pso_iris.fit(
x_train,
y_train,
epochs=200,
save=True,
save_path="./result/iris",
renewal="acc",
empirical_balance=False,
Dispersion=False,
check_point=25
)
gc.collect()

BIN
iris_relu_acc_200.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 97 KiB

32
mnist.py
View File

@@ -61,9 +61,7 @@ def make_model():
return model
# %%
model = make_model()
x_test, y_test = get_data_test()
# loss = 'binary_crossentropy'
@@ -73,16 +71,32 @@ x_test, y_test = get_data_test()
# loss = 'poisson'
# loss = 'cosine_similarity'
# loss = 'log_cosh'
loss = 'huber_loss'
# loss = 'huber_loss'
# loss = 'mean_absolute_error'
# loss = 'mean_absolute_percentage_error'
# loss = 'mean_squared_error'
loss = 'mean_squared_error'
pso_mnist = Optimizer(
model,
loss=loss,
n_particles=50,
c0=0.35,
c1=0.8,
w_min=0.7,
w_max=1.1,
negative_swarm=0.25
)
pso_mnist = Optimizer(model, loss=loss, n_particles=50, c0=0.4, c1=0.8, w_min=0.4, w_max=0.95, negative_swarm=0.3)
weight, score = pso_mnist.fit(
x_test, y_test, epochs=500, save=True, save_path="./result/mnist", renewal="acc", empirical_balance=True, Dispersion=False, check_point=10)
best_score = pso_mnist.fit(
x_test,
y_test,
epochs=200,
save=True,
save_path="./result/mnist",
renewal="acc",
empirical_balance=False,
Dispersion=False,
check_point=25
)
# pso_mnist.model_save("./result/mnist")
# pso_mnist.save_info("./result/mnist")
gc.collect()

33
plt.ipynb
View File

File diff suppressed because one or more lines are too long

43
pso/optimizer.py
View File

@@ -16,6 +16,13 @@ from copy import copy, deepcopy
from pso.particle import Particle
gpus = tf.config.experimental.list_physical_devices("GPU")
if gpus:
try:
# tf.config.experimental.set_visible_devices(gpus[0], "GPU")
tf.config.experimental.set_memory_growth(gpus[0], True)
except RuntimeError as e:
print(e)
class Optimizer:
"""
@@ -90,7 +97,6 @@ class Optimizer:
w_gpu = np.append(w_gpu, w_)
del weights
gc.collect()
return w_gpu, shape, lenght
"""
@@ -116,7 +122,6 @@ class Optimizer:
del weight
del shape
del lenght
gc.collect()
return weights
@@ -124,8 +129,6 @@ class Optimizer:
self.model.set_weights(weights)
self.model.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"])
score = self.model.evaluate(x, y, verbose=0)[1]
gc.collect()
if score > 0:
return 1 / (1 + score)
else:
@@ -163,6 +166,7 @@ class Optimizer:
self.g_best_score = 0
elif renewal == "loss":
self.g_best_score = np.inf
try:
if save:
if save_path is None:
@@ -175,9 +179,9 @@ class Optimizer:
except ValueError as e:
print(e)
sys.exit(1)
# for i, p in enumerate(self.particles):
for i in tqdm(range(self.n_particles), desc="Initializing Particles"):
p = copy(self.particles[i])
p = self.particles[i]
local_score = p.get_score(x, y, renewal=renewal)
if renewal == "acc":
@@ -190,8 +194,24 @@ class Optimizer:
self.g_best_score = local_score[0]
self.g_best = p.get_best_weights()
self.g_best_ = p.get_best_weights()
if local_score[0] == None:
local_score[0] = np.inf
if local_score[1] == None:
local_score[1] = 0
if save:
with open(
f"./{save_path}/{self.day}_{self.n_particles}_{epochs}_{self.c0}_{self.c1}_{self.w_min}_{renewal}.csv",
"a",
) as f:
f.write(f"{local_score[0]}, {local_score[1]}")
if i != self.n_particles - 1:
f.write(", ")
else:
f.write("\n")
del local_score
del p
gc.collect()
print(f"initial g_best_score : {self.g_best_score}")
@@ -266,6 +286,10 @@ class Optimizer:
self.g_best_score = score[0]
self.g_best = self.particles[i].get_best_weights()
if score[0] == None:
score[0] = np.inf
if score[1] == None:
score[1] = 0
loss = loss + score[0]
acc = acc + score[1]
if score[0] < min_loss:
@@ -295,7 +319,6 @@ class Optimizer:
f"loss min : {round(min_loss, 4)} | acc max : {round(max_score, 4)} | Best {renewal} : {self.g_best_score}"
)
gc.collect()
if check_point is not None:
if _ % check_point == 0:
@@ -303,6 +326,8 @@ class Optimizer:
self._check_point_save(f"./{save_path}/{self.day}/ckpt-{_}")
self.avg_score = acc / self.n_particles
gc.collect()
except KeyboardInterrupt:
print("Ctrl + C : Stop Training")
except MemoryError:
@@ -315,7 +340,7 @@ class Optimizer:
self.save_info(save_path)
print("save info")
return self.g_best, self.g_best_score
return self.g_best_score
def get_best_model(self):
model = keras.models.model_from_json(self.model.to_json())

14
pso/particle.py
View File

@@ -12,7 +12,7 @@ class Particle:
self.loss = loss
init_weights = self.model.get_weights()
i_w_, s_, l_ = self._encode(init_weights)
i_w_ = np.random.rand(len(i_w_)) / 5 - 0.10
i_w_ = np.random.rand(len(i_w_)) / 2 - 0.25
self.velocities = self._decode(i_w_, s_, l_)
self.negative = negative
self.best_score = 0
@@ -40,7 +40,7 @@ class Particle:
lenght.append(len(w_))
# w_gpu = cp.append(w_gpu, w_)
w_gpu = np.append(w_gpu, w_)
gc.collect()
return w_gpu, shape, lenght
"""
@@ -62,7 +62,7 @@ class Particle:
del start, end, w_
del shape, lenght
del weight
gc.collect()
return weights
def get_score(self, x, y, renewal: str = "acc"):
@@ -77,7 +77,7 @@ class Particle:
if score[0] < self.best_score:
self.best_score = score[0]
self.best_weights = self.model.get_weights()
gc.collect()
return score
def _update_velocity(self, local_rate, global_rate, w, g_best):
@@ -105,7 +105,6 @@ class Particle:
del encode_p, p_sh, p_len
del encode_g, g_sh, g_len
del r0, r1
gc.collect()
def _update_velocity_w(self, local_rate, global_rate, w, w_p, w_g, g_best):
encode_w, w_sh, w_len = self._encode(weights=self.model.get_weights())
@@ -132,7 +131,6 @@ class Particle:
del encode_p, p_sh, p_len
del encode_g, g_sh, g_len
del r0, r1
gc.collect()
def _update_weights(self):
encode_w, w_sh, w_len = self._encode(weights=self.model.get_weights())
@@ -141,12 +139,10 @@ class Particle:
self.model.set_weights(self._decode(new_w, w_sh, w_len))
del encode_w, w_sh, w_len
del encode_v, v_sh, v_len
gc.collect()
def f(self, x, y, weights):
self.model.set_weights(weights)
score = self.model.evaluate(x, y, verbose=0)[1]
gc.collect()
if score > 0:
return 1 / (1 + score)
else:
@@ -155,7 +151,6 @@ class Particle:
def step(self, x, y, local_rate, global_rate, w, g_best, renewal: str = "acc"):
self._update_velocity(local_rate, global_rate, w, g_best)
self._update_weights()
gc.collect()
return self.get_score(x, y, renewal)
def step_w(
@@ -163,7 +158,6 @@ class Particle:
):
self._update_velocity_w(local_rate, global_rate, w, w_p, w_g, g_best)
self._update_weights()
gc.collect()
return self.get_score(x, y, renewal)
def get_best_score(self):

90
readme.md
View File

@@ -74,6 +74,96 @@ pso 알고리즘을 이용하여 오차역전파 함수를 최적화 하는 방
위의 아이디어는 원래의 목표와 다른 방향으로 가고 있습니다. 따라서 다른 방법을 모색해야할 것 같습니다
<br>
## 3. PSO 알고리즘을 이용하여 풀이한 문제들의 정확도
### 1. xor 문제
``` python
loss = 'mean_squared_error'
pso_xor = Optimizer(
model,
loss=loss,
n_particles=75,
c0=0.35,
c1=0.8,
w_min=0.6,
w_max=1.2,
negative_swarm=0.25
)
best_score = pso_xor.fit(
x_test,
y_test,
epochs=200,
save=True,
save_path="./result/xor",
renewal="acc",
empirical_balance=False,
Dispersion=False,
check_point=25
)
```
위의 파라미터 기준 40 세대 이후부터 정확도가 100%가 나오는 것을 확인하였습니다
![xor](./xor_sigmoid_2_acc_40.png)
2. iris 문제
``` python
loss = 'categorical_crossentropy'
pso_iris = Optimizer(
model,
loss=loss,
n_particles=50,
c0=0.4,
c1=0.8,
w_min=0.7,
w_max=1.0,
negative_swarm=0.2
)
best_score = pso_iris.fit(
x_train,
y_train,
epochs=200,
save=True,
save_path="./result/iris",
renewal="acc",
empirical_balance=False,
Dispersion=False,
check_point=25
)
```
위의 파라미터 기준 2 세대에 94%의 정확도를, 7 세대에 96%, 106 세대에 99.16%의 정확도를 보였습니다
![iris](./iris_relu_acc_200.png)
3. mnist 문제
``` python
loss = 'mean_squared_error'
pso_mnist = Optimizer(
model,
loss=loss,
n_particles=50,
c0=0.35,
c1=0.8,
w_min=0.7,
w_max=1.0,
negative_swarm=0.2
)
best_score = pso_mnist.fit(
x_test,
y_test,
epochs=200,
save=True,
save_path="./result/mnist",
renewal="acc",
empirical_balance=False,
Dispersion=False,
check_point=25
)
```
### Trouble Shooting
> 1. 딥러닝 알고리즘 특성상 weights는 처음 컴파일시 무작위하게 생성된다. weights의 각 지점의 중요도는 매번 무작위로 정해지기에 전역 최적값으로 찾아갈 때 값이 높은 loss를 향해서 상승하는 현상이 나타난다.<br>

118
test.ipynb
View File

@@ -275,10 +275,122 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": []
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1/1 [==============================] - 0s 452ms/step\n",
"[[0.0000000e+00 1.0000000e+00 8.5117706e-28]\n",
" [0.0000000e+00 1.0000000e+00 0.0000000e+00]\n",
" [1.0000000e+00 3.3700031e-35 0.0000000e+00]\n",
" [1.0000000e+00 1.3158974e-19 0.0000000e+00]\n",
" [0.0000000e+00 1.0000000e+00 0.0000000e+00]\n",
" [0.0000000e+00 0.0000000e+00 1.0000000e+00]\n",
" [1.0000000e+00 1.4602315e-27 0.0000000e+00]\n",
" [0.0000000e+00 0.0000000e+00 1.0000000e+00]\n",
" [1.0000000e+00 2.4845295e-16 0.0000000e+00]\n",
" [0.0000000e+00 1.0000000e+00 1.6942224e-33]\n",
" [1.0000000e+00 0.0000000e+00 0.0000000e+00]\n",
" [0.0000000e+00 1.0000000e+00 0.0000000e+00]\n",
" [1.0000000e+00 9.0455008e-36 0.0000000e+00]\n",
" [1.0000000e+00 0.0000000e+00 0.0000000e+00]\n",
" [0.0000000e+00 1.8117375e-33 1.0000000e+00]\n",
" [0.0000000e+00 1.0000000e+00 6.7984806e-36]\n",
" [0.0000000e+00 1.7472901e-25 1.0000000e+00]\n",
" [0.0000000e+00 6.2991115e-37 1.0000000e+00]\n",
" [0.0000000e+00 0.0000000e+00 1.0000000e+00]\n",
" [0.0000000e+00 1.0598510e-30 1.0000000e+00]\n",
" [1.0000000e+00 1.7519910e-30 0.0000000e+00]\n",
" [0.0000000e+00 1.0000000e+00 0.0000000e+00]\n",
" [0.0000000e+00 1.0000000e+00 0.0000000e+00]\n",
" [1.0000000e+00 7.4562871e-27 0.0000000e+00]\n",
" [0.0000000e+00 0.0000000e+00 1.0000000e+00]\n",
" [0.0000000e+00 0.0000000e+00 1.0000000e+00]\n",
" [0.0000000e+00 0.0000000e+00 1.0000000e+00]\n",
" [0.0000000e+00 1.0000000e+00 0.0000000e+00]\n",
" [0.0000000e+00 1.0000000e+00 0.0000000e+00]\n",
" [1.0000000e+00 0.0000000e+00 0.0000000e+00]]\n",
"[[0. 1. 0.]\n",
" [0. 1. 0.]\n",
" [1. 0. 0.]\n",
" [1. 0. 0.]\n",
" [0. 1. 0.]\n",
" [0. 0. 1.]\n",
" [1. 0. 0.]\n",
" [0. 0. 1.]\n",
" [1. 0. 0.]\n",
" [0. 1. 0.]\n",
" [1. 0. 0.]\n",
" [0. 1. 0.]\n",
" [1. 0. 0.]\n",
" [1. 0. 0.]\n",
" [0. 0. 1.]\n",
" [0. 1. 0.]\n",
" [0. 0. 1.]\n",
" [0. 0. 1.]\n",
" [0. 0. 1.]\n",
" [0. 0. 1.]\n",
" [1. 0. 0.]\n",
" [0. 1. 0.]\n",
" [0. 1. 0.]\n",
" [1. 0. 0.]\n",
" [0. 0. 1.]\n",
" [0. 0. 1.]\n",
" [0. 0. 1.]\n",
" [0. 1. 0.]\n",
" [0. 1. 0.]\n",
" [1. 0. 0.]]\n",
"1/1 [==============================] - 0s 88ms/step - loss: 0.0000e+00 - accuracy: 1.0000\n",
"[0.0, 1.0]\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2023-06-02 14:34:49.851147: I tensorflow/stream_executor/cuda/cuda_blas.cc:1614] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n"
]
}
],
"source": [
"import numpy as np\n",
"import tensorflow as tf\n",
"from tensorflow import keras\n",
"from tensorflow.keras import layers\n",
"from tensorflow.keras.models import Sequential\n",
"\n",
"from sklearn.datasets import load_iris\n",
"from sklearn.model_selection import train_test_split\n",
"\n",
"def get_xor():\n",
" x = np.array([[0,0],[0,1],[1,0],[1,1]])\n",
" y = np.array([[0],[1],[1],[0]])\n",
"\n",
" return x,y\n",
"\n",
"def get_iris():\n",
" iris = load_iris()\n",
" x = iris.data\n",
" y = iris.target\n",
"\n",
" y = keras.utils.to_categorical(y, 3)\n",
"\n",
" x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, shuffle=True, stratify=y)\n",
"\n",
" return x_train, x_test, y_train, y_test\n",
"\n",
"# model = keras.models.load_model(\"./result/xor/06-02-13-31/75_0.35_0.8_0.6.h5\")\n",
"model = keras.models.load_model(\"./result/iris/06-02-13-48/50_0.4_0.8_0.7.h5\")\n",
"# x,y = get_xor()\n",
"x_train, x_test, y_train, y_test = get_iris()\n",
"\n",
"print(model.predict(x_test))\n",
"print(y_test)\n",
"print(model.evaluate(x_test,y_test))"
]
}
],
"metadata": {

1650
xor.ipynb
View File

File diff suppressed because it is too large Load Diff

59
xor.py Normal file
View File

@@ -0,0 +1,59 @@
# %%
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
import tensorflow as tf
tf.random.set_seed(777) # for reproducibility
# from pso_tf import PSO
from pso import Optimizer
from tensorflow import keras
import numpy as np
from tensorflow import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras import layers
from datetime import datetime
print(tf.__version__)
print(tf.config.list_physical_devices())
def get_data():
x = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
y = np.array([[0], [1], [1], [0]])
return x, y
def make_model():
leyer = []
leyer.append(layers.Dense(2, activation='sigmoid', input_shape=(2,)))
# leyer.append(layers.Dense(2, activation='sigmoid'))
leyer.append(layers.Dense(1, activation='sigmoid'))
model = Sequential(leyer)
return model
# %%
model = make_model()
x_test, y_test = get_data()
# loss = 'binary_crossentropy'
# loss = 'categorical_crossentropy'
# loss = 'sparse_categorical_crossentropy'
# loss = 'kullback_leibler_divergence'
# loss = 'poisson'
# loss = 'cosine_similarity'
# loss = 'log_cosh'
# loss = 'huber_loss'
# loss = 'mean_absolute_error'
# loss = 'mean_absolute_percentage_error'
loss = 'mean_squared_error'
pso_xor = Optimizer(model,
loss=loss, n_particles=75, c0=0.35, c1=0.8, w_min=0.6, w_max=1.2, negative_swarm=0.25)
best_score = pso_xor.fit(
x_test, y_test, epochs=200, save=True, save_path="./result/xor", renewal="acc", empirical_balance=False, Dispersion=False, check_point=25)
# %%

BIN
xor_sigmoid_2_acc_40.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 207 KiB

BIN
xor_sigmoid_3_acc_40.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 139 KiB