23-06-03

tensorflow gpu 의 메모리 용량 제한을 추가
readme에 분류 문제별 해결 현황 추가

iris.py

@@ -39,9 +39,27 @@ x_train, x_test, y_train, y_test = load_data()
 
 loss = 'categorical_crossentropy'
 
-pso_iris = Optimizer(model, loss=loss, n_particles=50, c0=0.4, c1=0.8, w_min=0.7, w_max=1.0, random=0.2)
+pso_iris = Optimizer(
+    model,
+    loss=loss, 
+    n_particles=75, 
+    c0=0.4, 
+    c1=0.8, 
+    w_min=0.7,
+    w_max=1.0, 
+    negative_swarm=0.25
+    )
 
-weight, score = pso_iris.fit(
+best_score = pso_iris.fit(
-    x_train, y_train, epochs=500, save=True, save_path="./result/iris", renewal="acc", empirical_balance=False, Dispersion=False, check_point=25)
+    x_train, 
+    y_train, 
+    epochs=200, 
+    save=True, 
+    save_path="./result/iris", 
+    renewal="acc", 
+    empirical_balance=False, 
+    Dispersion=False, 
+    check_point=25
+    )
 
 gc.collect()

mnist.py

@@ -61,9 +61,7 @@ def make_model():
 
     return model
 
-
 # %%
-
 model = make_model()
 x_test, y_test = get_data_test()
 # loss = 'binary_crossentropy'
@@ -73,16 +71,32 @@ x_test, y_test = get_data_test()
 # loss = 'poisson'
 # loss = 'cosine_similarity'
 # loss = 'log_cosh'
-loss = 'huber_loss' 
+# loss = 'huber_loss' 
 # loss = 'mean_absolute_error'
 # loss = 'mean_absolute_percentage_error'
-# loss = 'mean_squared_error'
+loss = 'mean_squared_error'
 
+pso_mnist = Optimizer(
+    model,
+    loss=loss, 
+    n_particles=50,
+    c0=0.35, 
+    c1=0.8, 
+    w_min=0.7,
+    w_max=1.1,
+    negative_swarm=0.25
+    )
 
-pso_mnist = Optimizer(model, loss=loss, n_particles=50, c0=0.4, c1=0.8, w_min=0.4, w_max=0.95, negative_swarm=0.3)
+best_score = pso_mnist.fit(
-weight, score = pso_mnist.fit(
+    x_test,
-    x_test, y_test, epochs=500, save=True, save_path="./result/mnist", renewal="acc", empirical_balance=True, Dispersion=False, check_point=10)
+    y_test,
+    epochs=200,
+    save=True,
+    save_path="./result/mnist", 
+    renewal="acc", 
+    empirical_balance=False,
+    Dispersion=False, 
+    check_point=25
+    )
 # pso_mnist.model_save("./result/mnist")
 # pso_mnist.save_info("./result/mnist")
-
-gc.collect()

pso/optimizer.py

@@ -16,6 +16,13 @@ from copy import copy, deepcopy
 
 from pso.particle import Particle
 
+gpus = tf.config.experimental.list_physical_devices("GPU")
+if gpus:
+    try:
+        # tf.config.experimental.set_visible_devices(gpus[0], "GPU")
+        tf.config.experimental.set_memory_growth(gpus[0], True)
+    except RuntimeError as e:
+        print(e)
 
 class Optimizer:
     """
@@ -90,7 +97,6 @@ class Optimizer:
             w_gpu = np.append(w_gpu, w_)
 
         del weights
-        gc.collect()
         return w_gpu, shape, lenght
 
     """
@@ -116,7 +122,6 @@ class Optimizer:
         del weight
         del shape
         del lenght
-        gc.collect()
 
         return weights
 
@@ -124,8 +129,6 @@ class Optimizer:
         self.model.set_weights(weights)
         self.model.compile(loss=self.loss, optimizer="sgd", metrics=["accuracy"])
         score = self.model.evaluate(x, y, verbose=0)[1]
-
-        gc.collect()
         if score > 0:
             return 1 / (1 + score)
         else:
@@ -163,6 +166,7 @@ class Optimizer:
             self.g_best_score = 0
         elif renewal == "loss":
             self.g_best_score = np.inf
+
         try:
             if save:
                 if save_path is None:
@@ -175,9 +179,9 @@ class Optimizer:
         except ValueError as e:
             print(e)
             sys.exit(1)
-        # for i, p in enumerate(self.particles):
+            
         for i in tqdm(range(self.n_particles), desc="Initializing Particles"):
-            p = copy(self.particles[i])
+            p = self.particles[i]
             local_score = p.get_score(x, y, renewal=renewal)
 
             if renewal == "acc":
@@ -190,8 +194,24 @@ class Optimizer:
                     self.g_best_score = local_score[0]
                     self.g_best = p.get_best_weights()
                     self.g_best_ = p.get_best_weights()
+
+            if local_score[0] == None:
+                local_score[0] = np.inf
+
+            if local_score[1] == None:
+                local_score[1] = 0
+                
+            if save:
+                with open(
+                    f"./{save_path}/{self.day}_{self.n_particles}_{epochs}_{self.c0}_{self.c1}_{self.w_min}_{renewal}.csv",
+                    "a",
+                ) as f:
+                    f.write(f"{local_score[0]}, {local_score[1]}")
+                    if i != self.n_particles - 1:
+                        f.write(", ")
+                    else:
+                        f.write("\n")
             del local_score
-            del p
             gc.collect()
 
         print(f"initial g_best_score : {self.g_best_score}")
@@ -266,6 +286,10 @@ class Optimizer:
                             self.g_best_score = score[0]
                             self.g_best = self.particles[i].get_best_weights()
 
+                    if score[0] == None:
+                        score[0] = np.inf
+                    if score[1] == None:
+                        score[1] = 0
                     loss = loss + score[0]
                     acc = acc + score[1]
                     if score[0] < min_loss:
@@ -295,7 +319,6 @@ class Optimizer:
                     f"loss min : {round(min_loss, 4)} | acc max : {round(max_score, 4)} | Best {renewal} : {self.g_best_score}"
                 )
 
-                gc.collect()
 
                 if check_point is not None:
                     if _ % check_point == 0:
@@ -303,6 +326,8 @@ class Optimizer:
                         self._check_point_save(f"./{save_path}/{self.day}/ckpt-{_}")
                 self.avg_score = acc / self.n_particles
 
+                gc.collect()
+                
         except KeyboardInterrupt:
             print("Ctrl + C : Stop Training")
         except MemoryError:
@@ -315,7 +340,7 @@ class Optimizer:
             self.save_info(save_path)
             print("save info")
 
-            return self.g_best, self.g_best_score
+            return self.g_best_score
 
     def get_best_model(self):
         model = keras.models.model_from_json(self.model.to_json())

pso/particle.py

@@ -12,7 +12,7 @@ class Particle:
         self.loss = loss
         init_weights = self.model.get_weights()
         i_w_, s_, l_ = self._encode(init_weights)
-        i_w_ = np.random.rand(len(i_w_)) / 5 - 0.10
+        i_w_ = np.random.rand(len(i_w_)) / 2 - 0.25
         self.velocities = self._decode(i_w_, s_, l_)
         self.negative = negative
         self.best_score = 0
@@ -40,7 +40,7 @@ class Particle:
             lenght.append(len(w_))
             # w_gpu = cp.append(w_gpu, w_)
             w_gpu = np.append(w_gpu, w_)
-        gc.collect()
+
         return w_gpu, shape, lenght
 
     """
@@ -62,7 +62,7 @@ class Particle:
         del start, end, w_
         del shape, lenght
         del weight
-        gc.collect()
+
         return weights
 
     def get_score(self, x, y, renewal: str = "acc"):
@@ -77,7 +77,7 @@ class Particle:
             if score[0] < self.best_score:
                 self.best_score = score[0]
                 self.best_weights = self.model.get_weights()
-        gc.collect()
+
         return score
 
     def _update_velocity(self, local_rate, global_rate, w, g_best):
@@ -105,7 +105,6 @@ class Particle:
         del encode_p, p_sh, p_len
         del encode_g, g_sh, g_len
         del r0, r1
-        gc.collect()
 
     def _update_velocity_w(self, local_rate, global_rate, w, w_p, w_g, g_best):
         encode_w, w_sh, w_len = self._encode(weights=self.model.get_weights())
@@ -132,7 +131,6 @@ class Particle:
         del encode_p, p_sh, p_len
         del encode_g, g_sh, g_len
         del r0, r1
-        gc.collect()
 
     def _update_weights(self):
         encode_w, w_sh, w_len = self._encode(weights=self.model.get_weights())
@@ -141,12 +139,10 @@ class Particle:
         self.model.set_weights(self._decode(new_w, w_sh, w_len))
         del encode_w, w_sh, w_len
         del encode_v, v_sh, v_len
-        gc.collect()
 
     def f(self, x, y, weights):
         self.model.set_weights(weights)
         score = self.model.evaluate(x, y, verbose=0)[1]
-        gc.collect()
         if score > 0:
             return 1 / (1 + score)
         else:
@@ -155,7 +151,6 @@ class Particle:
     def step(self, x, y, local_rate, global_rate, w, g_best, renewal: str = "acc"):
         self._update_velocity(local_rate, global_rate, w, g_best)
         self._update_weights()
-        gc.collect()
         return self.get_score(x, y, renewal)
 
     def step_w(
@@ -163,7 +158,6 @@ class Particle:
     ):
         self._update_velocity_w(local_rate, global_rate, w, w_p, w_g, g_best)
         self._update_weights()
-        gc.collect()
         return self.get_score(x, y, renewal)
 
     def get_best_score(self):

readme.md

@@ -74,6 +74,96 @@ pso 알고리즘을 이용하여 오차역전파 함수를 최적화 하는 방
 위의 아이디어는 원래의 목표와 다른 방향으로 가고 있습니다. 따라서 다른 방법을 모색해야할 것 같습니다
 <br>
 
+## 3. PSO 알고리즘을 이용하여 풀이한 문제들의 정확도
+
+### 1. xor 문제
+``` python
+ loss = 'mean_squared_error'
+
+ pso_xor = Optimizer(
+    model,
+    loss=loss, 
+    n_particles=75, 
+    c0=0.35, 
+    c1=0.8, 
+    w_min=0.6, 
+    w_max=1.2, 
+    negative_swarm=0.25
+    )
+
+ best_score = pso_xor.fit(
+    x_test, 
+    y_test, 
+    epochs=200, 
+    save=True, 
+    save_path="./result/xor", 
+    renewal="acc", 
+    empirical_balance=False, 
+    Dispersion=False, 
+    check_point=25
+    )
+```
+위의 파라미터 기준 40 세대 이후부터 정확도가 100%가 나오는 것을 확인하였습니다
+![xor](./xor_sigmoid_2_acc_40.png)
+
+2. iris 문제
+``` python
+loss = 'categorical_crossentropy'
+
+pso_iris = Optimizer(
+    model,
+    loss=loss, 
+    n_particles=50, 
+    c0=0.4, 
+    c1=0.8, 
+    w_min=0.7,
+    w_max=1.0, 
+    negative_swarm=0.2
+    )
+
+best_score = pso_iris.fit(
+    x_train, 
+    y_train, 
+    epochs=200, 
+    save=True, 
+    save_path="./result/iris", 
+    renewal="acc", 
+    empirical_balance=False, 
+    Dispersion=False, 
+    check_point=25
+    )
+```
+위의 파라미터 기준 2 세대에 94%의 정확도를, 7 세대에 96%, 106 세대에 99.16%의 정확도를 보였습니다
+![iris](./iris_relu_acc_200.png)
+
+3. mnist 문제
+``` python
+loss = 'mean_squared_error'
+
+pso_mnist = Optimizer(
+    model,
+    loss=loss, 
+    n_particles=50,
+    c0=0.35, 
+    c1=0.8, 
+    w_min=0.7,
+    w_max=1.0,
+    negative_swarm=0.2
+    )
+
+best_score = pso_mnist.fit(
+    x_test,
+    y_test,
+    epochs=200,
+    save=True,
+    save_path="./result/mnist", 
+    renewal="acc", 
+    empirical_balance=False,
+    Dispersion=False, 
+    check_point=25
+    )
+```
+
 ### Trouble Shooting
 
 > 1. 딥러닝 알고리즘 특성상 weights는 처음 컴파일시 무작위하게 생성된다. weights의 각 지점의 중요도는 매번 무작위로 정해지기에 전역 최적값으로 찾아갈 때 값이 높은 loss를 향해서 상승하는 현상이 나타난다.<br>

test.ipynb

@@ -275,10 +275,122 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
-   "source": []
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1/1 [==============================] - 0s 452ms/step\n",
+      "[[0.0000000e+00 1.0000000e+00 8.5117706e-28]\n",
+      " [0.0000000e+00 1.0000000e+00 0.0000000e+00]\n",
+      " [1.0000000e+00 3.3700031e-35 0.0000000e+00]\n",
+      " [1.0000000e+00 1.3158974e-19 0.0000000e+00]\n",
+      " [0.0000000e+00 1.0000000e+00 0.0000000e+00]\n",
+      " [0.0000000e+00 0.0000000e+00 1.0000000e+00]\n",
+      " [1.0000000e+00 1.4602315e-27 0.0000000e+00]\n",
+      " [0.0000000e+00 0.0000000e+00 1.0000000e+00]\n",
+      " [1.0000000e+00 2.4845295e-16 0.0000000e+00]\n",
+      " [0.0000000e+00 1.0000000e+00 1.6942224e-33]\n",
+      " [1.0000000e+00 0.0000000e+00 0.0000000e+00]\n",
+      " [0.0000000e+00 1.0000000e+00 0.0000000e+00]\n",
+      " [1.0000000e+00 9.0455008e-36 0.0000000e+00]\n",
+      " [1.0000000e+00 0.0000000e+00 0.0000000e+00]\n",
+      " [0.0000000e+00 1.8117375e-33 1.0000000e+00]\n",
+      " [0.0000000e+00 1.0000000e+00 6.7984806e-36]\n",
+      " [0.0000000e+00 1.7472901e-25 1.0000000e+00]\n",
+      " [0.0000000e+00 6.2991115e-37 1.0000000e+00]\n",
+      " [0.0000000e+00 0.0000000e+00 1.0000000e+00]\n",
+      " [0.0000000e+00 1.0598510e-30 1.0000000e+00]\n",
+      " [1.0000000e+00 1.7519910e-30 0.0000000e+00]\n",
+      " [0.0000000e+00 1.0000000e+00 0.0000000e+00]\n",
+      " [0.0000000e+00 1.0000000e+00 0.0000000e+00]\n",
+      " [1.0000000e+00 7.4562871e-27 0.0000000e+00]\n",
+      " [0.0000000e+00 0.0000000e+00 1.0000000e+00]\n",
+      " [0.0000000e+00 0.0000000e+00 1.0000000e+00]\n",
+      " [0.0000000e+00 0.0000000e+00 1.0000000e+00]\n",
+      " [0.0000000e+00 1.0000000e+00 0.0000000e+00]\n",
+      " [0.0000000e+00 1.0000000e+00 0.0000000e+00]\n",
+      " [1.0000000e+00 0.0000000e+00 0.0000000e+00]]\n",
+      "[[0. 1. 0.]\n",
+      " [0. 1. 0.]\n",
+      " [1. 0. 0.]\n",
+      " [1. 0. 0.]\n",
+      " [0. 1. 0.]\n",
+      " [0. 0. 1.]\n",
+      " [1. 0. 0.]\n",
+      " [0. 0. 1.]\n",
+      " [1. 0. 0.]\n",
+      " [0. 1. 0.]\n",
+      " [1. 0. 0.]\n",
+      " [0. 1. 0.]\n",
+      " [1. 0. 0.]\n",
+      " [1. 0. 0.]\n",
+      " [0. 0. 1.]\n",
+      " [0. 1. 0.]\n",
+      " [0. 0. 1.]\n",
+      " [0. 0. 1.]\n",
+      " [0. 0. 1.]\n",
+      " [0. 0. 1.]\n",
+      " [1. 0. 0.]\n",
+      " [0. 1. 0.]\n",
+      " [0. 1. 0.]\n",
+      " [1. 0. 0.]\n",
+      " [0. 0. 1.]\n",
+      " [0. 0. 1.]\n",
+      " [0. 0. 1.]\n",
+      " [0. 1. 0.]\n",
+      " [0. 1. 0.]\n",
+      " [1. 0. 0.]]\n",
+      "1/1 [==============================] - 0s 88ms/step - loss: 0.0000e+00 - accuracy: 1.0000\n",
+      "[0.0, 1.0]\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2023-06-02 14:34:49.851147: I tensorflow/stream_executor/cuda/cuda_blas.cc:1614] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n"
+     ]
+    }
+   ],
+   "source": [
+    "import numpy as np\n",
+    "import tensorflow as tf\n",
+    "from tensorflow import keras\n",
+    "from tensorflow.keras import layers\n",
+    "from tensorflow.keras.models import Sequential\n",
+    "\n",
+    "from sklearn.datasets import load_iris\n",
+    "from sklearn.model_selection import train_test_split\n",
+    "\n",
+    "def get_xor():\n",
+    "    x = np.array([[0,0],[0,1],[1,0],[1,1]])\n",
+    "    y = np.array([[0],[1],[1],[0]])\n",
+    "\n",
+    "    return x,y\n",
+    "\n",
+    "def get_iris():\n",
+    "    iris = load_iris()\n",
+    "    x = iris.data\n",
+    "    y = iris.target\n",
+    "\n",
+    "    y = keras.utils.to_categorical(y, 3)\n",
+    "\n",
+    "    x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, shuffle=True, stratify=y)\n",
+    "\n",
+    "    return x_train, x_test, y_train, y_test\n",
+    "\n",
+    "# model = keras.models.load_model(\"./result/xor/06-02-13-31/75_0.35_0.8_0.6.h5\")\n",
+    "model = keras.models.load_model(\"./result/iris/06-02-13-48/50_0.4_0.8_0.7.h5\")\n",
+    "# x,y = get_xor()\n",
+    "x_train, x_test, y_train, y_test = get_iris()\n",
+    "\n",
+    "print(model.predict(x_test))\n",
+    "print(y_test)\n",
+    "print(model.evaluate(x_test,y_test))"
+   ]
   }
  ],
  "metadata": {

xor.py

@@ -0,0 +1,59 @@
+# %%
+import os
+os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
+
+import tensorflow as tf
+tf.random.set_seed(777)  # for reproducibility
+
+# from pso_tf import PSO
+from pso import Optimizer
+from tensorflow import keras
+
+import numpy as np
+
+from tensorflow import keras
+from tensorflow.keras.models import Sequential
+from tensorflow.keras import layers
+
+from datetime import datetime
+
+print(tf.__version__)
+print(tf.config.list_physical_devices())
+
+def get_data():
+    x = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
+    y = np.array([[0], [1], [1], [0]])
+    return x, y
+
+def make_model():
+    leyer = []
+    leyer.append(layers.Dense(2, activation='sigmoid', input_shape=(2,)))
+    # leyer.append(layers.Dense(2, activation='sigmoid'))
+    leyer.append(layers.Dense(1, activation='sigmoid'))
+
+    model = Sequential(leyer)
+
+    return model
+
+# %%
+model = make_model()
+x_test, y_test = get_data()
+# loss = 'binary_crossentropy'
+# loss = 'categorical_crossentropy'
+# loss = 'sparse_categorical_crossentropy'
+# loss = 'kullback_leibler_divergence'
+# loss = 'poisson'
+# loss = 'cosine_similarity'
+# loss = 'log_cosh'
+# loss = 'huber_loss' 
+# loss = 'mean_absolute_error'
+# loss = 'mean_absolute_percentage_error'
+loss = 'mean_squared_error'
+
+pso_xor = Optimizer(model,
+    loss=loss, n_particles=75, c0=0.35, c1=0.8, w_min=0.6, w_max=1.2, negative_swarm=0.25)
+best_score = pso_xor.fit(
+    x_test, y_test, epochs=200, save=True, save_path="./result/xor", renewal="acc", empirical_balance=False, Dispersion=False, check_point=25)
+
+# %%
+