Notebooks/ActorCritic MountainCar LunarLander Continuous.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.2"
    },
    "colab": {
      "name": "Continuous ActorCritic MountainCar LunarLander.ipynb",
      "provenance": [],
      "toc_visible": true
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "kNMYiCgL649p"
      },
      "source": [
        "import tensorflow.python.keras.backend as K\n",
        "import tensorflow.compat.v1 as tf\n",
        "tf.disable_v2_behavior()\n",
        "\n",
        "from tensorflow.keras.layers import Dense, Input, Add\n",
        "from tensorflow.keras.models import  Model\n",
        "from tensorflow.keras.optimizers import Adam\n",
        "\n",
        "import random\n",
        "import gym\n",
        "from collections import deque\n",
        "import numpy as np\n",
        "\n",
        "import os\n",
        "import pickle\n",
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "from os import path\n",
        "!pip3 install box2d-py"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kZYWDJZC6492"
      },
      "source": [
        "if not path.isdir('test'):\n",
        "    os.mkdir('test/')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bYlThXfj649_"
      },
      "source": [
        "# DDPG"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9_-nDYs864-B"
      },
      "source": [
        "class Actor:\n",
        "    def __init__(self, env,sess, learning_rate, tau):\n",
        "        self.sess = sess\n",
        "        K.set_session(sess)\n",
        "        self.env = env\n",
        "\n",
        "        self.learning_rate = learning_rate\n",
        "        self.tau = tau\n",
        "\n",
        "        self.model, self.state_input = self.create_model(self.env)\n",
        "        self.target,_ = self.create_model(self.env)\n",
        "\n",
        "        self.target.set_weights(self.model.get_weights())\n",
        "\n",
        "        self.critic_grads = tf.placeholder(tf.float32,[None, self.env.action_space.shape[0]])\n",
        "\n",
        "        self.params_grads = tf.gradients(self.model.output, self.model.trainable_weights,-self.critic_grads)\n",
        "\n",
        "        self.grads = zip(self.params_grads, self.model.trainable_weights)\n",
        "\n",
        "        self.optimize = tf.train.AdamOptimizer(self.learning_rate).apply_gradients\\\n",
        "        (zip(self.params_grads, self.model.trainable_weights))\n",
        "        self.sess.run(tf.initialize_all_variables())\n",
        "\n",
        "    def create_model(self, env):\n",
        "        state_input = Input(shape=env.observation_space.shape)\n",
        "        h1 = Dense(32, activation='relu')(state_input)\n",
        "        h2 = Dense(256, activation='relu')(h1)\n",
        "        h3 = Dense(32, activation='relu')(h2)\n",
        "        output = Dense(env.action_space.shape[0], activation='tanh')(h3)\n",
        "\n",
        "        model = Model(state_input,output)\n",
        "        adam = Adam(lr=0.00001)\n",
        "        model.compile(loss=\"mse\", optimizer=adam)\n",
        "        return model, state_input\n",
        "\n",
        "    def predict_action(self, state):\n",
        "        action = self.model.predict(state)\n",
        "        return action\n",
        "\n",
        "    def predict_next_action(self, next_state):\n",
        "        next_action = self.target.predict(next_state)\n",
        "        return next_action\n",
        "\n",
        "    def train(self, batch, critic_grads):\n",
        "        current_states,_,_,_,_,_ = batch\n",
        "        self.sess.run(self.optimize, feed_dict={self.state_input: current_states,self.critic_grads: critic_grads})\n",
        "        grads = self.sess.run(self.params_grads, feed_dict={self.state_input: current_states,self.critic_grads: critic_grads})\n",
        "        return grads\n",
        "\n",
        "    def update_target(self):\n",
        "        actor_weights = self.model.get_weights()\n",
        "        target_weights = self.target.get_weights()\n",
        "        for i in range(len(actor_weights)):\n",
        "            target_weights[i] = self.tau * actor_weights[i] + (1 - self.tau) * target_weights[i]\n",
        "        self.target.set_weights(target_weights)\n",
        "\n",
        "    def save_model_architecture(self, file_name):\n",
        "        model_json = self.model.to_json()\n",
        "        with open(file_name, \"w\") as json_file:\n",
        "            json_file.write(model_json)\n",
        "\n",
        "    def save_weights(self, file_name):\n",
        "        self.model.save_weights(file_name)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DbX9ocOO64-H"
      },
      "source": [
        "class Critic:\n",
        "    def __init__(self,env,sess, learning_rate, tau, discount_factor):\n",
        "       self.sess = sess\n",
        "       K.set_session(sess)\n",
        "       self.env = env\n",
        "\n",
        "       self.gamma = discount_factor\n",
        "       self.tau = tau\n",
        "       self.learning_rate = learning_rate\n",
        "       self.model, self.state_input, self.action_input = self.create_model(self.env)\n",
        "\n",
        "       self.target, _,_ = self.create_model(self.env)\n",
        "       self.target.set_weights(self.model.get_weights())\n",
        "\n",
        "       self.gradients = tf.gradients(self.model.output, self.action_input)\n",
        "       self.sess.run(tf.initialize_all_variables())\n",
        "\n",
        "    def create_model(self,env):\n",
        "        state_input = Input(shape=env.observation_space.shape)\n",
        "        state_h1 = Dense(32, activation='relu')(state_input)\n",
        "        state_h2 = Dense(64)(state_h1)\n",
        "\n",
        "        action_input = Input(shape=env.action_space.shape)\n",
        "        action_h1 = Dense(64)(action_input)\n",
        "\n",
        "        merged = Add()([state_h2, action_h1])\n",
        "        m2 = Dense(256,activation='relu')(merged)\n",
        "        merged_h1 = Dense(32, activation='relu')(m2)\n",
        "        output = Dense(1, activation='linear')(merged_h1)\n",
        "        model = Model([state_input, action_input], output)\n",
        "\n",
        "        adam = Adam(lr=self.learning_rate)\n",
        "        model.compile(loss=\"mse\", optimizer=adam)\n",
        "        return model, state_input, action_input\n",
        "\n",
        "    def predict_qvalue(self,state,action):\n",
        "        qvalue = self.model.predict([state, action])\n",
        "        return qvalue\n",
        "\n",
        "    def predict_next_qvalue(self,next_state, next_action):\n",
        "        next_qvalue = self.target.predict([next_state, next_action])\n",
        "        return next_qvalue\n",
        "\n",
        "    def train(self, batch):\n",
        "        current_states, actions, rewards, next_states, next_actions, dones = batch\n",
        "\n",
        "        next_qvalues = self.predict_next_qvalue(next_states, next_actions)\n",
        "        next_qvalues[dones] = 0\n",
        "\n",
        "        td_targets = rewards + self.gamma*next_qvalues\n",
        "\n",
        "        self.model.fit([current_states,actions], td_targets, verbose=0)\n",
        "\n",
        "    def calc_grads(self,batch):\n",
        "        current_states,actions,_,_,_,_ = batch\n",
        "        grads = self.sess.run(self.gradients, feed_dict={self.state_input: current_states,self.action_input: actions})[0]\n",
        "        return grads\n",
        "\n",
        "    def update_target(self):\n",
        "        critic_weights = self.model.get_weights()\n",
        "        target_weights = self.target.get_weights()\n",
        "        for i in range(len(critic_weights)):\n",
        "            target_weights[i] = self.tau * critic_weights[i] + (1 - self.tau) * target_weights[i]\n",
        "        self.target.set_weights(target_weights)\n",
        "\n",
        "    def save_model_architecture(self, file_name):\n",
        "        model_json = self.model.to_json()\n",
        "        with open(file_name, \"w\") as json_file:\n",
        "            json_file.write(model_json)\n",
        "\n",
        "    def save_weights(self, file_name):\n",
        "        self.model.save_weights(file_name)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WHmXi3j964-N"
      },
      "source": [
        "class ReplayMemory:\n",
        "    def __init__(self, size):\n",
        "        self.memory = deque(maxlen=size)\n",
        "\n",
        "    def append(self,state, action, reward, next_state, next_action, done):\n",
        "        self.memory.append([state,action,reward,next_state, next_action, done])\n",
        "\n",
        "    def sample(self,batch_size):\n",
        "        batch = random.sample(self.memory,batch_size)\n",
        "        current_states = []\n",
        "        rewards = []\n",
        "        actions = []\n",
        "        next_states = []\n",
        "        next_actions = []\n",
        "        dones = []\n",
        "\n",
        "        for sample in batch:\n",
        "            state, action, reward, next_state, next_action, done = sample\n",
        "            current_states.append(state)\n",
        "            rewards.append(reward)\n",
        "            actions.append(action)\n",
        "            next_states.append(next_state)\n",
        "            next_actions.append(next_action)\n",
        "            dones.append(done)\n",
        "        current_states = np.array(current_states)\n",
        "        next_states = np.array(next_states)\n",
        "        actions = np.array(actions)\n",
        "        next_actions = np.array(next_actions)\n",
        "\n",
        "        rewards = np.array(rewards).reshape((batch_size,1))\n",
        "        dones = np.array(dones).reshape((batch_size,1))\n",
        "\n",
        "        return [current_states,actions, rewards, next_states, next_actions, dones]\n",
        "\n",
        "    def count(self):\n",
        "        return len(self.memory)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TrlOs-7964-S"
      },
      "source": [
        "class OUNoise:\n",
        "    def __init__(self, action_space, mu=0.0, theta=0.05, max_sigma=0.25, min_sigma=0.25, decay_period=100000):\n",
        "        self.mu = mu\n",
        "        self.theta = theta\n",
        "        self.sigma = max_sigma\n",
        "        self.max_sigma = max_sigma\n",
        "        self.min_sigma = min_sigma\n",
        "        self.decay_period = decay_period\n",
        "        self.action_dim = action_space.shape[0]\n",
        "        self.low = action_space.low\n",
        "        self.high = action_space.high\n",
        "        self.reset()\n",
        "\n",
        "    def reset(self):\n",
        "        self.state = np.ones(self.action_dim) * self.mu\n",
        "\n",
        "    def evolve_state(self):\n",
        "        x = self.state\n",
        "        dx = self.theta * (self.mu - x) + self.sigma * np.random.randn(self.action_dim)\n",
        "        self.state = x + dx\n",
        "        return self.state\n",
        "\n",
        "    def get_action(self, action, t=0):\n",
        "        ou_state = self.evolve_state()\n",
        "        self.sigma = self.max_sigma - (self.max_sigma - self.min_sigma) * max(1.0, t / self.decay_period)\n",
        "        return np.clip(action + ou_state, self.low, self.high)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9oSL6yVc64-Y"
      },
      "source": [
        "class GaussNoise:\n",
        "    def __init__(self,mu=0,std=0.25,sigma=0.25):\n",
        "        self.mu = mu\n",
        "        self.std = std\n",
        "        self.reset()\n",
        "        self.sigma=sigma\n",
        "    def reset(self):\n",
        "        self.state = self.mu\n",
        "    \n",
        "    def noise(self):\n",
        "        return np.random.normal(self.mu,self.std)\n",
        "    \n",
        "    def get_action(self,action,t=0):\n",
        "        return np.clip(action+self.sigma*self.noise(),-1,1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "msxHvL7464-f"
      },
      "source": [
        "\n",
        "def main(env,noise=0):\n",
        "    sess = tf.Session()\n",
        "    K.set_session(sess)\n",
        "    memory_size = 100000\n",
        "    batch_size = 64\n",
        "    tau = 0.001\n",
        "    lr_actor = 0.00001\n",
        "    lr_critic = 0.0001\n",
        "    discount_factor = 0.99\n",
        "    episodes = 3000\n",
        "    time_steps = 1000\n",
        "    collect_experience = 50000\n",
        "    save_frequency = 250\n",
        "    ep_reward = []\n",
        "    training = False\n",
        "    name='LLC'\n",
        "    size = env.observation_space.shape[0]\n",
        "    if noise==0:\n",
        "        noise = OUNoise(env.action_space)\n",
        "    else:\n",
        "        noise = GaussNoise()\n",
        "    actor = Actor(env,sess, lr_actor, tau)\n",
        "\n",
        "    critic = Critic(env,sess, lr_critic, tau, discount_factor)\n",
        "    if env.observation_space.shape[0]==2:\n",
        "        win_score=90\n",
        "        episodes = 1500\n",
        "        lr_actor = 0.0001\n",
        "        lr_critic = 0.001\n",
        "        batch_size=32\n",
        "        name='MCC'\n",
        "    elif env.observation_space.shape[0]==8:\n",
        "        win_score=200\n",
        "    replay_memory = ReplayMemory(memory_size)\n",
        "    steps_taken=[]\n",
        "    avgrewards=[]\n",
        "    for episode in range (episodes):\n",
        "        state = env.reset()\n",
        "        noise.reset()\n",
        "        episode_reward = 0\n",
        "        done = False\n",
        "        for time in range(1000):\n",
        "            action = actor.predict_action(state.reshape((1,size)))[0]\n",
        "            exploratory_action = noise.get_action(action,time)\n",
        "            next_state ,reward, done, _ = env.step(exploratory_action)\n",
        "            episode_reward+=reward\n",
        "            next_action = actor.predict_next_action(next_state.reshape((1,size)))[0]\n",
        "            replay_memory.append(state,exploratory_action,reward,next_state, next_action,done)\n",
        "            if episode==batch_size:\n",
        "                training = True\n",
        "            if training:\n",
        "                batch = replay_memory.sample(batch_size)\n",
        "                grads = critic.calc_grads(batch)\n",
        "                t_grads = actor.train(batch,grads)\n",
        "                critic.train(batch)\n",
        "                actor.update_target()\n",
        "                critic.update_target()\n",
        "            state = next_state\n",
        "            if done and episode_reward>90:\n",
        "                if episode > 2*batch_size:\n",
        "                    actor.save_weights('test/DDPG_actor_model_{}-win-{}-{}-noise-{}.h5'.format(episode,time_steps,name,noise))\n",
        "                break\n",
        "            steps_taken.append(time)\n",
        "        ep_reward.append(episode_reward)\n",
        "        avgrewards.append(np.mean(ep_reward[-10:]))\n",
        "        if done and np.mean(ep_reward[-100:])>win_score and episode>128:\n",
        "            actor.save_weights('test/DDPG_actor_model_{}-win-{}-{}-noise-{}.h5'.format(episode,time_steps,name,noise))\n",
        "            print('That was a good enough model')\n",
        "            print(np.mean(ep_reward[-10:]), ep_reward[-10:])\n",
        "            break\n",
        "        print(\"Episode: {},\\t steps {},\\t episode reward: {:.2f} \\t Running avg(100) {:.2f}\"\n",
        "              .format( episode,time, episode_reward,np.mean(ep_reward[-100:])))\n",
        "\n",
        "        if training and (episode % save_frequency) == 0:\n",
        "            print('Data saved at epsisode:', episode)\n",
        "            actor.save_weights('test/DDPG_actor_model_{}-win-{}-{}-noise-{}.h5'.format(episode,time_steps,name,noise))\n",
        "            \n",
        "    env.close()\n",
        "    plt.plot(ep_reward,label='rewards')\n",
        "    plt.xlabel('Episode')\n",
        "    plt.ylabel('Reward')\n",
        "    plt.show()\n",
        "    plt.plot(avgrewards,label='rewards')\n",
        "    plt.xlabel('Episode')\n",
        "    plt.ylabel('Reward')\n",
        "    plt.show()\n",
        "    return actor"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KaKXdmh464-6"
      },
      "source": [
        "# Lunar Lander Continuous"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y8ucbKfC64-7"
      },
      "source": [
        "env = gym.make('LunarLanderContinuous-v2')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KNFLdFSW64_A"
      },
      "source": [
        "actorllc_ou = main(env,0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kNFw8sRF64_J"
      },
      "source": [
        "actorllc_ou = main(env,1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "swMgWUgH64-k"
      },
      "source": [
        "# Mountain Car Continuous"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "plh2kfYd64-m"
      },
      "source": [
        "env = gym.make('MountainCarContinuous-v0')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E2QpmchW64-s"
      },
      "source": [
        "actor_ou=main(env,0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LukPy12E64-z"
      },
      "source": [
        "actor_gauss=main(env,1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Test"
      ],
      "metadata": {
        "id": "ztmhWzNB961u"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h3--POWG64_Z"
      },
      "source": [
        "def test(model,episodes,steps,size):\n",
        "    wins=0\n",
        "    rewards=[]\n",
        "    testenv = gym.make('MountainCarContinuous-v0')\n",
        "    for ep in range(episodes):\n",
        "        total=0\n",
        "        state=testenv.reset()\n",
        "        for step in range(steps):\n",
        "            action = model.predict(state.reshape((1,size)))[0]\n",
        "            new_state, reward, done,_=testenv.step(action)\n",
        "            \n",
        "            if ep%10==0:\n",
        "                env.render()\n",
        "            \n",
        "            state=new_state\n",
        "            \n",
        "            total+=reward\n",
        "            if done:\n",
        "                wins+=1\n",
        "                print(f'Won in episode {ep} Steps {step} Reward{total}')\n",
        "        print(f'Episode {ep} Reward {total}')\n",
        "        rewards.append(reward)\n",
        "    print(f'Win {wins} percent {(wins/episodes)*100} mean rew {np.mean(rewards)}')\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}