pushing model

README.md

@@ -16,7 +16,7 @@ model-index:
       type: CartPole-v1
     metrics:
     - type: mean_reward
-      value: 162.50 +/- 11.47
+      value: 500.00 +/- 0.00
       name: mean_reward
       verified: false
 ---
@@ -46,32 +46,33 @@ curl -OL https://huggingface.co/pfunk/CartPole-v1-DQN_baseline-seed1/raw/main/dq
 curl -OL https://huggingface.co/pfunk/CartPole-v1-DQN_baseline-seed1/raw/main/pyproject.toml
 curl -OL https://huggingface.co/pfunk/CartPole-v1-DQN_baseline-seed1/raw/main/poetry.lock
 poetry install --all-extras
-python dqn.py --exp-name DQN_baseline --track --wandb-entity pfunk --wandb-project-name dqpn --save-model true --upload-model true --hf-entity pfunk --env-id CartPole-v1 --seed 1 --total-timesteps 100000
+python dqn.py --exp-name DQN_baseline --seed 1 --track --wandb-entity pfunk --wandb-project-name dqpn --capture-video true --save-model true --upload-model true --hf-entity pfunk
 ```
 
 # Hyperparameters
 ```python
-{'batch_size': 128,
- 'buffer_size': 10000,
- 'capture_video': False,
+{'alg_type': 'dqn.py',
+ 'batch_size': 256,
+ 'buffer_size': 300000,
+ 'capture_video': True,
  'cuda': True,
- 'end_e': 0.05,
+ 'end_e': 0.1,
  'env_id': 'CartPole-v1',
  'exp_name': 'DQN_baseline',
- 'exploration_fraction': 0.5,
- 'gamma': 0.99,
+ 'exploration_fraction': 0.2,
+ 'gamma': 1.0,
  'hf_entity': 'pfunk',
- 'learning_rate': 0.00025,
- 'learning_starts': 10000,
+ 'learning_rate': 0.0001,
+ 'learning_starts': 1000,
  'save_model': True,
  'seed': 1,
- 'start_e': 1,
- 'target_network_frequency': 500,
- 'tau': 1.0,
+ 'start_e': 1.0,
+ 'target_network_frequency': 100,
+ 'target_tau': 1.0,
  'torch_deterministic': True,
- 'total_timesteps': 100000,
+ 'total_timesteps': 500000,
  'track': True,
- 'train_frequency': 10,
+ 'train_frequency': 1,
  'upload_model': True,
  'wandb_entity': 'pfunk',
  'wandb_project_name': 'dqpn'}

dqn.py

@@ -46,27 +46,27 @@ def parse_args():
         help="the id of the environment")
     parser.add_argument("--total-timesteps", type=int, default=500000,
         help="total timesteps of the experiments")
-    parser.add_argument("--learning-rate", type=float, default=2.5e-4,
+    parser.add_argument("--learning-rate", type=float, default=0.0001,
         help="the learning rate of the optimizer")
-    parser.add_argument("--buffer-size", type=int, default=10000,
+    parser.add_argument("--buffer-size", type=int, default=300000,
         help="the replay memory buffer size")
-    parser.add_argument("--gamma", type=float, default=0.99,
+    parser.add_argument("--gamma", type=float, default=1.0,
         help="the discount factor gamma")
-    parser.add_argument("--tau", type=float, default=1.,
+    parser.add_argument("--target-tau", type=float, default=1.0,
         help="the target network update rate")
-    parser.add_argument("--target-network-frequency", type=int, default=500,
+    parser.add_argument("--target-network-frequency", type=int, default=100,
         help="the timesteps it takes to update the target network")
-    parser.add_argument("--batch-size", type=int, default=128,
+    parser.add_argument("--batch-size", type=int, default=256,
         help="the batch size of sample from the reply memory")
-    parser.add_argument("--start-e", type=float, default=1,
+    parser.add_argument("--start-e", type=float, default=1.0,
         help="the starting epsilon for exploration")
-    parser.add_argument("--end-e", type=float, default=0.05,
+    parser.add_argument("--end-e", type=float, default=0.1,
         help="the ending epsilon for exploration")
-    parser.add_argument("--exploration-fraction", type=float, default=0.5,
+    parser.add_argument("--exploration-fraction", type=float, default=0.2,
         help="the fraction of `total-timesteps` it takes from start-e to go end-e")
-    parser.add_argument("--learning-starts", type=int, default=10000,
+    parser.add_argument("--learning-starts", type=int, default=1000,
         help="timestep to start learning")
-    parser.add_argument("--train-frequency", type=int, default=10,
+    parser.add_argument("--train-frequency", type=int, default=1,
         help="the frequency of training")
     args = parser.parse_args()
     # fmt: on
@@ -93,11 +93,11 @@ class QNetwork(nn.Module):
     def __init__(self, env):
         super().__init__()
         self.network = nn.Sequential(
-            nn.Linear(np.array(env.single_observation_space.shape).prod(), 120),
+            nn.Linear(np.array(env.single_observation_space.shape).prod(), 512),
             nn.ReLU(),
-            nn.Linear(120, 84),
+            nn.Linear(512, 128),
             nn.ReLU(),
-            nn.Linear(84, env.single_action_space.n),
+            nn.Linear(128, env.single_action_space.n),
         )
 
     def forward(self, x):
@@ -115,14 +115,16 @@ if __name__ == "__main__":
     if args.track:
         import wandb
 
-        wandb.init(
+        args.alg_type = os.path.basename(__file__)
+        wandb_sess = wandb.init(
             project=args.wandb_project_name,
             entity=args.wandb_entity,
-            sync_tensorboard=True,
             config=vars(args),
+            save_code=True,
+            # group='string',
             name=run_name,
+            sync_tensorboard=False,
             monitor_gym=True,
-            save_code=True,
         )
     writer = SummaryWriter(f"runs/{run_name}")
     writer.add_text(
@@ -130,6 +132,10 @@ if __name__ == "__main__":
         "|param|value|\n|-|-|\n%s" % ("\n".join([f"|{key}|{value}|" for key, value in vars(args).items()])),
     )
 
+    def log_value(name: str, x: float, y: int):
+        # writer.add_scalar(name, x, y)
+        wandb.log({name: x, "global_step": y})
+
     # TRY NOT TO MODIFY: seeding
     random.seed(args.seed)
     np.random.seed(args.seed)
@@ -141,9 +147,10 @@ if __name__ == "__main__":
     # env setup
     envs = gym.vector.SyncVectorEnv([make_env(args.env_id, args.seed, 0, args.capture_video, run_name)])
     assert isinstance(envs.single_action_space, gym.spaces.Discrete), "only discrete action space is supported"
+    envs.seed(args.seed)
 
     q_network = QNetwork(envs).to(device)
-    optimizer = optim.Adam(q_network.parameters(), lr=args.learning_rate)
+    optimizer = optim.RMSprop(q_network.parameters(), lr=args.learning_rate)
     target_network = QNetwork(envs).to(device)
     target_network.load_state_dict(q_network.state_dict())
 
@@ -152,15 +159,19 @@ if __name__ == "__main__":
         envs.single_observation_space,
         envs.single_action_space,
         device,
+        optimize_memory_usage=True,
         handle_timeout_termination=True,
     )
     start_time = time.time()
+    policy_update_counter = 0
+    episode_returns = []
 
     # TRY NOT TO MODIFY: start the game
     obs = envs.reset()
     for global_step in range(args.total_timesteps):
         # ALGO LOGIC: put action logic here
         epsilon = linear_schedule(args.start_e, args.end_e, args.exploration_fraction * args.total_timesteps, global_step)
+
         if random.random() < epsilon:
             actions = np.array([envs.single_action_space.sample() for _ in range(envs.num_envs)])
         else:
@@ -173,10 +184,14 @@ if __name__ == "__main__":
         # TRY NOT TO MODIFY: record rewards for plotting purposes
         for info in infos:
             if "episode" in info.keys():
-                print(f"global_step={global_step}, episodic_return={info['episode']['r']}")
-                writer.add_scalar("charts/episodic_return", info["episode"]["r"], global_step)
-                writer.add_scalar("charts/episodic_length", info["episode"]["l"], global_step)
-                writer.add_scalar("charts/epsilon", epsilon, global_step)
+                episode_returns.append(info['episode']['r'])
+                episode_returns = episode_returns[-100:]
+                print(f"step={global_step}, return={info['episode']['r']}, sps={int(global_step / (time.time() - start_time))}")
+                log_value("perf/episodic_return", info["episode"]["r"], global_step)
+                log_value("perf/episodic_return_mean_100", np.mean(episode_returns), global_step)
+                log_value("perf/episodic_return_std_100", np.std(episode_returns), global_step)
+                log_value("debug/episodic_length", info["episode"]["l"], global_step)
+                log_value("ex2/epsilon", epsilon, global_step)
                 break
 
         # TRY NOT TO MODIFY: save data to reply buffer; handle `terminal_observation`
@@ -200,10 +215,43 @@ if __name__ == "__main__":
                 loss = F.mse_loss(td_target, old_val)
 
                 if global_step % 100 == 0:
-                    writer.add_scalar("losses/td_loss", loss, global_step)
-                    writer.add_scalar("losses/q_values", old_val.mean().item(), global_step)
-                    print("SPS:", int(global_step / (time.time() - start_time)))
-                    writer.add_scalar("charts/SPS", int(global_step / (time.time() - start_time)), global_step)
+
+                    prev = old_val.detach().cpu().numpy()
+                    new = td_target.detach().cpu().numpy()
+                    diff, a_diff = new-prev, np.abs(new-prev)
+
+                    mean, a_mean = np.mean(diff), np.mean(a_diff)
+                    median, a_median = np.median(diff), np.median(a_diff)
+                    maximum, a_maximum = np.max(diff), np.max(a_diff)
+                    minimum, a_minimum = np.min(diff), np.min(a_diff)
+                    std, a_std = np.std(diff), np.std(a_diff)
+                    below, a_below = mean - std, a_mean - a_std
+                    above, a_above = mean + std, a_mean + a_std
+                    pu_scalar, a_pu_scalar = 2 * mean / maximum, 2 * a_mean / a_maximum
+                    policy_frequency_scalar_ratio = 1.0 * pu_scalar
+                    a_policy_frequency_scalar_ratio = 1.0 * a_pu_scalar
+
+                    log_value("losses/td_loss", loss, global_step)
+                    log_value("losses/q_values", old_val.mean().item(), global_step)
+                    log_value("td/mean", mean, global_step)
+                    log_value("td/a_mean", a_mean, global_step)
+                    log_value("td/median", median, global_step)
+                    log_value("td/a_median", a_median, global_step)
+                    log_value("td/max", maximum, global_step)
+                    log_value("td/a_max", a_maximum, global_step)
+                    log_value("td/min", minimum, global_step)
+                    log_value("td/a_min", a_minimum, global_step)
+                    log_value("td/std", std, global_step)
+                    log_value("td/a_std", a_std, global_step)
+                    log_value("td/below", below, global_step)
+                    log_value("td/a_below", a_below, global_step)
+                    log_value("td/above", above, global_step)
+                    log_value("td/a_above", a_above, global_step)
+                    log_value("pu/pu_scalar", pu_scalar, global_step)
+                    log_value("pu/a_pu_scalar", a_pu_scalar, global_step)
+                    log_value("pu/policy_frequency_scalar_ratio", policy_frequency_scalar_ratio, global_step)
+                    log_value("pu/a_policy_frequency_scalar_ratio", a_policy_frequency_scalar_ratio, global_step)
+                    log_value("debug/steps_per_second", int(global_step / (time.time() - start_time)), global_step)
 
                 # optimize the model
                 optimizer.zero_grad()
@@ -214,8 +262,11 @@ if __name__ == "__main__":
             if global_step % args.target_network_frequency == 0:
                 for target_network_param, q_network_param in zip(target_network.parameters(), q_network.parameters()):
                     target_network_param.data.copy_(
-                        args.tau * q_network_param.data + (1.0 - args.tau) * target_network_param.data
+                        args.target_tau * q_network_param.data + (1.0 - args.target_tau) * target_network_param.data
                     )
+            policy_update_counter += 1
+            if global_step % 100 == 0:
+                log_value("pu/n_policy_update", policy_update_counter, global_step)
 
     if args.save_model:
         model_path = f"runs/{run_name}/{args.exp_name}.cleanrl_model"
@@ -234,14 +285,15 @@ if __name__ == "__main__":
             epsilon=0.05,
         )
         for idx, episodic_return in enumerate(episodic_returns):
-            writer.add_scalar("eval/episodic_return", episodic_return, idx)
+            log_value("eval/episodic_return", episodic_return, idx)
 
         if args.upload_model:
             from cleanrl_utils.huggingface import push_to_hub
 
             repo_name = f"{args.env_id}-{args.exp_name}-seed{args.seed}"
             repo_id = f"{args.hf_entity}/{repo_name}" if args.hf_entity else repo_name
-            push_to_hub(args, episodic_returns, repo_id, "DQN", f"runs/{run_name}", f"videos/{run_name}-eval")
+            push_to_hub(args, np.mean(episode_returns), repo_id, "DQN", f"runs/{run_name}", f"videos/{run_name}-eval")
 
+    wandb_sess.finish()
     envs.close()
     writer.close()

events.out.tfevents.1676244619.wycliffeduncan-Victus-by-HP-Gaming-Laptop-15-fa0xxx.28826.0 → events.out.tfevents.1678647860.portal.3094185.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79e40fe681630b40511392b298c0cb7fe7110ec107bc9cab0ff04484f9cc28a3
-size 365881
+oid sha256:9423c8373dbc5b7a9a951d2520c14b332177a31609844d931aa8865fe2761480
+size 634