Update configs. Fix benchmark script

Author: amacati

README.md

@@ -146,7 +146,7 @@ The config folder contains settings for progressively harder scenarios:
 |                               |             |                           |                             |                        |                       |
 |           sim2real            |   **Yes**   |    Real-life hardware     |           **Yes**           |          *No*          |   Sim2real transfer   |
 
-> **Note:** "Rand. Between Episodes" (governed by argument `reseed_on_reset`) states whether randomized properties and positions vary or are kept constant (by re-seeding the random number generator on each `env.reset()`) across episodes
+> **Note:** "Rand. Between Episodes" (governed by argument `random_resets`) states whether randomized properties and positions vary or are kept constant (by re-seeding the random number generator on each `env.reset()`) across episodes
 
 ### Switching between configurations
 You can choose which configuration to use by changing the `--config` command line option. To e.g. run the example controller on the hardest scenario, you can use the following command

benchmarks/config/test.toml

@@ -1,23 +1,14 @@
 [sim]
-physics = "pyb"
+physics = "analytical"
 camera_view = [5.0, -40.0, -40.0, 0.5, -1.0, 0.5]
 sim_freq = 500                                    # Simulation frequency, in Hz
-ctrl_freq = 500                                   # Controller frequency, in Hz. This frequency is used to simulate the onboard controller, NOT for the environment's step function
+attitude_freq = 500                               # Controller frequency, in Hz. This frequency is used to simulate the onboard controller, NOT for the environment's step function
 gui = false                                       # Enable/disable PyBullet's GUI
 
-[sim.disturbances.action]
-type = "GaussianNoise"
-std = 0.001
-
-[sim.disturbances.dynamics]
-type = "UniformNoise"
-low = [-0.1, -0.1, -0.1]
-high = [0.1, 0.1, 0.1]
-
 [env]
-reseed = false   # Whether to re-seed the random number generator between episodes
+random_resets = false   # Whether to re-seed the random number generator between episodes
 seed = 1337      # Random seed
-freq = 60        # Frequency of the environment's step function, in Hz
+freq = 50        # Frequency of the environment's step function, in Hz
 symbolic = false # Whether to include symbolic expressions in the info dict. Note: This can interfere with multiprocessing! If you want to parallelize your training, set this to false.
 
 [env.track]
@@ -50,7 +41,17 @@ pos = [0.0, 1.0, 1.05]
 pos = [1.0, 1.0, 0.05]
 rpy = [0, 0, 0]
 vel = [0, 0, 0]
-ang_vel = [0, 0, 0]
+rpy_rates = [0, 0, 0]
+
+[env.disturbances.action]
+fn = "normal"
+scale = 0.001
+
+[env.disturbances.dynamics]
+fn = "uniform"
+[env.disturbances.dynamics.kwargs]
+minval = [-0.1, -0.1, -0.1]
+maxval = [0.1, 0.1, 0.1]
 
 [env.randomization.drone_pos]
 type = "uniform" # Everything that can be used as a distribution in numpy.random

benchmarks/main.py

@@ -19,15 +19,9 @@ def print_benchmark_results(name: str, timings: list[float]):
     print_benchmark_results(name="Sim reset", timings=timings)
     timings = time_sim_step(n_tests=n_tests, sim_steps=sim_steps)
     print_benchmark_results(name="Sim steps", timings=timings / sim_steps)
-    timings = time_sim_step(n_tests=n_tests, sim_steps=sim_steps, physics_mode="dyn")
-    print_benchmark_results(name="Sim steps (dyn backend)", timings=timings / sim_steps)
-    timings = time_sim_step(n_tests=n_tests, sim_steps=sim_steps, physics_mode="pyb_gnd")
-    print_benchmark_results(name="Sim steps (pyb_gnd backend)", timings=timings / sim_steps)
-    timings = time_sim_step(n_tests=n_tests, sim_steps=sim_steps, physics_mode="pyb_drag")
-    print_benchmark_results(name="Sim steps (pyb_drag backend)", timings=timings / sim_steps)
-    timings = time_sim_step(n_tests=n_tests, sim_steps=sim_steps, physics_mode="pyb_dw")
-    print_benchmark_results(name="Sim steps (pyb_dw backend)", timings=timings / sim_steps)
-    timings = time_sim_step(n_tests=n_tests, sim_steps=sim_steps, physics_mode="pyb_gnd_drag_dw")
-    print_benchmark_results(name="Sim steps (pyb_gnd_drag_dw backend)", timings=timings / sim_steps)
+    timings = time_sim_step(n_tests=n_tests, sim_steps=sim_steps, physics_mode="sys_id")
+    print_benchmark_results(name="Sim steps (sys_id backend)", timings=timings / sim_steps)
+    timings = time_sim_step(n_tests=n_tests, sim_steps=sim_steps, physics_mode="mujoco")
+    print_benchmark_results(name="Sim steps (mujoco backend)", timings=timings / sim_steps)
     timings = time_sim_attitude_step(n_tests=n_tests, sim_steps=sim_steps)
     print_benchmark_results(name="Sim steps (sys_id backend)", timings=timings / sim_steps)

benchmarks/sim.py

@@ -23,14 +23,19 @@
 import lsy_drone_racing
 
 env = gymnasium.make('DroneRacing-v0', config=config)
-
+env.reset()
+env.step(env.action_space.sample())  # JIT compile
+env.reset()
 """
 attitude_env_setup_code = """
 import gymnasium
 
 import lsy_drone_racing
 
 env = gymnasium.make('DroneRacingThrust-v0', config=config)
+env.reset()
+env.step(env.action_space.sample())  # JIT compile
+env.reset()
 """
 
 
@@ -41,7 +46,7 @@ def time_sim_reset(n_tests: int = 10) -> NDArray[np.floating]:
 
 
 def time_sim_step(
-    n_tests: int = 10, sim_steps: int = 100, physics_mode: str = "pyb"
+    n_tests: int = 10, sim_steps: int = 100, physics_mode: str = "analytical"
 ) -> NDArray[np.floating]:
     modify_config_code = f"""config.sim.physics = '{physics_mode}'\n"""
     setup = load_config_code + modify_config_code + env_setup_code + "\nenv.reset()"

config/level0.toml

@@ -38,7 +38,7 @@ high = [0.1, 0.1, 0.1]
 
 [env]
 id = "DroneRacing-v0" # Either "DroneRacing-v0" or "DroneRacingThrust-v0". If using "DroneRacingThrust-v0", the drone will use the thrust controller instead of the position controller.
-reseed = true         # Whether to re-seed the random number generator between episodes
+random_resets = true         # Whether to re-seed the random number generator between episodes
 seed = 1337           # Random seed
 freq = 50             # Frequency of the environment's step function, in Hz
 symbolic = false      # Whether to include symbolic expressions in the info dict. Note: This can interfere with multiprocessing! If you want to parallelize your training, set this to false.

config/level1.toml

@@ -16,33 +16,18 @@ check_drone_start_pos = true
 practice_without_track_objects = false
 
 [sim]
-# Physics options:
-# "pyb": PyBullet
-# "dyn": Mathematical dynamics model 
-# "pyb_gnd" PyBullet with ground effect
-# "pyb_drag": PyBullet with drag
-# "pyb_dw": PyBullet with downwash
-# "pyb_gnd_drag_dw": PyBullet with ground effect, drag, and downwash.
-physics = "pyb"
+# Physics options: analytical, sys_id, mujoco
+physics = "analytical"
 camera_view = [5.0, -40.0, -40.0, 0.5, -1.0, 0.5]
 sim_freq = 500                                    # Simulation frequency, in Hz
-ctrl_freq = 500                                   # Controller frequency, in Hz. This frequency is used to simulate the onboard controller, NOT for the environment's step function
+attitude_freq = 500                               # Controller frequency, in Hz. This frequency is used to simulate the onboard controller, NOT for the environment's step function
 gui = false                                       # Enable/disable PyBullet's GUI
 
-[sim.disturbances.action]
-type = "GaussianNoise"
-std = 0.001
-
-[sim.disturbances.dynamics]
-type = "UniformNoise"
-low = [-0.1, -0.1, -0.1]
-high = [0.1, 0.1, 0.1]
-
 [env]
 id = "DroneRacing-v0" # Either "DroneRacing-v0" or "DroneRacingThrust-v0". If using "DroneRacingThrust-v0", the drone will use the thrust controller instead of the position controller.
-reseed = true         # Whether to re-seed the random number generator between episodes
+random_resets = true         # Whether to re-seed the random number generator between episodes
 seed = 1337           # Random seed
-freq = 60             # Frequency of the environment's step function, in Hz
+freq = 50             # Frequency of the environment's step function, in Hz
 symbolic = false      # Whether to include symbolic expressions in the info dict. Note: This can interfere with multiprocessing! If you want to parallelize your training, set this to false.
 sensor_range = 0.45   # Range at which the exact location of gates and obstacles become visible to the drone. Objects that are not in the drone's sensor range report their nominal position.
 
@@ -76,25 +61,38 @@ pos = [0.0, 1.0, 1.4]
 pos = [1.0, 1.0, 0.05]
 rpy = [0, 0, 0]
 vel = [0, 0, 0]
-ang_vel = [0, 0, 0]
+rpy_rates = [0, 0, 0]
+
+[env.disturbances.action]
+fn = "normal"
+scale = 0.001
+
+[env.disturbances.dynamics]
+fn = "uniform"
+[env.disturbances.dynamics.kwargs]
+minval = [-0.1, -0.1, -0.1]
+maxval = [0.1, 0.1, 0.1]
 
 [env.randomization.drone_pos]
-type = "uniform" # Everything that can be used as a distribution in numpy.random
-# Kwargs that are permissable in the np random function
-low = [-0.1, -0.1, 0.0]
-high = [0.1, 0.1, 0.02]
+fn = "uniform"
+[env.randomization.drone_pos.kwargs]
+minval = [-0.1, -0.1, 0.0]
+maxval = [0.1, 0.1, 0.02]
 
 [env.randomization.drone_rpy]
-type = "uniform"
-low = [-0.1, -0.1, -0.1]
-high = [0.1, 0.1, 0.1]
+fn = "uniform"
+[env.randomization.drone_rpy.kwargs]
+minval = [-0.1, -0.1, -0.1]
+maxval = [0.1, 0.1, 0.1]
 
 [env.randomization.drone_mass]
-type = "uniform"
-low = -0.01
-high = 0.01
+fn = "uniform"
+[env.randomization.drone_mass.kwargs]
+minval = -0.01
+maxval = 0.01
 
 [env.randomization.drone_inertia]
-type = "uniform"
-low = [-0.000001, -0.000001, -0.000001]
-high = [0.000001, 0.000001, 0.000001]
+fn = "uniform"
+[env.randomization.drone_inertia.kwargs]
+minval = [-0.000001, -0.000001, -0.000001]
+maxval = [0.000001, 0.000001, 0.000001]

config/level2.toml

@@ -16,33 +16,18 @@ check_drone_start_pos = true
 practice_without_track_objects = false
 
 [sim]
-# Physics options:
-# "pyb": PyBullet
-# "dyn": Mathematical dynamics model 
-# "pyb_gnd" PyBullet with ground effect
-# "pyb_drag": PyBullet with drag
-# "pyb_dw": PyBullet with downwash
-# "pyb_gnd_drag_dw": PyBullet with ground effect, drag, and downwash.
-physics = "pyb"
+# Physics options: analytical, sys_id, mujoco
+physics = "analytical"
 camera_view = [5.0, -40.0, -40.0, 0.5, -1.0, 0.5]
 sim_freq = 500                                    # Simulation frequency, in Hz
-ctrl_freq = 500                                   # Controller frequency, in Hz. This frequency is used to simulate the onboard controller, NOT for the environment's step function
+attitude_freq = 500                               # Controller frequency, in Hz. This frequency is used to simulate the onboard controller, NOT for the environment's step function
 gui = false                                       # Enable/disable PyBullet's GUI
 
-[sim.disturbances.action]
-type = "GaussianNoise"
-std = 0.001
-
-[sim.disturbances.dynamics]
-type = "UniformNoise"
-low = [-0.1, -0.1, -0.1]
-high = [0.1, 0.1, 0.1]
-
 [env]
 id = "DroneRacing-v0" # Either "DroneRacing-v0" or "DroneRacingThrust-v0". If using "DroneRacingThrust-v0", the drone will use the thrust controller instead of the position controller.
-reseed = true         # Whether to re-seed the random number generator between episodes
+random_resets = true         # Whether to re-seed the random number generator between episodes
 seed = 1337           # Random seed
-freq = 60             # Frequency of the environment's step function, in Hz
+freq = 50             # Frequency of the environment's step function, in Hz
 symbolic = false      # Whether to include symbolic expressions in the info dict. Note: This can interfere with multiprocessing! If you want to parallelize your training, set this to false.
 sensor_range = 0.45   # Range at which the exact location of gates and obstacles become visible to the drone. Objects that are not in the drone's sensor range report their nominal position.
 
@@ -76,40 +61,56 @@ pos = [0.0, 1.0, 1.4]
 pos = [1.0, 1.0, 0.05]
 rpy = [0, 0, 0]
 vel = [0, 0, 0]
-ang_vel = [0, 0, 0]
+rpy_rates = [0, 0, 0]
+
+[env.disturbances.action]
+fn = "normal"
+scale = 0.001
+
+[env.disturbances.dynamics]
+fn = "uniform"
+[env.disturbances.dynamics.kwargs]
+minval = [-0.1, -0.1, -0.1]
+maxval = [0.1, 0.1, 0.1]
 
 [env.randomization.drone_pos]
-type = "uniform" # Everything that can be used as a distribution in numpy.random
-# Kwargs that are permissable in the np random function
-low = [-0.1, -0.1, 0.0]
-high = [0.1, 0.1, 0.02]
+fn = "uniform"
+[env.randomization.drone_pos.kwargs]
+minval = [-0.1, -0.1, 0.0]
+maxval = [0.1, 0.1, 0.02]
 
 [env.randomization.drone_rpy]
-type = "uniform"
-low = [-0.1, -0.1, -0.1]
-high = [0.1, 0.1, 0.1]
+fn = "uniform"
+[env.randomization.drone_rpy.kwargs]
+minval = [-0.1, -0.1, -0.1]
+maxval = [0.1, 0.1, 0.1]
 
 [env.randomization.drone_mass]
-type = "uniform"
-low = -0.01
-high = 0.01
+fn = "uniform"
+[env.randomization.drone_mass.kwargs]
+minval = -0.01
+maxval = 0.01
 
 [env.randomization.drone_inertia]
-type = "uniform"
-low = [-0.000001, -0.000001, -0.000001]
-high = [0.000001, 0.000001, 0.000001]
+fn = "uniform"
+[env.randomization.drone_inertia.kwargs]
+minval = [-0.000001, -0.000001, -0.000001]
+maxval = [0.000001, 0.000001, 0.000001]
 
 [env.randomization.gate_pos]
-type = "uniform"
-low = [-0.15, -0.15, 0.0]
-high = [0.15, 0.15, 0.0]
+fn = "uniform"
+[env.randomization.gate_pos.kwargs]
+minval = [-0.15, -0.15, -0.1]
+maxval = [0.15, 0.15, 0.1]
 
 [env.randomization.gate_rpy]
-type = "uniform"
-low = [0.0, 0.0, -0.1]
-high = [0.0, 0.0, 0.1]
+fn = "uniform"
+[env.randomization.gate_rpy.kwargs]
+minval = [0.0, 0.0, -0.1]
+maxval = [0.0, 0.0, 0.1]
 
 [env.randomization.obstacle_pos]
-type = "uniform"
-low = -0.15
-high = 0.15
+fn = "uniform"
+[env.randomization.obstacle_pos.kwargs]
+minval = [-0.15, -0.15, -0.05]
+maxval = [0.15, 0.15, 0.05]

config/level3.toml

@@ -16,7 +16,7 @@ check_drone_start_pos = true
 practice_without_track_objects = false
 
 [sim]
-# Physics options:
+# Physics options: analytical, sys_id, mujoco
 physics = "analytical"
 camera_view = [5.0, -40.0, -40.0, 0.5, -1.0, 0.5]
 sim_freq = 500                                    # Simulation frequency, in Hz
@@ -25,7 +25,7 @@ gui = false                                       # Enable/disable PyBullet's GU
 
 [env]
 id = "DroneRacing-v0" # Either "DroneRacing-v0" or "DroneRacingThrust-v0". If using "DroneRacingThrust-v0", the drone will use the thrust controller instead of the position controller.
-reseed = false        # Whether to re-seed the random number generator between episodes
+random_resets = false        # Whether to re-seed the random number generator between episodes
 seed = 1337           # Random seed
 freq = 50             # Frequency of the environment's step function, in Hz
 symbolic = false      # Whether to include symbolic expressions in the info dict. Note: This can interfere with multiprocessing! If you want to parallelize your training, set this to false.

docs/challenge/simulation.rst

@@ -50,7 +50,7 @@ The challenge is divided into different difficulty levels, each specified by a T
      - Sim2real transfer
 
 .. note::
-   "Rand. Between Episodes" (governed by argument `reseed_on_reset`) determines whether randomized properties and positions vary or are kept constant (by re-seeding the random number generator on each `env.reset()`) across episodes.
+   "Rand. Between Episodes" (governed by argument `random_resets`) determines whether randomized properties and positions vary or are kept constant (by re-seeding the random number generator on each `env.reset()`) across episodes.
 
 You may use the easier scenarios to develop and debug your controller. However, the final evaluation will be on the hardest scenario (Level 3).