81 lines
2.6 KiB
Plaintext
81 lines
2.6 KiB
Plaintext
|
#!/usr/bin/env python
|
||
|
|
#
|
||
|
|
# Run all the tasks on a benchmark using a random agent.
|
||
|
|
#
|
||
|
|
# This script assumes you have set an OPENAI_GYM_API_KEY environment
|
||
|
|
# variable. You can find your API key in the web interface:
|
||
|
|
# https://gym.openai.com/settings/profile.
|
||
|
|
#
|
||
|
|
import argparse
|
||
|
|
import logging
|
||
|
|
import os
|
||
|
|
import sys
|
||
|
|
|
||
|
|
import gym
|
||
|
|
# In modules, use `logger = logging.getLogger(__name__)`
|
||
|
|
from gym import wrappers
|
||
|
|
from gym.scoreboard.scoring import benchmark_score_from_local
|
||
|
|
|
||
|
|
import openai_benchmark
|
||
|
|
|
||
|
|
logger = logging.getLogger()
|
||
|
|
|
||
|
|
def main():
|
||
|
|
parser = argparse.ArgumentParser(description=None)
|
||
|
|
parser.add_argument('-b', '--benchmark-id', help='id of benchmark to run e.g. Atari7Ram-v0')
|
||
|
|
parser.add_argument('-v', '--verbose', action='count', dest='verbosity', default=0, help='Set verbosity.')
|
||
|
|
parser.add_argument('-f', '--force', action='store_true', dest='force', default=False)
|
||
|
|
parser.add_argument('-t', '--training-dir', default="/tmp/gym-results", help='What directory to upload.')
|
||
|
|
args = parser.parse_args()
|
||
|
|
|
||
|
|
if args.verbosity == 0:
|
||
|
|
logger.setLevel(logging.INFO)
|
||
|
|
elif args.verbosity >= 1:
|
||
|
|
logger.setLevel(logging.DEBUG)
|
||
|
|
|
||
|
|
benchmark_id = args.benchmark_id
|
||
|
|
if benchmark_id is None:
|
||
|
|
logger.info("Must supply a valid benchmark")
|
||
|
|
return 1
|
||
|
|
|
||
|
|
try:
|
||
|
|
benchmark = gym.benchmark_spec(benchmark_id)
|
||
|
|
except Exception:
|
||
|
|
logger.info("Invalid benchmark")
|
||
|
|
return 1
|
||
|
|
|
||
|
|
# run benchmark tasks
|
||
|
|
for task in benchmark.tasks:
|
||
|
|
logger.info("Running on env: {}".format(task.env_id))
|
||
|
|
for trial in range(task.trials):
|
||
|
|
env = gym.make(task.env_id)
|
||
|
|
training_dir_name = "{}/{}-{}".format(args.training_dir, task.env_id, trial)
|
||
|
|
env = wrappers.Monitor(env, training_dir_name, video_callable=False, force=args.force)
|
||
|
|
env.reset()
|
||
|
|
for _ in range(task.max_timesteps):
|
||
|
|
o, r, done, _ = env.step(env.action_space.sample())
|
||
|
|
if done:
|
||
|
|
env.reset()
|
||
|
|
env.close()
|
||
|
|
|
||
|
|
logger.info("""Computing statistics for this benchmark run...
|
||
|
|
{{
|
||
|
|
score: {score},
|
||
|
|
num_envs_solved: {num_envs_solved},
|
||
|
|
summed_training_seconds: {summed_training_seconds},
|
||
|
|
start_to_finish_seconds: {start_to_finish_seconds},
|
||
|
|
}}
|
||
|
|
|
||
|
|
""".rstrip().format(**benchmark_score_from_local(benchmark_id, args.training_dir)))
|
||
|
|
|
||
|
|
logger.info("""Done running, upload results using the following command:
|
||
|
|
|
||
|
|
python -c "import gym; gym.upload('{}', benchmark_id='{}', algorithm_id='(unknown)')"
|
||
|
|
|
||
|
|
""".rstrip().format(args.training_dir, benchmark_id))
|
||
|
|
|
||
|
|
return 0
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
sys.exit(main())
|