@Snehil-Shah: refactor to allow arbitrary args to experiments (#885)

Running experiments for #854, an amazing work contributed by
@Snehil-Shah.

---------

Signed-off-by: Snehil Shah <[email protected]>
Co-authored-by: Snehil Shah <[email protected]>

Author: DonggeLiu

Dockerfile

@@ -65,4 +65,4 @@ RUN apt-get update && \
 COPY . /experiment
 WORKDIR /experiment
 RUN /venv/bin/pip install --disable-pip-version-check -r requirements.txt
-ENTRYPOINT ["./report/docker_run.sh"]
+ENTRYPOINT ["/venv/bin/python3", "./report/docker_run.py"]

ci/ci_trial_build.py

@@ -68,7 +68,7 @@ def exec_command_from_github(pull_request_number):
 
   # Set the branch so that the trial_build builds the projects from the PR
   # branch.
-  command.extend(['-p', str(pull_request_number)])
+  command = ['-p', str(pull_request_number)] + command
   command = [c for c in command if c]
   logging.info('Command: %s.', command)
   return request_pr_exp.main(command)

ci/k8s/large-pr-exp.yaml

@@ -30,7 +30,11 @@ spec:
         # Modify the follow command to customize one-off experiments.
         # For benchmark sets that need more disk, increase the results volume
         # size too.
-        command: ["/bin/bash", "report/docker_run.sh", "${GKE_EXP_BENCHMARK}", "${GKE_EXP_NAME}", "${GKE_EXP_FUZZING_TIMEOUT}", "ofg-pr", "${GKE_EXP_LLM}", "${GKE_EXP_DELAY}", "${GKE_EXP_LOCAL_INTROSPECTOR}", "${GKE_EXP_NUM_SAMPLES}", "${GKE_EXP_LLM_FIX_LIMIT}", "${GKE_EXP_VARY_TEMPERATURE}", "${GKE_EXP_AGENT}"]
+        command: [
+          "/bin/bash",
+          "-c",
+          "/venv/bin/python3 report/docker_run.py --benchmark-set=${GKE_EXP_BENCHMARK} --frequency-label=${GKE_EXP_NAME} --run-timeout=${GKE_EXP_FUZZING_TIMEOUT} --sub-dir=ofg-pr --model=${GKE_EXP_LLM} --delay=${GKE_EXP_DELAY} --num-samples=${GKE_EXP_NUM_SAMPLES} --llm-fix-limit=${GKE_EXP_LLM_FIX_LIMIT} --max-round=${GKE_EXP_MAX_ROUND} --local-introspector=${GKE_EXP_LOCAL_INTROSPECTOR} --vary-temperature=${GKE_EXP_VARY_TEMPERATURE} --agent=${GKE_EXP_AGENT} --redirect-outs=${GKE_REDIRECT_OUTS} ${GKE_EXP_ADDITIONAL_ARGS}"
+        ]
         securityContext:
           privileged: true
         resources:
@@ -52,8 +56,6 @@ spec:
           value: 'us-west1'
         - name: GCB_BUILDPOOL_NAME
           value: projects/oss-fuzz/locations/us-west1/workerPools/buildpool-large-pr
-        - name: REDIRECT_OUTS
-          value: '${GKE_REDIRECT_OUTS}'
       # imagePullSecrets:
       # - name: oss-fuzz-base-artifect
       volumes:

ci/k8s/pr-exp.yaml

@@ -31,7 +31,11 @@ spec:
         # Modify the follow command to customize one-off experiments.
         # For benchmark sets that need more disk, increase the results volume
         # size too.
-        command: ["/bin/bash", "report/docker_run.sh", "${GKE_EXP_BENCHMARK}", "${GKE_EXP_NAME}", "${GKE_EXP_FUZZING_TIMEOUT}", "ofg-pr", "${GKE_EXP_LLM}", "${GKE_EXP_DELAY}", "${GKE_EXP_LOCAL_INTROSPECTOR}", "${GKE_EXP_NUM_SAMPLES}", "${GKE_EXP_LLM_FIX_LIMIT}", "${GKE_EXP_VARY_TEMPERATURE}", "${GKE_EXP_AGENT}", "${GKE_EXP_MAX_ROUND}"]
+        command: [
+          "/bin/bash",
+          "-c",
+          "/venv/bin/python3 report/docker_run.py --benchmark-set=${GKE_EXP_BENCHMARK} --frequency-label=${GKE_EXP_NAME} --run-timeout=${GKE_EXP_FUZZING_TIMEOUT} --sub-dir=ofg-pr --model=${GKE_EXP_LLM} --delay=${GKE_EXP_DELAY} --num-samples=${GKE_EXP_NUM_SAMPLES} --llm-fix-limit=${GKE_EXP_LLM_FIX_LIMIT} --max-round=${GKE_EXP_MAX_ROUND} --local-introspector=${GKE_EXP_LOCAL_INTROSPECTOR} --vary-temperature=${GKE_EXP_VARY_TEMPERATURE} --agent=${GKE_EXP_AGENT} --redirect-outs=${GKE_REDIRECT_OUTS} ${GKE_EXP_ADDITIONAL_ARGS}"
+        ]
         securityContext:
           privileged: true
         resources:
@@ -53,8 +57,6 @@ spec:
           value: 'us-west2'
         - name: GCB_BUILDPOOL_NAME
           value: projects/oss-fuzz/locations/us-west2/workerPools/buildpool-llm-agents
-        - name: REDIRECT_OUTS
-          value: '${GKE_REDIRECT_OUTS}'
       # imagePullSecrets:
       # - name: oss-fuzz-base-artifect
       volumes:
@@ -82,4 +84,4 @@ spec:
           values: [0]
       - action: Ignore             # One of: Ignore, FailJob, Count
         onPodConditions:
-        - type: DisruptionTarget   # Indicates Pod disruption
+        - type: DisruptionTarget   # Indicates Pod disruption

ci/request_pr_exp.py

@@ -18,6 +18,9 @@
   python -m report.request_pr_exp -p <PR-ID> -n <YOUR-NAME>
 e.g.,
   python -m report.request_pr_exp -p 73 -n dg
+
+You can also pass arbitrary flags to experiments after -- separator:
+  python -m report.request_pr_exp -p 73 -n dg -- --context --debug
 """
 
 import argparse
@@ -204,7 +207,10 @@ def _parse_args(cmd) -> argparse.Namespace:
                       action='store_true',
                       default=False,
                       help='Redirects experiments stdout/stderr to file')
-  args = parser.parse_args(cmd)
+
+  # Allow piping arbitrary args to run_all_experiments.py
+  args, additional_args = parser.parse_known_args(cmd)
+  args.additional_args = additional_args
 
   assert os.path.isfile(
       args.gke_template), (f'GKE template does not exist: {args.gke_template}')
@@ -232,6 +238,10 @@ def _parse_args(cmd) -> argparse.Namespace:
   if (args.max_round == 100 and
       any(args.name_suffix.startswith(suffix) for suffix in ['ascc-', 'dgk-'])):
     args.max_round = 10
+
+  if args.additional_args:
+    logging.info("Additional args: %s", args.additional_args)
+
   return args
 
 
@@ -342,15 +352,18 @@ def _fill_template(args: argparse.Namespace) -> str:
   exp_env_vars['GKE_EXP_NAME'] = args.experiment_name
   exp_env_vars['GKE_EXP_REQ_CPU'] = args.request_cpus
   exp_env_vars['GKE_EXP_REQ_MEM'] = f'{args.request_memory}Gi'
-  if args.local_introspector:
-    exp_env_vars['GKE_EXP_LOCAL_INTROSPECTOR'] = 'true'
+  exp_env_vars[
+      'GKE_EXP_LOCAL_INTROSPECTOR'] = f'{args.local_introspector}'.lower()
   exp_env_vars['GKE_EXP_NUM_SAMPLES'] = f'{args.num_samples}'
   exp_env_vars['GKE_EXP_LLM_FIX_LIMIT'] = f'{args.llm_fix_limit}'
   exp_env_vars['GKE_EXP_VARY_TEMPERATURE'] = f'{args.vary_temperature}'.lower()
   exp_env_vars['GKE_EXP_AGENT'] = f'{args.agent}'.lower()
-  exp_env_vars['GKE_REDIRECT_OUTS'] = 'true' if args.redirect_outs else ''
+  exp_env_vars['GKE_REDIRECT_OUTS'] = f'{args.redirect_outs}'.lower()
   exp_env_vars['GKE_EXP_MAX_ROUND'] = args.max_round
 
+  # Add additional args as a space-separated string
+  exp_env_vars['GKE_EXP_ADDITIONAL_ARGS'] = ' '.join(args.additional_args)
+
   with open(args.gke_template, 'r') as file:
     yaml_template = file.read()
 

report/README.md

@@ -11,7 +11,7 @@
 
 ## Trends Report
 
-1. After each experiment is finished, `docker_run.sh` uploads a summary json
+1. After each experiment is finished, `docker_run.py` uploads a summary json
    file to `gs://oss-fuzz-gcb-experiment-run-logs/trend-reports/`.
 2. Upload of the summary json triggers a
    [Cloud Run Function](https://pantheon.corp.google.com/functions/details/us-central1/llm-trends-report-index?env=gen1&project=oss-fuzz)