various

app.py

@@ -1,18 +1,24 @@
 import os
+import subprocess
+import importlib.util
 
-nvidia_available = os.system("nvidia-smi") == 0
-if nvidia_available:
-    os.system(
-        "pip install optimum-benchmark[onnxruntime-gpu,openvino,neural-compressor,diffusers,peft]@git+https://github.com/huggingface/optimum-benchmark.git"
-    )
-    os.system("pip uninstall onnxruntime onnxruntime-gpu -y")
-    os.system("pip install onnxruntime-gpu")
+NVIDIA_AVAILABLE = subprocess.check_output("nvidia-smi").decode("utf-8").strip() != ""
+
+if NVIDIA_AVAILABLE:
     DEVICES = ["cpu", "cuda"]
+    if importlib.util.find_spec("optimum_benchmark") is None:
+        os.system(
+            "pip install optimum-benchmark[onnxruntime-gpu,openvino,neural-compressor,diffusers,peft]@git+https://github.com/huggingface/optimum-benchmark.git"
+        )
+        os.system("pip uninstall onnxruntime onnxruntime-gpu -y")
+        os.system("pip install onnxruntime-gpu")
 else:
-    os.system(
-        "pip install optimum-benchmark[onnxruntime,openvino,neural-compressor,diffusers,peft]@git+https://github.com/huggingface/optimum-benchmark.git"
-    )
     DEVICES = ["cpu"]
+    if importlib.util.find_spec("optimum_benchmark") is None:
+        os.system(
+            "pip install optimum-benchmark[onnxruntime,openvino,neural-compressor,diffusers,peft]@git+https://github.com/huggingface/optimum-benchmark.git"
+        )
+
 
 BACKENDS = ["pytorch", "onnxruntime", "openvino", "neural-compressor"]
 BENCHMARKS = ["inference", "training"]
@@ -48,7 +54,8 @@ with gr.Blocks() as demo:
         "Zero code Gradio interface of <a href='https://github.com/huggingface/optimum-benchmark.git'>Optimum-Benchmark</a><br>"
         "</h3>"
         "<p style='text-align: center'>"
-        "Note: Duplicate the space and change the hardware settings to an Nvidia machine to target CUDA devices."
+        "Note: <a href='https://huggingface.co/spaces/optimum/optimum-benchmark-ui?duplicate=true'>Duplicate this space</a> and change its hardware to enable CUDA device<br>"
+        "or <a href='https://huggingface.co/spaces/optimum/optimum-benchmark-ui?docker=true'>Run with Docker</a> locally to target your own hardware."
         "</p>"
     )
 
@@ -71,7 +78,7 @@ with gr.Blocks() as demo:
     )
     experiment = gr.Textbox(
         label="experiment_name",
-        value=f"awesome-experiment-{random.randint(0, 1000)}",
+        value=f"awesome-experiment-{random.randint(0, 100000)}",
         info="Name of the experiment. Will be used to create a folder where results are stored.",
     )
     model.submit(fn=infer_task_from_model_name_or_path, inputs=model, outputs=task)
@@ -87,13 +94,13 @@ with gr.Blocks() as demo:
                 )
 
             with gr.Row() as backend_configs:
-                with gr.Accordion(label="Pytorch Config", open=False, visible=True):
+                with gr.Accordion(label="backend options", open=False, visible=True):
                     pytorch_config = get_pytorch_config()
-                with gr.Accordion(label="OnnxRunTime Config", open=False, visible=False):
+                with gr.Accordion(label="backend config", open=False, visible=False):
                     onnxruntime_config = get_onnxruntime_config()
-                with gr.Accordion(label="OpenVINO Config", open=False, visible=False):
+                with gr.Accordion(label="backend config", open=False, visible=False):
                     openvino_config = get_openvino_config()
-                with gr.Accordion(label="Neural Compressor Config", open=False, visible=False):
+                with gr.Accordion(label="backend config", open=False, visible=False):
                     neural_compressor_config = get_neural_compressor_config()
 
         # hide backend configs based on backend
@@ -113,9 +120,9 @@ with gr.Blocks() as demo:
                 )
 
             with gr.Row() as benchmark_configs:
-                with gr.Accordion(label="Inference Config", open=False, visible=True):
+                with gr.Accordion(label="benchmark Config", open=False, visible=True):
                     inference_config = get_inference_config()
-                with gr.Accordion(label="Training Config", open=False, visible=False):
+                with gr.Accordion(label="benchmark Config", open=False, visible=False):
                     training_config = get_training_config()
 
         # hide benchmark configs based on benchmark
@@ -127,8 +134,8 @@ with gr.Blocks() as demo:
 
     baseline = gr.Checkbox(
         value=False,
-        label="Compare to Baseline",
-        info="If checked, will run two experiments: one with the given configuration, and another with a a baseline pytorch configuration.",
+        label="compare_to_baseline",
+        info="Check this box to compare your chosen configuration to the baseline configuration.",
     )
 
     button = gr.Button(value="Run Benchmark", variant="primary")
@@ -156,5 +163,11 @@ with gr.Blocks() as demo:
         outputs=[html_output, button, table_output],
         queue=True,
     )
+    button.click(
+        fn=lambda: f"awesome-experiment-{random.randint(0, 100000)}",
+        inputs=[],
+        outputs=experiment,
+        queue=True,
+    )
 
 demo.queue().launch()

run.py

@@ -8,13 +8,12 @@ ansi2html_converter = Ansi2HTMLConverter(inline=True)
 
 def run_benchmark(kwargs):
     for key, value in kwargs.copy().items():
-        if key.label == "Compare to Baseline":
+        if key.label == "compare_to_baseline":
             baseline = value
             kwargs.pop(key)
         elif key.label == "experiment_name":
             experiment_name = value
             kwargs.pop(key)
-
         elif key.label == "model":
             model = value
             kwargs.pop(key)
@@ -45,7 +44,7 @@ def run_benchmark(kwargs):
             f"model={model}",
             f"device={device}",
             f"benchmark={benchmark}",
-            f"experiment_name=baseline",
+            f"experiment_name=baseline_{experiment_name}",
         ]
         for component, value in kwargs.items():
             if f"{benchmark}." in component.label:
@@ -93,7 +92,7 @@ def run_benchmark(kwargs):
         return
 
     if baseline:
-        baseline_table = pd.read_csv(f"runs/baseline/{benchmark}_results.csv", index_col=0)
+        baseline_table = pd.read_csv(f"runs/baseline_{experiment_name}/{benchmark}_results.csv", index_col=0)
         table = pd.read_csv(f"runs/{experiment_name}/{benchmark}_results.csv", index_col=0)
         # concat tables
         table = pd.concat([baseline_table, table], axis=0)
@@ -121,6 +120,9 @@ def run_experiment(args, html_text=""):
 
     curr_ansi_text = ""
     for ansi_line in iter(process.stdout.readline, ""):
+        if process.returncode is not None and process.returncode != 0:
+            break
+
         # stream process output to stdout
         print(ansi_line, end="")
         # skip torch.distributed.nn.jit.instantiator messages
@@ -164,7 +166,7 @@ def postprocess_table(table, experiment_name):
         table["forward.peak_memory.reduction(%)"] = (
             table["forward.peak_memory(MB)"] / table["forward.peak_memory(MB)"].iloc[0] - 1
         ) * 100
-        table["forward.peak_memory.reduction(%)"] = table["forward.peak_memory.savings(%)"].round(2)
+        table["forward.peak_memory.reduction(%)"] = table["forward.peak_memory.reduction(%)"].round(2)
 
     if "generate.latency(s)" in table.columns:
         table["generate.latency.reduction(%)"] = (