test

app.py

@@ -3,12 +3,14 @@ import time
 from huggingface_hub import create_repo, whoami
 import gradio as gr
 from config_store import (
+    get_process_config,
     get_inference_config,
     get_onnxruntime_config,
     get_openvino_config,
     get_pytorch_config,
-    get_process_config,
+    get_ipex_config,
 )
+from optimum_benchmark.launchers.base import Launcher  # noqa
 from optimum_benchmark.backends.openvino.utils import TASKS_TO_OVMODEL
 from optimum_benchmark.backends.transformers_utils import TASKS_TO_MODEL_LOADERS
 from optimum_benchmark.backends.onnxruntime.utils import TASKS_TO_ORTMODELS
@@ -25,15 +27,17 @@ from optimum_benchmark import (
 )
 from optimum_benchmark.logging_utils import setup_logging
 
-os.environ["LOG_TO_FILE"] = "0"
-os.environ["LOG_LEVEL"] = "INFO"
-setup_logging(level="INFO", prefix="MAIN-PROCESS")
 
 DEVICE = "cpu"
-BACKENDS = ["pytorch", "onnxruntime", "openvino", "ipex"]
-
-CHOSEN_MODELS = ["bert-base-uncased", "gpt2"]
-CHOSEN_TASKS = (
+LAUNCHER = "process"
+SCENARIO = "inference"
+BACKENDS = ["onnxruntime", "openvino", "pytorch", "ipex"]
+MODELS = [
+    "hf-internal-testing/tiny-random-bert",
+    "google-bert/bert-base-uncased",
+    "openai-community/gpt2",
+]
+TASKS = (
     set(TASKS_TO_OVMODEL.keys())
     & set(TASKS_TO_ORTMODELS.keys())
     & set(TASKS_TO_IPEXMODEL.keys())
@@ -67,14 +71,16 @@ def run_benchmark(kwargs, oauth_token: gr.OAuthToken):
             model = value
         elif key.label == "task":
             task = value
+        elif key.label == "backends":
+            backends = value
         elif "." in key.label:
             backend, argument = key.label.split(".")
             configs[backend][argument] = value
         else:
             continue
 
-    process_config = ProcessConfig(**configs.pop("process"))
-    inference_config = InferenceConfig(**configs.pop("inference"))
+    configs["process"] = ProcessConfig(**configs.pop("process"))
+    configs["inference"] = InferenceConfig(**configs.pop("inference"))
 
     configs["onnxruntime"] = ORTConfig(
         task=task,
@@ -101,17 +107,31 @@ def run_benchmark(kwargs, oauth_token: gr.OAuthToken):
         **configs["ipex"],
     )
 
-    for backend in configs:
-        benchmark_name = (
-            f"{model}-{task}-{backend}-{time.strftime('%Y-%m-%d-%H-%M-%S')}"
-        )
+    html_output = f"<h3>Running benchmark for model {model} on task {task} with backends {backends}</h3>"
+
+    yield html_output
+
+    timestamp = time.strftime("%Y-%m-%d-%H-%M-%S")
+
+    for backend in backends:
+        benchmark_name = f"{timestamp}/{backend}"
         benchmark_config = BenchmarkConfig(
             name=benchmark_name,
-            launcher=process_config,
-            scenario=inference_config,
             backend=configs[backend],
+            launcher=configs[LAUNCHER],
+            scenario=configs[SCENARIO],
+        )
+        benchmark_config.push_to_hub(
+            repo_id=f"{username}/benchmarks",
+            subfolder=benchmark_name,
+            token=oauth_token.token,
+        )
+        benchmark_report = Benchmark.launch(benchmark_config)
+        benchmark_report.push_to_hub(
+            repo_id=f"{username}/benchmarks",
+            subfolder=benchmark_name,
+            token=oauth_token.token,
         )
-        benchmark_report = Benchmark.run(benchmark_config)
         benchmark = Benchmark(config=benchmark_config, report=benchmark_report)
         benchmark.push_to_hub(
             repo_id=f"{username}/benchmarks",
@@ -119,79 +139,112 @@ def run_benchmark(kwargs, oauth_token: gr.OAuthToken):
             token=oauth_token.token,
         )
 
-    return f"🚀 Benchmark {benchmark_name} has been pushed to {username}/benchmarks"
+        html_output += f"<br>📊 Benchmark report for {backend} backend in the folder {benchmark_name} of your benchmarks dataset"
 
+        yield html_output
 
-with gr.Blocks() as demo:
-    # add login button
-    gr.LoginButton(min_width=250)
 
-    # add image
-    gr.Markdown(
-        """<img src="https://huggingface.co/spaces/optimum/optimum-benchmark-ui/resolve/main/huggy_bench.png" style="display: block; margin-left: auto; margin-right: auto; width: 30%;">"""
-    )
+def build_demo():
+    with gr.Blocks() as demo:
+        # add login button
+        gr.LoginButton(min_width=250)
 
-    # title text
-    gr.Markdown("<h1 style='text-align: center'>🤗 Optimum-Benchmark Interface 🏋️</h1>")
-
-    # explanation text
-    gr.HTML(
-        "<h3 style='text-align: center'>"
-        "Zero code Gradio interface of "
-        "<a href='https://github.com/huggingface/optimum-benchmark.git'>"
-        "Optimum-Benchmark"
-        "</a>"
-        "<br>"
-        "</h3>"
-    )
+        # add image
+        gr.Markdown(
+            """<img src="https://huggingface.co/spaces/optimum/optimum-benchmark-ui/resolve/main/huggy_bench.png" style="display: block; margin-left: auto; margin-right: auto; width: 30%;">"""
+        )
 
-    model = gr.Dropdown(
-        label="model",
-        choices=CHOSEN_MODELS,
-        value="bert-base-uncased",
-        info="Model to run the benchmark on.",
-    )
-    task = gr.Dropdown(
-        label="task",
-        choices=CHOSEN_TASKS,
-        value="feature-extraction",
-        info="Task to run the benchmark on.",
-    )
+        # title text
+        gr.Markdown(
+            "<h1 style='text-align: center'>🤗 Optimum-Benchmark Interface 🏋️</h1>"
+        )
+
+        # explanation text
+        gr.HTML(
+            "<h3 style='text-align: center'>"
+            "Zero code Gradio interface of "
+            "<a href='https://github.com/huggingface/optimum-benchmark.git'>"
+            "Optimum-Benchmark"
+            "</a>"
+            "<br>"
+            "</h3>"
+        )
+
+        model = gr.Dropdown(
+            label="model",
+            choices=MODELS,
+            value=MODELS[0],
+            info="Model to run the benchmark on.",
+        )
+        task = gr.Dropdown(
+            label="task",
+            choices=TASKS,
+            value="feature-extraction",
+            info="Task to run the benchmark on.",
+        )
+        backends = gr.CheckboxGroup(
+            interactive=True,
+            label="backends",
+            choices=BACKENDS,
+            value=BACKENDS,
+            info="Backends to run the benchmark on.",
+        )
+
+        with gr.Row():
+            with gr.Accordion(label="Process Config", open=False, visible=True):
+                process_config = get_process_config()
+
+        with gr.Row() as backend_configs:
+            with gr.Accordion(label="OnnxRuntime Config", open=False, visible=True):
+                onnxruntime_config = get_onnxruntime_config()
+            with gr.Accordion(label="OpenVINO Config", open=False, visible=True):
+                openvino_config = get_openvino_config()
+            with gr.Accordion(label="PyTorch Config", open=False, visible=True):
+                pytorch_config = get_pytorch_config()
+            with gr.Accordion(label="IPEX Config", open=False, visible=True):
+                ipex_config = get_ipex_config()
+
+        backends.change(
+            inputs=backends,
+            outputs=backend_configs.children,
+            fn=lambda values: [
+                gr.update(visible=value in values) for value in BACKENDS
+            ],
+        )
+
+        with gr.Row():
+            with gr.Accordion(label="Scenario Config", open=False, visible=True):
+                inference_config = get_inference_config()
+
+        button = gr.Button(value="Run Benchmark", variant="primary")
+
+        with gr.Row():
+            html_output = gr.HTML(label="Output", value="")
+
+        button.click(
+            fn=run_benchmark,
+            inputs={
+                task,
+                model,
+                backends,
+                *process_config.values(),
+                *inference_config.values(),
+                *onnxruntime_config.values(),
+                *openvino_config.values(),
+                *pytorch_config.values(),
+                *ipex_config.values(),
+            },
+            outputs=[html_output],
+            concurrency_limit=1,
+        )
+
+    return demo
 
-    with gr.Row():
-        with gr.Accordion(label="Process Config", open=False, visible=True):
-            process_config = get_process_config()
-
-    with gr.Row():
-        with gr.Accordion(label="PyTorch Config", open=True, visible=True):
-            pytorch_config = get_pytorch_config()
-        with gr.Accordion(label="OpenVINO Config", open=True, visible=True):
-            openvino_config = get_openvino_config()
-        with gr.Accordion(label="OnnxRuntime Config", open=True, visible=True):
-            onnxruntime_config = get_onnxruntime_config()
-
-    with gr.Row():
-        with gr.Accordion(label="Scenario Config", open=False, visible=True):
-            inference_config = get_inference_config()
-
-    button = gr.Button(value="Run Benchmark", variant="primary")
-
-    html_output = gr.HTML()
-
-    button.click(
-        fn=run_benchmark,
-        inputs={
-            task,
-            model,
-            *process_config.values(),
-            *inference_config.values(),
-            *onnxruntime_config.values(),
-            *openvino_config.values(),
-            *pytorch_config.values(),
-        },
-        outputs=[html_output],
-        concurrency_limit=1,
-    )
 
+if __name__ == "__main__":
+    os.environ["LOG_TO_FILE"] = "0"
+    os.environ["LOG_LEVEL"] = "INFO"
+    setup_logging(level="INFO", prefix="MAIN-PROCESS")
 
-demo.queue(max_size=10).launch()
+    demo = build_demo()
+    demo.queue(max_size=10).launch()

benchmark.log

@@ -0,0 +1,41 @@
+[ISOLATED-PROCESS][2024-09-25 18:53:27,439][pytorch][INFO] - Allocating pytorch backend
+[ISOLATED-PROCESS][2024-09-25 18:53:27,439][pytorch][INFO] - 	+ Seeding backend with 42
+[ISOLATED-PROCESS][2024-09-25 18:53:27,440][pytorch][INFO] - 	+ Benchmarking a Transformers model
+[ISOLATED-PROCESS][2024-09-25 18:53:28,549][inference][INFO] - Allocating inference scenario
+[ISOLATED-PROCESS][2024-09-25 18:53:28,549][inference][INFO] - 	+ Creating input generator
+[ISOLATED-PROCESS][2024-09-25 18:53:28,550][inference][INFO] - 	+ Generating Inference inputs
+[ISOLATED-PROCESS][2024-09-25 18:53:28,550][inference][INFO] - 	+ Initializing Inference report
+[ISOLATED-PROCESS][2024-09-25 18:53:28,551][inference][INFO] - 	+ Preparing input shapes for Inference
+[ISOLATED-PROCESS][2024-09-25 18:53:28,551][inference][INFO] - 	+ Running model loading tracking
+[ISOLATED-PROCESS][2024-09-25 18:53:28,551][latency][INFO] - 	+ Tracking latency using CPU performance counter
+[ISOLATED-PROCESS][2024-09-25 18:53:28,551][inference][INFO] - 	+ Loading model for Inference
+[ISOLATED-PROCESS][2024-09-25 18:53:28,552][pytorch][INFO] - 	+ Creating backend temporary directory
+[ISOLATED-PROCESS][2024-09-25 18:53:28,553][pytorch][INFO] - 	+ Loading model with pretrained weights
+[ISOLATED-PROCESS][2024-09-25 18:53:28,554][pytorch][INFO] - 	+ Loading Transformers model
+[ISOLATED-PROCESS][2024-09-25 18:53:28,881][pytorch][INFO] - 	+ Enabling eval mode
+[ISOLATED-PROCESS][2024-09-25 18:53:28,883][pytorch][INFO] - 	+ Cleaning up backend temporary directory
+[ISOLATED-PROCESS][2024-09-25 18:53:28,884][inference][INFO] - 	+ Preparing inputs for Inference
+[ISOLATED-PROCESS][2024-09-25 18:53:28,885][inference][INFO] - 	+ Warming up backend for Inference
+[ISOLATED-PROCESS][2024-09-25 18:53:29,252][inference][INFO] - 	+ Running Inference latency tracking
+[ISOLATED-PROCESS][2024-09-25 18:53:29,252][latency][INFO] - 	+ Tracking latency using CPU performance counter
+[ISOLATED-PROCESS][2024-09-25 18:53:39,282][latency][INFO] - 		+ load latency:
+[ISOLATED-PROCESS][2024-09-25 18:53:39,282][latency][INFO] - 			- count: 1
+[ISOLATED-PROCESS][2024-09-25 18:53:39,283][latency][INFO] - 			- total: 0.331631 s
+[ISOLATED-PROCESS][2024-09-25 18:53:39,283][latency][INFO] - 			- mean: 0.331631 s
+[ISOLATED-PROCESS][2024-09-25 18:53:39,283][latency][INFO] - 			- stdev: 0.000000 s (0.00%)
+[ISOLATED-PROCESS][2024-09-25 18:53:39,284][latency][INFO] - 			- p50: 0.331631 s
+[ISOLATED-PROCESS][2024-09-25 18:53:39,284][latency][INFO] - 			- p90: 0.331631 s
+[ISOLATED-PROCESS][2024-09-25 18:53:39,284][latency][INFO] - 			- p95: 0.331631 s
+[ISOLATED-PROCESS][2024-09-25 18:53:39,284][latency][INFO] - 			- p99: 0.331631 s
+[ISOLATED-PROCESS][2024-09-25 18:53:39,285][latency][INFO] - 		+ forward latency:
+[ISOLATED-PROCESS][2024-09-25 18:53:39,285][latency][INFO] - 			- count: 266
+[ISOLATED-PROCESS][2024-09-25 18:53:39,285][latency][INFO] - 			- total: 10.021863 s
+[ISOLATED-PROCESS][2024-09-25 18:53:39,285][latency][INFO] - 			- mean: 0.037676 s
+[ISOLATED-PROCESS][2024-09-25 18:53:39,286][latency][INFO] - 			- stdev: 0.008922 s (23.68%)
+[ISOLATED-PROCESS][2024-09-25 18:53:39,286][latency][INFO] - 			- p50: 0.035970 s
+[ISOLATED-PROCESS][2024-09-25 18:53:39,286][latency][INFO] - 			- p90: 0.043994 s
+[ISOLATED-PROCESS][2024-09-25 18:53:39,286][latency][INFO] - 			- p95: 0.046884 s
+[ISOLATED-PROCESS][2024-09-25 18:53:39,287][latency][INFO] - 			- p99: 0.073021 s
+[ISOLATED-PROCESS][2024-09-25 18:53:39,288][latency][INFO] - 		+ forward throughput: 53.083941 samples/s
+[ISOLATED-PROCESS][2024-09-25 18:53:39,296][process][INFO] - 	+ Sending report to main process
+[ISOLATED-PROCESS][2024-09-25 18:53:39,297][process][INFO] - 	+ Exiting isolated process

config_store.py

@@ -88,6 +88,10 @@ def get_openvino_config():
     }
 
 
+def get_ipex_config():
+    return {}
+
+
 def get_inference_config():
     return {
         "inference.warmup_runs": gr.Slider(

pyproject.toml

@@ -1,3 +0,0 @@
-[tool.black]
-line-length = 119
-target-version = ['py37']