add function to generate confidence curves

det-metrics.py

@@ -13,13 +13,12 @@
 # limitations under the License.
 """TODO: Add a description here."""
 
-from typing import List, Tuple, Literal
-from deprecated import deprecated
+from typing import List, Literal, Tuple
 
-import evaluate
 import datasets
+import evaluate
 import numpy as np
-
+from deprecated import deprecated
 from seametrics.detection import PrecisionRecallF1Support
 from seametrics.payload import Payload
 
@@ -200,7 +199,7 @@ class DetectionMetric(evaluate.Metric):
 
     def _compute(self, *, predictions, references, **kwargs):
         """Called within the evaluate.Metric.compute() method"""
-        return self.coco_metric.compute()["metrics"]
+        return self.coco_metric.compute()
 
     def add_payload(self, payload: Payload, model_name: str = None):
         """Converts the payload to the format expected by the metric"""
@@ -236,3 +235,55 @@ class DetectionMetric(evaluate.Metric):
             elif isinstance(v, list):
                 d[k] = np.array(v)
         return d
+    
+    def compute_for_multiple_models(self, payload):
+        results = {}
+        for model_name in payload.models:
+            self.add_payload(payload, model_name)
+            results[model_name] = self._compute()
+
+        return results
+
+
+    def generate_confidence_curves(self, results, models, confidence_config = {"T":0,
+                                                                       "R":0,
+                                                                       "K":0,
+                                                                       "A":0,
+                                                                       "M":0}):
+        
+        import plotly.graph_objects as go
+        from seametrics.detection.utils import get_confidence_metric_vals
+
+        # Create traces
+        fig = go.Figure()
+        metrics = ['precision', 'recall', 'f1']
+        for model in models:
+            plot_data = get_confidence_metric_vals(
+                cocoeval=results[model['name']]['eval'],
+                T=confidence_config['T'], 
+                R=confidence_config['R'], 
+                K=confidence_config['K'], 
+                A=confidence_config['A'], 
+                M=confidence_config['M']
+            )
+
+            for metric in metrics:
+                fig.add_trace(
+                    go.Scatter(
+                        x=plot_data['conf'],
+                        y=plot_data[metric],
+                        mode='lines',
+                        name=f"{model['name'].split('_')[0]} {metric}",
+                        line=dict(dash=None if metric == 'f1' else 'dash'),
+                    )
+                )
+
+        fig.update_layout(
+            title="Metric vs Confidence",
+            hovermode='x unified',
+            xaxis_title="Confidence",
+            yaxis_title="Metric value")
+        fig.show()
+
+        return fig
+        

requirements.txt

@@ -1,3 +1,4 @@
 git+https://github.com/huggingface/evaluate@main
 git+https://github.com/SEA-AI/seametrics@develop
-fiftyone
+fiftyone
+plotly