diff --git a/agenta-backend/agenta_backend/services/evaluators_service.py b/agenta-backend/agenta_backend/services/evaluators_service.py
index 53afb2629..42db9a2da 100644
--- a/agenta-backend/agenta_backend/services/evaluators_service.py
+++ b/agenta-backend/agenta_backend/services/evaluators_service.py
@@ -621,9 +621,25 @@ def auto_json_diff(
     lm_providers_keys: Dict[str, Any],  # pylint: disable=unused-argument
 ) -> Result:
     try:
+        output = output.get("data", "") if isinstance(output, dict) else output
+
+        if isinstance(output, dict):
+            output = json.dumps(output)
+        elif isinstance(output, str):
+            try:
+                json.loads(output)
+            except:
+                raise Exception(
+                    f"Evaluator 'auto_json_diff' requires string outputs to be JSON strings."
+                )
+        else:
+            raise Exception(
+                f"Evaluator 'auto_json_diff' requires the output to be either a JSON string or a JSON object, but received {type(output).__name__} instead."
+            )
+
         correct_answer = get_correct_answer(data_point, settings_values)
         average_score = compare_jsons(
-            ground_truth=correct_answer,
+            ground_truth=json.loads(correct_answer),
             app_output=json.loads(output),
             settings_values=settings_values,
         )
diff --git a/agenta-backend/agenta_backend/tests/unit/test_evaluators.py b/agenta-backend/agenta_backend/tests/unit/test_evaluators.py
index 809ab9427..d22b38563 100644
--- a/agenta-backend/agenta_backend/tests/unit/test_evaluators.py
+++ b/agenta-backend/agenta_backend/tests/unit/test_evaluators.py
@@ -186,12 +186,7 @@ def test_auto_contains_json(output, expected):
     [
         (
             {
-                "correct_answer": {
-                    "user": {
-                        "name": "John",
-                        "details": {"age": 30, "location": "New York"},
-                    }
-                }
+                "correct_answer": '{"user": {"name": "John", "details": {"age": 30, "location": "New York"}}}'
             },
             '{"user": {"name": "John", "details": {"age": 30, "location": "New York"}}}',
             {
@@ -205,12 +200,7 @@ def test_auto_contains_json(output, expected):
         ),
         (
             {
-                "correct_answer": {
-                    "user": {
-                        "name": "John",
-                        "details": {"age": 30, "location": "New York"},
-                    }
-                }
+                "correct_answer": '{"user": {"name": "John", "details": {"age": "30", "location": "New York"}}}'
             },
             '{"user": {"name": "John", "details": {"age": "30", "location": "New York"}}}',
             {
@@ -224,12 +214,7 @@ def test_auto_contains_json(output, expected):
         ),
         (
             {
-                "correct_answer": {
-                    "user": {
-                        "name": "John",
-                        "details": {"age": 30, "location": "New York"},
-                    }
-                }
+                "correct_answer": '{"user": {"name": "John", "details": {"age": 30, "location": "New York"}}}'
             },
             '{"USER": {"NAME": "John", "DETAILS": {"AGE": 30, "LOCATION": "New York"}}}',
             {