Add a total report

run_tests.py

@@ -53,6 +53,7 @@ class VeiledResultPlugin:
     def __init__(self) -> None:
         # exercise name -> outcomes -> reports
         self.reports: dict[str, Outcomes] = collections.defaultdict(Outcomes)
+        self.total_reports = Outcomes()
         self.exitstatus: pytest.ExitCode | None = None
 
     def _exercise_name(self, report: pytest.TestReport) -> str:
@@ -105,6 +106,7 @@ class VeiledResultPlugin:
             raise UnexpectedReportError(report)
 
         self.reports[self._exercise_name(report)].add_report(result, report)
+        self.total_reports.add_report(result, report)
 
     def result(self) -> dict[str, Any]:
         # exercise -> outcome -> percentage
@@ -117,6 +119,7 @@ class VeiledResultPlugin:
         return {
             "status": self.exitstatus,
             "tests": result,
+            "total": self.total_reports.to_perc_dict(),
         }
 
 

tests/test_run_tests.py

@@ -27,6 +27,7 @@ class TestVeiledResultPlugin:
         return {
             "status": None,
             "tests": collections.defaultdict(dict),
+            "total": {},
         }
 
     @pytest.fixture
@@ -55,6 +56,7 @@ class TestVeiledResultPlugin:
 
         expected_result["status"] = pytest.ExitCode.OK
         expected_result["tests"]["passing"]["passed"] = "100%"
+        expected_result["total"]["passed"] = "100%"
 
         assert vrp.result() == expected_result
 
@@ -119,16 +121,16 @@ class TestVeiledResultPlugin:
 
         expected_result["status"] = pytest.ExitCode.TESTS_FAILED
         # FIXME should we really count test_error_teardown as passed?
-        expected_result["tests"] = {
-            "outcomes": {
-                "error": "20%",
-                "failed": "10%",
-                "passed": "20%",
-                "skipped": "30%",
-                "xfailed": "10%",
-                "xpassed": "10%",
-            }
+        outcomes = {
+            "error": "20%",
+            "failed": "10%",
+            "passed": "20%",
+            "skipped": "30%",
+            "xfailed": "10%",
+            "xpassed": "10%",
         }
+        expected_result["tests"] = {"outcomes": outcomes}
+        expected_result["total"] = outcomes
         assert vrp.result() == expected_result
 
     # Rest gets filled by nbformat's upgrade mechanism
@@ -165,6 +167,7 @@ class TestVeiledResultPlugin:
         expected_result["tests"] = {
             "cell tag not found setup": {"celltag_not_found": "100%"}
         }
+        expected_result["total"] = {"celltag_not_found": "100%"}
 
         assert vrp.result() == expected_result
 
@@ -195,6 +198,7 @@ class TestVeiledResultPlugin:
         expected_result["tests"] = {
             "cell tag not found call": {"celltag_not_found": "100%"}
         }
+        expected_result["total"] = {"celltag_not_found": "100%"}
 
         assert vrp.result() == expected_result
 
@@ -234,6 +238,7 @@ class TestVeiledResultPlugin:
         expected_result["tests"] = {
             "rounded percent": {"passed": passed_perc, "failed": failed_perc}
         }
+        expected_result["total"] = {"passed": passed_perc, "failed": failed_perc}
 
         assert vrp.result() == expected_result
 
@@ -391,6 +396,7 @@ class TestRun:
             "commit": repo.head_short_sha(),
             "status": 0,
             "tests": {"nothing": {"passed": "100%"}},
+            "total": {"passed": "100%"},
         }
         assert bare_clone_repo.head_short_sha() == repo.head_short_sha()
 
@@ -410,6 +416,7 @@ class TestRun:
             "commit": repo.head_short_sha(),
             "status": None,
             "tests": {},
+            "total": {},
         }
 
 
@@ -439,13 +446,16 @@ def test_main(
     print(out)
     assert any("create mode 100644 notebook.ipynb" in msg for msg in caplog.messages)
 
-    line = out.splitlines()[-1]
-    assert line.startswith("{") and line.endswith("}")
-    data = eval(line)  # eh, it's a test, it's fine
+    lines = out.splitlines()
+    data_start = lines.index("{")
+    data_end = lines.index("}")
+    data_str = "\n".join(lines[data_start : data_end + 1])
+    data = eval(data_str)  # eh, it's a test, it's fine
 
     del data["commit"]  # not fixed
     expected = {
         "status": 0,
         "tests": {"nothing": {"passed": "100%"}},
+        "total": {"passed": "100%"},
     }
     assert data == expected