Print weblog crash logs (#2912)

Co-authored-by: Charles de Beauchesne <[email protected]>

utils/_context/_scenarios/endtoend.py

@@ -420,11 +420,11 @@ def _wait_and_stop_containers(self):
                 try:
                     r = self.weblog_container.request("GET", "/flush", timeout=10)
                     assert r.status_code == 200
-                except Exception as e:
-                    self.weblog_container.collect_logs()
-                    raise Exception(
-                        f"Failed to flush weblog, please check {self.host_log_folder}/docker/weblog/stdout.log"
-                    ) from e
+                except:
+                    self.weblog_container.healthy = False
+                    logger.stdout(
+                        f"Warning: Failed to flush weblog, please check {self.host_log_folder}/docker/weblog/stdout.log"
+                    )
 
             self.weblog_container.stop()
             interfaces.library.check_deserialization_errors()

utils/_context/containers.py

@@ -312,37 +312,52 @@ def _fix_host_pwd_in_volumes(self):
         self.kwargs["volumes"] = result
 
     def stop(self):
+        self._starting_thread = None
+
         if self._container:
+            self._container.reload()
+            if self._container.status != "running":
+                self.healthy = False
+                pytest.exit(f"Container {self.name} is not running, please check logs", 1)
+
             self._container.stop()
-        self._starting_thread = None
+
+            if not self.healthy:
+                pytest.exit(f"Container {self.name} is not healthy, please check logs", 1)
 
     def collect_logs(self):
-        stdout = self._container.logs(stdout=True, stderr=False)
-        stderr = self._container.logs(stdout=False, stderr=True)
+        TAIL_LIMIT = 50
+        SEP = "=" * 30
 
         keys = [
             bytearray(os.environ["DD_API_KEY"], "utf-8"),
         ]
         if "DD_APP_KEY" in os.environ:
             keys.append(bytearray(os.environ["DD_APP_KEY"], "utf-8"))
 
-        for key in keys:
-            stdout = stdout.replace(key, b"***")
-            stderr = stderr.replace(key, b"***")
+        data = (
+            ("stdout", self._container.logs(stdout=True, stderr=False)),
+            ("stderr", self._container.logs(stdout=False, stderr=True)),
+        )
+
+        for output_name, output in data:
+            filename = f"{self.log_folder_path}/{output_name}.log"
+
+            for key in keys:
+                output = output.replace(key, b"***")
 
-        with open(f"{self.log_folder_path}/stdout.log", "wb") as f:
-            f.write(stdout)
+            with open(filename, "wb") as f:
+                f.write(output)
 
-        with open(f"{self.log_folder_path}/stderr.log", "wb") as f:
-            f.write(stderr)
+            if not self.healthy:
+                decoded_output = output.decode("utf-8")
 
-        if not self.healthy:
-            sep = "=" * 30
-            logger.stdout(f"\n{sep} {self.name} STDERR {sep}")
-            logger.stdout(stderr.decode("utf-8"))
-            logger.stdout(f"\n{sep} {self.name} STDOUT {sep}")
-            logger.stdout(stdout.decode("utf-8"))
-            logger.stdout("")
+                logger.stdout(f"\n{SEP} {self.name} {output_name.upper()} last {TAIL_LIMIT} lines {SEP}")
+                logger.stdout(f"-> See {filename} for full logs")
+                logger.stdout("")
+                # print last <tail> lines in stdout
+                logger.stdout("\n".join(decoded_output.splitlines()[-TAIL_LIMIT:]))
+                logger.stdout("")
 
     def remove(self):
         logger.debug(f"Removing container {self.name}")
@@ -352,7 +367,7 @@ def remove(self):
                 # collect logs before removing
                 self.collect_logs()
                 self._container.remove(force=True)
-            except:
+            except Exception:
                 # Sometimes, the container does not exists.
                 # We can safely ignore this, because if it's another issue
                 # it will be killed at startup