diff --git a/utils/_context/_scenarios/endtoend.py b/utils/_context/_scenarios/endtoend.py index 99852749f8..3fdbde4ab7 100644 --- a/utils/_context/_scenarios/endtoend.py +++ b/utils/_context/_scenarios/endtoend.py @@ -420,11 +420,11 @@ def _wait_and_stop_containers(self): try: r = self.weblog_container.request("GET", "/flush", timeout=10) assert r.status_code == 200 - except Exception as e: - self.weblog_container.collect_logs() - raise Exception( - f"Failed to flush weblog, please check {self.host_log_folder}/docker/weblog/stdout.log" - ) from e + except: + self.weblog_container.healthy = False + logger.stdout( + f"Warning: Failed to flush weblog, please check {self.host_log_folder}/docker/weblog/stdout.log" + ) self.weblog_container.stop() interfaces.library.check_deserialization_errors() diff --git a/utils/_context/containers.py b/utils/_context/containers.py index 0bbfd3e2e1..9ec5223c0b 100644 --- a/utils/_context/containers.py +++ b/utils/_context/containers.py @@ -312,13 +312,22 @@ def _fix_host_pwd_in_volumes(self): self.kwargs["volumes"] = result def stop(self): + self._starting_thread = None + if self._container: + self._container.reload() + if self._container.status != "running": + self.healthy = False + pytest.exit(f"Container {self.name} is not running, please check logs", 1) + self._container.stop() - self._starting_thread = None + + if not self.healthy: + pytest.exit(f"Container {self.name} is not healthy, please check logs", 1) def collect_logs(self): - stdout = self._container.logs(stdout=True, stderr=False) - stderr = self._container.logs(stdout=False, stderr=True) + TAIL_LIMIT = 50 + SEP = "=" * 30 keys = [ bytearray(os.environ["DD_API_KEY"], "utf-8"), @@ -326,23 +335,29 @@ def collect_logs(self): if "DD_APP_KEY" in os.environ: keys.append(bytearray(os.environ["DD_APP_KEY"], "utf-8")) - for key in keys: - stdout = stdout.replace(key, b"***") - stderr = stderr.replace(key, b"***") + data = ( + ("stdout", self._container.logs(stdout=True, stderr=False)), + ("stderr", self._container.logs(stdout=False, stderr=True)), + ) + + for output_name, output in data: + filename = f"{self.log_folder_path}/{output_name}.log" + + for key in keys: + output = output.replace(key, b"***") - with open(f"{self.log_folder_path}/stdout.log", "wb") as f: - f.write(stdout) + with open(filename, "wb") as f: + f.write(output) - with open(f"{self.log_folder_path}/stderr.log", "wb") as f: - f.write(stderr) + if not self.healthy: + decoded_output = output.decode("utf-8") - if not self.healthy: - sep = "=" * 30 - logger.stdout(f"\n{sep} {self.name} STDERR {sep}") - logger.stdout(stderr.decode("utf-8")) - logger.stdout(f"\n{sep} {self.name} STDOUT {sep}") - logger.stdout(stdout.decode("utf-8")) - logger.stdout("") + logger.stdout(f"\n{SEP} {self.name} {output_name.upper()} last {TAIL_LIMIT} lines {SEP}") + logger.stdout(f"-> See {filename} for full logs") + logger.stdout("") + # print last lines in stdout + logger.stdout("\n".join(decoded_output.splitlines()[-TAIL_LIMIT:])) + logger.stdout("") def remove(self): logger.debug(f"Removing container {self.name}") @@ -352,7 +367,7 @@ def remove(self): # collect logs before removing self.collect_logs() self._container.remove(force=True) - except: + except Exception: # Sometimes, the container does not exists. # We can safely ignore this, because if it's another issue # it will be killed at startup