From f128d7ffc14987d9b316ca55ffc107aa16a203e6 Mon Sep 17 00:00:00 2001 From: Quinten Date: Sat, 13 Apr 2024 00:39:08 -0700 Subject: [PATCH] increase test coverage --- README.md | 2 +- pyproject.toml | 2 +- src/revnets/context/context.py | 22 +---- src/revnets/data/mnist1d.py | 16 +++- .../evaluations/analysis/activations.py | 2 +- src/revnets/evaluations/attack/network.py | 4 +- src/revnets/evaluations/base.py | 2 +- src/revnets/evaluations/evaluation.py | 12 +-- src/revnets/evaluations/weights/__init__.py | 2 +- src/revnets/evaluations/weights/mse.py | 2 +- .../evaluations/weights/named_layers_mae.py | 2 +- .../weights/standardize/network.py | 9 +- src/revnets/evaluations/weights/visualizer.py | 85 ------------------- src/revnets/models/__init__.py | 2 +- src/revnets/models/experiment.py | 4 - src/revnets/models/path.py | 6 -- src/revnets/models/split.py | 12 --- src/revnets/networks/base.py | 6 +- src/revnets/pipelines/base.py | 4 +- .../pipelines/mininet/mininet_untrained.py | 2 +- src/revnets/pipelines/train.py | 13 +-- src/revnets/reconstructions/base.py | 10 +-- src/revnets/reconstructions/queries/base.py | 4 +- src/revnets/reconstructions/queries/data.py | 4 - .../reconstructions/queries/iterative/base.py | 2 +- .../queries/iterative/difficult_inputs.py | 4 +- src/revnets/training/network.py | 2 +- .../training/reconstructions/metrics.py | 4 +- .../training/reconstructions/network.py | 4 - src/revnets/training/trainer.py | 18 ---- src/revnets/utils/__init__.py | 1 - src/revnets/utils/colors.py | 4 +- src/revnets/utils/named_class.py | 7 +- src/revnets/utils/pl_logger.py | 30 ------- src/revnets/utils/table.py | 20 ----- tests/conftest.py | 4 +- tests/{ => evaluations}/test_evaluations.py | 7 +- tests/evaluations/test_standardize.py | 8 ++ tests/evaluations/verifier.py | 3 + 39 files changed, 83 insertions(+), 264 deletions(-) delete mode 100644 src/revnets/evaluations/weights/visualizer.py delete mode 100644 src/revnets/utils/pl_logger.py delete mode 100644 src/revnets/utils/table.py rename tests/{ => evaluations}/test_evaluations.py (88%) diff --git a/README.md b/README.md index 78d9088..e8f1a41 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Revnets ![Python version](https://img.shields.io/badge/python-3.10+-brightgreen) ![Operating system](https://img.shields.io/badge/os-linux%20%7c%20macOS-brightgreen) -![Coverage](https://img.shields.io/badge/coverage-92%25-brightgreen) +![Coverage](https://img.shields.io/badge/coverage-100%25-brightgreen) Reverse engineer internal parameters of black box neural networks diff --git a/pyproject.toml b/pyproject.toml index 7f49cd9..f7c5b60 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ command_line = "-m pytest" [tool.coverage.report] precision = 4 -fail_under = 80 +fail_under = 100 [tool.mypy] strict = true diff --git a/src/revnets/context/context.py b/src/revnets/context/context.py index 0757889..28fa284 100644 --- a/src/revnets/context/context.py +++ b/src/revnets/context/context.py @@ -3,31 +3,15 @@ import torch from package_utils.context import Context as Context_ -from ..models import Config, HyperParameters, Options, Path +from ..models import Config, Options, Path class Context(Context_[Options, Config, None]): - @property - def training(self) -> HyperParameters: - return ( - self.config.reconstruction_training_debug - if self.config.debug - else self.config.reconstruction_training - ) - - @property - def number_of_epochs(self) -> int: - return self.training.epochs - - @property - def batch_size(self) -> int: - return self.training.batch_size - @cached_property def is_running_in_notebook(self) -> bool: try: get_ipython() # type: ignore[name-defined] - is_in_notebook = True + is_in_notebook = True # pragma: nocover except NameError: is_in_notebook = False return is_in_notebook @@ -56,7 +40,7 @@ def device(self) -> torch.device: @property def dtype(self) -> torch.dtype: - match self.config.precision: + match self.config.precision: # pragma: nocover case 32: dtype = torch.float32 case 64: diff --git a/src/revnets/data/mnist1d.py b/src/revnets/data/mnist1d.py index 2691cf7..9241ad7 100644 --- a/src/revnets/data/mnist1d.py +++ b/src/revnets/data/mnist1d.py @@ -2,12 +2,14 @@ import pickle from dataclasses import dataclass +from typing import cast import numpy as np import requests import torch from numpy.typing import NDArray from package_utils.dataclasses import SerializationMixin +from simple_classproperty import classproperty from sklearn.preprocessing import StandardScaler from torch.utils.data import TensorDataset @@ -46,8 +48,18 @@ def extract_test(self) -> TensorDataset: @dataclass class DataModule(base.DataModule): - path: Path = Path.data / "mnist_1D" - raw_path: Path = Path.data / "mnist_1D.pkl" + @classmethod + @classproperty + def path(cls) -> Path: + path = Path.data / "mnist_1D" + return cast(Path, path) + + @classmethod + @classproperty + def raw_path(cls) -> Path: + path = Path.data / "mnist_1D.pkl" + return cast(Path, path) + download_url: str = ( "https://github.com/greydanus/mnist1d/raw/master/mnist1d_data.pkl" ) diff --git a/src/revnets/evaluations/analysis/activations.py b/src/revnets/evaluations/analysis/activations.py index 6272a2e..c9a22f2 100644 --- a/src/revnets/evaluations/analysis/activations.py +++ b/src/revnets/evaluations/analysis/activations.py @@ -51,7 +51,7 @@ def visualize_model_outputs(self, model: Module, name: str) -> None: inputs = Reconstructor(self.pipeline).create_queries(self.n_inputs) outputs = QueryDataSet(model).compute_targets(inputs) if self.activation: - outputs = F.relu(outputs) + outputs = F.relu(outputs) # pragma: nocover ActivationsVisualizer(outputs, name).run() diff --git a/src/revnets/evaluations/attack/network.py b/src/revnets/evaluations/attack/network.py index a897ec7..6d4a811 100644 --- a/src/revnets/evaluations/attack/network.py +++ b/src/revnets/evaluations/attack/network.py @@ -80,8 +80,10 @@ def show_comparison( ) -> None: length = len(inputs[0]) indices = np.flip(np.arange(length)) + inputs_numpy = inputs.cpu().numpy()[:10] + adversarial_inputs_numpy = adversarial_inputs.cpu().numpy() - for image, adversarial in zip(inputs, adversarial_inputs): + for image, adversarial in zip(inputs_numpy, adversarial_inputs_numpy): plt.plot(image, indices, color="green", label="original") plt.plot(adversarial, indices, color="red", label="adversarial") plt.legend() diff --git a/src/revnets/evaluations/base.py b/src/revnets/evaluations/base.py index 14e92e4..273dfd5 100644 --- a/src/revnets/evaluations/base.py +++ b/src/revnets/evaluations/base.py @@ -37,4 +37,4 @@ def format_evaluation( return result def evaluate(self) -> Any: - raise NotImplementedError + raise NotImplementedError # pragma: nocover diff --git a/src/revnets/evaluations/evaluation.py b/src/revnets/evaluations/evaluation.py index 0238a3a..8a9ec7d 100644 --- a/src/revnets/evaluations/evaluation.py +++ b/src/revnets/evaluations/evaluation.py @@ -4,8 +4,10 @@ from dataclasses import asdict, dataclass, fields from typing import TYPE_CHECKING +import cli + if TYPE_CHECKING: - from ..utils.table import Table + from rich.table import Table # pragma: nocover @dataclass @@ -40,10 +42,9 @@ def dict(self) -> dict[str, str]: def values(self) -> Iterator[str]: yield from self.dict().values() - @property - def table(self) -> Table: + def create_table(self) -> Table: # slow import - from ..utils.table import Table + from rich.table import Table table = Table(show_lines=True) table.add_column("Metric", style="cyan", max_width=20, overflow="fold") @@ -54,4 +55,5 @@ def table(self) -> Table: return table def show(self) -> None: - self.table.show() + table = self.create_table() + cli.console.print(table) diff --git a/src/revnets/evaluations/weights/__init__.py b/src/revnets/evaluations/weights/__init__.py index d7f82d4..40b2f8a 100644 --- a/src/revnets/evaluations/weights/__init__.py +++ b/src/revnets/evaluations/weights/__init__.py @@ -1 +1 @@ -from . import layers_mae, mae, max_ae, mse, named_layers_mae, visualizer +from . import layers_mae, mae, max_ae, mse, named_layers_mae diff --git a/src/revnets/evaluations/weights/mse.py b/src/revnets/evaluations/weights/mse.py index c4a8548..a7e2502 100644 --- a/src/revnets/evaluations/weights/mse.py +++ b/src/revnets/evaluations/weights/mse.py @@ -18,7 +18,7 @@ def standardize_networks(self) -> bool: standardized = self.has_same_architecture() if standardized: if context.config.evaluation.use_align: - align(self.original, self.reconstruction) + align(self.original, self.reconstruction) # pragma: nocover else: for network in (self.original, self.reconstruction): Standardizer(network).standardize_scale() diff --git a/src/revnets/evaluations/weights/named_layers_mae.py b/src/revnets/evaluations/weights/named_layers_mae.py index d1b92ab..a09b444 100644 --- a/src/revnets/evaluations/weights/named_layers_mae.py +++ b/src/revnets/evaluations/weights/named_layers_mae.py @@ -30,5 +30,5 @@ def format_evaluation(cls, value: dict[str, float], precision: int = 3) -> str: } formatted_value = json.dumps(values, indent=4) else: - formatted_value = "/" + formatted_value = "/" # pragma: nocover return formatted_value diff --git a/src/revnets/evaluations/weights/standardize/network.py b/src/revnets/evaluations/weights/standardize/network.py index ac71d9a..30ef5b1 100644 --- a/src/revnets/evaluations/weights/standardize/network.py +++ b/src/revnets/evaluations/weights/standardize/network.py @@ -8,6 +8,7 @@ from revnets.models import InternalNeurons from . import order, scale +from .utils import extract_linear_layer_weights T = TypeVar("T") @@ -15,6 +16,7 @@ @dataclass class Standardizer: model: Module + optimize_mae: bool = False def run(self) -> None: """ @@ -23,12 +25,14 @@ def run(self) -> None: self.standardize_scale() for neurons in self.internal_neurons: order.Standardizer(neurons).run() + if self.optimize_mae: + self.apply_optimize_mae() def standardize_scale(self) -> None: for neurons in self.internal_neurons: scale.Standardizer(neurons).run() - def optimize_mae(self) -> None: + def apply_optimize_mae(self) -> None: # optimize mae by distributing last layer scale factor over all layers if all(neuron.has_norm_isomorphism for neuron in self.internal_neurons): desired_scale = self.calculate_average_scale_per_layer() @@ -37,7 +41,8 @@ def optimize_mae(self) -> None: scale.Standardizer(neurons).run() def calculate_average_scale_per_layer(self) -> float: - last_neuron_scales = self.internal_neurons[-1].outgoing.norm(dim=1, p=2) + weights = extract_linear_layer_weights(self.internal_neurons[-1].outgoing) + last_neuron_scales = weights.norm(dim=1, p=2) last_neuron_scale = sum(last_neuron_scales) / len(last_neuron_scales) num_internal_layers = len(self.internal_neurons) average_scale = last_neuron_scale ** (1 / num_internal_layers) diff --git a/src/revnets/evaluations/weights/visualizer.py b/src/revnets/evaluations/weights/visualizer.py deleted file mode 100644 index e399331..0000000 --- a/src/revnets/evaluations/weights/visualizer.py +++ /dev/null @@ -1,85 +0,0 @@ -from collections.abc import Iterator -from typing import Any - -import matplotlib.pyplot as plt -import torch -from torch.nn import Module - -from ...utils.colors import get_colors -from . import layers_mae -from .standardize import extract_layer_weights, generate_layers - -cpu = torch.device("cpu") - - -class Evaluator(layers_mae.Evaluator): - def evaluate(self) -> None: - self.standardize_networks() - networks = {"original": self.original, "reconstruction": self.reconstruction} - for name, network in networks.items(): - self.visualize_network_weights(network, name) - if self.has_same_architecture(): - self.visualize_network_differences() - - def visualize_network_weights(self, network: Module, name: str) -> None: - layers = generate_layers(network) - for i, layer in enumerate(layers): - weights = self.extract_layer_weights(layer) - if weights is not None: - title = f"{name} layer {i + 1} weights".capitalize() - self.visualize_weights(weights, title) - - @classmethod - def visualize_weights( - cls, weights: torch.Tensor, title: str, n_show: int | None = None - ) -> None: - weights = weights[:n_show] - - # weights = torch.transpose(weights, 0, 1) - - n_neurons = len(weights) - colors = get_colors(number_of_colors=n_neurons) - ax = cls.create_figure() - - for i, (neuron, color) in enumerate(zip(weights, colors)): - label = f"Neuron {i+1}" - ax.plot(neuron, color=color, label=label) - - n_features = len(weights[0]) - interval = n_features // 4 - x_ticks = list(range(0, n_features, interval)) + [n_features - 1] - if n_features - 2 not in x_ticks and False: - x_ticks.insert(-1, n_features - 2) - x_tick_labels = [str(xtick) for xtick in x_ticks[:-1]] + ["Bias weight"] - plt.xticks(x_ticks, x_tick_labels) - - plt.title(title) - plt.xlabel("Weight index") - plt.ylabel("Weight Value") - cls.show() - - def visualize_network_differences(self) -> None: - layers = self.iterate_compared_layers() - for i, (original, reconstructed) in enumerate(layers): - weights = original - reconstructed - title = f"Layer {i+1} weight differences" - self.visualize_weights(weights, title) - - @classmethod - def create_figure(cls) -> Any: - _, ax = plt.subplots(figsize=(20, 10)) - return ax - - @classmethod - def show(cls) -> None: - plt.legend(loc="upper left", bbox_to_anchor=(1, 1)) - plt.show() - - def iterate_compared_layers( - self, device: torch.device | None = cpu - ) -> Iterator[tuple[torch.Tensor, torch.Tensor]]: - return super().iterate_compared_layers(device=device) - - @classmethod - def extract_layer_weights(cls, layer: Module) -> torch.Tensor | None: - return extract_layer_weights(layer, device=cpu) diff --git a/src/revnets/models/__init__.py b/src/revnets/models/__init__.py index 3e379a3..9c66fb0 100644 --- a/src/revnets/models/__init__.py +++ b/src/revnets/models/__init__.py @@ -1,4 +1,4 @@ -from .config import Activation, Config, HyperParameters +from .config import Activation, Config, Evaluation, HyperParameters from .experiment import Experiment from .internal_neurons import InternalNeurons from .options import Options diff --git a/src/revnets/models/experiment.py b/src/revnets/models/experiment.py index 2e2274b..25e6158 100644 --- a/src/revnets/models/experiment.py +++ b/src/revnets/models/experiment.py @@ -24,10 +24,6 @@ def names(self) -> tuple[str, ...]: pipeline = "_".join(self.pipeline) return (reconstruction, pipeline, seeds) - @property - def name(self) -> str: - return "_".join(self.names) - @property def title(self) -> str: parts = self.generate_title_parts() diff --git a/src/revnets/models/path.py b/src/revnets/models/path.py index 9f9cdd7..53fc766 100644 --- a/src/revnets/models/path.py +++ b/src/revnets/models/path.py @@ -39,12 +39,6 @@ def weights(cls: type[T]) -> T: path = cls.assets / "weights" return typing.cast(T, path) - @classmethod - @classproperty - def outputs(cls: type[T]) -> T: - path = cls.assets / "outputs" - return typing.cast(T, path) - @classmethod @classproperty def results(cls: type[T]) -> T: diff --git a/src/revnets/models/split.py b/src/revnets/models/split.py index 23acfa9..3830f0b 100644 --- a/src/revnets/models/split.py +++ b/src/revnets/models/split.py @@ -6,15 +6,3 @@ class Split(Enum): valid = "valid" test = "test" train_val = "train_val" - - @property - def is_train(self) -> bool: - return self == Split.train - - @property - def is_valid(self) -> bool: - return self == Split.valid - - @property - def is_train_or_valid(self) -> bool: - return self.is_train or self.is_valid diff --git a/src/revnets/networks/base.py b/src/revnets/networks/base.py index da98826..719db71 100644 --- a/src/revnets/networks/base.py +++ b/src/revnets/networks/base.py @@ -34,8 +34,4 @@ def create_network(self, seed: int | None = None) -> Sequential: return Sequential(*layers) def create_layers(self) -> Iterable[torch.nn.Module]: - raise NotImplementedError - - @classmethod - def get_base_name(cls) -> str: - return NetworkFactory.__module__ + raise NotImplementedError # pragma: nocover diff --git a/src/revnets/pipelines/base.py b/src/revnets/pipelines/base.py index ecc9085..ea45dc7 100644 --- a/src/revnets/pipelines/base.py +++ b/src/revnets/pipelines/base.py @@ -5,10 +5,10 @@ class Pipeline(NamedClass): def create_target_network(self) -> Sequential: - raise NotImplementedError + raise NotImplementedError # pragma: nocover def create_initialized_network(self) -> Sequential: - raise NotImplementedError + raise NotImplementedError # pragma: nocover @classmethod def get_base_name(cls) -> str: diff --git a/src/revnets/pipelines/mininet/mininet_untrained.py b/src/revnets/pipelines/mininet/mininet_untrained.py index 4846f51..8b0252e 100644 --- a/src/revnets/pipelines/mininet/mininet_untrained.py +++ b/src/revnets/pipelines/mininet/mininet_untrained.py @@ -5,4 +5,4 @@ class Pipeline(mininet.Pipeline): def train(self, model: torch.nn.Module) -> None: - pass + pass # pragma: nocover diff --git a/src/revnets/pipelines/train.py b/src/revnets/pipelines/train.py index 0952411..60538c2 100644 --- a/src/revnets/pipelines/train.py +++ b/src/revnets/pipelines/train.py @@ -1,6 +1,5 @@ from abc import ABC from dataclasses import dataclass -from typing import cast import torch from pytorch_lightning import LightningModule @@ -52,7 +51,7 @@ def run_training(cls, network: LightningModule, data: DataModule) -> None: @classmethod def load_data(cls) -> DataModule: - raise NotImplementedError + raise NotImplementedError # pragma: nocover @classmethod def load_prepared_data(cls) -> DataModule: @@ -61,16 +60,6 @@ def load_prepared_data(cls) -> DataModule: data.setup("train") return data - def calculate_output_size(self) -> int: - data = self.load_data() - data.prepare_data() - sample = data.train_validation[0][0] - inputs = sample.unsqueeze(0) - model = self.create_initialized_network() - outputs = model(inputs)[0] - size = outputs.shape[-1] - return cast(int, size) - def load_weights(self, model: torch.nn.Module) -> None: state = torch.load(self.weights_path_str) model.load_state_dict(state) diff --git a/src/revnets/reconstructions/base.py b/src/revnets/reconstructions/base.py index 3970f2e..c2c0cd4 100644 --- a/src/revnets/reconstructions/base.py +++ b/src/revnets/reconstructions/base.py @@ -21,25 +21,25 @@ def __post_init__(self) -> None: def create_reconstruction(self) -> Sequential: if self.downscale_factor is not None: - self.scale_weights() + self.scale_weights() # pragma: nocover if context.config.start_reconstruction_with_zero_biases: - self.set_biases() + self.set_biases() # pragma: nocover self.reconstruct_weights() return self.reconstruction - def scale_weights(self) -> None: + def scale_weights(self) -> None: # pragma: nocover layers = generate_layers(self.reconstruction) for layer in layers: layer.weight.data /= self.downscale_factor - def set_biases(self) -> None: + def set_biases(self) -> None: # pragma: nocover layers = generate_layers(self.reconstruction) for layer in layers: bias = torch.zeros_like(layer.bias, dtype=layer.bias.dtype) layer.bias = torch.nn.Parameter(bias) def reconstruct_weights(self) -> None: - raise NotImplementedError + raise NotImplementedError # pragma: nocover @classmethod def get_base_name(cls) -> str: diff --git a/src/revnets/reconstructions/queries/base.py b/src/revnets/reconstructions/queries/base.py index a0a07f0..7c30f1c 100644 --- a/src/revnets/reconstructions/queries/base.py +++ b/src/revnets/reconstructions/queries/base.py @@ -27,7 +27,7 @@ def trained_weights_path(self) -> Path: def reconstruct_weights(self) -> None: weights_saved = self.trained_weights_path.exists() if context.config.reconstruct_from_checkpoint and weights_saved: - self.load_weights() + self.load_weights() # pragma: nocover if context.config.always_train or not weights_saved: self.start_training() self.save_weights() @@ -74,4 +74,4 @@ def add_queries(self, data: DataModule) -> None: dataset.add(validation_queries) def create_queries(self, num_samples: int) -> torch.Tensor: - raise NotImplementedError + raise NotImplementedError # pragma: nocover diff --git a/src/revnets/reconstructions/queries/data.py b/src/revnets/reconstructions/queries/data.py index 443bfd5..cba9cbd 100644 --- a/src/revnets/reconstructions/queries/data.py +++ b/src/revnets/reconstructions/queries/data.py @@ -66,7 +66,3 @@ def __post_init__(self) -> None: self.train = QueryDataSet(target, self.evaluation_batch_size) self.validation = QueryDataSet(target, self.evaluation_batch_size) self.test = QueryDataSet(target, self.evaluation_batch_size) - - @property - def has_validation_data(self) -> bool: - return self.validation_ratio is not None and self.validation_ratio > 0 diff --git a/src/revnets/reconstructions/queries/iterative/base.py b/src/revnets/reconstructions/queries/iterative/base.py index ac60125..4774c96 100644 --- a/src/revnets/reconstructions/queries/iterative/base.py +++ b/src/revnets/reconstructions/queries/iterative/base.py @@ -43,4 +43,4 @@ def add_difficult_inputs(self, data: DataModule) -> None: data.train.add(queries) def create_difficult_samples(self) -> torch.Tensor: - raise NotImplementedError + raise NotImplementedError # pragma: nocover diff --git a/src/revnets/reconstructions/queries/iterative/difficult_inputs.py b/src/revnets/reconstructions/queries/iterative/difficult_inputs.py index 61a0ba6..4463d43 100644 --- a/src/revnets/reconstructions/queries/iterative/difficult_inputs.py +++ b/src/revnets/reconstructions/queries/iterative/difficult_inputs.py @@ -31,7 +31,7 @@ def __init__( def on_train_start(self) -> None: if self.verbose: - print("\nAverage pairwise distances: ", end="\n\t") + print("\nAverage pairwise distances: ", end="\n\t") # pragma: nocover @classmethod def get_input_embeddings(cls, shape: tuple[int, int]) -> torch.nn.Embedding: @@ -53,7 +53,7 @@ def calculate_loss(self, outputs: torch.Tensor) -> torch.Tensor: distance_total = pairwise_distances.mean() loss = -distance_total if self.verbose: - print(f"{distance_total.item():.3f}", end=" ") + print(f"{distance_total.item():.3f}", end=" ") # pragma: nocover return loss def training_step(self, batch: torch.Tensor, batch_idx: int) -> torch.Tensor: diff --git a/src/revnets/training/network.py b/src/revnets/training/network.py index 3ab17ef..29813c6 100644 --- a/src/revnets/training/network.py +++ b/src/revnets/training/network.py @@ -46,7 +46,7 @@ def obtain_metrics(self, batch: torch.Tensor, phase: Phase) -> Metrics: return metrics def calculate_metrics(self, outputs: torch.Tensor, labels: torch.Tensor) -> Metrics: - raise NotImplementedError + raise NotImplementedError # pragma: nocover def configure_optimizers(self) -> optim.Optimizer: return optim.Adam(self.parameters(), lr=self.learning_rate) diff --git a/src/revnets/training/reconstructions/metrics.py b/src/revnets/training/reconstructions/metrics.py index 89c7712..0f9b990 100644 --- a/src/revnets/training/reconstructions/metrics.py +++ b/src/revnets/training/reconstructions/metrics.py @@ -30,12 +30,12 @@ def l1_loss(self) -> torch.Tensor: return self.l1_loss_sum / self.size @property - def l2_loss(self) -> torch.Tensor: + def l2_loss(self) -> torch.Tensor: # pragma: nocover return self.l2_loss_sum / self.size @property def loss_sum(self) -> torch.Tensor: - match context.config.loss_criterion: + match context.config.loss_criterion: # pragma: nocover # case "smooth_l1": # loss = self.smooth_l1_loss case "l1": diff --git a/src/revnets/training/reconstructions/network.py b/src/revnets/training/reconstructions/network.py index 34a8eed..42131c0 100644 --- a/src/revnets/training/reconstructions/network.py +++ b/src/revnets/training/reconstructions/network.py @@ -20,7 +20,3 @@ def calculate_metrics( targets: torch.Tensor, ) -> Metrics: return Metrics.from_results(outputs, targets) - - @classmethod - def extract_loss(cls, metrics: Metrics) -> torch.Tensor: - return metrics.loss diff --git a/src/revnets/training/trainer.py b/src/revnets/training/trainer.py index 6f80fbb..90e5359 100644 --- a/src/revnets/training/trainer.py +++ b/src/revnets/training/trainer.py @@ -52,21 +52,3 @@ def generate_callbacks( metrics_format=".3e", metrics_text_delimiter="\n" ) yield RichProgressBar(theme=theme) - - @property - def log_message(self) -> str: - from ..utils.table import Table - - table = Table() - table.add_column("Test metric", style="cyan", no_wrap=True) - table.add_column("Value", style="magenta") - - for name, value in self.logged_metrics.items(): - if name != "step": - name = name.replace("test ", "") - value_float = value.item() - display_format = ".3%" if "accuracy" in name else ".3f" - value_str = f"{value_float :{display_format}}".replace("%", " %") - table.add_row(name, value_str) - - return table.text diff --git a/src/revnets/utils/__init__.py b/src/revnets/utils/__init__.py index bf250f9..e1dade6 100644 --- a/src/revnets/utils/__init__.py +++ b/src/revnets/utils/__init__.py @@ -1,2 +1 @@ -from . import pl_logger from .named_class import NamedClass diff --git a/src/revnets/utils/colors.py b/src/revnets/utils/colors.py index 30ca648..17b87c4 100644 --- a/src/revnets/utils/colors.py +++ b/src/revnets/utils/colors.py @@ -31,5 +31,5 @@ def generate_color_maps( for number, color_map in mappers.items(): if number_of_colors <= number: yield number_of_colors, color_map - color_maps = list(mappers[60]) * (number_of_colors // 60 + 1) - yield number_of_colors, tuple(color_maps) + color_maps = list(mappers[60]) * (number_of_colors // 60 + 1) # pragma: nocover + yield number_of_colors, tuple(color_maps) # pragma: nocover diff --git a/src/revnets/utils/named_class.py b/src/revnets/utils/named_class.py index c28b84c..b1ca5e6 100644 --- a/src/revnets/utils/named_class.py +++ b/src/revnets/utils/named_class.py @@ -4,12 +4,7 @@ class NamedClass: @classmethod def get_base_name(cls) -> str: - raise NotImplementedError - - @classmethod - @classproperty - def title(cls) -> str: - return " ".join(cls.relative_module).capitalize() + raise NotImplementedError # pragma: nocover @classmethod @classproperty diff --git a/src/revnets/utils/pl_logger.py b/src/revnets/utils/pl_logger.py deleted file mode 100644 index d13a5ef..0000000 --- a/src/revnets/utils/pl_logger.py +++ /dev/null @@ -1,30 +0,0 @@ -import logging -import warnings -from collections.abc import Sequence -from types import TracebackType - - -class Quiet: - def __init__(self, names: Sequence[str] | None = None) -> None: - if names is None: - names = ("pytorch_lightning", "lightning") - self.names = names - self.old_loglevel = None - - def __enter__(self) -> None: - self.old_log_levels = [ - logging.getLogger(name).getEffectiveLevel() for name in self.names - ] - for name in self.names: - logger = logging.getLogger(name) - logger.setLevel(0) - warnings.filterwarnings("ignore", ".*does not have many workers.*") - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - pass - # always keep level 0 diff --git a/src/revnets/utils/table.py b/src/revnets/utils/table.py deleted file mode 100644 index 8d80b13..0000000 --- a/src/revnets/utils/table.py +++ /dev/null @@ -1,20 +0,0 @@ -import cli -from rich import table - -from ..models import Path - - -class Table(table.Table): - @property - def text(self) -> str: - from rich.console import Console # noqa: autoimport - - with Path.tempfile() as tmp: - with tmp.open("w") as fp: - console = Console(record=True, file=fp, force_terminal=True) - console.print(self) - message = console.export_text(styles=True) - return message - - def show(self) -> None: - cli.console.print(self, highlight=False) diff --git a/tests/conftest.py b/tests/conftest.py index fa51b90..75f9346 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,7 @@ import pytest from revnets.context import context as context_ from revnets.context.context import Context -from revnets.models import Config, HyperParameters, Path +from revnets.models import Config, Evaluation, HyperParameters, Path @pytest.fixture(scope="session") @@ -26,11 +26,13 @@ def mocked_assets_path() -> Iterator[None]: def test_context(context: Context, mocked_assets_path: None) -> Iterator[None]: config = context.config hyperparameters = HyperParameters(epochs=1, learning_rate=1.0e-2, batch_size=32) + evaluation = Evaluation(visualize_attack=True) context.loaders.config.value = Config( target_network_training=hyperparameters, reconstruction_training=hyperparameters, max_difficult_inputs_epochs=1, run_analysis=True, + evaluation=evaluation, ) mock = patch("matplotlib.pyplot.show") with mock: diff --git a/tests/test_evaluations.py b/tests/evaluations/test_evaluations.py similarity index 88% rename from tests/test_evaluations.py rename to tests/evaluations/test_evaluations.py index 3685c91..709e86a 100644 --- a/tests/test_evaluations.py +++ b/tests/evaluations/test_evaluations.py @@ -4,6 +4,7 @@ import pytest from revnets import evaluations, pipelines, reconstructions from revnets.evaluations import analysis, attack, outputs, weights +from revnets.evaluations.evaluate import format_percentage from revnets.pipelines import Pipeline evaluation_modules = ( @@ -50,4 +51,8 @@ def test_evaluations( ) -> None: reconstructor = reconstructions.empty.Reconstructor(pipeline) reconstruction = reconstructor.create_reconstruction() - evaluation_module.Evaluator(reconstruction, pipeline).evaluate() + evaluation_module.Evaluator(reconstruction, pipeline).get_evaluation() + + +def test_format_percentage() -> None: + format_percentage(0.1) diff --git a/tests/evaluations/test_standardize.py b/tests/evaluations/test_standardize.py index 8b6f48a..1a58a9e 100644 --- a/tests/evaluations/test_standardize.py +++ b/tests/evaluations/test_standardize.py @@ -71,3 +71,11 @@ def test_second_standardize_no_effect( ) -> None: tester = Verifier(network_module, activation, standardization_type) tester.test_second_standardize_no_effect() + + +@pytest.mark.parametrize("network_module", network_modules) +@pytest.mark.parametrize("activation", activations) +def test_optimize_mae(network_module: ModuleType, activation: Activation) -> None: + standardization_type = StandardizationType.standardize + tester = Verifier(network_module, activation, standardization_type) + tester.test_optimize_mae() diff --git a/tests/evaluations/verifier.py b/tests/evaluations/verifier.py index 7edccbb..4fa9bf7 100644 --- a/tests/evaluations/verifier.py +++ b/tests/evaluations/verifier.py @@ -104,6 +104,9 @@ def test_second_standardize_no_effect(self) -> None: is_close = torch.isclose(value, second_value) assert torch.all(is_close) + def test_optimize_mae(self) -> None: + Standardizer(self.network, optimize_mae=True).run() + def verify_scale_standardized(neurons: InternalNeurons) -> None: neurons_standardizer = standardize.scale.Standardizer(neurons)