diff --git a/file_retriever/_clients.py b/file_retriever/_clients.py index 2ff3ac3..c1769b9 100644 --- a/file_retriever/_clients.py +++ b/file_retriever/_clients.py @@ -8,6 +8,7 @@ import ftplib import os import paramiko +import sys from typing import List, Union, Optional from file_retriever.file import File @@ -35,7 +36,7 @@ def __init__( port: port number for server """ - self.connection = self.__create_ftp_connection( + self.connection = self._create_ftp_connection( username=username, password=password, host=host, port=int(port) ) @@ -55,7 +56,7 @@ def __exit__(self, *args): """ self.connection.close() - def __create_ftp_connection( + def _create_ftp_connection( self, username: str, password: str, host: str, port: int ) -> ftplib.FTP: """ @@ -102,18 +103,21 @@ def get_remote_file_data(self, file: str, remote_dir: Optional[str] = None) -> F """ if remote_dir is not None: remote_file = f"{remote_dir}/{file}" - file_name = file else: remote_file = file - file_name = os.path.basename(remote_file) + file_name = os.path.basename(remote_file) try: + permissions = None def get_file_permissions(data): nonlocal permissions - permissions = File.parse_permissions(data) + permissions = File.parse_permissions(permissions_str=data) self.connection.retrlines(f"LIST {remote_file}", get_file_permissions), + if permissions is None: + raise ftplib.error_perm("File not found on server.") + return File( file_name=file_name, file_size=self.connection.size(remote_file), @@ -123,7 +127,13 @@ def get_file_permissions(data): file_mode=permissions, ) except ftplib.error_reply: - raise + raise ftplib.error_reply( + f"Received unexpected response from server: {sys.exc_info()[1]}" + ) + except ftplib.error_perm: + raise ftplib.error_perm( + f"Unable to retrieve file data: {sys.exc_info()[1]}" + ) def list_file_data(self, remote_dir: str) -> List[File]: """ @@ -173,7 +183,7 @@ def download_file(self, file: str, remote_dir: str, local_dir: str) -> None: try: with open(local_file, "wb") as f: self.connection.retrbinary(f"RETR {remote_file}", f.write) - except OSError: + except (OSError, ftplib.error_reply): raise def upload_file(self, file: str, remote_dir: str, local_dir: str) -> File: @@ -229,7 +239,7 @@ def __init__( port: port number for server """ - self.connection = self.__create_sftp_connection( + self.connection = self._create_sftp_connection( username=username, password=password, host=host, port=int(port) ) @@ -249,7 +259,7 @@ def __exit__(self, *args): """ self.connection.close() - def __create_sftp_connection( + def _create_sftp_connection( self, username: str, password: str, host: str, port: int ) -> paramiko.SFTPClient: """ diff --git a/file_retriever/connect.py b/file_retriever/connect.py index b9c3ff1..c9eb4da 100644 --- a/file_retriever/connect.py +++ b/file_retriever/connect.py @@ -64,6 +64,26 @@ def _create_client( case _: raise ValueError(f"Invalid port number: {self.port}") + def check_file(self, file: str, check_dir: str, remote: bool) -> bool: + """ + Check if `file` exists in `check_dir`. If `remote` is True then check will + be performed on server, otherwise check will be performed locally. + + Args: + file: name of file to check + check_dir: directory to check for file + remote: whether to check file on server (True) or locally (False) + + Returns: + bool indicating if file exists in `check_dir` + """ + with self.session as session: + if remote: + remote_file = session.get_remote_file_data(file, check_dir) + return remote_file.file_name == file + else: + return os.path.exists(os.path.join(check_dir, file)) + def get_file_data(self, file: str, remote_dir: Optional[str] = None) -> List[File]: """ Retrieve metadata for file in `remote_dir` on server. If `remote_dir` is not @@ -120,16 +140,19 @@ def get_file( file: str, remote_dir: Optional[str] = None, local_dir: str = ".", + check: bool = True, ) -> File: """ Downloads `file` from `remote_dir` on server to `local_dir`. If `remote_dir` is not provided then file will be downloaded from `self.remote_dir`. If - `local_dir` is not provided then file will be downloaded to cwd. + `local_dir` is not provided then file will be downloaded to cwd. If `check` is + True, then `local_dir` will be checked for file before downloading. Args: file: name of file to download remote_dir: directory on server to download file from local_dir: local directory to download file to + check: check if file exists in `local_dir` before downloading Returns: file downloaded to `local_dir` as `File` object @@ -137,6 +160,11 @@ def get_file( if not remote_dir or remote_dir is None: remote_dir = self.remote_dir with self.session as session: + if check and self.check_file(file, check_dir=local_dir, remote=False): + raise FileExistsError( + f"File {file} already exists in {local_dir}. File not downloaded." + ) + session.download_file(file=file, remote_dir=remote_dir, local_dir=local_dir) local_file = os.path.normpath(os.path.join(local_dir, file)) @@ -147,16 +175,19 @@ def put_file( file: str, local_dir: str = ".", remote_dir: Optional[str] = None, + check: bool = True, ) -> File: """ Uploads file from local directory to server. If `remote_dir` is not provided then file will be uploaded to `self.remote_dir`. If `local_dir` - is not provided then file will be uploaded from cwd. + is not provided then file will be uploaded from cwd. If `check` is + True, then `remote_dir` will be checked for file before downloading. Args: file: name of file to upload local_dir: local directory to upload file from remote_dir: remote directory to upload file to + check: check if file exists in `remote_dir` before uploading Returns: file uploaded to `remote_dir` as `File` object @@ -164,6 +195,10 @@ def put_file( if remote_dir is None: remote_dir = self.remote_dir with self.session as session: + if check and self.check_file(file, check_dir=remote_dir, remote=True): + raise FileExistsError( + f"File {file} already exists in {remote_dir}. File not uploaded." + ) uploaded_file = session.upload_file( file=file, remote_dir=remote_dir, local_dir=local_dir ) diff --git a/file_retriever/file.py b/file_retriever/file.py index f65e021..03563c3 100644 --- a/file_retriever/file.py +++ b/file_retriever/file.py @@ -73,7 +73,7 @@ def parse_mdtm_time(mdtm_time: str) -> int: ) @staticmethod - def parse_permissions(permissions: str) -> int: + def parse_permissions(permissions_str: str) -> int: """ parse permissions string to decimal value. @@ -91,9 +91,9 @@ def parse_permissions(permissions: str) -> int: (filetype * 8^5) + (0 * 8^4) + (0 * 8^3) + (owner * 8^2) + (group * 8^1) + (others * 8^0) = decimal value """ - file_type = permissions[0].replace("d", "4").replace("-", "1") + file_type = permissions_str[0].replace("d", "4").replace("-", "1") file_perm = ( - permissions[1:10] + permissions_str[1:10] .replace("-", "0") .replace("r", "4") .replace("w", "2") diff --git a/tests/conftest.py b/tests/conftest.py index ca82d01..7d28ad3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,9 +2,11 @@ import ftplib import os import paramiko -from typing import Dict, List, Optional +from typing import Dict, List import yaml import pytest +from file_retriever._clients import _ftpClient, _sftpClient +from file_retriever.connect import Client class FakeUtcNow(datetime.datetime): @@ -13,50 +15,43 @@ def now(cls, tzinfo=datetime.timezone.utc): return cls(2024, 6, 1, 1, 0, 0, 0, tzinfo=datetime.timezone.utc) -class MockSFTPAttributes: +class MockFileData: """File properties for a mock file object.""" def __init__(self): - pass - - @property - def st_mtime(self, *args, **kwargs): - """1704070800 is equivalent to 2024-01-01 01:00:00""" - return 1704070800 - - @property - def st_size(self, *args, **kwargs): - return 140401 - - @property - def st_uid(self, *args, **kwargs): - return 0 - - @property - def st_gid(self, *args, **kwargs): - return 0 - - @property - def st_atime(self, *args, **kwargs): - return None - - @property - def st_mode(self, *args, **kwargs): - return 33188 + self.file_name = "foo.mrc" + self.st_mtime = 1704070800 + self.st_mode = 33188 + self.st_atime = None + self.st_gid = 0 + self.st_uid = 0 + self.st_size = 140401 @pytest.fixture def mock_file_data(monkeypatch): def mock_stat(*args, **kwargs): - return MockSFTPAttributes() + return MockFileData() monkeypatch.setattr(os, "stat", mock_stat) +class Mock_SFTPAttributes(paramiko.SFTPAttributes): + + def __init__(self): + self.filename = "foo.mrc" + self.st_mtime = 1704070800 + self.st_size = 140401 + self.st_uid = 0 + self.st_gid = 0 + self.st_mode = 33188 + self.st_atime = None + + class MockFTP: """Mock response from FTP server for a successful login""" - def __init__(self): + def __init__(self, *args, **kwargs): pass def __enter__(self): @@ -65,25 +60,19 @@ def __enter__(self): def __exit__(self, *args) -> None: self.close() - def connect(self, *args, **kwargs) -> None: - pass - def close(self, *args, **kwargs) -> None: pass def getwelcome(self, *args, **kwargs) -> str: return "220 (vsFTPd 3.0.5)" - def login(self, *args, **kwargs) -> None: - pass - def nlst(self, *args, **kwargs) -> List[str]: return ["foo.mrc"] def retrbinary(self, *args, **kwargs) -> None: pass - def retrlines(self, *args, **kwargs) -> List[str]: + def retrlines(self, *args, **kwargs) -> str: files = "-rw-r--r-- 1 0 0 140401 Jan 1 00:01 foo.mrc" return args[1](files) @@ -93,11 +82,8 @@ def size(self, *args, **kwargs) -> int: def storbinary(self, *args, **kwargs) -> None: pass - def voidcmd(self, *args, **kwargs) -> Optional[str]: - if "MDTM" in args[0]: - return "213 20240101010000" - else: - return None + def voidcmd(self, *args, **kwargs) -> str: + return "213 20240101010000" class MockSFTPClient: @@ -112,30 +98,20 @@ def __enter__(self): def __exit__(self, *args) -> None: self.close() - def chdir(self, *args, **kwargs) -> None: - pass - def close(self, *args, **kwargs) -> None: pass def get(self, *args, **kwargs) -> None: open(args[1], "x+") - def listdir(self, *args, **kwargs) -> List[str]: - return ["foo.mrc"] - def listdir_attr(self, *args, **kwargs) -> List[paramiko.SFTPAttributes]: - file = paramiko.SFTPAttributes.from_stat(os.stat("foo.mrc"), filename="foo.mrc") - file.longname = ( - "-rw-r--r-- 1 0 0 140401 Jan 1 00:01 foo.mrc" - ) - return [file] + return [Mock_SFTPAttributes()] def put(self, *args, **kwargs) -> paramiko.SFTPAttributes: - return paramiko.SFTPAttributes.from_stat(os.stat(args[0]), filename=args[0]) + return Mock_SFTPAttributes() def stat(self, *args, **kwargs) -> paramiko.SFTPAttributes: - return paramiko.SFTPAttributes.from_stat(os.stat(args[0]), filename=args[0]) + return Mock_SFTPAttributes() class MockSSHClient: @@ -156,21 +132,35 @@ def set_missing_host_key_policy(self, *args, **kwargs) -> None: @pytest.fixture def mock_client(monkeypatch): - def mock_ftp_client(*args, **kwargs): - return MockFTP() + def mock_login(*args, **kwargs): + pass - def mock_ssh_client(*args, **kwargs): - return MockSSHClient() + def mock_file_data(*args, **kwargs): + return MockFileData() - def mock_SFTPAttributes_from_stat(*args, **kwargs): - return MockSFTPAttributes() + def mock_file_exists(*args, **kwargs): + return False - monkeypatch.setattr(os, "stat", mock_SFTPAttributes_from_stat) - monkeypatch.setattr(ftplib, "FTP", mock_ftp_client) - monkeypatch.setattr(paramiko, "SSHClient", mock_ssh_client) + monkeypatch.setattr(os.path, "exists", mock_file_exists) + monkeypatch.setattr(os, "stat", mock_file_data) + monkeypatch.setattr(ftplib.FTP, "login", mock_login) + monkeypatch.setattr(ftplib.FTP, "connect", mock_login) + monkeypatch.setattr(paramiko.SSHClient, "connect", mock_login) + monkeypatch.setattr(paramiko.SSHClient, "open_sftp", MockSFTPClient) monkeypatch.setattr(datetime, "datetime", FakeUtcNow) +@pytest.fixture +def stub_sftp_client(monkeypatch, mock_open_file, mock_client): + def mock_sftp_client(*args, **kwargs): + return MockSFTPClient() + + monkeypatch.setattr(_sftpClient, "_create_sftp_connection", mock_sftp_client) + monkeypatch.setattr(paramiko.SFTPClient, "get", mock_sftp_client) + monkeypatch.setattr(paramiko.SFTPClient, "listdir_attr", mock_sftp_client) + monkeypatch.setattr(paramiko.SFTPClient, "put", mock_sftp_client) + + @pytest.fixture def mock_open_file(mocker): m = mocker.mock_open() @@ -180,7 +170,53 @@ def mock_open_file(mocker): @pytest.fixture def stub_client(monkeypatch, mock_open_file, mock_client): - pass + def mock_ftp_client(*args, **kwargs): + return MockFTP() + + def mock_sftp_client(*args, **kwargs): + return MockSFTPClient() + + def mock_file_not_found(*args, **kwargs): + return False + + monkeypatch.setattr(_ftpClient, "_create_ftp_connection", mock_ftp_client) + monkeypatch.setattr(_sftpClient, "_create_sftp_connection", mock_sftp_client) + monkeypatch.setattr(Client, "check_file", mock_file_not_found) + + +@pytest.fixture +def stub_client_tmp_path(monkeypatch, mock_client): + def mock_ftp_client(*args, **kwargs): + return MockFTP() + + def mock_sftp_client(*args, **kwargs): + return MockSFTPClient() + + monkeypatch.setattr(_ftpClient, "_create_ftp_connection", mock_ftp_client) + monkeypatch.setattr(_sftpClient, "_create_sftp_connection", mock_sftp_client) + + +@pytest.fixture +def stub_client_file_exists(monkeypatch, mock_open_file, mock_client): + def mock_ftp_client(*args, **kwargs): + return MockFTP() + + def mock_sftp_client(*args, **kwargs): + return MockSFTPClient() + + def mock_file_exists(*args, **kwargs): + file = MockFileData() + file.file_name = "bar.mrc" + return file + + def path_exists(*args, **kwargs): + return True + + monkeypatch.setattr(os.path, "exists", path_exists) + monkeypatch.setattr(_ftpClient, "_create_ftp_connection", mock_ftp_client) + monkeypatch.setattr(_sftpClient, "_create_sftp_connection", mock_sftp_client) + # monkeypatch.setattr(_ftpClient, "get_remote_file_data", mock_file_exists) + # monkeypatch.setattr(_sftpClient, "get_remote_file_data", mock_file_exists) class MockOSError: @@ -220,50 +256,68 @@ def __init__(self): @pytest.fixture -def client_file_error(monkeypatch, mock_open_file, mock_client): - def mock_os_error(*args, **kwargs): - return MockOSError() +def mock_auth_error(monkeypatch, mock_client): + def mock_ssh_auth_error(*args, **kwargs): + return MockAuthenticationException() - monkeypatch.setattr(os, "stat", mock_os_error) - monkeypatch.setattr(MockFTP, "retrbinary", mock_os_error) - monkeypatch.setattr(MockFTP, "storbinary", mock_os_error) - monkeypatch.setattr(MockSFTPClient, "get", mock_os_error) - monkeypatch.setattr(MockSFTPClient, "listdir", mock_os_error) + def mock_ftp_error_perm(*args, **kwargs): + return MockErrorPerm() + + monkeypatch.setattr(ftplib.FTP, "login", mock_ftp_error_perm) + monkeypatch.setattr(paramiko.SSHClient, "connect", mock_ssh_auth_error) @pytest.fixture -def client_error_reply(monkeypatch, mock_client, mock_open_file): - def mock_ftp_error_reply(*args, **kwargs): - return MockErrorReply() +def mock_login_connection_error(monkeypatch, mock_client): + def mock_ftp_error_perm(*args, **kwargs): + return MockErrorTemp() - monkeypatch.setattr(MockFTP, "retrlines", mock_ftp_error_reply) - monkeypatch.setattr(MockFTP, "retrbinary", mock_ftp_error_reply) - monkeypatch.setattr(MockFTP, "storbinary", mock_ftp_error_reply) - monkeypatch.setattr(MockFTP, "voidcmd", mock_ftp_error_reply) + def mock_ssh_error(*args, **kwargs): + return MockSSHException() + + monkeypatch.setattr(paramiko.SSHClient, "connect", mock_ssh_error) + monkeypatch.setattr(ftplib.FTP, "login", mock_ftp_error_perm) @pytest.fixture -def client_auth_error(monkeypatch, stub_client): - def mock_ssh_auth_error(*args, **kwargs): - return MockAuthenticationException() +def mock_permissions_error(monkeypatch, mock_open_file, mock_client): + def mock_retrlines(*args, **kwargs): + return None + + monkeypatch.setattr(ftplib.FTP, "retrlines", mock_retrlines) + + +@pytest.fixture +def mock_file_error(monkeypatch, mock_open_file, stub_client): + def mock_os_error(*args, **kwargs): + return MockOSError() - def mock_ftp_auth_error(*args, **kwargs): + def mock_ftp_error_perm(*args, **kwargs): return MockErrorPerm() - monkeypatch.setattr(MockSSHClient, "connect", mock_ssh_auth_error) - monkeypatch.setattr(MockFTP, "login", mock_ftp_auth_error) + def mock_retrlines(*args, **kwargs): + return None + monkeypatch.setattr(MockFTP, "voidcmd", mock_ftp_error_perm) + monkeypatch.setattr(MockFTP, "size", mock_ftp_error_perm) + monkeypatch.setattr(MockFTP, "retrlines", mock_retrlines) + monkeypatch.setattr(MockFTP, "retrbinary", mock_os_error) + monkeypatch.setattr(MockFTP, "storbinary", mock_os_error) + monkeypatch.setattr(MockSFTPClient, "stat", mock_os_error) + monkeypatch.setattr(MockSFTPClient, "get", mock_os_error) + monkeypatch.setattr(MockSFTPClient, "put", mock_os_error) + monkeypatch.setattr(MockSFTPClient, "listdir_attr", mock_os_error) -@pytest.fixture -def client_other_error(monkeypatch, stub_client): - def mock_ssh_error(*args, **kwargs): - return MockSSHException() - def mock_ftp_error_temp(*args, **kwargs): - return MockErrorTemp() +@pytest.fixture +def mock_connection_error_reply(monkeypatch, mock_open_file, mock_client): + def mock_ftp_error_reply(*args, **kwargs): + return MockErrorReply() - monkeypatch.setattr(MockSSHClient, "connect", mock_ssh_error) - monkeypatch.setattr(MockFTP, "__init__", mock_ftp_error_temp) + monkeypatch.setattr(ftplib.FTP, "storbinary", mock_ftp_error_reply) + monkeypatch.setattr(ftplib.FTP, "retrbinary", mock_ftp_error_reply) + monkeypatch.setattr(ftplib.FTP, "retrlines", mock_ftp_error_reply) + monkeypatch.setattr(ftplib.FTP, "nlst", mock_ftp_error_reply) @pytest.fixture diff --git a/tests/test_clients.py b/tests/test_clients.py index cd60f7a..adacf37 100644 --- a/tests/test_clients.py +++ b/tests/test_clients.py @@ -32,12 +32,12 @@ def test_ftpClient_no_creds(self, stub_client): with pytest.raises(TypeError): _ftpClient(**creds) - def test_ftpClient_auth_error(self, client_auth_error, stub_creds): + def test_ftpClient_error_perm(self, mock_auth_error, stub_creds): stub_creds["port"] = "21" with pytest.raises(ftplib.error_perm): _ftpClient(**stub_creds) - def test_ftpClient_other_error(self, client_other_error, stub_creds): + def test_ftpClient_error_temp(self, mock_login_connection_error, stub_creds): stub_creds["port"] = "21" with pytest.raises(ftplib.error_temp): _ftpClient(**stub_creds) @@ -57,13 +57,29 @@ def test_ftpClient_get_remote_file_data(self, stub_client, stub_creds): ) def test_ftpClient_get_remote_file_data_connection_error( - self, client_error_reply, stub_creds + self, mock_connection_error_reply, stub_creds ): stub_creds["port"] = "21" ftp = _ftpClient(**stub_creds) with pytest.raises(ftplib.error_reply): ftp.get_remote_file_data("foo.mrc", "testdir") + def test_ftpClient_get_remote_file_data_not_found( + self, mock_file_error, stub_creds + ): + stub_creds["port"] = "21" + ftp = _ftpClient(**stub_creds) + with pytest.raises(ftplib.error_perm): + ftp.get_remote_file_data("foo.mrc", "testdir") + + def test_ftpClient_get_remote_file_data_non_permissions( + self, mock_permissions_error, stub_creds + ): + stub_creds["port"] = "21" + ftp = _ftpClient(**stub_creds) + with pytest.raises(ftplib.error_perm): + ftp.get_remote_file_data("foo.mrc", "testdir") + def test_ftpClient_list_file_data(self, stub_client, stub_creds): stub_creds["port"] = "21" ftp = _ftpClient(**stub_creds) @@ -81,7 +97,7 @@ def test_ftpClient_list_file_data(self, stub_client, stub_creds): ] def test_ftpClient_list_file_data_connection_error( - self, client_error_reply, stub_creds + self, mock_connection_error_reply, stub_creds ): stub_creds["port"] = "21" ftp = _ftpClient(**stub_creds) @@ -89,7 +105,7 @@ def test_ftpClient_list_file_data_connection_error( ftp.list_file_data("testdir") @pytest.mark.tmpdir - def test_ftpClient_download_file(self, tmp_path, mock_client, stub_creds): + def test_ftpClient_download_file(self, tmp_path, stub_client_tmp_path, stub_creds): stub_creds["port"] = "21" path = tmp_path / "test" path.mkdir() @@ -103,13 +119,15 @@ def test_ftpClient_download_mock_file(self, stub_client, stub_creds): ftp = _ftpClient(**stub_creds) ftp.download_file(file="foo.mrc", remote_dir="bar", local_dir="test") - def test_ftpClient_download_file_not_found(self, client_file_error, stub_creds): + def test_ftpClient_download_file_not_found(self, mock_file_error, stub_creds): stub_creds["port"] = "21" with pytest.raises(OSError): ftp = _ftpClient(**stub_creds) ftp.download_file(file="foo.mrc", remote_dir="bar", local_dir="test") - def test_ftpClient_download_connection_error(self, client_error_reply, stub_creds): + def test_ftpClient_download_connection_error( + self, mock_connection_error_reply, stub_creds + ): stub_creds["port"] = "21" with pytest.raises(ftplib.error_reply): ftp = _ftpClient(**stub_creds) @@ -121,13 +139,15 @@ def test_ftpClient_upload_file(self, stub_client, stub_creds): file = ftp.upload_file(file="foo.mrc", local_dir="foo", remote_dir="bar") assert file.file_mtime == 1704070800 - def test_ftpClient_upload_file_not_found(self, client_file_error, stub_creds): + def test_ftpClient_upload_file_not_found(self, mock_file_error, stub_creds): stub_creds["port"] = "21" with pytest.raises(OSError): ftp = _ftpClient(**stub_creds) ftp.upload_file(file="foo.mrc", remote_dir="foo", local_dir="bar") - def test_ftpClient_upload_connection_error(self, client_error_reply, stub_creds): + def test_ftpClient_upload_connection_error( + self, mock_connection_error_reply, stub_creds + ): stub_creds["port"] = "21" with pytest.raises(ftplib.error_reply): ftp = _ftpClient(**stub_creds) @@ -137,33 +157,33 @@ def test_ftpClient_upload_connection_error(self, client_error_reply, stub_creds) class TestMock_sftpClient: """Test the _sftpClient class with mock responses.""" - def test_sftpClient(self, stub_client, stub_creds): + def test_sftpClient(self, mock_client, stub_creds): stub_creds["port"] = "22" sftp = _sftpClient(**stub_creds) assert sftp.connection is not None @pytest.mark.parametrize("port", [None, [], {}]) - def test_sftpClient_port_TypeError(self, stub_client, stub_creds, port): + def test_sftpClient_port_TypeError(self, mock_client, stub_creds, port): stub_creds["port"] = port with pytest.raises(TypeError): _sftpClient(**stub_creds) - def test_sftpClient_no_creds(self, stub_client): + def test_sftpClient_no_creds(self, mock_client): creds = {} with pytest.raises(TypeError): _sftpClient(**creds) - def test_sftpClient_auth_error(self, client_auth_error, stub_creds): + def test_sftpClient_auth_error(self, mock_auth_error, stub_creds): stub_creds["port"] = "22" with pytest.raises(paramiko.AuthenticationException): _sftpClient(**stub_creds) - def test_sftpclient_error_reply(self, client_other_error, stub_creds): + def test_sftpclient_error_reply(self, mock_login_connection_error, stub_creds): stub_creds["port"] = "22" with pytest.raises(paramiko.SSHException): _sftpClient(**stub_creds) - def test_sftpClient_context_manager(self, stub_client, stub_creds): + def test_sftpClient_context_manager(self, mock_client, stub_creds): stub_creds["port"] = "22" with _sftpClient(**stub_creds) as client: assert client.connection is not None @@ -183,7 +203,7 @@ def test_sftpClient_get_remote_file_data(self, stub_client, stub_creds): ) def test_sftpClient_get_remote_file_data_not_found( - self, client_file_error, stub_creds + self, mock_file_error, stub_creds ): stub_creds["port"] = "22" sftp = _sftpClient(**stub_creds) @@ -206,28 +226,28 @@ def test_sftpClient_list_file_data(self, stub_client, stub_creds): ) ] - def test_sftpClient_list_file_data_not_found(self, client_file_error, stub_creds): + def test_sftpClient_list_file_data_not_found(self, mock_file_error, stub_creds): stub_creds["port"] = "22" sftp = _sftpClient(**stub_creds) with pytest.raises(OSError): sftp.list_file_data("testdir") @pytest.mark.tmpdir - def test_sftpClient_download_file(self, tmp_path, mock_client, stub_creds): + def test_sftpClient_download_file(self, tmp_path, stub_client_tmp_path, stub_creds): stub_creds["port"] = "22" path = tmp_path / "test" path.mkdir() - sftp = _ftpClient(**stub_creds) + sftp = _sftpClient(**stub_creds) sftp.download_file(file="foo.mrc", remote_dir="bar", local_dir=str(path)) assert "foo.mrc" in os.listdir(path) def test_sftpClient_download_mock_file(self, stub_client, stub_creds): stub_creds["port"] = "22" with does_not_raise(): - sftp = _ftpClient(**stub_creds) + sftp = _sftpClient(**stub_creds) sftp.download_file(file="foo.mrc", remote_dir="bar", local_dir="test") - def test_sftpClient_download_file_not_found(self, client_file_error, stub_creds): + def test_sftpClient_download_file_not_found(self, mock_file_error, stub_creds): stub_creds["port"] = "22" sftp = _sftpClient(**stub_creds) with pytest.raises(OSError): @@ -239,7 +259,7 @@ def test_sftpClient_upload_file(self, stub_client, stub_creds): file = sftp.upload_file(file="foo.mrc", local_dir="foo", remote_dir="bar") assert file.file_mtime == 1704070800 - def test_sftpClient_upload_file_not_found(self, client_file_error, stub_creds): + def test_sftpClient_upload_file_not_found(self, mock_file_error, stub_creds): stub_creds["port"] = "22" sftp = _sftpClient(**stub_creds) with pytest.raises(OSError): diff --git a/tests/test_connect.py b/tests/test_connect.py index f2dae3f..7255dc7 100644 --- a/tests/test_connect.py +++ b/tests/test_connect.py @@ -43,7 +43,7 @@ def test_Client_invalid_port(self, stub_client, stub_creds): Client(**stub_creds) assert f"Invalid port number: {stub_creds['port']}" in str(e) - def test_Client_ftp_auth_error(self, client_auth_error, stub_creds): + def test_Client_ftp_auth_error(self, mock_auth_error, stub_creds): ( stub_creds["port"], stub_creds["remote_dir"], @@ -52,7 +52,7 @@ def test_Client_ftp_auth_error(self, client_auth_error, stub_creds): with pytest.raises(ftplib.error_perm): Client(**stub_creds) - def test_Client_sftp_auth_error(self, client_auth_error, stub_creds): + def test_Client_sftp_auth_error(self, mock_auth_error, stub_creds): ( stub_creds["port"], stub_creds["remote_dir"], @@ -61,6 +61,34 @@ def test_Client_sftp_auth_error(self, client_auth_error, stub_creds): with pytest.raises(paramiko.AuthenticationException): Client(**stub_creds) + @pytest.mark.parametrize( + "port", + [21, 22], + ) + def test_Client_check_file_local(self, stub_client_file_exists, stub_creds, port): + ( + stub_creds["port"], + stub_creds["remote_dir"], + stub_creds["vendor"], + ) = (port, "testdir", "test") + connect = Client(**stub_creds) + file_exists = connect.check_file(file="foo.mrc", check_dir="bar", remote=False) + assert file_exists is True + + @pytest.mark.parametrize( + "port", + [21, 22], + ) + def test_Client_check_file_remote(self, stub_client_file_exists, stub_creds, port): + ( + stub_creds["port"], + stub_creds["remote_dir"], + stub_creds["vendor"], + ) = (port, "testdir", "test") + connect = Client(**stub_creds) + file_exists = connect.check_file(file="foo.mrc", check_dir="bar", remote=True) + assert file_exists is True + @pytest.mark.parametrize( "port, dir, uid_gid", [(21, "testdir", None), (21, None, None), (22, "testdir", 0), (22, None, 0)], @@ -83,23 +111,27 @@ def test_Client_get_file_data(self, stub_client, stub_creds, port, dir, uid_gid) file_atime=None, ) - def test_Client_ftp_get_file_data_not_found(self, client_error_reply, stub_creds): + def test_Client_ftp_get_file_data_not_found( + self, mock_connection_error_reply, stub_creds + ): ( stub_creds["port"], stub_creds["remote_dir"], stub_creds["vendor"], ) = (21, "testdir", "test") + connect = Client(**stub_creds) with pytest.raises(ftplib.error_reply): - Client(**stub_creds).get_file_data("foo.mrc", "testdir") + connect.get_file_data("foo.mrc", "testdir") - def test_Client_sftp_get_file_data_not_found(self, client_file_error, stub_creds): + def test_Client_sftp_get_file_data_not_found(self, mock_file_error, stub_creds): ( stub_creds["port"], stub_creds["remote_dir"], stub_creds["vendor"], ) = (22, "testdir", "test") + connect = Client(**stub_creds) with pytest.raises(OSError): - Client(**stub_creds).get_file_data("foo.mrc", "testdir") + connect.get_file_data("foo.mrc", "testdir") @pytest.mark.parametrize("port, uid_gid", [(21, None), (22, 0)]) def test_Client_list_files(self, stub_client, stub_creds, port, uid_gid): @@ -124,23 +156,27 @@ def test_Client_list_files(self, stub_client, stub_creds, port, uid_gid): ] assert recent_files == [] - def test_Client_list_ftp_file_not_found(self, client_error_reply, stub_creds): + def test_Client_list_ftp_file_not_found( + self, mock_connection_error_reply, stub_creds + ): ( stub_creds["port"], stub_creds["remote_dir"], stub_creds["vendor"], ) = (21, "testdir", "test") + connect = Client(**stub_creds) with pytest.raises(ftplib.error_reply): - Client(**stub_creds).list_files() + connect.list_files() - def test_Client_list_sftp_file_not_found(self, client_file_error, stub_creds): + def test_Client_list_sftp_file_not_found(self, mock_file_error, stub_creds): ( stub_creds["port"], stub_creds["remote_dir"], stub_creds["vendor"], ) = (22, "testdir", "test") + connect = Client(**stub_creds) with pytest.raises(OSError): - Client(**stub_creds).list_files() + connect.list_files() @pytest.mark.parametrize( "port, dir", @@ -154,7 +190,7 @@ def test_Client_get_file(self, stub_client, stub_creds, port, dir): ) = (port, "testdir", "test") connect = Client(**stub_creds) downloaded_file = connect.get_file( - "foo.mrc", remote_dir=dir, local_dir="baz_dir" + "foo.mrc", remote_dir=dir, local_dir="baz_dir", check=False ) assert downloaded_file == File( file_name="foo.mrc", @@ -167,7 +203,7 @@ def test_Client_get_file(self, stub_client, stub_creds, port, dir): ) @pytest.mark.parametrize("port", [21, 22]) - def test_Client_get_file_not_found(self, client_file_error, stub_creds, port): + def test_Client_get_file_not_found(self, mock_file_error, stub_creds, port): ( stub_creds["port"], stub_creds["remote_dir"], @@ -175,7 +211,51 @@ def test_Client_get_file_not_found(self, client_file_error, stub_creds, port): ) = (port, "testdir", "test") connect = Client(**stub_creds) with pytest.raises(OSError): - connect.get_file("foo.mrc", remote_dir="bar_dir", local_dir="baz_dir") + connect.get_file( + "foo.mrc", remote_dir="bar_dir", local_dir="baz_dir", check=False + ) + + @pytest.mark.parametrize( + "port", + [21, 22], + ) + def test_Client_get_check_file_exists_true( + self, stub_client_file_exists, stub_creds, port + ): + ( + stub_creds["port"], + stub_creds["remote_dir"], + stub_creds["vendor"], + ) = (port, "testdir", "test") + connect = Client(**stub_creds) + with pytest.raises(FileExistsError): + connect.get_file("foo.mrc", "testdir", check=True) + + @pytest.mark.parametrize( + "port, dir", + [(21, "testdir"), (21, None), (22, "testdir"), (22, None)], + ) + def test_Client_get_check_file_exists_false( + self, stub_client, stub_creds, port, dir + ): + ( + stub_creds["port"], + stub_creds["remote_dir"], + stub_creds["vendor"], + ) = (port, "testdir", "test") + connect = Client(**stub_creds) + downloaded_file = connect.get_file( + "foo.mrc", remote_dir=dir, local_dir="baz_dir", check=True + ) + assert downloaded_file == File( + file_name="foo.mrc", + file_mtime=1704070800, + file_size=140401, + file_mode=33188, + file_uid=0, + file_gid=0, + file_atime=None, + ) @pytest.mark.parametrize( "port, dir, uid_gid", @@ -188,7 +268,9 @@ def test_Client_put_file(self, stub_client, stub_creds, port, dir, uid_gid): stub_creds["vendor"], ) = (port, "foo", "test") connect = Client(**stub_creds) - put_file = connect.put_file("foo.mrc", remote_dir=dir, local_dir="baz_dir") + put_file = connect.put_file( + "foo.mrc", remote_dir=dir, local_dir="baz_dir", check=False + ) assert put_file == File( file_name="foo.mrc", file_mtime=1704070800, @@ -200,28 +282,76 @@ def test_Client_put_file(self, stub_client, stub_creds, port, dir, uid_gid): ) @pytest.mark.parametrize("port", [21, 22]) - def test_Client_put_file_not_found(self, client_file_error, stub_creds, port): + def test_Client_put_file_not_found(self, mock_file_error, stub_creds, port): ( stub_creds["port"], stub_creds["remote_dir"], stub_creds["vendor"], ) = (port, "testdir", "test") + connect = Client(**stub_creds) with pytest.raises(OSError): - Client(**stub_creds).put_file( - "foo.mrc", remote_dir="bar_dir", local_dir="baz_dir" + connect.put_file( + "foo.mrc", remote_dir="bar_dir", local_dir="baz_dir", check=False ) - def test_Client_put_client_error_reply(self, client_error_reply, stub_creds): + def test_Client_put_client_error_reply( + self, mock_connection_error_reply, stub_creds + ): ( stub_creds["port"], stub_creds["remote_dir"], stub_creds["vendor"], ) = (21, "testdir", "test") + client = Client(**stub_creds) with pytest.raises(ftplib.error_reply): - Client(**stub_creds).put_file( - "foo.mrc", remote_dir="bar_dir", local_dir="baz_dir" + client.put_file( + "foo.mrc", remote_dir="bar_dir", local_dir="baz_dir", check=False + ) + + @pytest.mark.parametrize( + "port", + [21, 22], + ) + def test_Client_put_check_file_exists_true( + self, stub_client_file_exists, stub_creds, port + ): + ( + stub_creds["port"], + stub_creds["remote_dir"], + stub_creds["vendor"], + ) = (port, "testdir", "test") + connect = Client(**stub_creds) + with pytest.raises(FileExistsError): + connect.put_file( + "foo.mrc", remote_dir="bar_dir", local_dir="baz_dir", check=True ) + @pytest.mark.parametrize( + "port, uid_gid", + [(21, None), (22, 0)], + ) + def test_Client_put_check_file_exists_false( + self, stub_client, stub_creds, port, uid_gid + ): + ( + stub_creds["port"], + stub_creds["remote_dir"], + stub_creds["vendor"], + ) = (port, "testdir", "test") + connect = Client(**stub_creds) + uploaded_file = connect.put_file( + "foo.mrc", remote_dir="bar_dir", local_dir="baz_dir", check=True + ) + assert uploaded_file == File( + file_name="foo.mrc", + file_mtime=1704070800, + file_size=140401, + file_mode=33188, + file_gid=uid_gid, + file_uid=uid_gid, + file_atime=None, + ) + @pytest.mark.livetest def test_Client_ftp_live_test(live_ftp_creds):