Run pyupgrade --py310-plus

tools/backups2datalad/__main__.py

@@ -1,14 +1,14 @@
 from __future__ import annotations
 
-from collections.abc import Callable
+from collections.abc import AsyncGenerator, Awaitable, Callable, Sequence
 from contextlib import aclosing
 from functools import partial, wraps
 import json
 import logging
 from pathlib import Path
 import re
 import sys
-from typing import AsyncGenerator, Awaitable, Concatenate, Optional, ParamSpec, Sequence
+from typing import Concatenate, ParamSpec
 
 import asyncclick as click
 from dandi.consts import DANDISET_ID_REGEX
@@ -58,12 +58,12 @@
 @click.pass_context
 async def main(
     ctx: click.Context,
-    jobs: Optional[int],
+    jobs: int | None,
     log_level: str,
     pdb: bool,
     quiet_debug: bool,
     backup_root: Path,
-    config: Optional[Path],
+    config: Path | None,
 ) -> None:
     if config is None:
         cfg = BackupConfig()
@@ -180,14 +180,14 @@ async def wrapped(
 async def update_from_backup(
     datasetter: DandiDatasetter,
     dandisets: Sequence[str],
-    exclude: Optional[re.Pattern[str]],
-    tags: Optional[bool],
-    asset_filter: Optional[re.Pattern[str]],
-    force: Optional[str],
-    workers: Optional[int],
-    gc_assets: Optional[bool],
-    mode: Optional[Mode],
-    zarr_mode: Optional[ZarrMode],
+    exclude: re.Pattern[str] | None,
+    tags: bool | None,
+    asset_filter: re.Pattern[str] | None,
+    force: str | None,
+    workers: int | None,
+    gc_assets: bool | None,
+    mode: Mode | None,
+    zarr_mode: ZarrMode | None,
 ) -> None:
     async with datasetter:
         if asset_filter is not None:
@@ -221,8 +221,8 @@ async def update_from_backup(
 async def backup_zarrs(
     datasetter: DandiDatasetter,
     dandiset: str,
-    workers: Optional[int],
-    partial_dir: Optional[Path],
+    workers: int | None,
+    partial_dir: Path | None,
 ) -> None:
     async with datasetter:
         if datasetter.config.zarrs is None:
@@ -248,7 +248,7 @@ async def backup_zarrs(
 async def update_github_metadata(
     datasetter: DandiDatasetter,
     dandisets: Sequence[str],
-    exclude: Optional[re.Pattern[str]],
+    exclude: re.Pattern[str] | None,
 ) -> None:
     """
     Update the homepages and descriptions for the GitHub repositories for the
@@ -285,10 +285,10 @@ async def release(
     datasetter: DandiDatasetter,
     dandiset: str,
     version: str,
-    commitish: Optional[str],
+    commitish: str | None,
     push: bool,
-    asset_filter: Optional[re.Pattern[str]],
-    force: Optional[str],
+    asset_filter: re.Pattern[str] | None,
+    force: str | None,
 ) -> None:
     async with datasetter:
         if asset_filter is not None:
@@ -323,8 +323,8 @@ async def release(
 async def populate_cmd(
     datasetter: DandiDatasetter,
     dandisets: Sequence[str],
-    exclude: Optional[re.Pattern[str]],
-    workers: Optional[int],
+    exclude: re.Pattern[str] | None,
+    workers: int | None,
 ) -> None:
     async with datasetter:
         if (r := datasetter.config.dandisets.remote) is not None:
@@ -367,7 +367,7 @@ async def populate_cmd(
 @click.pass_obj
 @print_logfile
 async def populate_zarrs(
-    datasetter: DandiDatasetter, zarrs: Sequence[str], workers: Optional[int]
+    datasetter: DandiDatasetter, zarrs: Sequence[str], workers: int | None
 ) -> None:
     async with datasetter:
         zcfg = datasetter.config.zarrs

tools/backups2datalad/adandi.py

@@ -1,13 +1,14 @@
 from __future__ import annotations
 
 from abc import abstractmethod
+from collections.abc import AsyncGenerator, Sequence
 from contextlib import aclosing
 from dataclasses import dataclass
 from datetime import datetime
 import json
 import re
 from time import time
-from typing import Any, AsyncGenerator, Dict, Optional, Sequence, Type, cast
+from typing import Any, cast
 
 import anyio
 from anyio.abc import AsyncResource
@@ -32,7 +33,7 @@ class AsyncDandiClient(AsyncResource):
 
     @classmethod
     def for_dandi_instance(
-        cls, instance: str, token: Optional[str] = None
+        cls, instance: str, token: str | None = None
     ) -> AsyncDandiClient:
         headers = {"User-Agent": USER_AGENT}
         if token is not None:
@@ -67,8 +68,8 @@ async def delete(self, path: str, **kwargs: Any) -> None:
     async def paginate(
         self,
         path: str,
-        page_size: Optional[int] = None,
-        params: Optional[dict] = None,
+        page_size: int | None = None,
+        params: dict | None = None,
         **kwargs: Any,
     ) -> AsyncGenerator:
         """
@@ -90,7 +91,7 @@ async def paginate(
                 break
 
     async def get_dandiset(
-        self, dandiset_id: str, version_id: Optional[str] = None
+        self, dandiset_id: str, version_id: str | None = None
     ) -> RemoteDandiset:
         data = await self.get(f"/dandisets/{dandiset_id}/")
         d = RemoteDandiset.from_data(self, data)
@@ -166,7 +167,7 @@ async def aget_version(self, version_id: str) -> Version:
         )
 
     async def aget_versions(
-        self, include_draft: bool = True, order: Optional[str] = None
+        self, include_draft: bool = True, order: str | None = None
     ) -> AsyncGenerator[Version, None]:
         async with aclosing(
             self.aclient.paginate(f"{self.api_path}versions/", params={"order": order})
@@ -295,7 +296,7 @@ class RemoteAsset(BaseModel):
     #: The date at which the asset was last modified
     modified: datetime
     #: The asset's raw metadata
-    metadata: Dict[str, Any]
+    metadata: dict[str, Any]
 
     class Config:
         """Configuration for pydantic"""
@@ -307,13 +308,13 @@ class Config:
         arbitrary_types_allowed = True
 
     @classmethod
-    def from_data(cls, dandiset: RemoteDandiset, data: Dict[str, Any]) -> RemoteAsset:
+    def from_data(cls, dandiset: RemoteDandiset, data: dict[str, Any]) -> RemoteAsset:
         """
         Construct a `RemoteAsset` instance from a `RemoteDandiset` and a `dict`
         of raw data in the same format as returned by the API's pagination
         endpoints (including metadata).
         """
-        klass: Type[RemoteAsset]
+        klass: type[RemoteAsset]
         if data.get("blob") is not None:
             klass = RemoteBlobAsset
             if data.pop("zarr", None) is not None:
@@ -340,7 +341,7 @@ def version_id(self) -> str:
 
     def json_dict(self) -> dict[str, Any]:
         return cast(
-            Dict[str, Any],
+            dict[str, Any],
             json.loads(self.json(exclude={"aclient", "dandiset"}, by_alias=True)),
         )
 
@@ -363,7 +364,7 @@ def base_download_url(self) -> str:
         """
         return self.aclient.get_url(f"/assets/{self.identifier}/download/")
 
-    def get_digest_value(self, algorithm: Optional[DigestType] = None) -> Optional[str]:
+    def get_digest_value(self, algorithm: DigestType | None = None) -> str | None:
         if algorithm is None:
             algorithm = self.digest_type
         try:

tools/backups2datalad/adataset.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from collections import Counter
+from collections.abc import AsyncGenerator, Sequence
 from contextlib import aclosing
 from dataclasses import InitVar, dataclass, field, replace
 from datetime import datetime
@@ -11,7 +12,7 @@
 import re
 import subprocess
 import textwrap
-from typing import Any, AsyncGenerator, ClassVar, Optional, Sequence, cast
+from typing import Any, ClassVar, cast
 
 import anyio
 from datalad.api import Dataset
@@ -49,10 +50,10 @@ def pathobj(self) -> Path:
     async def ensure_installed(
         self,
         desc: str,
-        commit_date: Optional[datetime] = None,
-        backup_remote: Optional[Remote] = None,
+        commit_date: datetime | None = None,
+        backup_remote: Remote | None = None,
         backend: str = "SHA256E",
-        cfg_proc: Optional[str] = "text2git",
+        cfg_proc: str | None = "text2git",
     ) -> bool:
         # Returns True if the dataset was freshly created
         if self.ds.is_installed():
@@ -105,7 +106,7 @@ def _unclean() -> bool:
 
         return await anyio.to_thread.run_sync(_unclean)
 
-    async def get_repo_config(self, key: str) -> Optional[str]:
+    async def get_repo_config(self, key: str) -> str | None:
         try:
             return await self.read_git("config", "--get", key)
         except subprocess.CalledProcessError as e:
@@ -144,7 +145,7 @@ async def save(
         self,
         message: str,
         path: Sequence[str | Path] = (),
-        commit_date: Optional[datetime] = None,
+        commit_date: datetime | None = None,
     ) -> None:
         # TODO: Improve
         await aruncmd(
@@ -162,7 +163,7 @@ async def save(
     async def commit(
         self,
         message: str,
-        commit_date: Optional[datetime] = None,
+        commit_date: datetime | None = None,
         paths: Sequence[str | Path] = (),
         check_dirty: bool = True,
     ) -> None:
@@ -188,7 +189,7 @@ async def commit(
                 "  Please check if all changes were staged."
             )
 
-    async def push(self, to: str, jobs: int, data: Optional[str] = None) -> None:
+    async def push(self, to: str, jobs: int, data: str | None = None) -> None:
         waits = exp_wait(attempts=6, base=2.1)
         while True:
             try:
@@ -286,7 +287,7 @@ async def remove(self, path: str) -> None:
                         )
                         await anyio.sleep(delay)
 
-    async def update(self, how: str, sibling: Optional[str] = None) -> None:
+    async def update(self, how: str, sibling: str | None = None) -> None:
         await anyio.to_thread.run_sync(
             partial(self.ds.update, how=how, sibling=sibling)
         )
@@ -359,7 +360,7 @@ async def create_github_sibling(
         self,
         owner: str,
         name: str,
-        backup_remote: Optional[Remote],
+        backup_remote: Remote | None,
         *,
         existing: str = "reconfigure",
     ) -> bool:
@@ -450,7 +451,7 @@ async def get_zarr_sub_stats(
         zarr_stat = await zarr_ds.get_stats(config=config)
         return zarr_id, zarr_stat
 
-    async def get_stored_stats(self) -> Optional[DatasetStats]:
+    async def get_stored_stats(self) -> DatasetStats | None:
         if (stored_stats := self.ds.config.get("dandi.stats", None)) is not None:
             try:
                 stored_commit, files_str, size_str = stored_stats.split(",")
@@ -482,7 +483,7 @@ def assert_no_duplicates_in_gitmodules(self) -> None:
         dupped = [name for (name, count) in qtys.most_common() if count > 1]
         assert not dupped, f"Duplicates found in {filepath}: {dupped}"
 
-    def get_assets_state(self) -> Optional[AssetsState]:
+    def get_assets_state(self) -> AssetsState | None:
         try:
             return AssetsState.parse_file(self.pathobj / AssetsState.PATH)
         except FileNotFoundError:
@@ -541,7 +542,7 @@ async def update_submodule(self, path: str, commit_hash: str) -> None:
             "-z",
             # apparently must be the last argument!
             "--index-info",
-            input=f"160000 commit {commit_hash}\t{path}\0".encode("utf-8"),
+            input=f"160000 commit {commit_hash}\t{path}\0".encode(),
         )
 
     async def populate_up_to_date(self) -> bool:
@@ -566,7 +567,7 @@ class ObjectType(Enum):
 class FileStat:
     path: str
     type: ObjectType
-    size: Optional[int]
+    size: int | None
 
     @classmethod
     def from_entry(cls, entry: str) -> FileStat:

tools/backups2datalad/aioutil.py

@@ -1,6 +1,13 @@
 from __future__ import annotations
 
-from collections.abc import AsyncGenerator, AsyncIterator, Callable, Container, Mapping
+from collections.abc import (
+    AsyncGenerator,
+    AsyncIterator,
+    Awaitable,
+    Callable,
+    Container,
+    Mapping,
+)
 from contextlib import aclosing, asynccontextmanager
 from dataclasses import dataclass, field
 import logging
@@ -10,7 +17,7 @@
 import ssl
 import subprocess
 import textwrap
-from typing import Any, Awaitable, Generic, Optional, TypeVar
+from typing import Any, Generic, TypeVar
 
 import anyio
 from anyio.streams.memory import MemoryObjectReceiveStream
@@ -83,7 +90,7 @@ async def send_eof(self) -> None:
 
 async def open_git_annex(
     *args: str,
-    path: Optional[Path] = None,
+    path: Path | None = None,
     warn_on_fail: bool = True,
 ) -> TextProcess:
     # This is strictly for spawning git-annex processes that data will be both
@@ -225,7 +232,7 @@ async def areadcmd(*args: str | Path, strip: bool = True, **kwargs: Any) -> str:
 
 
 async def stream_null_command(
-    *args: str | Path, cwd: Optional[Path] = None
+    *args: str | Path, cwd: Path | None = None
 ) -> AsyncGenerator[str, None]:
     argstrs = [str(a) for a in args]
     desc = f"`{shlex.join(argstrs)}`"
@@ -254,7 +261,7 @@ async def stream_null_command(
 
 
 async def stream_lines_command(
-    *args: str | Path, cwd: Optional[Path] = None
+    *args: str | Path, cwd: Path | None = None
 ) -> AsyncGenerator[str, None]:
     argstrs = [str(a) for a in args]
     desc = f"`{shlex.join(argstrs)}`"
@@ -324,7 +331,7 @@ class LineReceiveStream(anyio.abc.ObjectReceiveStream[str]):
     def __init__(
         self,
         transport_stream: anyio.abc.ObjectReceiveStream[str],
-        newline: Optional[str] = None,
+        newline: str | None = None,
     ) -> None:
         """
         :param transport_stream: any `str`-based receive stream