diff --git a/README.md b/README.md index 8d4a355d..9f5f8516 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,7 @@ Available notebooks provided as examples: - [DEM](https://eoreader.readthedocs.io/en/latest/notebooks/dem.html) - [Custom stacks](https://eoreader.readthedocs.io/en/latest/notebooks/custom.html) - [Methods to clean optical bands](https://eoreader.readthedocs.io/en/latest/notebooks/optical_cleaning_methods.html) +- [AWS storage](https://eoreader.readthedocs.io/en/latest/notebooks/aws.html) - [S3 Compatible Storage](https://eoreader.readthedocs.io/en/latest/notebooks/s3_compatible_storage.html) - [Dask](https://eoreader.readthedocs.io/en/latest/notebooks/dask.html) - [STAC](https://eoreader.readthedocs.io/en/latest/notebooks/stac.html) diff --git a/docs/_build/.jupyter_cache/global.db b/docs/_build/.jupyter_cache/global.db index 7440cbc0..81f6340d 100644 Binary files a/docs/_build/.jupyter_cache/global.db and b/docs/_build/.jupyter_cache/global.db differ diff --git a/docs/notebooks/advanced.md b/docs/notebooks/advanced.md index cd474538..28659f0e 100644 --- a/docs/notebooks/advanced.md +++ b/docs/notebooks/advanced.md @@ -10,4 +10,6 @@ water_detection optical_cleaning_methods windowed_reading + aws + s3_compatible_storage ``` diff --git a/docs/notebooks/aws.ipynb b/docs/notebooks/aws.ipynb index dc761b23..2c2302a9 100644 --- a/docs/notebooks/aws.ipynb +++ b/docs/notebooks/aws.ipynb @@ -8,15 +8,19 @@ "# AWS\n", "Let's read Sentinel-2 data stored on AWS.\n", "\n", + "This tutorial is a bit tricky as EOReader uses `rasterio` and `cloudpathlib` libraries and they are currently handling S3 buckets differently.\n", + "That's why we'll use the helper context manager `temp_s3` from `sertit.s3`, that will do the right connections for us.\n", + "\n", "
\n", " \n", " Note: These products are not stored in the `.SAFE` format.\n", " \n", "
\n", "\n", - "## Let's read data processed by Element84: Sentinel-2 L2A as COGs\n", + "## Cloud data processed by Element84: Sentinel-2 L2A as COGs\n", "\n", - "See this [registry](https://registry.opendata.aws/sentinel-2-l2a-cogs) (`arn:aws:s3:::sentinel-cogs`)" + "See this [registry](https://registry.opendata.aws/sentinel-2-l2a-cogs) (`arn:aws:s3:::sentinel-cogs`).\n", + "This registry is open access, so you don't have to sign-in. " ] }, { @@ -72,10 +76,8 @@ "source": [ "with tempenv.TemporaryEnvironment({\n", " \"AWS_S3_ENDPOINT\": \"s3.us-west-2.amazonaws.com\",\n", - " \"AWS_SECRET_ACCESS_KEY\": os.getenv(\"AMAZON_AWS_SECRET_ACCESS_KEY\"),\n", - " \"AWS_ACCESS_KEY_ID\": os.getenv(\"AMAZON_AWS_ACCESS_KEY_ID\"),\n", "}):\n", - " with s3.temp_s3():\n", + " with s3.temp_s3(no_sign_request=True):\n", " logs.init_logger(logging.getLogger(\"eoreader\"), logging.DEBUG)\n", " path = r\"s3://sentinel-cogs/sentinel-s2-l2a-cogs/39/K/ZU/2023/10/S2A_39KZU_20231031_0_L2A\"\n", " prod = Reader().open(path)\n", @@ -96,9 +98,10 @@ { "cell_type": "markdown", "source": [ - "## Let's read data processed by Sinergise: Sentinel-2 L1C\n", + "## Cloudprocessed by Sinergise: Sentinel-2 L1C\n", "\n", - "See this [registry](https://registry.opendata.aws/sentinel-2/) (`arn:aws:s3:::sentinel-s2-l1c`)\n", + "See this [registry](https://registry.opendata.aws/sentinel-2/) (`arn:aws:s3:::sentinel-s2-l1c`).\n", + "This registry needs authentication.\n", "\n", "NB: L2A would have been the same (`arn:aws:s3:::sentinel-s2-l2a`)\n", "\n", diff --git a/docs/notebooks/experimental.md b/docs/notebooks/experimental.md index 0e400b6a..d32aa770 100644 --- a/docs/notebooks/experimental.md +++ b/docs/notebooks/experimental.md @@ -4,8 +4,6 @@ :maxdepth: 1 :caption: Experimental - s3_compatible_storage - aws dask stac ``` \ No newline at end of file diff --git a/docs/notebooks/s3_compatible_storage.ipynb b/docs/notebooks/s3_compatible_storage.ipynb index 9a373d24..211263ef 100644 --- a/docs/notebooks/s3_compatible_storage.ipynb +++ b/docs/notebooks/s3_compatible_storage.ipynb @@ -9,18 +9,11 @@ "Let's use EOReader with products stored in a S3 compatible storage cloud.\n", "\n", "This tutorial is a bit tricky as EOReader uses `rasterio` and `cloudpathlib` libraries and they are currently handling S3 buckets differently.\n", - "\n", - "
\n", - " \n", - "Note: This is experimental for now, use it at your own risk !\n", - " \n", - "
\n", + "That's why we'll use the helper context manager `temp_s3` from `sertit.s3`, that will do the right connections for us.\n", "\n", "
\n", " \n", - "Warning:\n", - "
  • S3 compatible storage is not well handled by rasterio for now, you should use Linux or Docker instead !\n", - "
  • This tutorial assumes that you have correctly set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY as environment variables\n", + "Warning: This tutorial assumes that you have correctly set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY as environment variables\n", " \n", "
    • " ] @@ -42,37 +35,15 @@ "from cloudpathlib import S3Client, AnyPath\n", "import matplotlib.pyplot as plt\n", "\n", - "import rasterio\n", + "from sertit import s3\n", "\n", "from eoreader.reader import Reader\n", "from eoreader.bands import MNDWI, CLOUDS\n", "\n", - "# Other\n", - "AWS_S3_ENDPOINT = os.getenv(\"AWS_S3_ENDPOINT\")" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "4d2a6657", - "metadata": { - "ExecuteTime": { - "start_time": "2023-05-31T13:47:01.052799Z", - "end_time": "2023-05-31T13:47:01.405195Z" - } - }, - "outputs": [], - "source": [ - "# Create your S3 compatible storage path with cloudpathlib\n", - "# See here for more insights about this awesome lib: https://cloudpathlib.drivendata.org/\n", - "client = S3Client(\n", - " endpoint_url=f\"https://{AWS_S3_ENDPOINT}\",\n", - " aws_access_key_id=os.getenv(\"AWS_ACCESS_KEY_ID\"),\n", - " aws_secret_access_key=os.getenv(\"AWS_SECRET_ACCESS_KEY\"),\n", - ")\n", - "client.set_as_default_client()\n", - "path = AnyPath(\"s3://sertit-eoreader-ci\").joinpath(\n", - " \"optical/S2B_MSIL2A_20200114T065229_N0213_R020_T40REQ_20200114T094749.SAFE/\")" + "# This endpoint points to your S3 compatible bucket\n", + "custom_s3_compatible_endpoint = os.getenv(\"AWS_S3_ENDPOINT\")\n", + "\n", + "# Be sure to have AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables correctly pointing to the credentials of your S3-compatible storage" ] }, { @@ -82,13 +53,10 @@ "metadata": {}, "outputs": [], "source": [ - "# Create a rasterio env to enable S3 compatible storage\n", - "with rasterio.Env(\n", - " CPL_CURL_VERBOSE=False,\n", - " AWS_VIRTUAL_HOSTING=False,\n", - " AWS_S3_ENDPOINT=AWS_S3_ENDPOINT,\n", - " GDAL_DISABLE_READDIR_ON_OPEN=False,\n", - "):\n", + "with s3.temp_s3(default_endpoint=custom_s3_compatible_endpoint):\n", + " path = AnyPath(\"s3://sertit-eoreader-ci\").joinpath(\n", + " \"optical/S2B_MSIL2A_20200114T065229_N0213_R020_T40REQ_20200114T094749.SAFE/\")\n", + " \n", " # Create the reader\n", " reader = Reader()\n", "\n", diff --git a/eoreader/products/custom_product.py b/eoreader/products/custom_product.py index 69ee7ac9..91332a91 100644 --- a/eoreader/products/custom_product.py +++ b/eoreader/products/custom_product.py @@ -90,7 +90,7 @@ def __init__( """Custom kwargs""" # Initialization from the super class - # (Custom products arte managing constellation on their own) + # (Custom products are managing constellation on their own) super_kwargs = kwargs.copy() super_kwargs.pop("constellation", None) super().__init__( diff --git a/requirements.txt b/requirements.txt index d89b3573..19a88868 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,7 +40,7 @@ spyndex>=0.3.0 pystac[validation] # SERTIT libs -sertit[full]>=1.32.0 +sertit[full]>=1.32.1 # Optimizations dask[complete]>=2021.10.0