.serialize Returns an Error

[nishsilva]

Hello there,

I am new to using intake-esm. I have been following the documentation to read data from a data catalog file and subset it to a new data catalog file.

To do this when I use the .serialize function, it does not write the .jsonc file telling that I do not have write access. ButI do have rite access to the location mentioned in the error message.

Can someone help me figure out this issue.

I tried using the method for NCAR managed data catalogs as well as the datastore url. Both returns the same error.

Thank you!
Nish

Below is the error that I recieve.

> ---------------------------------------------------------------------------
> FileNotFoundError                         Traceback (most recent call last)
> Cell In[5], line 1
> ----> 1 col_subset.serialize(directory='/glade/work/netige/mdtf_Apr24_2/data_catalogs', name="my_catalog_subset")
> 
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/pydantic/deprecated/decorator.py:55, in validate_arguments.<locals>.validate.<locals>.wrapper_function(*args, **kwargs)
>      53 @wraps(_func)
>      54 def wrapper_function(*args: Any, **kwargs: Any) -> Any:
> ---> 55     return vd.call(*args, **kwargs)
> 
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/pydantic/deprecated/decorator.py:150, in ValidatedFunction.call(self, *args, **kwargs)
>     148 def call(self, *args: Any, **kwargs: Any) -> Any:
>     149     m = self.init_model_instance(*args, **kwargs)
> --> 150     return self.execute(m)
> 
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/pydantic/deprecated/decorator.py:222, in ValidatedFunction.execute(self, m)
>     220     return self.raw_function(*args_, **kwargs, **var_kwargs)
>     221 else:
> --> 222     return self.raw_function(**d, **var_kwargs)
> 
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/intake_esm/core.py:492, in esm_datastore.serialize(self, name, directory, catalog_type, to_csv_kwargs, json_dump_kwargs, storage_options)
>     446 @pydantic.validate_arguments
>     447 def serialize(
>     448     self,
>    (...)
>     454     storage_options: typing.Optional[dict[str, typing.Any]] = None,
>     455 ) -> None:
>     456     """Serialize catalog to corresponding json and csv files.
>     457 
>     458     Parameters
>    (...)
>     489     >>> cat_subset.serialize(name="cmip6_bcc_esm1", catalog_type="file")
>     490     """
> --> 492     self.esmcat.save(
>     493         name,
>     494         directory=directory,
>     495         catalog_type=catalog_type,
>     496         to_csv_kwargs=to_csv_kwargs,
>     497         json_dump_kwargs=json_dump_kwargs,
>     498         storage_options=storage_options,
>     499     )
> 
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/intake_esm/cat.py:207, in ESMCatalogModel.save(self, name, directory, catalog_type, to_csv_kwargs, json_dump_kwargs, storage_options)
>     204 else:
>     205     data['catalog_dict'] = self.df.to_dict(orient='records')
> --> 207 with fs.open(json_file_name, 'w') as outfile:
>     208     json_kwargs = {'indent': 2}
>     209     json_kwargs.update(json_dump_kwargs or {})
> 
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/fsspec/spec.py:1295, in AbstractFileSystem.open(self, path, mode, block_size, cache_options, compression, **kwargs)
>    1287     mode = mode.replace("t", "") + "b"
>    1289     text_kwargs = {
>    1290         k: kwargs.pop(k)
>    1291         for k in ["encoding", "errors", "newline"]
>    1292         if k in kwargs
>    1293     }
>    1294     return io.TextIOWrapper(
> -> 1295         self.open(
>    1296             path,
>    1297             mode,
>    1298             block_size=block_size,
>    1299             cache_options=cache_options,
>    1300             compression=compression,
>    1301             **kwargs,
>    1302         ),
>    1303         **text_kwargs,
>    1304     )
>    1305 else:
>    1306     ac = kwargs.pop("autocommit", not self._intrans)
> 
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/fsspec/spec.py:1307, in AbstractFileSystem.open(self, path, mode, block_size, cache_options, compression, **kwargs)
>    1305 else:
>    1306     ac = kwargs.pop("autocommit", not self._intrans)
> -> 1307     f = self._open(
>    1308         path,
>    1309         mode=mode,
>    1310         block_size=block_size,
>    1311         autocommit=ac,
>    1312         cache_options=cache_options,
>    1313         **kwargs,
>    1314     )
>    1315     if compression is not None:
>    1316         from fsspec.compression import compr
> 
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/fsspec/implementations/local.py:180, in LocalFileSystem._open(self, path, mode, block_size, **kwargs)
>     178 if self.auto_mkdir and "w" in mode:
>     179     self.makedirs(self._parent(path), exist_ok=True)
> --> 180 return LocalFileOpener(path, mode, fs=self, **kwargs)
> 
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/fsspec/implementations/local.py:302, in LocalFileOpener.__init__(self, path, mode, autocommit, fs, compression, **kwargs)
>     300 self.compression = get_compression(path, compression)
>     301 self.blocksize = io.DEFAULT_BUFFER_SIZE
> --> 302 self._open()
> 
> File /glade/work/netige/conda-envs/mdtf_env/lib/python3.9/site-packages/fsspec/implementations/local.py:307, in LocalFileOpener._open(self)
>     305 if self.f is None or self.f.closed:
>     306     if self.autocommit or "w" not in self.mode:
> --> 307         self.f = open(self.path, mode=self.mode)
>     308         if self.compression:
>     309             compress = compr[self.compression]
> 
> FileNotFoundError: [Errno 2] No such file or directory: "/glade/work/netige/mdtf_Apr24_2/scripts/('file', 'local'):///glade/work/netige/mdtf_Apr24_2/data_catalogs/my_catalog_subset.json"