modos.storage#

Attributes#

ZARR_ROOT

Classes#

Storage

Abstract base class for modos storage backends.

LocalStorage

Abstract base class for modos storage backends.

S3Path

Pydantic Model for S3 URLs. Performs validation against amazon's official naming rules [1] [2]

S3Storage

Abstract base class for modos storage backends.

Functions#

init_zarr(zarr_store)

Initialize object's directory and metadata structure.

connect_s3(path, endpoint[, s3_kwargs])

add_metadata_group(parent_group, metadata)

Add input metadata dictionary to an existing zarr group.

add_data(group, data)

Add a numpy array to an existing zarr group.

list_zarr_items(group)

Recursively list all zarr groups and arrays

Module Contents#

modos.storage.ZARR_ROOT[source]#
class modos.storage.Storage[source]#

Bases: abc.ABC

Abstract base class for modos storage backends. Paths in a Storage are always interpreted relative to self.path.

property path: pathlib.Path[source]#
Abstractmethod:

Return type:

pathlib.Path

property zarr: zarr.Group[source]#
Abstractmethod:

Return type:

zarr.Group

abstract exists(target)[source]#
Parameters:

target (pathlib.Path)

Return type:

bool

abstract list(target=None)[source]#

List files in the storage.

Parameters:

target (Optional[pathlib.Path]) – path to a directory to list relative to storage.path. If None, list all files in storage.

Return type:

Generator[pathlib.Path, None, None]

abstract move(rel_source, target)[source]#

Move a file within storage.

Parameters:

  • rel_source (pathlib.Path) – source path to the file to move relative to storage.path

  • target (pathlib.Path) – target path within storage

abstract open(target)[source]#
Parameters:

target (pathlib.Path)

Return type:

io.BufferedReader

abstract put(source, target)[source]#
Parameters:
abstract remove(target)[source]#
Parameters:

target (pathlib.Path)

transfer(other)[source]#

Transfer all contents of one storage to another one.

Parameters:

other (Storage)

empty()[source]#
Return type:

bool

class modos.storage.LocalStorage(path)[source]#

Bases: Storage

Abstract base class for modos storage backends. Paths in a Storage are always interpreted relative to self.path.

Parameters:

path (pathlib.Path)

_path[source]#
property zarr: zarr.Group[source]#
Return type:

zarr.Group

property path: pathlib.Path[source]#
Return type:

pathlib.Path

exists(target)[source]#
Parameters:

target (pathlib.Path)

Return type:

bool

list(target=None)[source]#

List files in the storage.

Parameters:

target (Optional[pathlib.Path]) – path to a directory to list relative to storage.path. If None, list all files in storage.

Return type:

Generator[pathlib.Path, None, None]

move(rel_source, target)[source]#

Move a file within storage.

Parameters:

  • rel_source (pathlib.Path) – source path to the file to move relative to storage.path

  • target (pathlib.Path) – target path within storage

open(target)[source]#
Parameters:

target (pathlib.Path)

Return type:

io.BufferedReader

put(source, target)[source]#
Parameters:
remove(target)[source]#
Parameters:

target (pathlib.Path)

class modos.storage.S3Path[source]#

Pydantic Model for S3 URLs. Performs validation against amazon’s official naming rules [1] [2]

Examples

>>> S3Path(url="s3://test/ex")
S3Path(url='s3://test/ex')
>>> S3Path(url='s3://?invalid-bucket-name!/def') 
Traceback (most recent call last):
  ...
pydantic_core._pydantic_core.ValidationError: 1 validation error for S3Path
_s3_pattern: ClassVar[re.Pattern[str]][source]#
url: str = None[source]#
__truediv__(other)[source]#
__str__()[source]#
s3_url_parts()[source]#
property bucket: str[source]#
Return type:

str

property key: str[source]#
Return type:

str

class modos.storage.S3Storage(path, s3_endpoint, s3_kwargs=None)[source]#

Bases: Storage

Abstract base class for modos storage backends. Paths in a Storage are always interpreted relative to self.path.

Parameters:

  • path (str)

  • s3_endpoint (pydantic.HttpUrl)

  • s3_kwargs (Optional[dict[str, Any]])

_path[source]#
endpoint[source]#
store[source]#
zarr_store[source]#
property path: pathlib.Path[source]#
Return type:

pathlib.Path

property zarr: zarr.Group[source]#
Return type:

zarr.Group

exists(target)[source]#

Checks whether target, or objects in target directory exist in the store. The target should be relative to self.path.

Parameters:

target (pathlib.Path)

Return type:

bool

list(target=None)[source]#

Lists objects in target directory. The target should be relative to self.path.

Parameters:

target (pathlib.Path | None)

Return type:

Generator[pathlib.Path, None, None]

open(target)[source]#
Parameters:

target (pathlib.Path)

Return type:

io.BufferedReader

remove(target)[source]#
Parameters:

target (pathlib.Path)

put(source, target)[source]#
Parameters:
move(rel_source, target)[source]#

Move a file within storage.

Parameters:

  • rel_source (pathlib.Path) – source path to the file to move relative to storage.path

  • target (pathlib.Path) – target path within storage

modos.storage.init_zarr(zarr_store)[source]#

Initialize object’s directory and metadata structure.

Parameters:

zarr_store (zarr.storage.StoreLike)

Return type:

zarr.Group

modos.storage.connect_s3(path, endpoint, s3_kwargs=None)[source]#
Parameters:

  • path (str)

  • endpoint (pydantic.HttpUrl)

  • s3_kwargs (dict[str, Any] | None)

Return type:

obstore.store.S3Store

modos.storage.add_metadata_group(parent_group, metadata)[source]#

Add input metadata dictionary to an existing zarr group.

Parameters:
Return type:

None

modos.storage.add_data(group, data)[source]#

Add a numpy array to an existing zarr group.

Parameters:

group (zarr.Group)

Return type:

None

modos.storage.list_zarr_items(group)[source]#

Recursively list all zarr groups and arrays

Parameters:

group (zarr.Group)

Return type:

tuple[tuple[str, zarr.Group | zarr.Array], Ellipsis]