Source code for synthesizer.pipeline.pipeline_io

"""A module for handling I/O operations in the pipeline.

This module contains classes and functions for reading and writing data
in the pipeline. This includes reading and writing HDF5 files, as well as
handling the MPI communications for parallel I/O operations.

Example usage:

    # Write data to an HDF5 file
    writer = PipelineIO("output.hdf5")
    writer.write_data(data, key)


"""

import os
import time
from contextlib import contextmanager

import h5py
import numpy as np
from unyt import unyt_array

from synthesizer._version import __version__
from synthesizer.instruments import InstrumentCollection
from synthesizer.pipeline.pipeline_utils import (
    get_dataset_properties,
    unify_dict_structure_across_ranks,
)
from synthesizer.synth_warnings import warn
from synthesizer.utils.operation_timers import timed

# These live under Galaxies alongside the outputs, but describe the whole
# run rather than one row per galaxy. Rank 0 writes them once.
_SHARED_AXIS_GROUPS = (
    "/Galaxies/Stars/SFZHAxes",
    "/Galaxies/Stars/SFHAxes",
)


@contextmanager
def _hdf5_replaced_on_success(path):
    """Open an HDF5 file for writing which only replaces path on success.

    The file is written to a temporary path next to path, so a failure while
    writing leaves any existing file at path untouched, rather than an
    incomplete file which looks valid.

    Args:
        path (str): The path the file should end up at.

    Yields:
        h5py.File: The open file to write to.
    """
    temp_path = f"{path}.tmp"
    try:
        with h5py.File(temp_path, "w") as hdf:
            yield hdf

        # Everything was written (and the file closed), so swap it in
        os.replace(temp_path, path)
    except BaseException:
        # Clean up the incomplete file before re-raising
        if os.path.exists(temp_path):
            os.remove(temp_path)
        raise


[docs] class PipelineIO: """A class for writing data to an HDF5 file. This class provides methods for writing data to an HDF5 file. It can handle writing data in parallel using MPI if the h5py library has been built with parallel support. Attributes: hdf (h5py.File): The HDF5 file to write to. comm (mpi.Comm): The MPI communicator. num_galaxies (int): The total number of galaxies. rank (int): The rank of the MPI process. is_parallel (bool): Whether the writer is running in parallel. is_root (bool): Whether the writer is running on the root process. is_collective (bool): Whether the writer is running in collective mode. verbose (bool): Whether to print verbose output. _start_time (float): The start time of the pipeline. """ # Is our h5py build parallel? if hasattr(h5py, "get_config"): PARALLEL = h5py.get_config().mpi else: PARALLEL = False @timed("PipelineIO.__init__") def __init__( self, filepath, comm=None, ngalaxies_local=None, start_time=None, verbose=1, parallel_io=False, ): """Initialize the HDF5Writer class. Args: filepath (str): The path to the HDF5 file to write to. comm (mpi.Comm, optional): The MPI communicator. ngalaxies_local (int, optional): The local number of galaxies. start_time (float, optional): The start time of the pipeline, used for timing information. verbose (int, optional): How verbose the output should be. 1 will only print on rank 0, 2 will print on all ranks, 0 will be silent. Defaults to 1. parallel_io (bool, optional): Whether to use parallel I/O. This requires h5py to be built with parallel support. Defaults to False. """ # Store the file path self.filepath = filepath # Store the communicator and its properties self.comm = comm self.size = comm.Get_size() if comm is not None else 1 self.rank = comm.Get_rank() if comm is not None else 0 # Make sure we can write in parallel if requested if parallel_io and not self.PARALLEL: raise RuntimeError( "h5py not built with parallel support. " "Cannot use parallel_io option." ) # Treat a communicator of size 1 as a serial run so single-rank jobs do # not create synthetic per-rank filenames such as ``_0``. self.is_parallel = comm is not None and self.size > 1 self.is_root = self.rank == 0 self.is_collective = self.is_parallel and self.PARALLEL and parallel_io # TODO: fix collective I/O if self.is_collective: self.is_collective = False warn( "Collective I/O currently locks up. " "Writing a file per rank instead. " "Feel free to contribute a fix!" ) # Store the start time if start_time is None: self._start_time = time.perf_counter() else: self._start_time = start_time # Are we talking? self.verbose = verbose # Time how long we have to wait for everyone to get here start = time.perf_counter() if self.is_parallel: self.comm.Barrier() self._took(start, "Waiting for all ranks to get to I/O") # If we are writing in parallel but not using collective I/O we # need to write a file per rank. Modify the file path to include # the rank. if not self.is_collective: ext = filepath.split(".")[-1] self.filepath = filepath.replace( f".{ext}", f"_{self.rank}.{ext}" ) # Report some useful information if self.is_collective: self._print( f"Writing in parallel to {filepath} " f"with {comm.Get_size()} ranks, and collective I/O." ) elif self.is_parallel: self._print( f"Writing to {filepath} " f"with {comm.Get_size()} ranks, and a file per rank." ) else: self._print(f"Writing to {filepath}.") # For collective I/O we need the counts on each rank so we know where # to write the data if self.is_collective or self.is_parallel: rank_gal_counts = self.comm.allgather(ngalaxies_local) self._ngalaxies_local = ngalaxies_local self.num_galaxies = sum(rank_gal_counts) self.start = sum(rank_gal_counts[: self.rank]) self.end = self.start + ngalaxies_local else: self.num_galaxies = ngalaxies_local self._ngalaxies_local = ngalaxies_local self.start = 0 self.end = ngalaxies_local def _print(self, *args, **kwargs): """Print a message to the screen with extra information. The prints behave differently depending on whether we are using MPI or not. We can also set the verbosity level at the Pipeline level which will control the verbosity of the print statements. Verbosity: 0: No output beyond hello and goodbye. 1: Outputs with timings but only on rank 0 (when using MPI). 2: Outputs with timings on all ranks (when using MPI). Args: *args: The message to print. **kwargs: Additional arguments to pass to the print function. """ # At verbosity 0 we are silent if self.verbose == 0: return # Get the current time code in seconds with 0 padding and 2 # decimal places now = time.perf_counter() - self._start_time int_now = str(int(now)).zfill( len(str(int(now))) + 1 if now > 9999 else 5 ) decimal = str(now).split(".")[-1][:2] now_str = f"{int_now}.{decimal}" # Create the prefix for the print, theres extra info to output if # we are using MPI if self.is_parallel: # Only print on rank 0 if we are using MPI and have verbosity 1 if self.verbose == 1 and self.rank != 0: return prefix = ( f"[{str(self.rank).zfill(len(str(self.size)) + 1)}]" f"[{now_str}]:" ) else: prefix = f"[{now_str}]:" print(prefix, *args, **kwargs) def _took(self, start, message): """Print a message with the time taken since the start time. Args: start (float): The start time of the process. message (str): The message to print. """ elapsed = time.perf_counter() - start # Report in sensible units if elapsed < 1: elapsed *= 1000 units = "ms" elif elapsed < 60: units = "s" else: elapsed /= 60 units = "mins" # Report how blazingly fast we are self._print(f"{message} took {elapsed:.3f} {units}.")
[docs] @timed("PipelineIO.create_file_with_metadata") def create_file_with_metadata( self, instruments, emission_model, include_lines=False, ): """Write metadata to the HDF5 file. This writes useful metadata to the root group of the HDF5 file and outputs the instruments and emission model to the appropriate groups. Args: instruments (dict): A dictionary of instrument objects. emission_model (dict): A dictionary of emission model objects. include_lines (bool): Whether to create the root-level line metadata group. """ start = time.perf_counter() # Unpack the instruments into a single instrument collection containing # all unique instruments unique_instruments = InstrumentCollection() for inst in instruments: if inst.label in unique_instruments: continue unique_instruments.add_instruments(inst) instruments = unique_instruments # Only write this metadata once if self.is_root: with h5py.File(self.filepath, "w") as hdf: # Write out some top level metadata hdf.attrs["synthesizer_version"] = __version__ # Create groups for the instruments, emission model, and # galaxies. inst_group = hdf.create_group("Instruments") model_group = hdf.create_group("EmissionModel") if include_lines: hdf.create_group("Lines") hdf.create_group("Galaxies") # we'll use this in a mo # Write out the instruments inst_start = time.perf_counter() inst_group.attrs["ninstruments"] = instruments.ninstruments for label, instrument in instruments.items(): instrument.to_hdf5(inst_group.create_group(label)) self._took(inst_start, "Writing instruments") # Write out the emission model model_start = time.perf_counter() for label, model in emission_model.items(): model.to_hdf5(model_group.create_group(label)) self._took(model_start, "Writing emission model") self._took(start, "Writing metadata") # The other ranks only hold galaxy outputs. Start each file afresh # here too, including the Galaxies group: a rank with no galaxies # will never create it through write_data, but combine_files needs it. elif self.is_parallel and not self.is_collective: with h5py.File(self.filepath, "w") as hdf: hdf.create_group("Galaxies") if self.is_parallel: self.comm.Barrier()
[docs] def write_dataset(self, data, key): """Write a dataset to an HDF5 file. We handle various different cases here: - If the data is a unyt object, we write the value and units. - If the data is a string we'll convert it to a h5py compatible string and write it with dimensionless units. - If the data is a numpy array, we write the data and set the units to "dimensionless". Args: data (any): The data to write. key (str): The key to write the data to. """ # Ensure we are working with a unyt_array data = unyt_array(data) # Strip the units off the data and convert to a numpy array if hasattr(data, "units"): units = str(data.units) data = data.value else: units = "dimensionless" # If we have an array of strings, convert to a h5py compatible string if data.dtype.kind == "U": data = np.array([d.encode("utf-8") for d in data]) # Write the dataset with the appropriate units with h5py.File(self.filepath, "a") as hdf: dset = hdf.create_dataset(key, data=data, compression="gzip") dset.attrs["Units"] = units
[docs] def write_dataset_parallel(self, data, key): """Write a dataset to an HDF5 file in parallel. This function requires that h5py has been built with parallel support. Args: data (any): The data to write. key (str): The key to write the data to. """ # If we have an array of strings, convert to a h5py compatible string if data.dtype.kind == "U": data = np.array([d.encode("utf-8") for d in data]) # Write the data for our slice with h5py.File( self.filepath, "a", driver="mpio", comm=self.comm, compression="gzip", ) as hdf: dset = hdf[key] dset[self.start : self.end, ...] = data self._print(f"Wrote dataset {key} with shape {data.shape}")
[docs] def write_datasets_recursive(self, data, key): """Write a dictionary to an HDF5 file recursively. Args: data (dict): The data to write. key (str): The key to write the data to. """ # Early exit if data is None if data is None: return # If the data isn't a dictionary just write the dataset if not isinstance(data, dict): try: self.write_dataset(data, key) except TypeError as e: raise TypeError( f"Failed to write dataset {key} (type={type(data)}) - {e}" ) return # Loop over the data for k, v in data.items(): self.write_datasets_recursive(v, f"{key}/{k}")
[docs] def write_datasets_parallel(self, data, key, paths): """Write a dictionary to an HDF5 file recursively in parallel. This function requires that h5py has been built with parallel support. Args: data (dict): The data to write. key (str): The key to write the data to. paths (list): The paths (key structure) of the datasets to write. """ # Loop over each path and write the data for path in paths: d = data for k in path.split("/"): d = d[k] self.write_dataset_parallel(d, f"{key}/{path}")
[docs] def create_datasets_parallel(self, data, key): """Create datasets ready to be populated in parallel. This is only needed for collective I/O operations. We will first make the datasets here in serial so they can be written to in any order on any rank. Args: data (dict): The data to write. key (str): The key to write the data to. """ start = time.perf_counter() # Get the shapes and dtypes of the data shapes, dtypes, units, paths = get_dataset_properties(data, self.comm) # Create the datasets if self.is_root: with h5py.File(self.filepath, "a") as hdf: for k, shape in shapes.items(): dset = hdf.create_dataset( f"{key}/{k}", shape=shape, dtype=dtypes[k], ) dset.attrs["Units"] = units[k] self._took(start, f"Creating datasets for {key}") return paths
[docs] @timed("PipelineIO.write_data") def write_data(self, data, key, indexes=None, root=0): """Write data using the appropriate method based on the environment. Args: data (any): The data to write. key (str): The key to write the data to. indexes (list, optional): The indexes to write the data to. root (int, optional): The root rank for gathering and writing. """ try: start = time.perf_counter() # In parallel land we need to make sure we're on the same page with # the structure we are writing if self.is_parallel: data = unify_dict_structure_across_ranks(data, self.comm) # Early exit if data is empty. Zero-row per-galaxy arrays are # still written so empty ranks hold the dataset with the right # trailing shape. if data is None or ( len(data) == 0 and (isinstance(data, dict) or indexes is None) ): return # Root-level metadata should only be written once, even when using # MPI or collective I/O. if self.is_parallel and indexes is None: if self.rank != root: return self.write_datasets_recursive(data, key) self._took(start, f"Writing {key} (and subgroups)") return # Use the appropriate write method if self.is_collective: # For collective I/O we need to create the datasets first, then # write the data to them in parallel. paths = self.create_datasets_parallel(data, key) self.comm.barrier() self.write_datasets_parallel(data, key, paths) else: # Otherwise, we can just write everything recursively. Bear # in mind that when using MPI this will write a file per # rank ready for later combination into a virtual file. self.write_datasets_recursive(data, key) self._took(start, f"Writing {key} (and subgroups)") except Exception as e: raise OSError(f"Failed to write {key} - {e}") from e
[docs] @timed("PipelineIO.combine_rank_files") def combine_rank_files(self): """Combine the rank files into a single file. Args: output_file (str): The name of the output file. """ start = time.perf_counter() def _recursive_copy(src, dest, slice): """Recursively copy the contents of an HDF5 group. Args: src (h5py.Group): The source group. dest (h5py.Group): The destination group. slice: The slice (along the first axis) the data belongs to. """ # Copy over the attributes for attr in src.attrs: dest.attrs[attr] = src.attrs[attr] # Loop over the items in the source group for k, v in src.items(): # If we found a group we need to recurse and create the group # in the destination file if it doesn't exist. We also need to # copy the attributes. if isinstance(v, h5py.Group): # Bin axes have no galaxy dimension. Copy their group # once instead of slicing each dataset by rank. if v.name in _SHARED_AXIS_GROUPS: if k not in dest: src.file.copy(v, dest, name=k) continue # Create the group if it doesn't exist if k not in dest: dest.create_group(k) # Recurse _recursive_copy(v, dest[k], slice) elif slice is None: # Just copy the dataset directly dset = dest.create_dataset( k, data=v[...], dtype=v.dtype, ) # Copy the attributes for attr in v.attrs: dset.attrs[attr] = v.attrs[attr] else: # Each rank must hold exactly one row per galaxy if v.shape[0] != slice.stop - slice.start: raise ValueError( f"Cannot combine {v.name}: {v.shape[0]} rows " f"for {slice.stop - slice.start} galaxies" ) # Empty ranks only hold zero-row placeholders, which # must not decide the combined dataset's shape if v.shape[0] == 0: continue # If the dataset doesn't exist we need to create it if k not in dest: dset = dest.create_dataset( k, shape=(self.num_galaxies, *v.shape[1:]), dtype=v.dtype, ) else: dset = dest[k] # Copy the data into the slice dset[slice, ...] = v[...] # Copy the attributes for attr in v.attrs: dset.attrs[attr] = v.attrs[attr] # Get the number of galaxies on each rank starts = self.comm.gather(self.start, root=0) ends = self.comm.gather(self.end, root=0) # Only the root rank needs to do this work if not self.is_root: return # Define the new file path ext = self.filepath.split(".")[-1] path_no_ext = ".".join(self.filepath.split(".")[:-1]) new_path = "_".join(path_no_ext.split("_")[:-1]) + f".{ext}" temp_path = "_".join(path_no_ext.split("_")[:-1]) + "_<rank>.hdf5" # Open the output file (any existing file is only replaced once the # new one is complete) with _hdf5_replaced_on_success(new_path) as hdf: # Loop over each rank file for rank in range(self.size): # Open the rank file with h5py.File( temp_path.replace("<rank>", str(rank)), "r", ) as rank_hdf: # Copy root metadata once and only combine Galaxies if rank == 0: for attr in rank_hdf.attrs: hdf.attrs[attr] = rank_hdf.attrs[attr] for key, value in rank_hdf.items(): if key == "Galaxies": hdf.create_group("Galaxies") continue hdf.copy(value, key) _recursive_copy( rank_hdf["Galaxies"], hdf["Galaxies"], slice=slice(starts[rank], ends[rank]), ) for rank in range(self.size): os.remove(temp_path.replace("<rank>", str(rank))) self._took(start, "Combining files")
[docs] @timed("PipelineIO.combine_rank_files_virtual") def combine_rank_files_virtual(self): """Combine the rank files into a single virtual file. Note that the virtual file this produces requires the rank files to remain in the same location as when they were created. """ start = time.perf_counter() # Gather start/end indices from all ranks starts = self.comm.gather(self.start, root=0) ends = self.comm.gather(self.end, root=0) # Only the root rank (rank == 0) needs to create the virtual file if not self.is_root: return # Define file paths ext = self.filepath.split(".")[-1] path_no_ext = ".".join(self.filepath.split(".")[:-1]) temp_path = "_".join(path_no_ext.split("_")[:-1]) + "_<rank>.hdf5" new_path = "_".join(path_no_ext.split("_")[:-1]) + f".{ext}" # Remove any old virtual file. Unlike a copied file it can't be kept # if this combine fails, since it points at the rank files this run # has already overwritten if os.path.exists(new_path): os.remove(new_path) # Open the first rank file (rank 0 file) to discover the structure first_file_path = temp_path.replace("<rank>", "0") with h5py.File(first_file_path, "r") as f0: # Gather dataset information (for virtual datasets). A rank # without galaxies may hold empty placeholders so we take each # dataset's information from the first rank with rows for it datasets_info = {} def gather_datasets(group, group_path="/"): for k, v in group.items(): current_path = f"{group_path}{k}" # Axes are shared metadata, not virtual galaxy rows. # We'll copy them directly from rank 0 below. if current_path in _SHARED_AXIS_GROUPS: continue if isinstance(v, h5py.Group): gather_datasets(v, current_path + "/") elif current_path not in datasets_info or ( datasets_info[current_path][0][0] == 0 and v.shape[0] > 0 ): # Record dataset shape, dtype, and attributes datasets_info[current_path] = ( v.shape, v.dtype, dict(v.attrs), ) # Gather group structure (to replicate in the virtual file) groups_info = {} def gather_groups(group, group_path="/"): for k, v in group.items(): current_path = f"{group_path}{k}" if current_path in _SHARED_AXIS_GROUPS: continue if isinstance(v, h5py.Group): groups_info.setdefault(current_path, dict(v.attrs)) gather_groups(v, current_path + "/") gather_datasets(f0["Galaxies"], "/Galaxies/") gather_groups(f0["Galaxies"], "/Galaxies/") # A rank without galaxies may be missing datasets entirely, so # also gather the structure from the other rank files for rank in range(1, self.size): src_file = temp_path.replace("<rank>", str(rank)) with h5py.File(src_file, "r") as rank_f: gather_datasets(rank_f["Galaxies"], "/Galaxies/") gather_groups(rank_f["Galaxies"], "/Galaxies/") # Create the virtual file (any existing file is only replaced # once the new one is complete) with _hdf5_replaced_on_success(new_path) as hdf: for attr in f0.attrs: hdf.attrs[attr] = f0.attrs[attr] for key, value in f0.items(): if key == "Galaxies": continue hdf.copy(value, key) hdf.create_group("Galaxies") # Create empty group structure for gpath, gattrs in groups_info.items(): grp = hdf.create_group(gpath) for attr_name, attr_val in gattrs.items(): grp.attrs[attr_name] = attr_val # Preserve the axes at their usual paths without interpreting # their lengths as a number of galaxies. Only rank 0 writes # them, so it is also the source of truth here. for path in _SHARED_AXIS_GROUPS: if path in f0: hdf.require_group("Galaxies/Stars") f0.copy(path, hdf, name=path) # Construct the virtual datasets for each dataset for dpath, (shape, dtype, dattrs) in datasets_info.items(): final_shape = (self.num_galaxies,) + shape[1:] layout = h5py.VirtualLayout(shape=final_shape, dtype=dtype) # Loop over each rank and map the data to the layout, # keeping track of whether any rank had rows to map mapped = False for rank, (start_i, end_i) in enumerate(zip(starts, ends)): src_file = temp_path.replace("<rank>", str(rank)) with h5py.File(src_file, "r") as rank_f: # Nothing to map from a rank without this dataset if dpath not in rank_f: continue src_dset = rank_f[dpath] src_shape = src_dset.shape src_units = src_dset.attrs.get("Units") local_size = src_dset.shape[0] # Nothing to map from a rank without any rows if local_size == 0: continue # Ensure the local size is as expected expected_size = end_i - start_i if local_size != expected_size: raise ValueError( f"Cannot map {dpath}: rank {rank} has " f"{local_size} rows for {expected_size} " "galaxies" ) target_shape = (end_i - start_i,) + shape[1:] if ( len(src_shape) != len(shape) or src_shape[1:] != shape[1:] ): raise ValueError( "Cannot create virtual dataset mapping for " f"{dpath!r}: rank {rank} dataset in " f"{src_file!r} " f"has shape {src_shape}, but other ranks have " f"shape {shape}. Non-leading dimensions must " "match across rank files." ) # The units must also match, we don't convert them if src_units != dattrs.get("Units"): raise ValueError( "Cannot create virtual dataset mapping for " f"{dpath!r}: rank {rank} dataset in " f"{src_file!r} has units {src_units!r}, but " f"other ranks have units " f"{dattrs.get('Units')!r}." ) # Create a virtual source for this rank vsource = h5py.VirtualSource( src_file, dpath, shape=(local_size,) + shape[1:] ) # Map the portion of the layout to this source slice try: layout[start_i:end_i, ...] = vsource[...] except ValueError as exc: raise ValueError( "Failed to map rank dataset into virtual " f"dataset for {dpath!r}. Rank {rank}, source " f"file {src_file!r}, source shape " f"{src_shape}, virtual slice " f"[{start_i}:{end_i}] with target " f"shape {target_shape}, final virtual shape " f"{final_shape}. Original error: {exc}" ) from exc mapped = True # Skip datasets without rows on any rank, otherwise we'd # make a dataset full of fill values if not mapped: continue # Create the virtual dataset in the final file vds = hdf.create_virtual_dataset(dpath, layout) # Copy dataset attributes for attr_name, attr_val in dattrs.items(): vds.attrs[attr_name] = attr_val self._took(start, "Creating virtual file")