Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding and updating __init__ files for modules #14

Merged
merged 15 commits into from
Mar 11, 2024
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/mpi/global_timings.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import numpy as np
from mpi4py import MPI

from ndsl.performance.timer import Timer
from ndsl import Timer


@contextlib.contextmanager
Expand Down
11 changes: 7 additions & 4 deletions examples/mpi/zarr_monitor.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,14 @@
import zarr
from mpi4py import MPI

from ndsl.comm.partitioner import CubedSpherePartitioner, TilePartitioner
from ndsl import (
CubedSpherePartitioner,
QuantityFactory,
SubtileGridSizer,
TilePartitioner,
ZarrMonitor,
)
from ndsl.constants import X_DIM, Y_DIM, Z_DIM
from ndsl.initialization.allocator import QuantityFactory
from ndsl.initialization.sizer import SubtileGridSizer
from ndsl.monitor import ZarrMonitor


OUTPUT_PATH = "output/zarr_monitor.zarr"
Expand Down
61 changes: 61 additions & 0 deletions ndsl/__init__.py
FlorianDeconinck marked this conversation as resolved.
Show resolved Hide resolved
FlorianDeconinck marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
@@ -1 +1,62 @@
from .buffer import Buffer
from .comm.boundary import Boundary, SimpleBoundary
FlorianDeconinck marked this conversation as resolved.
Show resolved Hide resolved
from .comm.communicator import CubedSphereCommunicator, TileCommunicator
from .comm.local_comm import AsyncResult, ConcurrencyError, LocalComm
FlorianDeconinck marked this conversation as resolved.
Show resolved Hide resolved
from .comm.mpi import MPIComm
from .comm.null_comm import NullAsyncResult, NullComm
FlorianDeconinck marked this conversation as resolved.
Show resolved Hide resolved
from .comm.partitioner import CubedSpherePartitioner, TilePartitioner
from .constants import ConstantVersions
from .dsl.caches.codepath import FV3CodePath
from .dsl.dace.dace_config import DaceConfig, DaCeOrchestration, FrozenCompiledSDFG
from .dsl.dace.orchestration import orchestrate, orchestrate_function
from .dsl.dace.utils import (
ArrayReport,
DaCeProgress,
MaxBandwithBenchmarkProgram,
StorageReport,
)
from .dsl.dace.wrapped_halo_exchange import WrappedHaloUpdater
from .dsl.stencil import (
CompareToNumpyStencil,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if we need this at the top level

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is advanced system but for now it's the only one, so I'd leave it here

FlorianDeconinck marked this conversation as resolved.
Show resolved Hide resolved
FrozenStencil,
GridIndexing,
StencilFactory,
TimingCollector,
)
from .dsl.stencil_config import CompilationConfig, RunMode, StencilConfig
from .exceptions import OutOfBoundsError
from .halo.data_transformer import (
HaloDataTransformer,
HaloDataTransformerCPU,
HaloDataTransformerGPU,
HaloExchangeSpec,
)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure that these need to be here, but will defer to @FlorianDeconinck

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Spec is needed, the other should be down in halo

from .halo.updater import HaloUpdater, HaloUpdateRequest, VectorInterfaceHaloUpdater
from .initialization.allocator import QuantityFactory, StorageNumpy
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think StorageNumpy needs to be at ndsl root

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nope, hide it

from .initialization.sizer import GridSizer, SubtileGridSizer
from .logging import ndsl_log
from .monitor.netcdf_monitor import NetCDFMonitor
from .monitor.protocol import Protocol
from .monitor.zarr_monitor import ZarrMonitor
from .namelist import Namelist
from .optional_imports import RaiseWhenAccessed
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

RaiseWhenAccessed doesn't need to be here

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hide away

from .performance.collector import (
AbstractPerformanceCollector,
NullPerformanceCollector,
PerformanceCollector,
)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unsure about these

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Abstract into typing, the other two are required for pace.driver and any model wrapper

from .performance.config import PerformanceConfig
from .performance.profiler import NullProfiler, Profiler
from .performance.report import Experiment, Report, TimeReport
FlorianDeconinck marked this conversation as resolved.
Show resolved Hide resolved
from .performance.timer import NullTimer, Timer
from .quantity import (
BoundaryArrayView,
BoundedArrayView,
Quantity,
QuantityHaloSpec,
QuantityMetadata,
)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think we mostly just need Quantity here, everything else doesn't need to be at this level I think.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Quantity is the main thing. QuantityHaloSpec might be needed in the typing though right now we are not using it, so can be burried

from .testing.dummy_comm import DummyComm
from .types import Allocator, AsyncRequest, NumpyModule
from .units import UnitsError
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think these need to be exposed here either

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Those can be really useful for typing, especially NumpyModule. Move NumpyModule, AsyncRequest, UnitsError in typing

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Poke

from .utils import MetaEnumStr
3 changes: 1 addition & 2 deletions ndsl/checkpointer/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
from .base import Checkpointer
from .null import NullCheckpointer
from .snapshots import SnapshotCheckpointer
from .snapshots import SnapshotCheckpointer, _Snapshots
from .thresholds import (
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need _Snapshots? We should always try to have _X class or functions hidden, as this what the prefixed _ is suppose to hint at

InsufficientTrialsError,
SavepointThresholds,
Expand Down
7 changes: 0 additions & 7 deletions ndsl/checkpointer/base.py
FlorianDeconinck marked this conversation as resolved.
Show resolved Hide resolved

This file was deleted.

2 changes: 1 addition & 1 deletion ndsl/checkpointer/null.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from .base import Checkpointer
from ndsl.typing import Checkpointer


class NullCheckpointer(Checkpointer):
Expand Down
3 changes: 1 addition & 2 deletions ndsl/checkpointer/snapshots.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,7 @@

from ndsl.optional_imports import cupy as cp
from ndsl.optional_imports import xarray as xr

from .base import Checkpointer
from ndsl.typing import Checkpointer


def make_dims(savepoint_dim, label, data_list):
Expand Down
4 changes: 2 additions & 2 deletions ndsl/checkpointer/thresholds.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@

import numpy as np

from ..quantity import Quantity
from .base import Checkpointer
from ndsl.quantity import Quantity
from ndsl.typing import Checkpointer


try:
Expand Down
10 changes: 7 additions & 3 deletions ndsl/checkpointer/validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,14 @@

import numpy as np

from ndsl.checkpointer.thresholds import (
ArrayLike,
SavepointName,
SavepointThresholds,
cast_to_ndarray,
)
from ndsl.optional_imports import xarray as xr

from .base import Checkpointer
from .thresholds import ArrayLike, SavepointName, SavepointThresholds, cast_to_ndarray
from ndsl.typing import Checkpointer


def _clip_pace_array_to_target(
Expand Down
9 changes: 9 additions & 0 deletions ndsl/comm/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
from .caching_comm import (
CachingCommData,
CachingCommReader,
CachingCommWriter,
CachingRequestReader,
CachingRequestWriter,
NullRequest,
FlorianDeconinck marked this conversation as resolved.
Show resolved Hide resolved
)
from .comm_abc import Comm, Request
4 changes: 2 additions & 2 deletions ndsl/comm/boundary.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import dataclasses
from typing import Tuple

from ..quantity import Quantity, QuantityHaloSpec
from ._boundary_utils import get_boundary_slice
from ndsl.comm._boundary_utils import get_boundary_slice
from ndsl.quantity import Quantity, QuantityHaloSpec


@dataclasses.dataclass
Expand Down
Loading
Loading