Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for ASDF serialisation and deserialisation #776

Draft
wants to merge 49 commits into
base: main
Choose a base branch
from
Draft
Changes from 1 commit
Commits
Show all changes
49 commits
Select commit Hold shift + click to select a range
ac36722
initial commit
ViciousEagle03 May 20, 2024
6be61e7
version alignment
ViciousEagle03 May 20, 2024
6e619fc
Remove method to remove any delay and
ViciousEagle03 May 22, 2024
f505703
Update tag and
ViciousEagle03 May 22, 2024
af2f9c7
enable schema test
ViciousEagle03 May 22, 2024
adfc6b3
revert small change
ViciousEagle03 May 22, 2024
78eeb53
Add the converters for ExtraCoords and TimeTableCoordinate class
ViciousEagle03 May 30, 2024
0804f1d
Add ExtraCoords and TimeTableCoord Schema
ViciousEagle03 May 30, 2024
0f4a86f
Update manifest and entry_point.py
ViciousEagle03 May 30, 2024
86bfac9
Add the QuantityTableCoordinate and SkyCoordTableCoordinate converters
ViciousEagle03 May 31, 2024
da1c14e
Add the schema for QuantityTableCoordinate and SkyCoordTableCoordinate
ViciousEagle03 May 31, 2024
dc24d40
Update manifest and entry_point.py
ViciousEagle03 May 31, 2024
ef208ed
Add validation for schema and manifests
ViciousEagle03 Jun 4, 2024
c3d2606
pre-commit
Cadair Jun 5, 2024
86c2b6c
Update the tox.ini and CI workflow
ViciousEagle03 Jun 5, 2024
3a875ab
Update schemas
ViciousEagle03 Jun 5, 2024
ce5a14b
Add the converter and schema for GlobalCoords
ViciousEagle03 Jun 17, 2024
9247b46
Update converters
ViciousEagle03 Jun 17, 2024
c8a1bc4
Update entry_points,schemas and manifest
ViciousEagle03 Jun 17, 2024
c681cb2
minor change
ViciousEagle03 Jun 19, 2024
42c590e
lowercase schema URIs and use wildcards
ViciousEagle03 Jun 21, 2024
7f3a81b
Add tests and GWCS objects
ViciousEagle03 Jul 1, 2024
7622950
apply suggestions from code review
ViciousEagle03 Jul 6, 2024
f89e195
Remove mesh as a property validator
ViciousEagle03 Jul 11, 2024
8b519ae
Update the dependencies version
ViciousEagle03 Jul 11, 2024
37bf563
Style changes and add warnings to NDCube converter
ViciousEagle03 Jul 19, 2024
693e63e
env name update
ViciousEagle03 Jul 19, 2024
de6944e
Add asdf as an optional dep
ViciousEagle03 Aug 4, 2024
22a839e
Remove asdf as an optional dependency
ViciousEagle03 Aug 5, 2024
ec863ad
Add support for meta in converter and schema
ViciousEagle03 Aug 6, 2024
b2fd5e1
Add mask as a validator property
ViciousEagle03 Aug 21, 2024
6d81ee9
Add unit support to asdf
Cadair Aug 22, 2024
a6a33be
Add the serialization support for the ResampledLowlevelWCS ReorderedL…
ViciousEagle03 Aug 16, 2024
95486b0
Apply suggestions from code review
ViciousEagle03 Aug 23, 2024
4fb5288
Use Tabular1D, Tabular2D instead of tabular_model
ViciousEagle03 Aug 25, 2024
6cf9602
Add comment and update logic
ViciousEagle03 Aug 25, 2024
ddc75be
Apply minor suggestions from code review
DanRyanIrish Nov 4, 2024
e7cf14f
More minor tweaks
Cadair Nov 6, 2024
7ee5048
Merge pull request #751 from ViciousEagle03/wcs_wrapper_asdf_support
Cadair Nov 6, 2024
8610805
Add validation for schema and manifests
ViciousEagle03 Jun 4, 2024
149944f
revert small change
ViciousEagle03 Jul 4, 2024
98e633c
Add asdf as an optional dep
ViciousEagle03 Aug 4, 2024
789e841
Remove asdf as an optional dependency
ViciousEagle03 Aug 5, 2024
24af793
Add serialization logic for the NDCubeSequence and NDCollection
ViciousEagle03 Aug 24, 2024
494e2c5
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Nov 6, 2024
db8f22d
Minor tweaks
Cadair Nov 6, 2024
3876924
Merge pull request #756 from ViciousEagle03/asdf_ndcube_sequence_and_…
Cadair Nov 7, 2024
1b9679a
Merge branch 'main' into asdf-support
Cadair Nov 7, 2024
082d8c6
Merge branch 'main' into asdf-support
Cadair Nov 13, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
apply suggestions from code review
ViciousEagle03 authored and Cadair committed Nov 6, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
commit 76229509755731baf3116d31105bb221c266bb58
5 changes: 2 additions & 3 deletions ndcube/asdf/converters/extracoords_converter.py
Original file line number Diff line number Diff line change
@@ -10,7 +10,7 @@ def from_yaml_tree(self, node, tag, ctx):
extra_coords = ExtraCoords()
extra_coords._wcs = node.get("wcs")
extra_coords._mapping = node.get("mapping")
extra_coords._lookup_tables = node.get("lookup_tables")
extra_coords._lookup_tables = node.get("lookup_tables", [])
extra_coords._dropped_tables = node.get("dropped_tables")
extra_coords._ndcube = node.get("ndcube")
return extra_coords
@@ -23,7 +23,6 @@ def to_yaml_tree(self, extracoords, tag, ctx):
node["mapping"] = extracoords._mapping
if extracoords._lookup_tables:
node["lookup_tables"] = extracoords._lookup_tables
if extracoords._dropped_tables:
node["dropped_tables"] = extracoords._dropped_tables
node["dropped_tables"] = extracoords._dropped_tables
node["ndcube"] = extracoords._ndcube
return node
8 changes: 3 additions & 5 deletions ndcube/asdf/converters/globalcoords_converter.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
from typing import OrderedDict

from asdf.extension import Converter


@@ -11,8 +9,8 @@ def from_yaml_tree(self, node, tag, ctx):
from ndcube.global_coords import GlobalCoords

globalcoords = GlobalCoords()
if node.get("internal_coords") is not None:
globalcoords._internal_coords = OrderedDict(node.get("internal_coords"))
if "internal_coords" in node:
globalcoords._internal_coords = node["internal_coords"]
globalcoords._ndcube = node["ndcube"]

return globalcoords
@@ -21,6 +19,6 @@ def to_yaml_tree(self, globalcoords, tag, ctx):
node = {}
node["ndcube"] = globalcoords._ndcube
if globalcoords._internal_coords:
node["internal_coords"] = dict(globalcoords._internal_coords)
node["internal_coords"] = globalcoords._internal_coords

return node
4 changes: 1 addition & 3 deletions ndcube/asdf/converters/ndcube_converter.py
Original file line number Diff line number Diff line change
@@ -8,9 +8,7 @@ class NDCubeConverter(Converter):
def from_yaml_tree(self, node, tag, ctx):
from ndcube.ndcube import NDCube

data = node["data"]
wcs = node["wcs"]
ndcube = NDCube(data, wcs)
ndcube = NDCube(node["data"], node["wcs"])
ndcube._extra_coords = node["extra_coords"]
ndcube._global_coords = node["global_coords"]

12 changes: 4 additions & 8 deletions ndcube/asdf/converters/tablecoord_converter.py
Original file line number Diff line number Diff line change
@@ -8,12 +8,11 @@ class TimeTableCoordConverter(Converter):
def from_yaml_tree(self, node, tag, ctx):
from ndcube.extra_coords.table_coord import TimeTableCoordinate

table = node.get("table")
names = node.get("names")
physical_types = node.get("physical_types")
reference_time = node.get("reference_time")
timetablecoordinate = TimeTableCoordinate(
table, names=names, physical_types=physical_types, reference_time=reference_time)
node["table"], names=names, physical_types=physical_types, reference_time=reference_time)

return timetablecoordinate

@@ -36,14 +35,12 @@ class QuantityTableCoordinateConverter(Converter):
def from_yaml_tree(self, node, tag, ctx):
from ndcube.extra_coords.table_coord import QuantityTableCoordinate

unit = node.get("unit")
table = node.get("table")
names = node.get("names")
mesh = node.get("mesh")
physical_types = node.get("physical_types")
quantitytablecoordinate = QuantityTableCoordinate(*table,
quantitytablecoordinate = QuantityTableCoordinate(*node["table"],
names=names, physical_types=physical_types)
quantitytablecoordinate.unit = unit
quantitytablecoordinate.unit = node["unit"]
quantitytablecoordinate.mesh = mesh
return quantitytablecoordinate

@@ -66,11 +63,10 @@ class SkyCoordTableCoordinateConverter(Converter):
def from_yaml_tree(self, node, tag, ctx):
from ndcube.extra_coords.table_coord import SkyCoordTableCoordinate

table = node.get("table")
names = node.get("names")
mesh = node.get("mesh")
physical_types = node.get("physical_types")
skycoordinatetablecoordinate = SkyCoordTableCoordinate(table, mesh=mesh,
skycoordinatetablecoordinate = SkyCoordTableCoordinate(node["table"], mesh=mesh,
names=names, physical_types=physical_types)
return skycoordinatetablecoordinate

5 changes: 3 additions & 2 deletions ndcube/asdf/converters/tests/test_ndcube_converter.py
Original file line number Diff line number Diff line change
@@ -8,9 +8,10 @@

@pytest.mark.parametrize("ndc",[("ndcube_gwcs_2d_ln_lt"),
("ndcube_gwcs_3d_ln_lt_l"),
("ndcube_gwcs_3d_ln_lt_l_ec_gc"),
("ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim"),
("ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc"),
("ndcube_gwcs_3d_rotated"),
("ndcube_gwcs_4d_ln_lt_l_t"),
("ndcube_gwcs_4d_ln_lt_l_t")
], indirect=("ndc",))
def test_serialization(ndc, tmp_path):
file_path = tmp_path / "test.asdf"
17 changes: 8 additions & 9 deletions ndcube/asdf/resources/schemas/global_coords-0.1.0.yaml
Original file line number Diff line number Diff line change
@@ -13,15 +13,14 @@ type: object
properties:
internal_coords:
type: object
patternProperties:
.*:
type: array
items:
- type: string
- type: object
oneOf:
- tag: "tag:stsci.edu:asdf/unit/quantity-*"
- tag: "tag:astropy.org:astropy/coordinates/skycoord-*"
additionalProperties:
type: array
items:
- type: string
- type: object
oneOf:
- tag: "tag:stsci.edu:asdf/unit/quantity-*"
- tag: "tag:astropy.org:astropy/coordinates/skycoord-*"
ndcube:
tag: "tag:sunpy.org:ndcube/ndcube/ndcube-0.*"

2 changes: 1 addition & 1 deletion ndcube/asdf/resources/schemas/ndcube-0.1.0.yaml
Original file line number Diff line number Diff line change
@@ -20,6 +20,6 @@ properties:
global_coords:
tag: "tag:sunpy.org:ndcube/global_coords/globalcoords-0.*"

required: [data , wcs]
required: [data, wcs]
additionalProperties: true
...
18 changes: 16 additions & 2 deletions ndcube/conftest.py
Original file line number Diff line number Diff line change
@@ -39,6 +39,10 @@
# Helper Functions
################################################################################

def time_lut(shape):
base_time = Time('2000-01-01', format='fits', scale='utc')
timestamps = Time([base_time + TimeDelta(60 * i, format='sec') for i in range(shape[0])])
return timestamps

def skycoord_2d_lut(shape):
total_len = np.prod(shape)
@@ -477,14 +481,24 @@ def ndcube_gwcs_3d_rotated(gwcs_3d_lt_ln_l, simple_extra_coords_3d):
return cube

@pytest.fixture
def ndcube_gwcs_3d_ln_lt_l_ec_gc(gwcs_3d_lt_ln_l, simple_extra_coords_3d):
def ndcube_gwcs_3d_ln_lt_l_ec_dropped_dim(gwcs_3d_lt_ln_l, time_and_simple_extra_coords_2d):
shape = (2, 3, 4)
gwcs_3d_lt_ln_l.array_shape = shape
data_cube = data_nd(shape)
cube = NDCube(data_cube, wcs=gwcs_3d_lt_ln_l)
cube._extra_coords = time_and_simple_extra_coords_2d[0]
return cube

@pytest.fixture
def ndcube_gwcs_3d_ln_lt_l_ec_q_t_gc(gwcs_3d_lt_ln_l):
shape = (3, 3, 4)
gwcs_3d_lt_ln_l.array_shape = shape
data_cube = data_nd(shape)
cube = NDCube(data_cube, wcs=gwcs_3d_lt_ln_l)
coord1 = 1 * u.m
cube.global_coords.add('name1', 'custom:physical_type1', coord1)
cube._extra_coords = simple_extra_coords_3d
cube.extra_coords.add("time", 0, time_lut(shape))
cube.extra_coords.add("exposure_lut", 1, range(shape[1]) * u.s)
return cube

@pytest.fixture