Skip to content

[Backport 9.1] DSL: preserve the skip_empty setting in to_dict() recursive serializations #3047

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Aug 20, 2025
Merged

[Backport 9.1] DSL: preserve the skip_empty setting in to_dict() recursive serializations #3047

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 23 additions & 10 deletions elasticsearch/dsl/field.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,16 @@ def __init__(
def __getitem__(self, subfield: str) -> "Field":
return cast(Field, self._params.get("fields", {})[subfield])

def _serialize(self, data: Any) -> Any:
def _serialize(self, data: Any, skip_empty: bool) -> Any:
return data

def _safe_serialize(self, data: Any, skip_empty: bool) -> Any:
try:
return self._serialize(data, skip_empty)
except TypeError:
# older method signature, without skip_empty
return self._serialize(data) # type: ignore[call-arg]

def _deserialize(self, data: Any) -> Any:
return data

Expand All @@ -133,10 +140,16 @@ def empty(self) -> Optional[Any]:
return AttrList([])
return self._empty()

def serialize(self, data: Any) -> Any:
def serialize(self, data: Any, skip_empty: bool = True) -> Any:
if isinstance(data, (list, AttrList, tuple)):
return list(map(self._serialize, cast(Iterable[Any], data)))
return self._serialize(data)
return list(
map(
self._safe_serialize,
cast(Iterable[Any], data),
[skip_empty] * len(data),
)
)
return self._safe_serialize(data, skip_empty)

def deserialize(self, data: Any) -> Any:
if isinstance(data, (list, AttrList, tuple)):
Expand Down Expand Up @@ -186,7 +199,7 @@ def _deserialize(self, data: Any) -> Range["_SupportsComparison"]:
data = {k: self._core_field.deserialize(v) for k, v in data.items()} # type: ignore[union-attr]
return Range(data)

def _serialize(self, data: Any) -> Optional[Dict[str, Any]]:
def _serialize(self, data: Any, skip_empty: bool) -> Optional[Dict[str, Any]]:
if data is None:
return None
if not isinstance(data, collections.abc.Mapping):
Expand Down Expand Up @@ -550,7 +563,7 @@ def _deserialize(self, data: Any) -> "InnerDoc":
return self._wrap(data)

def _serialize(
self, data: Optional[Union[Dict[str, Any], "InnerDoc"]]
self, data: Optional[Union[Dict[str, Any], "InnerDoc"]], skip_empty: bool
) -> Optional[Dict[str, Any]]:
if data is None:
return None
Expand All @@ -559,7 +572,7 @@ def _serialize(
if isinstance(data, collections.abc.Mapping):
return data

return data.to_dict()
return data.to_dict(skip_empty=skip_empty)

def clean(self, data: Any) -> Any:
data = super().clean(data)
Expand Down Expand Up @@ -768,7 +781,7 @@ def clean(self, data: str) -> str:
def _deserialize(self, data: Any) -> bytes:
return base64.b64decode(data)

def _serialize(self, data: Any) -> Optional[str]:
def _serialize(self, data: Any, skip_empty: bool) -> Optional[str]:
if data is None:
return None
return base64.b64encode(data).decode()
Expand Down Expand Up @@ -2619,7 +2632,7 @@ def _deserialize(self, data: Any) -> Union["IPv4Address", "IPv6Address"]:
# the ipaddress library for pypy only accepts unicode.
return ipaddress.ip_address(unicode(data))

def _serialize(self, data: Any) -> Optional[str]:
def _serialize(self, data: Any, skip_empty: bool) -> Optional[str]:
if data is None:
return None
return str(data)
Expand Down Expand Up @@ -3367,7 +3380,7 @@ def __init__(
def _deserialize(self, data: Any) -> "Query":
return Q(data) # type: ignore[no-any-return]

def _serialize(self, data: Any) -> Optional[Dict[str, Any]]:
def _serialize(self, data: Any, skip_empty: bool) -> Optional[Dict[str, Any]]:
if data is None:
return None
return data.to_dict() # type: ignore[no-any-return]
Expand Down
2 changes: 1 addition & 1 deletion elasticsearch/dsl/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -603,7 +603,7 @@ def to_dict(self, skip_empty: bool = True) -> Dict[str, Any]:
# if this is a mapped field,
f = self.__get_field(k)
if f and f._coerce:
v = f.serialize(v)
v = f.serialize(v, skip_empty=skip_empty)

# if someone assigned AttrList, unwrap it
if isinstance(v, AttrList):
Expand Down
11 changes: 9 additions & 2 deletions test_elasticsearch/test_dsl/test_integration/_async/test_document.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -630,15 +630,22 @@ async def test_can_save_to_different_index(
async def test_save_without_skip_empty_will_include_empty_fields(
async_write_client: AsyncElasticsearch,
) -> None:
test_repo = Repository(field_1=[], field_2=None, field_3={}, meta={"id": 42})
test_repo = Repository(
field_1=[], field_2=None, field_3={}, owner={"name": None}, meta={"id": 42}
)
assert await test_repo.save(index="test-document", skip_empty=False)

assert_doc_equals(
{
"found": True,
"_index": "test-document",
"_id": "42",
"_source": {"field_1": [], "field_2": None, "field_3": {}},
"_source": {
"field_1": [],
"field_2": None,
"field_3": {},
"owner": {"name": None},
},
},
await async_write_client.get(index="test-document", id=42),
)
Expand Down
11 changes: 9 additions & 2 deletions test_elasticsearch/test_dsl/test_integration/_sync/test_document.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -624,15 +624,22 @@ def test_can_save_to_different_index(
def test_save_without_skip_empty_will_include_empty_fields(
write_client: Elasticsearch,
) -> None:
test_repo = Repository(field_1=[], field_2=None, field_3={}, meta={"id": 42})
test_repo = Repository(
field_1=[], field_2=None, field_3={}, owner={"name": None}, meta={"id": 42}
)
assert test_repo.save(index="test-document", skip_empty=False)

assert_doc_equals(
{
"found": True,
"_index": "test-document",
"_id": "42",
"_source": {"field_1": [], "field_2": None, "field_3": {}},
"_source": {
"field_1": [],
"field_2": None,
"field_3": {},
"owner": {"name": None},
},
},
write_client.get(index="test-document", id=42),
)
Expand Down
27 changes: 17 additions & 10 deletions utils/templates/field.py.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,16 @@ class Field(DslBase):
def __getitem__(self, subfield: str) -> "Field":
return cast(Field, self._params.get("fields", {})[subfield])

def _serialize(self, data: Any) -> Any:
def _serialize(self, data: Any, skip_empty: bool) -> Any:
return data

def _safe_serialize(self, data: Any, skip_empty: bool) -> Any:
try:
return self._serialize(data, skip_empty)
except TypeError:
# older method signature, without skip_empty
return self._serialize(data) # type: ignore[call-arg]

def _deserialize(self, data: Any) -> Any:
return data

Expand All @@ -133,10 +140,10 @@ class Field(DslBase):
return AttrList([])
return self._empty()

def serialize(self, data: Any) -> Any:
def serialize(self, data: Any, skip_empty: bool = True) -> Any:
if isinstance(data, (list, AttrList, tuple)):
return list(map(self._serialize, cast(Iterable[Any], data)))
return self._serialize(data)
return list(map(self._safe_serialize, cast(Iterable[Any], data), [skip_empty] * len(data)))
return self._safe_serialize(data, skip_empty)

def deserialize(self, data: Any) -> Any:
if isinstance(data, (list, AttrList, tuple)):
Expand Down Expand Up @@ -186,7 +193,7 @@ class RangeField(Field):
data = {k: self._core_field.deserialize(v) for k, v in data.items()} # type: ignore[union-attr]
return Range(data)

def _serialize(self, data: Any) -> Optional[Dict[str, Any]]:
def _serialize(self, data: Any, skip_empty: bool) -> Optional[Dict[str, Any]]:
if data is None:
return None
if not isinstance(data, collections.abc.Mapping):
Expand Down Expand Up @@ -318,7 +325,7 @@ class {{ k.name }}({{ k.parent }}):
return self._wrap(data)

def _serialize(
self, data: Optional[Union[Dict[str, Any], "InnerDoc"]]
self, data: Optional[Union[Dict[str, Any], "InnerDoc"]], skip_empty: bool
) -> Optional[Dict[str, Any]]:
if data is None:
return None
Expand All @@ -327,7 +334,7 @@ class {{ k.name }}({{ k.parent }}):
if isinstance(data, collections.abc.Mapping):
return data

return data.to_dict()
return data.to_dict(skip_empty=skip_empty)

def clean(self, data: Any) -> Any:
data = super().clean(data)
Expand Down Expand Up @@ -433,7 +440,7 @@ class {{ k.name }}({{ k.parent }}):
# the ipaddress library for pypy only accepts unicode.
return ipaddress.ip_address(unicode(data))

def _serialize(self, data: Any) -> Optional[str]:
def _serialize(self, data: Any, skip_empty: bool) -> Optional[str]:
if data is None:
return None
return str(data)
Expand All @@ -448,7 +455,7 @@ class {{ k.name }}({{ k.parent }}):
def _deserialize(self, data: Any) -> bytes:
return base64.b64decode(data)

def _serialize(self, data: Any) -> Optional[str]:
def _serialize(self, data: Any, skip_empty: bool) -> Optional[str]:
if data is None:
return None
return base64.b64encode(data).decode()
Expand All @@ -458,7 +465,7 @@ class {{ k.name }}({{ k.parent }}):
def _deserialize(self, data: Any) -> "Query":
return Q(data) # type: ignore[no-any-return]

def _serialize(self, data: Any) -> Optional[Dict[str, Any]]:
def _serialize(self, data: Any, skip_empty: bool) -> Optional[Dict[str, Any]]:
if data is None:
return None
return data.to_dict() # type: ignore[no-any-return]
Expand Down