Merge remote-tracking branch 'origin/main' into trio-support

Author: pquentin

.buildkite/functions/imports.sh

@@ -43,7 +43,7 @@ if [[ -z $es_node_name ]]; then
 
 fi
 
-  export script_path=$(dirname $(realpath -s $0))
+  export script_path=$(dirname $(realpath $0))
   source $script_path/functions/cleanup.sh
   source $script_path/functions/wait-for-container.sh
   trap "cleanup_trap ${network_name}" EXIT

.buildkite/run-elasticsearch.sh

@@ -21,7 +21,7 @@
 # - Moved ELASTIC_PASSWORD and xpack.security.enabled to the base arguments for "Security On by default"
 # - Use https only when TEST_SUITE is "platinum", when "free" use http
 
-script_path=$(dirname $(realpath -s $0))
+script_path=$(dirname $(realpath $0))
 source $script_path/functions/imports.sh
 set -euo pipefail
 

.buildkite/run-tests

@@ -10,7 +10,7 @@ export TEST_SUITE="${TEST_SUITE:=platinum}"
 export PYTHON_VERSION="${PYTHON_VERSION:=3.13}"
 export PYTHON_CONNECTION_CLASS="${PYTHON_CONNECTION_CLASS:=urllib3}"
 
-script_path=$(dirname $(realpath -s $0))
+script_path=$(dirname $(realpath $0))
 source $script_path/functions/imports.sh
 set -euo pipefail
 

docs/reference/connecting.md

@@ -277,13 +277,6 @@ client = Elasticsearch(
 ```
 
 
-## Enabling the Compatibility Mode [compatibility-mode]
-
-The {{es}} server version 8.0 is introducing a new compatibility mode that allows you a smoother upgrade experience from 7 to 8. In a nutshell, you can use the latest 7.x Python {{es}} {{es}} client with an 8.x {{es}} server, giving more room to coordinate the upgrade of your codebase to the next major version.
-
-If you want to leverage this functionality, please make sure that you are using the latest 7.x Python {{es}} client and set the environment variable `ELASTIC_CLIENT_APIVERSIONING` to `true`. The client is handling the rest internally. For every 8.0 and beyond Python {{es}} client, you’re all set! The compatibility mode is enabled by default.
-
-
 ## Using the Client in a Function-as-a-Service Environment [connecting-faas]
 
 This section illustrates the best practices for leveraging the {{es}} client in a Function-as-a-Service (FaaS) environment.

docs/reference/index.md

@@ -69,8 +69,14 @@ Compatibility does not imply full feature parity. New {{es}} features are suppor
 
 {{es}} language clients are also _backward compatible_ across minor versions — with default distributions and without guarantees. 
 
+### Major version upgrades
+
 :::{tip}
 To upgrade to a new major version, first upgrade {{es}}, then upgrade the Python {{es}} client.
 :::
 
-If you need to work with multiple client versions, note that older versions are also released as `elasticsearch7` and `elasticsearch8`.
+Since version 8.0, the {{es}} server supports a compatibility mode that allows smoother upgrade experiences. In a nutshell, this makes it possible to upgrade the {{es}} server to the next major version, while continuing to use the same client. This gives more room to coordinate the upgrade of your codebase to the next major version.
+
+For example, to upgrade a system that uses {{es}} 8.x you can upgrade the {{es}} server to 9.x first, and the 8.x Python {{es}} client will continue to work (aside from any breaking changes, which should be listed in the server release notes). You can continue using the 8.x client during the server migration, and only upgrade it once the server migration is complete. The process is described in detail in the [REST API compatibility workflow](https://www.elastic.co/docs/reference/elasticsearch/rest-apis/compatibility#_rest_api_compatibility_workflow) section of the {{es}} documentation.
+
+If you need to work with multiple client versions, note that older versions are also released with the `elasticsearch8` and `elasticsearch9` package names so that they can be installed together.

elasticsearch/dsl/field.py

@@ -572,7 +572,11 @@ def _serialize(
         if isinstance(data, collections.abc.Mapping):
             return data
 
-        return data.to_dict(skip_empty=skip_empty)
+        try:
+            return data.to_dict(skip_empty=skip_empty)
+        except TypeError:
+            # this would only happen if an AttrDict was given instead of an InnerDoc
+            return data.to_dict()
 
     def clean(self, data: Any) -> Any:
         data = super().clean(data)

test_elasticsearch/test_dsl/test_field.py

@@ -24,7 +24,14 @@
 from dateutil import tz
 
 from elasticsearch import dsl
-from elasticsearch.dsl import InnerDoc, Range, ValidationException, field
+from elasticsearch.dsl import (
+    AttrDict,
+    AttrList,
+    InnerDoc,
+    Range,
+    ValidationException,
+    field,
+)
 
 
 def test_date_range_deserialization() -> None:
@@ -235,6 +242,33 @@ class Inner(InnerDoc):
         field.Object(doc_class=Inner, dynamic=False)
 
 
+def test_dynamic_object() -> None:
+    f = field.Object(dynamic=True)
+    assert f.deserialize({"a": "b"}).to_dict() == {"a": "b"}
+    assert f.deserialize(AttrDict({"a": "b"})).to_dict() == {"a": "b"}
+    assert f.serialize({"a": "b"}) == {"a": "b"}
+    assert f.serialize(AttrDict({"a": "b"})) == {"a": "b"}
+
+
+def test_dynamic_nested() -> None:
+    f = field.Nested(dynamic=True)
+    assert f.deserialize([{"a": "b"}, {"c": "d"}]) == [{"a": "b"}, {"c": "d"}]
+    assert f.deserialize([AttrDict({"a": "b"}), {"c": "d"}]) == [
+        {"a": "b"},
+        {"c": "d"},
+    ]
+    assert f.deserialize(AttrList([AttrDict({"a": "b"}), {"c": "d"}])) == [
+        {"a": "b"},
+        {"c": "d"},
+    ]
+    assert f.serialize([{"a": "b"}, {"c": "d"}]) == [{"a": "b"}, {"c": "d"}]
+    assert f.serialize([AttrDict({"a": "b"}), {"c": "d"}]) == [{"a": "b"}, {"c": "d"}]
+    assert f.serialize(AttrList([AttrDict({"a": "b"}), {"c": "d"}])) == [
+        {"a": "b"},
+        {"c": "d"},
+    ]
+
+
 def test_all_fields_exported() -> None:
     """Make sure that all the generated field classes are exported at the top-level"""
     fields = [

test_elasticsearch/test_dsl/test_integration/_async/test_document.py

@@ -33,6 +33,7 @@
 from elasticsearch.dsl import (
     AsyncDocument,
     AsyncSearch,
+    AttrDict,
     Binary,
     Boolean,
     Date,
@@ -627,13 +628,17 @@ async def test_can_save_to_different_index(
 
 
 @pytest.mark.anyio
+@pytest.mark.parametrize("validate", (True, False))
 async def test_save_without_skip_empty_will_include_empty_fields(
     async_write_client: AsyncElasticsearch,
+    validate: bool,
 ) -> None:
     test_repo = Repository(
         field_1=[], field_2=None, field_3={}, owner={"name": None}, meta={"id": 42}
     )
-    assert await test_repo.save(index="test-document", skip_empty=False)
+    assert await test_repo.save(
+        index="test-document", skip_empty=False, validate=validate
+    )
 
     assert_doc_equals(
         {
@@ -650,6 +655,23 @@ async def test_save_without_skip_empty_will_include_empty_fields(
         await async_write_client.get(index="test-document", id=42),
     )
 
+    test_repo = Repository(owner=AttrDict({"name": None}), meta={"id": 43})
+    assert await test_repo.save(
+        index="test-document", skip_empty=False, validate=validate
+    )
+
+    assert_doc_equals(
+        {
+            "found": True,
+            "_index": "test-document",
+            "_id": "43",
+            "_source": {
+                "owner": {"name": None},
+            },
+        },
+        await async_write_client.get(index="test-document", id=43),
+    )
+
 
 @pytest.mark.anyio
 async def test_delete(async_write_client: AsyncElasticsearch) -> None:

test_elasticsearch/test_dsl/test_integration/_sync/test_document.py

@@ -31,6 +31,7 @@
 
 from elasticsearch import ConflictError, Elasticsearch, NotFoundError
 from elasticsearch.dsl import (
+    AttrDict,
     Binary,
     Boolean,
     Date,
@@ -621,13 +622,15 @@ def test_can_save_to_different_index(
 
 
 @pytest.mark.sync
+@pytest.mark.parametrize("validate", (True, False))
 def test_save_without_skip_empty_will_include_empty_fields(
     write_client: Elasticsearch,
+    validate: bool,
 ) -> None:
     test_repo = Repository(
         field_1=[], field_2=None, field_3={}, owner={"name": None}, meta={"id": 42}
     )
-    assert test_repo.save(index="test-document", skip_empty=False)
+    assert test_repo.save(index="test-document", skip_empty=False, validate=validate)
 
     assert_doc_equals(
         {
@@ -644,6 +647,21 @@ def test_save_without_skip_empty_will_include_empty_fields(
         write_client.get(index="test-document", id=42),
     )
 
+    test_repo = Repository(owner=AttrDict({"name": None}), meta={"id": 43})
+    assert test_repo.save(index="test-document", skip_empty=False, validate=validate)
+
+    assert_doc_equals(
+        {
+            "found": True,
+            "_index": "test-document",
+            "_id": "43",
+            "_source": {
+                "owner": {"name": None},
+            },
+        },
+        write_client.get(index="test-document", id=43),
+    )
+
 
 @pytest.mark.sync
 def test_delete(write_client: Elasticsearch) -> None:

utils/templates/field.py.tpl

@@ -334,7 +334,11 @@ class {{ k.name }}({{ k.parent }}):
         if isinstance(data, collections.abc.Mapping):
             return data
 
-        return data.to_dict(skip_empty=skip_empty)
+        try:
+            return data.to_dict(skip_empty=skip_empty)
+        except TypeError:
+            # this would only happen if an AttrDict was given instead of an InnerDoc
+            return data.to_dict()
 
     def clean(self, data: Any) -> Any:
         data = super().clean(data)