diff --git a/src/main/java/org/codelibs/elasticsearch/df/content/geojson/GeoJsonContent.java b/src/main/java/org/codelibs/elasticsearch/df/content/geojson/GeoJsonContent.java index a503c6d..7b66dce 100644 --- a/src/main/java/org/codelibs/elasticsearch/df/content/geojson/GeoJsonContent.java +++ b/src/main/java/org/codelibs/elasticsearch/df/content/geojson/GeoJsonContent.java @@ -6,6 +6,8 @@ import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Writer; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -118,7 +120,9 @@ protected OnLoadListener(final File outputFile, final ActionListener liste @Override public void onResponse(final SearchResponse response) { - final Gson gsonWriter = new GsonBuilder().create(); + final Gson gsonWriter = AccessController.doPrivileged((PrivilegedAction) () -> { + return new GsonBuilder().create(); + }); final String scrollId = response.getScrollId(); final SearchHits hits = response.getHits(); final int size = hits.getHits().length; @@ -137,12 +141,14 @@ public void onResponse(final SearchResponse response) { firstLine = false; } - final JsonElement propertiesJson = JsonParser.parseString(source); + final JsonElement propertiesJson = AccessController.doPrivileged((PrivilegedAction) () -> { + return JsonParser.parseString(source); + }); String geometryType = ""; JsonArray geometryCoordinates = new JsonArray(); if (!geometryCoordinatesField.isEmpty()){ - JsonElement jsonEltCoord = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesField); + final JsonElement jsonEltCoord = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesField); if (jsonEltCoord !=null && !jsonEltCoord.isJsonNull()){ geometryCoordinates = jsonEltCoord.getAsJsonArray​(); if (!geometryKeepGeoInfo){ @@ -150,7 +156,7 @@ public void onResponse(final SearchResponse response) { } } if (!geometryTypeField.isEmpty()){ - JsonElement jsonEltType = JsonUtils.getJsonElement(propertiesJson,geometryTypeField); + final JsonElement jsonEltType = JsonUtils.getJsonElement(propertiesJson,geometryTypeField); if (jsonEltType !=null && !jsonEltType.isJsonNull()){ geometryType = jsonEltType.getAsString(); if (!geometryKeepGeoInfo){ @@ -160,8 +166,8 @@ public void onResponse(final SearchResponse response) { } }else{ if (!geometryCoordinatesLonField.isEmpty() && !geometryCoordinatesLatField.isEmpty()){ - JsonElement jsonEltLon = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesLonField); - JsonElement jsonEltLat = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesLatField); + final JsonElement jsonEltLon = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesLonField); + final JsonElement jsonEltLat = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesLatField); if (jsonEltLon !=null && !jsonEltLon.isJsonNull() && jsonEltLat !=null && !jsonEltLat.isJsonNull()){ geometryCoordinates.add(jsonEltLon.getAsNumber()); geometryCoordinates.add(jsonEltLat.getAsNumber()); @@ -172,7 +178,7 @@ public void onResponse(final SearchResponse response) { } } if (!geometryCoordinatesAltField.isEmpty()){ - JsonElement jsonElt = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesAltField); + final JsonElement jsonElt = JsonUtils.getJsonElement(propertiesJson,geometryCoordinatesAltField); if (jsonElt !=null && !jsonElt.isJsonNull()){ geometryCoordinates.add(jsonElt.getAsNumber()); if (!geometryKeepGeoInfo) { @@ -186,7 +192,7 @@ public void onResponse(final SearchResponse response) { for (String excludeField : excludeFields) { JsonUtils.removeJsonElement(propertiesJson,excludeField); } - + JsonObject geometryObject = new JsonObject(); geometryObject.addProperty("type", geometryType); geometryObject.add("coordinates", geometryCoordinates); diff --git a/src/main/plugin-metadata/plugin-security.policy b/src/main/plugin-metadata/plugin-security.policy index 615a15b..d9ed2fe 100644 --- a/src/main/plugin-metadata/plugin-security.policy +++ b/src/main/plugin-metadata/plugin-security.policy @@ -1,3 +1,4 @@ grant { permission java.lang.RuntimePermission "getClassLoader"; + permission java.lang.RuntimePermission "accessDeclaredMembers"; }; diff --git a/src/test/java/org/codelibs/elasticsearch/df/DataFormatPluginTest.java b/src/test/java/org/codelibs/elasticsearch/df/DataFormatPluginTest.java index 3b244d4..1a995c5 100644 --- a/src/test/java/org/codelibs/elasticsearch/df/DataFormatPluginTest.java +++ b/src/test/java/org/codelibs/elasticsearch/df/DataFormatPluginTest.java @@ -65,6 +65,8 @@ public class DataFormatPluginTest { private final Map paramsGeoJson = new HashMap<>(); static { + // Doc number used for test + // -> max 9990 because elastic query size limited below 10000 docNumber = 20; csvTempFile = createTempFile("csvtest", ".csv"); @@ -168,14 +170,15 @@ public void dumpCsvWithQuery() throws IOException { assertLineContains(lines[0], "\"aaa\"", "\"bbb\"", "\"ccc\"", "\"eee.fff\"", "\"eee.ggg\""); assertLineContains(lines[1], "\"1\""); } - + // Download 10 docs as CSV clearParams(); prepareParams(); paramsCsv.put("q", "*:*"); paramsCsv.put("from", "5"); - try (CurlResponse response = createRequest(node, path, paramsCsv).execute()) { - assertEquals(16, response.getContentAsString().split("\n").length); + try (CurlResponse response = createRequest(node, path, paramsCsv) + .param("size", Integer.toString(docNumber)).execute()) { + assertEquals(docNumber - 4, response.getContentAsString().split("\n").length); } // Download all the docs from the 5th as CSV @@ -193,7 +196,9 @@ public void dumpCsvWithQuery() throws IOException { // Download All as CSV with Query and from try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json") - .param("format", "csv").body(queryWithFrom).execute()) { + .param("format", "csv") + .param("size", Integer.toString(docNumber)) + .body(queryWithFrom).execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); assertEquals(docNumber - 10 + 1, lines.length); @@ -204,6 +209,7 @@ public void dumpCsvWithQuery() throws IOException { .header("Content-Type", "application/json") .param("format", "csv").param("source", queryWithFrom) .param("source_content_type", "application/json") + .param("size", String.valueOf(docNumber)) .execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); @@ -214,6 +220,7 @@ public void dumpCsvWithQuery() throws IOException { try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json") .param("search_type", "query_then_fetch").param("format", "csv") + .param("size", String.valueOf(docNumber)) .execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); @@ -299,7 +306,9 @@ public void dumpExcel() throws IOException { // Download All as Excel with search_type try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json") - .param("search_type", "query_then_fetch").param("format", "xls") + .param("search_type", "query_then_fetch") + .param("format", "xls") + .param("size", Integer.toString(docNumber)) .execute()) { try (InputStream is = curlResponse.getContentAsStream()) { final POIFSFileSystem fs = new POIFSFileSystem(is); @@ -331,7 +340,8 @@ public void dumpJson() throws IOException { // Download All as JSON try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json") - .param("format", "json").execute()) { + .param("format", "json") + .param("size", Integer.toString(docNumber)).execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); assertEquals(docNumber * 2, lines.length); @@ -343,6 +353,7 @@ public void dumpJson() throws IOException { try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json") .param("format", "json").param("bulk.index", "dataset02") + .param("size", Integer.toString(docNumber)) .execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); @@ -366,15 +377,6 @@ public void dumpJson() throws IOException { assertTrue(lines[1].startsWith("{\"aaa\":\"test 1\",")); } - // Download 10 docs as JSON - try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") - .header("Content-Type", "application/json").param("q", "*:*") - .param("format", "json").param("from", "5").execute()) { - final String content = curlResponse.getContentAsString(); - final String[] lines = content.split("\n"); - assertEquals(30, lines.length); - } - // Download all the docs from the 5th as JSON try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json").param("q", "*:*") @@ -390,7 +392,9 @@ public void dumpJson() throws IOException { // Download All as JSON with Query and from try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json") - .param("format", "json").body(queryWithFrom).execute()) { + .param("format", "json") + .param("size", String.valueOf(docNumber)) + .body(queryWithFrom).execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); assertEquals((docNumber - 5) * 2, lines.length); @@ -401,6 +405,7 @@ public void dumpJson() throws IOException { .header("Content-Type", "application/json") .param("format", "json").param("source", queryWithFrom) .param("source_content_type", "application/json") + .param("size", String.valueOf(docNumber)) .execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); @@ -411,7 +416,8 @@ public void dumpJson() throws IOException { try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json") .param("search_type", "query_then_fetch") - .param("format", "json").execute()) { + .param("format", "json") + .param("size", String.valueOf(docNumber)).execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); assertEquals(docNumber * 2, lines.length); @@ -440,7 +446,7 @@ public void dumpJsonList() throws IOException { // Download All as JSON try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json") - .param("format", "jsonlist").execute()) { + .param("format", "jsonlist").param("size", Integer.toString(docNumber)).execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); assertEquals(docNumber + 2, lines.length); @@ -467,10 +473,11 @@ public void dumpJsonList() throws IOException { // Download 10 docs as JSON try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json").param("q", "*:*") - .param("format", "jsonlist").param("from", "5").execute()) { + .param("format", "jsonlist").param("from", "5") + .param("size", Integer.toString(docNumber)).execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); - assertEquals(15 + 2, lines.length); + assertEquals(docNumber - 5 + 2, lines.length); } // Download all the docs from the 5th as JSON @@ -488,7 +495,9 @@ public void dumpJsonList() throws IOException { // Download All as JSON with Query and from try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json") - .param("format", "jsonlist").body(queryWithFrom).execute()) { + .param("format", "jsonlist") + .param("size", String.valueOf(docNumber)) + .body(queryWithFrom).execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); assertEquals((docNumber - 5) + 2, lines.length); @@ -499,7 +508,7 @@ public void dumpJsonList() throws IOException { .header("Content-Type", "application/json") .param("format", "jsonlist").param("source", queryWithFrom) .param("source_content_type", "application/json") - .execute()) { + .param("size", String.valueOf(docNumber)).execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); assertEquals((docNumber - 5) + 2, lines.length); @@ -509,7 +518,8 @@ public void dumpJsonList() throws IOException { try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json") .param("search_type", "query_then_fetch") - .param("format", "jsonlist").execute()) { + .param("format", "jsonlist") + .param("size", String.valueOf(docNumber)).execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); assertEquals(docNumber + 2, lines.length); @@ -540,7 +550,7 @@ public void dumpGeoJson() throws IOException { // default call try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset1/_data") .header("Content-Type", "application/json") - .param("format", "geojson").execute()) { + .param("format", "geojson").param("size", Integer.toString(docNumber)).execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); assertEquals(docNumber + 2, lines.length); @@ -658,7 +668,9 @@ public void dumpSizeLimit() throws IOException { // Default try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json") - .param("format", "csv").execute()) { + .param("format", "csv") + .param("size", Integer.toString(docNumber)) + .execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); assertEquals(docNumber + 1, lines.length); @@ -667,7 +679,10 @@ public void dumpSizeLimit() throws IOException { // 50% try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json") - .param("format", "csv").param("limit", "50%").execute()) { + .param("format", "csv") + .param("size", Integer.toString(docNumber)) + .param("limit", "50%") + .execute()) { final String content = curlResponse.getContentAsString(); final String[] lines = content.split("\n"); assertEquals(docNumber + 1, lines.length); @@ -676,7 +691,10 @@ public void dumpSizeLimit() throws IOException { //0% try (CurlResponse curlResponse = EcrCurl.get(node, "/dataset0/_data") .header("Content-Type", "application/json") - .param("format", "csv").param("limit", "0").execute()) { + .param("format", "csv") + .param("size", Integer.toString(docNumber)) + .param("limit", "0") + .execute()) { assertEquals(500, curlResponse.getHttpStatusCode()); } } @@ -762,7 +780,7 @@ private static File createTempFile(String prefix, String suffix) { } private CurlRequest createRequest(Node node, String path, Map params) { - CurlRequest request = EcrCurl.get(node, path).header("Content-Type", "application/json"); + CurlRequest request = EcrCurl.get(node, path).header("Content-Type", "application/json").param("size", Integer.toString(docNumber)); for (final Map.Entry entry : params.entrySet()) { request.param(entry.getKey(), entry.getValue()); }