From 21057cf72071840e2a984450bf8845c6e6490ec9 Mon Sep 17 00:00:00 2001 From: Alex Dolski Date: Fri, 25 Mar 2022 13:37:37 -0500 Subject: [PATCH] Remove HttpSource.proxy.http.enabled and rename HttpSource.proxy.http.server to host (#502) --- CHANGES.md | 25 +++++++--- UPGRADING.md | 2 + cantaloupe.properties.sample | 9 ++-- .../library/cantaloupe/config/Key.java | 5 +- .../library/cantaloupe/source/HttpSource.java | 50 +++++++------------ src/main/resources/admin.vm | 28 +++++++++++ .../resource/admin/AdminResourceUITest.java | 6 +++ .../source/HTTPStreamFactoryTest.java | 46 +++++++++++++++++ .../cantaloupe/source/HttpSourceTest.java | 45 ++++++++++++----- 9 files changed, 158 insertions(+), 58 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2a5e83238..6146b708b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,23 +1,36 @@ # Change Log -# 6.0 +## 6.0 + +### Endpoints * Image and information responses include a `Last-Modified` header when possible. * The health endpoint is enabled via `endpoint.health.enabled` rather than `endpoint.api.enabled`. +* Added an HTTP API method to purge all infos from the derivative cache. +* Added a configuration option to automatically purge source-cached images + whose format cannot be inferred. + +### Sources + +* HttpSource supports a client HTTP proxy. (Thanks to @mightymax and + @mlindeman) * HttpSource can be configured to send a ranged GET request instead of a HEAD request, enabling it to work with pre-signed URLs that do not allow HEAD requests. * S3Source supports multiple endpoints when using ScriptLookupStrategy. -* Added a configuration option to automatically purge source-cached images - whose format cannot be inferred. -* Added an HTTP API method to purge all infos from the derivative cache. + +### Caches + +* S3Cache uses multipart uploads, which reduces memory usage when caching + derivatives larger than 5 MB. + +### Delegate Script + * The delegate script pathname can be set using the `-Dcantaloupe.delegate_script` VM argument, which takes precedence over the `delegate_script.pathname` configuration key. -* S3Cache uses multipart uploads, which reduces memory usage when caching - derivatives larger than 5 MB. * The delegate script's `metadata` context key contains a new field, `xmp_elements`, that provides a high-level key-value view of the XMP data. diff --git a/UPGRADING.md b/UPGRADING.md index 58261b77b..3e38801c7 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -7,6 +7,8 @@ current version. 1. Add the following keys from the sample configuration: * `endpoint.health.enabled` + * `HttpSource.proxy.http.host` + * `HttpSource.proxy.http.port` * `HttpSource.BasicLookupStrategy.send_head_requests` * `processor.purge_incompatible_from_source_cache` 2. Add the following methods from the sample delegate script: diff --git a/cantaloupe.properties.sample b/cantaloupe.properties.sample index e5450d649..e205ad760 100644 --- a/cantaloupe.properties.sample +++ b/cantaloupe.properties.sample @@ -175,6 +175,10 @@ HttpSource.allow_insecure = false # Request timeout in seconds. HttpSource.request_timeout = +# !! Client HTTP proxy. +HttpSource.proxy.http.host = +HttpSource.proxy.http.port = + # Tells HttpSource how to look up resources. Allowed values are # `BasicLookupStrategy` and `ScriptLookupStrategy`. ScriptLookupStrategy # uses a delegate method for dynamic lookups; see the user manual. @@ -212,11 +216,6 @@ HttpSource.chunking.cache.enabled = true # Max per-request chunk cache size. HttpSource.chunking.cache.max_size = 5M -# Enable HTTP Proxy for HttpSource -HttpSource.proxy.http.enabled = false -HttpSource.proxy.http.server = proxy.example.com -HttpSource.proxy.http.port = 8080 - #---------------------------------------- # S3Source #---------------------------------------- diff --git a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java index a315294a9..2ac19ce09 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/config/Key.java +++ b/src/main/java/edu/illinois/library/cantaloupe/config/Key.java @@ -108,6 +108,8 @@ public enum Key { HTTPSOURCE_CHUNK_SIZE("HttpSource.chunking.chunk_size"), HTTPSOURCE_CHUNK_CACHE_ENABLED("HttpSource.chunking.cache.enabled"), HTTPSOURCE_CHUNK_CACHE_MAX_SIZE("HttpSource.chunking.cache.max_size"), + HTTPSOURCE_HTTP_PROXY_HOST("HttpSource.proxy.http.host"), + HTTPSOURCE_HTTP_PROXY_PORT("HttpSource.proxy.http.port"), HTTPSOURCE_LOOKUP_STRATEGY("HttpSource.lookup_strategy"), HTTPSOURCE_REQUEST_TIMEOUT("HttpSource.request_timeout"), HTTPSOURCE_SEND_HEAD_REQUESTS("HttpSource.BasicLookupStrategy.send_head_requests"), @@ -120,9 +122,6 @@ public enum Key { HTTPS_KEY_STORE_PATH("https.key_store_path"), HTTPS_KEY_STORE_TYPE("https.key_store_type"), HTTPS_PORT("https.port"), - HTTPSOURCE_HTTP_PROXY_ENABLED("HttpSource.proxy.http.enabled"), - HTTPSOURCE_HTTP_PROXY_SERVER("HttpSource.proxy.http.server"), - HTTPSOURCE_HTTP_PROXY_PORT("HttpSource.proxy.http.port"), IIIF_1_ENDPOINT_ENABLED("endpoint.iiif.1.enabled"), IIIF_2_ENDPOINT_ENABLED("endpoint.iiif.2.enabled"), IIIF_3_ENDPOINT_ENABLED("endpoint.iiif.3.enabled"), diff --git a/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java b/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java index 5208a6314..e9c2af51d 100644 --- a/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java +++ b/src/main/java/edu/illinois/library/cantaloupe/source/HttpSource.java @@ -19,10 +19,10 @@ import javax.net.ssl.X509TrustManager; import javax.script.ScriptException; import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Proxy; import java.net.URI; import java.net.URISyntaxException; -import java.net.Proxy; -import java.net.InetSocketAddress; import java.nio.file.AccessDeniedException; import java.nio.file.NoSuchFileException; import java.security.KeyManagementException; @@ -410,27 +410,24 @@ static synchronized OkHttpClient getHTTPClient() { .connectTimeout(getRequestTimeout().getSeconds(), TimeUnit.SECONDS) .readTimeout(getRequestTimeout().getSeconds(), TimeUnit.SECONDS) .writeTimeout(getRequestTimeout().getSeconds(), TimeUnit.SECONDS); - final Configuration config = Configuration.getInstance(); - - final boolean httpProxyEnabled = config.getBoolean( - Key.HTTPSOURCE_HTTP_PROXY_ENABLED, false); - if (httpProxyEnabled) { - final String httpProxyServer = config.getString(Key.HTTPSOURCE_HTTP_PROXY_SERVER, ""); - if (httpProxyServer == "") { - throw new RuntimeException("proxy server setting HttpSource.proxy.http.server should not be empty"); - } - final int httpProxyPort = config.getInt(Key.HTTPSOURCE_HTTP_PROXY_PORT, 8080); - - LOGGER.trace("Using HTTP Proxy at server {} on port {}", httpProxyServer, httpProxyPort); - Proxy httpProxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress(httpProxyServer, httpProxyPort)); - builder.proxy(httpProxy); - } - final boolean allowInsecure = config.getBoolean( - Key.HTTPSOURCE_ALLOW_INSECURE, false); + final String proxyHost = + config.getString(Key.HTTPSOURCE_HTTP_PROXY_HOST, ""); + if (!proxyHost.isBlank()) { + final int proxyPort = + config.getInt(Key.HTTPSOURCE_HTTP_PROXY_PORT); + if (proxyPort == 0) { + throw new RuntimeException("Proxy port setting " + + Key.HTTPSOURCE_HTTP_PROXY_PORT + " must be set"); + } + LOGGER.debug("Using HTTP proxy: {}:{}", proxyHost, proxyPort); + Proxy httpProxy = new Proxy(Proxy.Type.HTTP, + new InetSocketAddress(proxyHost, proxyPort)); + builder.proxy(httpProxy); + } - if (allowInsecure) { + if (config.getBoolean(Key.HTTPSOURCE_ALLOW_INSECURE, false)) { try { X509TrustManager[] tm = new X509TrustManager[]{ new X509TrustManager() { @@ -470,17 +467,6 @@ private static Duration getRequestTimeout() { return Duration.ofSeconds(timeout); } - static String getUserAgent() { - return String.format("%s/%s (%s/%s; java/%s; %s/%s)", - HttpSource.class.getSimpleName(), - Application.getVersion(), - Application.getName(), - Application.getVersion(), - System.getProperty("java.version"), - System.getProperty("os.name"), - System.getProperty("os.version")); - } - /** * @see #request(HTTPRequestInfo, String, Map) */ @@ -505,7 +491,7 @@ static Response request(HTTPRequestInfo requestInfo, Request.Builder builder = new Request.Builder() .method(method, null) .url(requestInfo.getURI()) - .addHeader("User-Agent", getUserAgent()); + .addHeader("User-Agent", USER_AGENT); // Add credentials. if (requestInfo.getUsername() != null && requestInfo.getSecret() != null) { diff --git a/src/main/resources/admin.vm b/src/main/resources/admin.vm index 45a46327e..8890f1648 100644 --- a/src/main/resources/admin.vm +++ b/src/main/resources/admin.vm @@ -1721,6 +1721,34 @@ data-requires-restart="false"> + + + Proxy Host + ? + + + + + + + + Proxy Port + ? + + + + + Lookup Strategy 1000); + } + @Test void newInputStreamSendsCustomHeaders() throws Exception { server.setHandler(new DefaultHandler() { @@ -138,6 +162,28 @@ void newSeekableStreamWhenChunkingIsDisabled() throws Exception { } } + @Disabled + @Test + void newSeekableStreamWithProxy() throws Exception { + final int proxyPort = SocketUtils.getOpenPort(); + + // Set up the proxy + // TODO: write this + + // Set up HttpSource + final var config = Configuration.getInstance(); + config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_HOST, "127.0.0.1"); + config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_PORT, proxyPort); + + int length = 0; + try (ImageInputStream is = newInstance(true).newSeekableStream()) { + while (is.read() != -1) { + length++; + } + } + assertTrue(length > 1000); + } + @Test void newSeekableStreamSendsCustomHeaders() throws Exception { server.setHandler(new DefaultHandler() { diff --git a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java index ccbfda23b..a30046c2a 100644 --- a/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java +++ b/src/test/java/edu/illinois/library/cantaloupe/source/HttpSourceTest.java @@ -9,10 +9,12 @@ import edu.illinois.library.cantaloupe.delegate.DelegateProxy; import edu.illinois.library.cantaloupe.test.TestUtil; import edu.illinois.library.cantaloupe.test.WebServer; +import edu.illinois.library.cantaloupe.util.SocketUtils; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.DefaultHandler; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import jakarta.servlet.http.HttpServletRequest; @@ -117,16 +119,16 @@ void useScriptLookupStrategy() { "ScriptLookupStrategy"); } - /* checkAccess() */ + /* stat() */ @Test - void testCheckAccessUsingBasicLookupStrategyWithPresentUnreadableImage() + void testStatUsingBasicLookupStrategyWithPresentUnreadableImage() throws Exception { doTestCheckAccessWithPresentUnreadableImage(new Identifier("gif")); } @Test - void testCheckAccessUsingScriptLookupStrategyWithPresentReadableImage() + void testStatUsingScriptLookupStrategyWithPresentReadableImage() throws Exception { useScriptLookupStrategy(); Identifier identifier = new Identifier(getServerURI() + "/" + @@ -135,7 +137,7 @@ void testCheckAccessUsingScriptLookupStrategyWithPresentReadableImage() } @Test - void testCheckAccessUsingScriptLookupStrategyWithMissingImage() + void testStatUsingScriptLookupStrategyWithMissingImage() throws Exception { useScriptLookupStrategy(); Identifier identifier = new Identifier(getServerURI() + "/bogus"); @@ -143,7 +145,7 @@ void testCheckAccessUsingScriptLookupStrategyWithMissingImage() } @Test - void testCheckAccessUsingScriptLookupStrategyWithPresentUnreadableImage() + void testStatUsingScriptLookupStrategyWithPresentUnreadableImage() throws Exception { useScriptLookupStrategy(); Identifier identifier = new Identifier(getServerURI() + "/gif"); @@ -202,7 +204,7 @@ private void doTestCheckAccessWithMissingImage(Identifier identifier) } @Test - void testCheckAccessUsingScriptLookupStrategyWithValidAuthentication() + void testStatUsingScriptLookupStrategyWithValidAuthentication() throws Exception { useScriptLookupStrategy(); @@ -220,7 +222,7 @@ void testCheckAccessUsingScriptLookupStrategyWithValidAuthentication() } @Test - void testCheckAccessUsingScriptLookupStrategyWithInvalidAuthentication() + void testStatUsingScriptLookupStrategyWithInvalidAuthentication() throws Exception { useScriptLookupStrategy(); @@ -238,7 +240,7 @@ void testCheckAccessUsingScriptLookupStrategyWithInvalidAuthentication() } @Test - void testCheckAccessWith403Response() throws Exception { + void testStatWith403Response() throws Exception { server.setHandler(new DefaultHandler() { @Override public void handle(String target, @@ -261,7 +263,7 @@ public void handle(String target, } @Test - void testCheckAccessWith500Response() throws Exception { + void testStatWith500Response() throws Exception { server.setHandler(new DefaultHandler() { @Override public void handle(String target, @@ -283,8 +285,27 @@ public void handle(String target, } } + @Disabled @Test - void testCheckAccessSendsUserAgentHeader() throws Exception { + void testStatUsingProxy() throws Exception { + server.start(); + + final int proxyPort = SocketUtils.getOpenPort(); + + // Set up the proxy + // TODO; write this + + // Set up HttpSource + final Configuration config = Configuration.getInstance(); + config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_HOST, "127.0.0.1"); + config.setProperty(Key.HTTPSOURCE_HTTP_PROXY_PORT, proxyPort); + + // Expect no exception + instance.stat(); + } + + @Test + void testStatSendsUserAgentHeader() throws Exception { server.setHandler(new DefaultHandler() { @Override public void handle(String target, @@ -309,7 +330,7 @@ public void handle(String target, } @Test - void testCheckAccessSendsCustomHeaders() throws Exception { + void testStatSendsCustomHeaders() throws Exception { useScriptLookupStrategy(); server.setHandler(new DefaultHandler() { @@ -335,7 +356,7 @@ public void handle(String target, } @Test - void testCheckAccessWithMalformedURI() throws Exception { + void testStatWithMalformedURI() throws Exception { server.start(); Configuration config = Configuration.getInstance();