From 81d581f482c4aaaa4204976f3f452589c2ac520e Mon Sep 17 00:00:00 2001 From: Victor Johnston Date: Tue, 23 Jun 2026 08:27:46 +0200 Subject: [PATCH 1/4] Upgrade Solr dependencies to 9.10.1 and bump lucene to follow --- pom.xml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index 634d8610..48b0b45e 100644 --- a/pom.xml +++ b/pom.xml @@ -131,34 +131,33 @@ org.apache.solr solr-solrj - 9.7.0 + 9.10.1 org.apache.solr solr-test-framework - 9.7.0 + 9.10.1 test org.apache.solr solr-core - 9.7.0 + 9.10.1 test - + org.apache.logging.log4j log4j-slf4j-impl - - - + + org.apache.lucene lucene-core - 9.11.1 + 9.12.1 test From d7924776e6ab08402ca489fcc09daf645791a6bd Mon Sep 17 00:00:00 2001 From: Victor Johnston Date: Tue, 23 Jun 2026 08:28:37 +0200 Subject: [PATCH 2/4] Migrate from deprecated NoOpResponseParser to InputStreamResponseParser --- .../solr/NetarchiveSolrClient.java | 84 +++++++------------ 1 file changed, 31 insertions(+), 53 deletions(-) diff --git a/src/main/java/dk/kb/netarchivesuite/solrwayback/solr/NetarchiveSolrClient.java b/src/main/java/dk/kb/netarchivesuite/solrwayback/solr/NetarchiveSolrClient.java index 113d8678..ffdda615 100644 --- a/src/main/java/dk/kb/netarchivesuite/solrwayback/solr/NetarchiveSolrClient.java +++ b/src/main/java/dk/kb/netarchivesuite/solrwayback/solr/NetarchiveSolrClient.java @@ -1,6 +1,8 @@ package dk.kb.netarchivesuite.solrwayback.solr; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.LocalDate; @@ -21,7 +23,7 @@ import org.apache.solr.client.solrj.SolrRequest.METHOD; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.NoOpResponseParser; +import org.apache.solr.client.solrj.impl.InputStreamResponseParser; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.*; import org.apache.solr.client.solrj.response.FacetField.Count; @@ -1414,15 +1416,7 @@ public String searchJsonResponseOnlyFacets(String query, List fq, boolea } - NoOpResponseParser rawJsonResponseParser = new NoOpResponseParser(); - rawJsonResponseParser.setWriterType("json"); - - QueryRequest req = new QueryRequest(solrQuery); - req.setResponseParser(rawJsonResponseParser); - - NamedList resp = solrServer.request(req); - String jsonResponse = (String) resp.get("response"); - return jsonResponse; + return requestRawJson(solrQuery); } public String searchJsonResponseOnlyFacetsLoadMore( String query, List fq, String facetField, boolean revisits) throws Exception { @@ -1458,17 +1452,7 @@ public String searchJsonResponseOnlyFacetsLoadMore( String query, List f } } - - - NoOpResponseParser rawJsonResponseParser = new NoOpResponseParser(); - rawJsonResponseParser.setWriterType("json"); - - QueryRequest req = new QueryRequest(solrQuery); - req.setResponseParser(rawJsonResponseParser); - - NamedList resp = solrServer.request(req); - String jsonResponse = (String) resp.get("response"); - return jsonResponse; + return requestRawJson(solrQuery); } public String searchJsonResponseNoFacets(String query, List fq, boolean grouping, boolean revisits, Integer start, String sort) throws Exception { @@ -1516,15 +1500,7 @@ public String searchJsonResponseNoFacets(String query, List fq, boolean } - NoOpResponseParser rawJsonResponseParser = new NoOpResponseParser(); - rawJsonResponseParser.setWriterType("json"); - - QueryRequest req = new QueryRequest(solrQuery); - req.setResponseParser(rawJsonResponseParser); - - NamedList resp = solrServer.request(req); - String jsonResponse = (String) resp.get("response"); - return jsonResponse; + return requestRawJson(solrQuery); } /* @@ -1544,14 +1520,32 @@ public String idLookupResponse(String id, String fieldList) throws Exception { solrQuery.set("fl",fieldList); } - NoOpResponseParser rawJsonResponseParser = new NoOpResponseParser(); - rawJsonResponseParser.setWriterType("json"); + return requestRawJson(solrQuery); + } + /** + * Executes the given query and returns the raw JSON response body as a String. + *

+ * Uses {@link InputStreamResponseParser}, which streams the response body directly instead of + * buffering it into a parsed {@link NamedList}. The stream is fully read and closed here. + * + * @param solrQuery the query to execute against {@link #solrServer}. + * @return the raw response body produced by Solr's {@code wt=json} writer. + */ + private String requestRawJson(SolrQuery solrQuery) throws Exception { QueryRequest req = new QueryRequest(solrQuery); - req.setResponseParser(rawJsonResponseParser); + req.setResponseParser(new InputStreamResponseParser("json")); + NamedList resp = solrServer.request(req); - String jsonResponse = resp.get("response").toString(); - return jsonResponse; + int status = (int) resp.get("responseStatus"); + try (InputStream stream = (InputStream) resp.get("stream")) { + String jsonResponse = new String(stream.readAllBytes(), StandardCharsets.UTF_8); + if (status != 200) { + throw new SolrServerException( + "Solr returned HTTP status " + status + " for query " + solrQuery + ": " + jsonResponse); + } + return jsonResponse; + } } /* @@ -1671,15 +1665,7 @@ public String domainStatisticsForQuery(String query, List fq) throws Exc for (String filter : fq) { solrQuery.addFilterQuery(filter); } - NoOpResponseParser rawJsonResponseParser = new NoOpResponseParser(); - rawJsonResponseParser.setWriterType("json"); - - QueryRequest req = new QueryRequest(solrQuery); - req.setResponseParser(rawJsonResponseParser); - - NamedList resp = solrServer.request(req); - String jsonResponse = (String) resp.get("response"); - return jsonResponse; + return requestRawJson(solrQuery); } // returns JSON. Response not supported by SolrJ @@ -1708,15 +1694,7 @@ public String domainStatisticsForQuery(String query, List fq, String sta for (String filter : fq) { solrQuery.addFilterQuery(filter); } - NoOpResponseParser rawJsonResponseParser = new NoOpResponseParser(); - rawJsonResponseParser.setWriterType("json"); - - QueryRequest req = new QueryRequest(solrQuery); - req.setResponseParser(rawJsonResponseParser); - - NamedList resp = solrServer.request(req); - String jsonResponse = (String) resp.get("response"); - return jsonResponse; + return requestRawJson(solrQuery); } /** From 13e691b0b0b392ea0ae4d9e51e4177d2b437ecc1 Mon Sep 17 00:00:00 2001 From: Victor Johnston Date: Wed, 24 Jun 2026 14:22:52 +0200 Subject: [PATCH 3/4] Implement cache for the stream and use constants --- .../solr/NetarchiveSolrClient.java | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/main/java/dk/kb/netarchivesuite/solrwayback/solr/NetarchiveSolrClient.java b/src/main/java/dk/kb/netarchivesuite/solrwayback/solr/NetarchiveSolrClient.java index ffdda615..c4bdc045 100644 --- a/src/main/java/dk/kb/netarchivesuite/solrwayback/solr/NetarchiveSolrClient.java +++ b/src/main/java/dk/kb/netarchivesuite/solrwayback/solr/NetarchiveSolrClient.java @@ -55,6 +55,11 @@ public class NetarchiveSolrClient { private static final Logger log = LoggerFactory.getLogger(NetarchiveSolrClient.class); private static final long M = 1000000; // ns -> ms + /** Key under which {@link #requestRawJson} stashes the materialized JSON body into the + * (possibly cached) response NamedList, so repeated/cached queries don't re-read the + * already-consumed InputStream from {@link InputStreamResponseParser}. */ + private static final String RAW_JSON_KEY = "solrwaybackRawJson"; + protected static SolrClient solrServer; protected static SolrClient noCacheSolrServer; protected static NetarchiveSolrClient instance = null; @@ -1537,13 +1542,29 @@ private String requestRawJson(SolrQuery solrQuery) throws Exception { req.setResponseParser(new InputStreamResponseParser("json")); NamedList resp = solrServer.request(req); - int status = (int) resp.get("responseStatus"); - try (InputStream stream = (InputStream) resp.get("stream")) { - String jsonResponse = new String(stream.readAllBytes(), StandardCharsets.UTF_8); + + // CachingSolrClient caches and re-serves this NamedList for repeated queries, but + // InputStreamResponseParser puts a single-use InputStream ("stream") in it. The stream can + // only be read once, so on a cache hit it is already closed (-> IOException: closed). + // Materialize the body to a String once and stash it back into the NamedList, so subsequent + // cache hits return the String instead of re-reading the closed stream. Synchronize so + // concurrent callers that share the cached NamedList don't race on the one-shot stream. + synchronized (resp) { + Object cachedJson = resp.get(RAW_JSON_KEY); + if (cachedJson != null) { + return (String) cachedJson; + } + + int status = (int) resp.get(InputStreamResponseParser.HTTP_STATUS_KEY); + String jsonResponse; + try (InputStream stream = (InputStream) resp.get(InputStreamResponseParser.STREAM_KEY)) { + jsonResponse = new String(stream.readAllBytes(), StandardCharsets.UTF_8); + } if (status != 200) { throw new SolrServerException( "Solr returned HTTP status " + status + " for query " + solrQuery + ": " + jsonResponse); } + resp.add(RAW_JSON_KEY, jsonResponse); return jsonResponse; } } From b2d777b8c325ef7b675356f91d9445ed443c8fb8 Mon Sep 17 00:00:00 2001 From: Victor Johnston Date: Wed, 24 Jun 2026 14:26:02 +0200 Subject: [PATCH 4/4] Update changelog --- CHANGES.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index f69e97c9..ad773054 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -3,6 +3,8 @@ UNRELEASED * Four new fields (inactive) added to Solr: nsfw_probability,is_nsfw,is_virus And virus_description. Fields are inactive in default solrwayback bundle. * Navigation tracker auto poll increased from 2 seconds to 60 seconds. +* Upgraded Solr dependencies to 9.10.1 +* Switched from deprecated NoOpResponseParser to the new InputStreamResponseParser. Caching of SolrResponses is now handled by storing the response in the NetarchiveSolrClient. 5.4.2 -----