From c2bb53a2a81dcd863f6824a3605ec35862c01b05 Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Mon, 20 Jul 2026 15:18:00 +0530 Subject: [PATCH 01/16] Implemented fetcher for ScholarApi for List Api --- .github/workflows/tests-code.yml | 1 + docs/code-howtos/fetchers.md | 2 +- jablib/build.gradle.kts | 3 + .../logic/importer/ImporterPreferences.java | 2 + .../importer/fetcher/ScholarFetcher.java | 228 ++++++++++++++++++ .../ScholarApiQueryTransformer.java | 43 ++++ .../java/org/jabref/logic/util/BuildInfo.java | 2 + jablib/src/main/resources/build.properties | 1 + .../fetcher/ScholarApiFetcherTest.java | 133 ++++++++++ .../ScholarApiQueryTransformerTest.java | 18 ++ 10 files changed, 432 insertions(+), 1 deletion(-) create mode 100644 jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java create mode 100644 jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformer.java create mode 100644 jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarApiFetcherTest.java create mode 100644 jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformerTest.java diff --git a/.github/workflows/tests-code.yml b/.github/workflows/tests-code.yml index 38801c498a62..d91e560dd5e0 100644 --- a/.github/workflows/tests-code.yml +++ b/.github/workflows/tests-code.yml @@ -16,6 +16,7 @@ env: MedlineAPiKey: ${{ secrets.MedlineApiKey_FOR_TESTS }} OpenAlexApiKey: ${{ secrets.OpenAlexApiKey_FOR_TESTS }} SpringerNatureAPIKey: ${{ secrets.SPRINGERNATUREAPIKEY_FOR_TESTS }} + scholarApiKey: ${{ secrets.SCHOLARAPIKEY_FOR_TESTS }} GRADLE_OPTS: -Xmx4g JAVA_OPTS: -Xmx4g diff --git a/docs/code-howtos/fetchers.md b/docs/code-howtos/fetchers.md index fd6058e07d8b..aa58f1ee759a 100644 --- a/docs/code-howtos/fetchers.md +++ b/docs/code-howtos/fetchers.md @@ -18,7 +18,7 @@ Fetchers are the implementation of the [search using online services](https://do | [Springer Nature](https://docs.jabref.org/collect/import-using-online-bibliographic-database#springer) | [Springer Nature API portal](https://dev.springernature.com). Use the "Meta API" API key. | `SpringerNatureAPIKey` | 5000 calls/day | | [Wiley (TDM)](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) | [Wiley TDM portal](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) | `WileyTdmApiKey` | 3 articles/second **AND** 60 requests/10min | | [Zentralblatt Math](https://www.zbmath.org) | (none) | (none) | Depending on the current network | - +| [ScholarApi](https://scholarapi.net/) | [SchoarApi User Account](https://scholarapi.net/auth/register) | `ScholarApiKey` | 1000 free credits per key | | "Depending on the current network" means that it depends on whether your request is routed through a network having paid access. For instance, some universities have subscriptions to MathSciNet. On Windows, you have to log off and log on to let IntelliJ know about the environment variable change. Execute the gradle task `processResources` in the group "others" within IntelliJ to ensure the values have been correctly written. Now, the fetcher tests should run without issues. diff --git a/jablib/build.gradle.kts b/jablib/build.gradle.kts index 0e6b7b8aa565..b805731a76a5 100644 --- a/jablib/build.gradle.kts +++ b/jablib/build.gradle.kts @@ -184,6 +184,7 @@ val medlineApiKey = providers.environmentVariable("MedlineApiKey").orElse("") val openAlexApiKey = providers.environmentVariable("OpenAlexApiKey").orElse("") val scopusApiKey = providers.environmentVariable("ScopusApiKey").orElse("") val semanticScholarApiKey = providers.environmentVariable("SemanticScholarApiKey").orElse("") +val scholarApiKey = providers.environmentVariable("ScholarApiKey").orElse("") val springerNatureAPIKey = providers.environmentVariable("SpringerNatureAPIKey").orElse("") val unpaywallEmail = providers.environmentVariable("UNPAYWALL_EMAIL").orElse("") val wileyTdmApiKey = providers.environmentVariable("WileyTdmApiKey").orElse("") @@ -206,6 +207,7 @@ tasks.named("processResources") { inputs.property("medlineApiKey", medlineApiKey) inputs.property("openAlexApiKey", openAlexApiKey) inputs.property("springerNatureAPIKey", springerNatureAPIKey) + inputs.property("scholarApiKey", scholarApiKey) inputs.property("scopusApiKey", scopusApiKey) inputs.property("semanticScholarApiKey", semanticScholarApiKey) inputs.property("unpaywallEmail", unpaywallEmail) @@ -226,6 +228,7 @@ tasks.named("processResources") { "openAlexApiKey" to inputs.properties["openAlexApiKey"], "scopusApiKey" to inputs.properties["scopusApiKey"], "semanticScholarApiKey" to inputs.properties["semanticScholarApiKey"], + "scholarApiKey" to inputs.properties["scholarApiKey"], "springerNatureAPIKey" to inputs.properties["springerNatureAPIKey"], "unpaywallEmail" to inputs.properties["unpaywallEmail"], "wileyTdmApiKey" to inputs.properties["wileyTdmApiKey"], diff --git a/jablib/src/main/java/org/jabref/logic/importer/ImporterPreferences.java b/jablib/src/main/java/org/jabref/logic/importer/ImporterPreferences.java index 5ad00ec3c3b9..737e031dcaa9 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/ImporterPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/importer/ImporterPreferences.java @@ -23,6 +23,7 @@ import org.jabref.logic.importer.fetcher.BiodiversityLibrary; import org.jabref.logic.importer.fetcher.DBLPFetcher; import org.jabref.logic.importer.fetcher.IEEE; +import org.jabref.logic.importer.fetcher.ScholarFetcher; import org.jabref.logic.importer.fetcher.Scopus; import org.jabref.logic.importer.fetcher.SpringerNatureWebFetcher; import org.jabref.logic.importer.fetcher.WileyFetcher; @@ -111,6 +112,7 @@ private static Map getDefaultFetcherKeys() { Scopus.FETCHER_NAME, buildInfo.scopusApiKey, SemanticScholarCitationFetcher.FETCHER_NAME, buildInfo.semanticScholarApiKey, // SpringerLink uses the same key and fetcher name as SpringerFetcher + ScholarFetcher.FETCHER_NAME, buildInfo.scholarApiKey, SpringerNatureWebFetcher.FETCHER_NAME, buildInfo.springerNatureAPIKey, WileyFetcher.FETCHER_NAME, buildInfo.wileyTdmApiKey ); diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java new file mode 100644 index 000000000000..f68a6301582c --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java @@ -0,0 +1,228 @@ +package org.jabref.logic.importer.fetcher; + +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.ImporterPreferences; +import org.jabref.logic.importer.PagedSearchBasedFetcher; +import org.jabref.logic.importer.ParseException; +import org.jabref.logic.importer.fetcher.transformers.ScholarApiQueryTransformer; +import org.jabref.logic.importer.util.JsonReader; +import org.jabref.logic.net.URLDownload; +import org.jabref.logic.util.URLUtil; +import org.jabref.logic.util.strings.StringUtil; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.paging.Page; +import org.jabref.model.search.query.BaseQueryNode; + +import kong.unirest.core.UnirestException; +import kong.unirest.core.json.JSONArray; +import kong.unirest.core.json.JSONException; +import kong.unirest.core.json.JSONObject; +import org.apache.hc.core5.net.URIBuilder; +import org.jspecify.annotations.NonNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ScholarFetcher implements PagedSearchBasedFetcher, CustomizableKeyFetcher { + public static final String FETCHER_NAME = "Scholar"; + + private static final Logger LOGGER = LoggerFactory.getLogger(ScholarFetcher.class); + + private static final String LIST_URL = "https://scholarapi.net/api/v1/list"; + + private final Map cursorCacheMap = new ConcurrentHashMap<>(); + + private final ImporterPreferences importerPreferences; + + public ScholarFetcher(ImporterPreferences importerPreferences) { + this.importerPreferences = importerPreferences; + } + + /// Convert a JSONObject obtained from the Scholar API to a BibEntry + /// + /// @param scholarJsonEntry the JSONObject from search results + /// @return the converted BibEntry + public static BibEntry jsonItemToBibEntry(JSONObject scholarJsonEntry) throws ParseException { + try { + BibEntry entry = new BibEntry(StandardEntryType.Article); + + if (scholarJsonEntry.has("authors")) { + JSONArray authors = scholarJsonEntry.getJSONArray("authors"); + List authorsList = new ArrayList<>(); + for (int i = 0; i < authors.length(); i++) { + authorsList.add(authors.getString(i)); + } + if (!authorsList.isEmpty()) { + entry.setField(StandardField.AUTHOR, String.join(" and ", authorsList)); + } else { + LOGGER.info("Empty authors array."); + } + } else { + LOGGER.info("No authors found."); + } + + // direct accessible fields + entry.setField(StandardField.TITLE, scholarJsonEntry.getString("title")); + String publishedDate = scholarJsonEntry.getString("published_date"); + String publishedDateOnly = publishedDate.split("T")[0]; + entry.setField(StandardField.DATE, publishedDateOnly); + entry.setField(StandardField.YEAR, publishedDateOnly.split("-")[0]); + + if (scholarJsonEntry.has("id")) { + entry.setField(new UnknownField("scholarapi-id"), scholarJsonEntry.getString("id")); + } + // doi + if (scholarJsonEntry.has("doi")) { + entry.setField(StandardField.DOI, scholarJsonEntry.getString("doi")); + } + // Journal issue + if (scholarJsonEntry.has("journal_issue")) { + entry.setField(StandardField.NUMBER, scholarJsonEntry.getString("journal_issue")); + } + // Journal pages + if (scholarJsonEntry.has("journal_pages")) { + entry.setField(StandardField.PAGES, scholarJsonEntry.getString("journal_pages")); + } + // ISSN + Optional.ofNullable(scholarJsonEntry.optJSONArray("journal_issn")).filter(arr -> !arr.isEmpty()).ifPresent(arr -> entry.setField(StandardField.ISSN, arr.getString(0))); + // Journal + if (scholarJsonEntry.has("journal")) { + entry.setField(StandardField.JOURNAL, scholarJsonEntry.getString("journal")); + } + // Url + if (scholarJsonEntry.has("url")) { + entry.setField(StandardField.URL, scholarJsonEntry.getString("url")); + } + // Abstract + if (scholarJsonEntry.has("abstract")) { + entry.setField(StandardField.ABSTRACT, scholarJsonEntry.getString("abstract")); + } + // Journal publisher + if (scholarJsonEntry.has("journal_publisher")) { + entry.setField(StandardField.PUBLISHER, scholarJsonEntry.getString("journal_publisher")); + } + return entry; + } catch (JSONException exception) { + throw new ParseException("ScholarAPI JSON format has changed", exception); + } + } + + @Override + public Page performSearchPaged(BaseQueryNode queryNode, int pageNumber) throws FetcherException { + ScholarApiQueryTransformer transformer = new ScholarApiQueryTransformer(); + String transformedQuery = transformer.transformSearchQuery(queryNode).orElse(""); + return fetchPage(transformedQuery, pageNumber, transformer.getStartYear(), transformer.getEndYear()); + } + + @Override + public Page performRawSearchQueryPaged(String rawQuery, int pageNumber) throws FetcherException { + if (rawQuery.isBlank()) { + return new Page<>(rawQuery, pageNumber, List.of()); + } + return fetchPage(rawQuery, pageNumber, Optional.empty(), Optional.empty()); + } + + private Page fetchPage(String query, int pageNumber, Optional startYear, Optional endYear) throws FetcherException { + URL url; + try { + url = buildSearchUrl(query, pageNumber, startYear, endYear); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Invalid URL", e); + } + + JSONObject response = callListApi(url); + + try { + int count = response.optInt("count", 0); + boolean isLastPage = count < getPageSize(); + + if (isLastPage) { + String nextIndexedAfter = response.getString("next_indexed_after"); + cursorCacheMap.put(new PageKey(query, startYear, endYear, pageNumber + 1), nextIndexedAfter); + } + + JSONArray results = response.optJSONArray("results"); + List entries = new ArrayList<>(); + if (results != null) { + for (int i = 0; i < results.length(); i++) { + entries.add(jsonItemToBibEntry(results.getJSONObject(i))); + } + } + return new Page<>(query, pageNumber, entries); + } catch (JSONException e) { + throw new FetcherException(url, "ScholarAPI response was not in the expected format", e); + } catch (ParseException e) { + throw new FetcherException(url, "ScholarAPI response could not be parsed", e); + } + } + + private JSONObject callListApi(URL url) throws FetcherException { + URLDownload urlDownload = new URLDownload(url); + importerPreferences.getApiKey(getName()).ifPresent(key -> urlDownload.addHeader("X-API-Key", key)); + + try (InputStream stream = urlDownload.asInputStream()) { + return JsonReader.toJsonObject(stream); + } catch (IOException | ParseException e) { + throw new FetcherException(url, "ScholarAPI request failed", e); + } + } + + @Override + public String getName() { + return FETCHER_NAME; + } + + @Override + public boolean isValidKey(@NonNull String apiKey) { + try { + URLDownload urlDownload = new URLDownload(getTestUrl()); + urlDownload.addHeader("X-API-Key", apiKey); + int statusCode = ((HttpURLConnection) urlDownload.getSource().openConnection()).getResponseCode(); + return (statusCode >= 200) && (statusCode < 300); + } catch (IOException | UnirestException e) { + return false; + } + } + + private URL getTestUrl() throws MalformedURLException { + return URLUtil.create("https://scholarapi.net/api/v1/list?limit=1"); + } + + private URL buildSearchUrl(String query, int pageNumber, Optional startYear, Optional endYear) + throws URISyntaxException, MalformedURLException { + URIBuilder uriBuilder = new URIBuilder(LIST_URL); + if (StringUtil.isNotBlank(query)) { + uriBuilder.setParameter("q", query); + } + uriBuilder.setParameter("limit", String.valueOf(getPageSize())); + startYear.ifPresent(year -> uriBuilder.addParameter("published_after", year + "-01-01")); + endYear.ifPresent(year -> uriBuilder.addParameter("published_before", (year + 1) + "-01-01")); + + if (pageNumber > 0) { + String cursor = cursorCacheMap.get(new PageKey(query, startYear, endYear, pageNumber)); + if (cursor == null) { + throw new URISyntaxException(LIST_URL, + "Page " + pageNumber + " was requested before its cursor was available; pages must be fetched sequentially"); + } + uriBuilder.addParameter("indexed_after", cursor); + } + return uriBuilder.build().toURL(); + } + + private record PageKey(String query, Optional startYear, Optional endYear, int pageNumber) { + } +} diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformer.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformer.java new file mode 100644 index 000000000000..8f8c443fdfef --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformer.java @@ -0,0 +1,43 @@ +package org.jabref.logic.importer.fetcher.transformers; + +import org.jabref.logic.util.strings.StringUtil; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class ScholarApiQueryTransformer extends YearAndYearRangeByFilteringQueryTransformer { + private static final Logger LOGGER = LoggerFactory.getLogger(ScholarApiQueryTransformer.class); + + @Override + protected String getLogicalAndOperator() { + return " AND "; + } + + @Override + protected String getLogicalOrOperator() { + return " OR "; + } + + @Override + protected String getLogicalNotOperator() { + return " NOT "; + } + + @Override + protected String handleAuthor(String author) { + // ScholarApi does not support explicit author field search + return StringUtil.quoteStringIfSpaceIsContained(author); + } + + @Override + protected String handleTitle(String title) { + // ScholarApi does not support explicit title field search + return StringUtil.quoteStringIfSpaceIsContained(title); + } + + @Override + protected String handleJournal(String journalTitle) { + LOGGER.warn("ScholarAPI has no journal scoped search"); + return ""; + } +} diff --git a/jablib/src/main/java/org/jabref/logic/util/BuildInfo.java b/jablib/src/main/java/org/jabref/logic/util/BuildInfo.java index 00273e72229d..8489b451e813 100644 --- a/jablib/src/main/java/org/jabref/logic/util/BuildInfo.java +++ b/jablib/src/main/java/org/jabref/logic/util/BuildInfo.java @@ -44,6 +44,7 @@ public final class BuildInfo { public final String openAlexApiKey; public final String scopusApiKey; public final String semanticScholarApiKey; + public final String scholarApiKey; public final String springerNatureAPIKey; public final String unpaywallEmail; public final String wileyTdmApiKey; @@ -76,6 +77,7 @@ public BuildInfo(String path) { openAlexApiKey = BuildInfo.getValue(properties, "openAlexApiKey", ""); scopusApiKey = BuildInfo.getValue(properties, "scopusApiKey", "fb82f2e692b3c72dafe5f4f1fa0ac00b"); semanticScholarApiKey = BuildInfo.getValue(properties, "semanticScholarApiKey", ""); + scholarApiKey = BuildInfo.getValue(properties, "scholarApiKey", ""); springerNatureAPIKey = BuildInfo.getValue(properties, "springerNatureAPIKey", "118d90a519d0fc2a01ee9715400054d4"); unpaywallEmail = BuildInfo.getValue(properties, "unpaywallEmail", ""); wileyTdmApiKey = BuildInfo.getValue(properties, "wileyTdmApiKey", ""); diff --git a/jablib/src/main/resources/build.properties b/jablib/src/main/resources/build.properties index 6e606abeb651..f4c230614c33 100644 --- a/jablib/src/main/resources/build.properties +++ b/jablib/src/main/resources/build.properties @@ -9,5 +9,6 @@ ieeeAPIKey=${ieeeAPIKey} medlineApiKey=${medlineApiKey} openAlexApiKey=${openAlexApiKey} semanticScholarApiKey=${semanticScholarApiKey} +scholarApiKey=${scholarApiKey} springerNatureAPIKey=${springerNatureAPIKey} wileyTdmApiKey=${wileyTdmApiKey} diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarApiFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarApiFetcherTest.java new file mode 100644 index 000000000000..04e83ddd81e6 --- /dev/null +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarApiFetcherTest.java @@ -0,0 +1,133 @@ +package org.jabref.logic.importer.fetcher; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import javafx.collections.FXCollections; + +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.ImporterPreferences; +import org.jabref.logic.importer.PagedSearchBasedFetcher; +import org.jabref.logic.importer.ParseException; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.util.BuildInfo; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.paging.Page; +import org.jabref.support.DisabledOnCIServer; +import org.jabref.testutils.category.FetcherTest; + +import com.airhacks.afterburner.injection.Injector; +import kong.unirest.core.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@FetcherTest +public class ScholarApiFetcherTest implements SearchBasedFetcherCapabilityTest, PagedSearchFetcherTest { + + ImporterPreferences importerPreferences = mock(ImporterPreferences.class); + ScholarFetcher fetcher = new ScholarFetcher(importerPreferences); + + @BeforeEach + void setUp() { + BuildInfo buildInfo = Injector.instantiateModelOrService(BuildInfo.class); + fetcher = new ScholarFetcher(importerPreferences); + when(importerPreferences.getApiKeys()).thenReturn(FXCollections.emptyObservableSet()); + when(importerPreferences.getApiKey(fetcher.getName())).thenReturn(Optional.of(buildInfo.scholarApiKey)); + } + + @Test + void scholarApiJsonToBibtex() throws ParseException { + String jsonString = """ + {\r + "id": "846a45f",\r + "title": "Paraneoplastic pemphigus case study",\r + "authors": ["E. R. Novak"],\r + "abstract": "Case report on autoimmune blistering disorders...",\r + "journal": "Clinical Immunology",\r + "journal_issn": ["1521-6616"],\r + "journal_issue": "3",\r + "journal_pages": "12-18",\r + "doi": "10.1016/j.clim.2023.109245",\r + "published_date": "2023-09-14",\r + "published_date_raw": "2023-09-14",\r + "indexed_at": "2024-03-01T12:30:45.123Z",\r + "has_text": true,\r + "has_pdf": true,\r + "url": "https://clinical.example.com/paper/846a45f"\r + }"""; + + JSONObject jsonObject = new JSONObject(jsonString); + BibEntry bibEntry = ScholarFetcher.jsonItemToBibEntry(jsonObject); + + assertEquals(Optional.of("846a45f"), bibEntry.getField(new UnknownField("scholarapi-id"))); + assertEquals(Optional.of("2023-09-14"), bibEntry.getField(StandardField.DATE)); + assertEquals(Optional.of("2023"), bibEntry.getField(StandardField.YEAR)); + assertEquals(Optional.of("Paraneoplastic pemphigus case study"), bibEntry.getField(StandardField.TITLE)); + assertEquals(Optional.of("E. R. Novak"), bibEntry.getField(StandardField.AUTHOR)); + assertEquals(Optional.of("Case report on autoimmune blistering disorders..."), bibEntry.getField(StandardField.ABSTRACT)); + assertEquals(Optional.of("Clinical Immunology"), bibEntry.getField(StandardField.JOURNAL)); + assertEquals(Optional.of("1521-6616"), bibEntry.getField(StandardField.ISSN)); + assertEquals(Optional.of("3"), bibEntry.getField(StandardField.NUMBER)); + assertEquals(Optional.of("12-18"), bibEntry.getField(StandardField.PAGES)); + assertEquals(Optional.of("10.1016/j.clim.2023.109245"), bibEntry.getField(StandardField.DOI)); + assertEquals(Optional.of("https://clinical.example.com/paper/846a45f"), bibEntry.getField(StandardField.URL)); + } + + @Test + void performRawSearchQueryPagedWithBlankQueryReturnsEmptyPage() throws FetcherException { + Page result = fetcher.performRawSearchQueryPaged("", 0); + assertEquals(List.of(), new ArrayList<>(result.getContent())); + } + + @Test + void searchByEmptyQueryFindsNothing() throws FetcherException { + assertEquals(List.of(), fetcher.performSearch("")); + } + + @Test + @Disabled("ScholarAPI has no journal scoped search") + @Override + public void supportsJournalSearch() { + } + + @Test + @Disabled("ScholarAPI has no author scoped search") + @Override + public void supportsAuthorSearch() { + } + + @Override + public PagedSearchBasedFetcher getPagedFetcher() { + return fetcher; + } + + @Override + public SearchBasedFetcher getFetcher() { + return fetcher; + } + + @Override + public List getTestAuthors() { + return List.of(); + } + + @Override + public String getTestJournal() { + return ""; + } + + @Test + @Override + @DisabledOnCIServer("Unstable on CI") + public void pageSearchReturnsUniqueResultsPerPage() { + // Implementation is done in the interface + } +} diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformerTest.java new file mode 100644 index 000000000000..5ad60156ca04 --- /dev/null +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformerTest.java @@ -0,0 +1,18 @@ +package org.jabref.logic.importer.fetcher.transformers; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +class ScholarApiQueryTransformerTest extends YearAndYearRangeByFilteringQueryTransformerTest { + + @Override + public ScholarApiQueryTransformer getTransformer() { + return new ScholarApiQueryTransformer(); + } + + @Test + @Disabled("ScholarAPI has no journal scoped search") + @Override + public void convertJournalFieldPrefix() { + } +} From 67d10194d5d37fd4a65662879253a7e23f49a2e6 Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Mon, 20 Jul 2026 18:18:38 +0530 Subject: [PATCH 02/16] fixed the load more error in fetch page and removed overridden validKey method relying on the pre-defined method --- .../org/jabref/logic/importer/WebFetchers.java | 7 +++++-- .../logic/importer/fetcher/ScholarFetcher.java | 18 +----------------- 2 files changed, 6 insertions(+), 19 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/WebFetchers.java b/jablib/src/main/java/org/jabref/logic/importer/WebFetchers.java index 8d12311362e5..1022bd496c45 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/WebFetchers.java +++ b/jablib/src/main/java/org/jabref/logic/importer/WebFetchers.java @@ -43,6 +43,7 @@ import org.jabref.logic.importer.fetcher.ResearchGate; import org.jabref.logic.importer.fetcher.RfcFetcher; import org.jabref.logic.importer.fetcher.ScholarArchiveFetcher; +import org.jabref.logic.importer.fetcher.ScholarFetcher; import org.jabref.logic.importer.fetcher.ScienceDirect; import org.jabref.logic.importer.fetcher.Scopus; import org.jabref.logic.importer.fetcher.SemanticScholar; @@ -79,7 +80,7 @@ public class WebFetchers { private WebFetchers() { } - /// @implNote Needs to be consistent with [#getIdBasedFetcherFoIdentifier(Identifier, ImportFormatPreferences) ] + /// @implNote Needs to be consistent with [#getIdBasedFetcherFoIdentifier(Identifier, ImportFormatPreferences)] public static Optional getIdBasedFetcherForField(Field field, ImportFormatPreferences importFormatPreferences) { IdBasedFetcher fetcher; @@ -100,7 +101,7 @@ public static Optional getIdBasedFetcherForField(Field field, Im return Optional.of(fetcher); } - /// @implNote Needs to be consistent with [#getIdBasedFetcherForField(Field, ImportFormatPreferences) ] + /// @implNote Needs to be consistent with [#getIdBasedFetcherForField(Field, ImportFormatPreferences)] public static Optional getIdBasedFetcherForIdentifier(Identifier identifier, ImportFormatPreferences importFormatPreferences) { IdBasedFetcher fetcher; @@ -174,6 +175,7 @@ public static synchronized SortedSet getSearchBasedFetchers( // set.add(new CollectionOfComputerScienceBibliographiesFetcher(importFormatPreferences)); searchBasedFetchers.add(new DOABFetcher()); // set.add(new JstorFetcher(importFormatPreferences)); + searchBasedFetchers.add(new ScholarFetcher(importerPreferences)); searchBasedFetchers.add(new SemanticScholar(importerPreferences)); searchBasedFetchers.add(new ResearchGate(importFormatPreferences)); searchBasedFetchers.add(new BiodiversityLibrary(importerPreferences)); @@ -296,6 +298,7 @@ public static Set getCustomizableKeyFetchers(ImportForma new IEEE(importFormatPreferences, importerPreferences), new MedlineFetcher(importerPreferences), new OpenAlex(importerPreferences), + new ScholarFetcher(importerPreferences), new SemanticScholar(importerPreferences), new Scopus(importerPreferences), new SpringerNatureWebFetcher(importerPreferences), diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java index f68a6301582c..8c9330a043db 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java @@ -150,7 +150,7 @@ private Page fetchPage(String query, int pageNumber, Optional int count = response.optInt("count", 0); boolean isLastPage = count < getPageSize(); - if (isLastPage) { + if (!isLastPage) { String nextIndexedAfter = response.getString("next_indexed_after"); cursorCacheMap.put(new PageKey(query, startYear, endYear, pageNumber + 1), nextIndexedAfter); } @@ -186,22 +186,6 @@ public String getName() { return FETCHER_NAME; } - @Override - public boolean isValidKey(@NonNull String apiKey) { - try { - URLDownload urlDownload = new URLDownload(getTestUrl()); - urlDownload.addHeader("X-API-Key", apiKey); - int statusCode = ((HttpURLConnection) urlDownload.getSource().openConnection()).getResponseCode(); - return (statusCode >= 200) && (statusCode < 300); - } catch (IOException | UnirestException e) { - return false; - } - } - - private URL getTestUrl() throws MalformedURLException { - return URLUtil.create("https://scholarapi.net/api/v1/list?limit=1"); - } - private URL buildSearchUrl(String query, int pageNumber, Optional startYear, Optional endYear) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(LIST_URL); From 82755be60f4257434da82dc569b80a8422a9c159 Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Mon, 20 Jul 2026 21:11:10 +0530 Subject: [PATCH 03/16] fixed formatting error and un-used imports --- .github/workflows/tests-code-fetchers.yml | 1 + .github/workflows/tests-code.yml | 2 +- docs/code-howtos/fetchers.md | 2 +- .../importer/fetcher/ScholarFetcher.java | 44 ++++++++++++------- 4 files changed, 32 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests-code-fetchers.yml b/.github/workflows/tests-code-fetchers.yml index eaf897470da4..2ead13b581cd 100644 --- a/.github/workflows/tests-code-fetchers.yml +++ b/.github/workflows/tests-code-fetchers.yml @@ -36,6 +36,7 @@ env: IEEEAPIKey: ${{ secrets.IEEEAPIKey_FOR_TESTS }} MedlineApiKey: ${{ secrets.MedlineApiKey_FOR_TESTS }} OpenAlexApiKey: ${{ secrets.OpenAlexApiKey_FOR_TESTS }} + ScholarApiKey: ${{ secrets.SCHOLARAPIKEY_FOR_TESTS }} SpringerNatureAPIKey: ${{ secrets.SPRINGERNATUREAPIKEY_FOR_TESTS }} concurrency: diff --git a/.github/workflows/tests-code.yml b/.github/workflows/tests-code.yml index d91e560dd5e0..f65059b6567c 100644 --- a/.github/workflows/tests-code.yml +++ b/.github/workflows/tests-code.yml @@ -16,7 +16,7 @@ env: MedlineAPiKey: ${{ secrets.MedlineApiKey_FOR_TESTS }} OpenAlexApiKey: ${{ secrets.OpenAlexApiKey_FOR_TESTS }} SpringerNatureAPIKey: ${{ secrets.SPRINGERNATUREAPIKEY_FOR_TESTS }} - scholarApiKey: ${{ secrets.SCHOLARAPIKEY_FOR_TESTS }} + ScholarApiKey: ${{ secrets.SCHOLARAPIKEY_FOR_TESTS }} GRADLE_OPTS: -Xmx4g JAVA_OPTS: -Xmx4g diff --git a/docs/code-howtos/fetchers.md b/docs/code-howtos/fetchers.md index aa58f1ee759a..b8d22d919830 100644 --- a/docs/code-howtos/fetchers.md +++ b/docs/code-howtos/fetchers.md @@ -18,7 +18,7 @@ Fetchers are the implementation of the [search using online services](https://do | [Springer Nature](https://docs.jabref.org/collect/import-using-online-bibliographic-database#springer) | [Springer Nature API portal](https://dev.springernature.com). Use the "Meta API" API key. | `SpringerNatureAPIKey` | 5000 calls/day | | [Wiley (TDM)](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) | [Wiley TDM portal](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) | `WileyTdmApiKey` | 3 articles/second **AND** 60 requests/10min | | [Zentralblatt Math](https://www.zbmath.org) | (none) | (none) | Depending on the current network | -| [ScholarApi](https://scholarapi.net/) | [SchoarApi User Account](https://scholarapi.net/auth/register) | `ScholarApiKey` | 1000 free credits per key | | +| [ScholarApi](https://scholarapi.net/) | [ScholarApi User Account](https://scholarapi.net/auth/register) | `ScholarApiKey` | 1000 free credits per key | | "Depending on the current network" means that it depends on whether your request is routed through a network having paid access. For instance, some universities have subscriptions to MathSciNet. On Windows, you have to log off and log on to let IntelliJ know about the environment variable change. Execute the gradle task `processResources` in the group "others" within IntelliJ to ensure the values have been correctly written. Now, the fetcher tests should run without issues. diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java index 8c9330a043db..208d71f3a0f1 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.io.InputStream; -import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -28,7 +27,6 @@ import org.jabref.model.paging.Page; import org.jabref.model.search.query.BaseQueryNode; -import kong.unirest.core.UnirestException; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONException; import kong.unirest.core.json.JSONObject; @@ -69,51 +67,51 @@ public static BibEntry jsonItemToBibEntry(JSONObject scholarJsonEntry) throws Pa if (!authorsList.isEmpty()) { entry.setField(StandardField.AUTHOR, String.join(" and ", authorsList)); } else { - LOGGER.info("Empty authors array."); + LOGGER.debug("Empty authors array."); } } else { - LOGGER.info("No authors found."); + LOGGER.debug("No authors found."); } // direct accessible fields - entry.setField(StandardField.TITLE, scholarJsonEntry.getString("title")); + entry.withField(StandardField.TITLE, scholarJsonEntry.getString("title")); String publishedDate = scholarJsonEntry.getString("published_date"); String publishedDateOnly = publishedDate.split("T")[0]; - entry.setField(StandardField.DATE, publishedDateOnly); - entry.setField(StandardField.YEAR, publishedDateOnly.split("-")[0]); + entry.withField(StandardField.DATE, publishedDateOnly); + entry.withField(StandardField.YEAR, publishedDateOnly.split("-")[0]); if (scholarJsonEntry.has("id")) { entry.setField(new UnknownField("scholarapi-id"), scholarJsonEntry.getString("id")); } // doi if (scholarJsonEntry.has("doi")) { - entry.setField(StandardField.DOI, scholarJsonEntry.getString("doi")); + entry.withField(StandardField.DOI, scholarJsonEntry.getString("doi")); } // Journal issue if (scholarJsonEntry.has("journal_issue")) { - entry.setField(StandardField.NUMBER, scholarJsonEntry.getString("journal_issue")); + entry.withField(StandardField.NUMBER, scholarJsonEntry.getString("journal_issue")); } // Journal pages if (scholarJsonEntry.has("journal_pages")) { - entry.setField(StandardField.PAGES, scholarJsonEntry.getString("journal_pages")); + entry.withField(StandardField.PAGES, scholarJsonEntry.getString("journal_pages")); } // ISSN - Optional.ofNullable(scholarJsonEntry.optJSONArray("journal_issn")).filter(arr -> !arr.isEmpty()).ifPresent(arr -> entry.setField(StandardField.ISSN, arr.getString(0))); + Optional.ofNullable(scholarJsonEntry.optJSONArray("journal_issn")).filter(arr -> !arr.isEmpty()).ifPresent(arr -> entry.withField(StandardField.ISSN, arr.getString(0))); // Journal if (scholarJsonEntry.has("journal")) { - entry.setField(StandardField.JOURNAL, scholarJsonEntry.getString("journal")); + entry.withField(StandardField.JOURNAL, scholarJsonEntry.getString("journal")); } // Url if (scholarJsonEntry.has("url")) { - entry.setField(StandardField.URL, scholarJsonEntry.getString("url")); + entry.withField(StandardField.URL, scholarJsonEntry.getString("url")); } // Abstract if (scholarJsonEntry.has("abstract")) { - entry.setField(StandardField.ABSTRACT, scholarJsonEntry.getString("abstract")); + entry.withField(StandardField.ABSTRACT, scholarJsonEntry.getString("abstract")); } // Journal publisher if (scholarJsonEntry.has("journal_publisher")) { - entry.setField(StandardField.PUBLISHER, scholarJsonEntry.getString("journal_publisher")); + entry.withField(StandardField.PUBLISHER, scholarJsonEntry.getString("journal_publisher")); } return entry; } catch (JSONException exception) { @@ -181,6 +179,22 @@ private JSONObject callListApi(URL url) throws FetcherException { } } + @Override + public boolean isValidKey(@NonNull String apiKey) { + try { + URLDownload urlDownload = new URLDownload(getTestUrl()); + urlDownload.addHeader("X-API-Key", apiKey); + urlDownload.asInputStream().close(); + return true; + } catch (IOException | FetcherException e) { + return false; + } + } + + private URL getTestUrl() throws MalformedURLException { + return URLUtil.create(LIST_URL + "?limit=1"); + } + @Override public String getName() { return FETCHER_NAME; From e9e4739a2203a53c2d50e0dc85f4db79049b4c3d Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Mon, 20 Jul 2026 21:53:19 +0530 Subject: [PATCH 04/16] fixed the fetchers.md --- docs/code-howtos/fetchers.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/code-howtos/fetchers.md b/docs/code-howtos/fetchers.md index b8d22d919830..81aeb6476641 100644 --- a/docs/code-howtos/fetchers.md +++ b/docs/code-howtos/fetchers.md @@ -18,8 +18,7 @@ Fetchers are the implementation of the [search using online services](https://do | [Springer Nature](https://docs.jabref.org/collect/import-using-online-bibliographic-database#springer) | [Springer Nature API portal](https://dev.springernature.com). Use the "Meta API" API key. | `SpringerNatureAPIKey` | 5000 calls/day | | [Wiley (TDM)](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) | [Wiley TDM portal](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) | `WileyTdmApiKey` | 3 articles/second **AND** 60 requests/10min | | [Zentralblatt Math](https://www.zbmath.org) | (none) | (none) | Depending on the current network | -| [ScholarApi](https://scholarapi.net/) | [ScholarApi User Account](https://scholarapi.net/auth/register) | `ScholarApiKey` | 1000 free credits per key | | -"Depending on the current network" means that it depends on whether your request is routed through a network having paid access. For instance, some universities have subscriptions to MathSciNet. +| [ScholarApi](https://scholarapi.net/) | [ScholarApi User Account](https://scholarapi.net/auth/register) | `ScholarApiKey` | 1000 free credits per key | On Windows, you have to log off and log on to let IntelliJ know about the environment variable change. Execute the gradle task `processResources` in the group "others" within IntelliJ to ensure the values have been correctly written. Now, the fetcher tests should run without issues. From 2c02b7633aea74fc623d69fb46c3d476e4a2588c Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Tue, 21 Jul 2026 08:46:36 +0530 Subject: [PATCH 05/16] fixed thr jabgui errors --- .../org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java b/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java index 9bdcffa42659..75d973abdbb5 100644 --- a/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java +++ b/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java @@ -62,6 +62,7 @@ void emptyStudyConstructorFillsDatabasesCorrectly() { new StudyCatalogItem("OpenAlex", false), new StudyCatalogItem("ResearchGate", false), new StudyCatalogItem("SAO/NASA ADS", false), + new StudyCatalogItem("Scholar", false), new StudyCatalogItem("ScholarArchive", false), new StudyCatalogItem("Scopus", false), new StudyCatalogItem("SemanticScholar", false), @@ -95,6 +96,7 @@ void studyConstructorFillsDatabasesCorrectly(@TempDir Path tempDir) { new StudyCatalogItem("OpenAlex", false), new StudyCatalogItem("ResearchGate", false), new StudyCatalogItem("SAO/NASA ADS", false), + new StudyCatalogItem("Scholar", false), new StudyCatalogItem("ScholarArchive", false), new StudyCatalogItem("Scopus", false), new StudyCatalogItem("SemanticScholar", false), From 46a7c8fe0de463e63fd8580c4468279cfda6b384 Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Mon, 27 Jul 2026 21:58:36 +0530 Subject: [PATCH 06/16] Implemented all requested changes --- .../importer/fetcher/ScholarFetcher.java | 87 ++++++++++++++----- .../ScholarApiQueryTransformer.java | 6 +- ...tcherTest.java => ScholarFetcherTest.java} | 4 +- 3 files changed, 70 insertions(+), 27 deletions(-) rename jablib/src/test/java/org/jabref/logic/importer/fetcher/{ScholarApiFetcherTest.java => ScholarFetcherTest.java} (97%) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java index 208d71f3a0f1..ae2fdf992a0f 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.io.InputStream; +import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -10,6 +11,8 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImporterPreferences; @@ -27,6 +30,7 @@ import org.jabref.model.paging.Page; import org.jabref.model.search.query.BaseQueryNode; +import kong.unirest.core.UnirestException; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONException; import kong.unirest.core.json.JSONObject; @@ -36,16 +40,21 @@ import org.slf4j.LoggerFactory; public class ScholarFetcher implements PagedSearchBasedFetcher, CustomizableKeyFetcher { - public static final String FETCHER_NAME = "Scholar"; + public static final String FETCHER_NAME = "ScholarAPI"; private static final Logger LOGGER = LoggerFactory.getLogger(ScholarFetcher.class); private static final String LIST_URL = "https://scholarapi.net/api/v1/list"; + private static final int NO_YEAR_BOUND = Integer.MIN_VALUE; + private final Map cursorCacheMap = new ConcurrentHashMap<>(); private final ImporterPreferences importerPreferences; + private static final Pattern JOURNAL_VOLUME = Pattern.compile("Volume\\s+([^,]+)", Pattern.CASE_INSENSITIVE); + private static final Pattern JOURNAL_ISSUE_NUMBER = Pattern.compile("Issue\\s+([^,]+)", Pattern.CASE_INSENSITIVE); + public ScholarFetcher(ImporterPreferences importerPreferences) { this.importerPreferences = importerPreferences; } @@ -65,7 +74,7 @@ public static BibEntry jsonItemToBibEntry(JSONObject scholarJsonEntry) throws Pa authorsList.add(authors.getString(i)); } if (!authorsList.isEmpty()) { - entry.setField(StandardField.AUTHOR, String.join(" and ", authorsList)); + entry.withField(StandardField.AUTHOR, String.join(" and ", authorsList)); } else { LOGGER.debug("Empty authors array."); } @@ -73,43 +82,62 @@ public static BibEntry jsonItemToBibEntry(JSONObject scholarJsonEntry) throws Pa LOGGER.debug("No authors found."); } - // direct accessible fields entry.withField(StandardField.TITLE, scholarJsonEntry.getString("title")); String publishedDate = scholarJsonEntry.getString("published_date"); String publishedDateOnly = publishedDate.split("T")[0]; entry.withField(StandardField.DATE, publishedDateOnly); entry.withField(StandardField.YEAR, publishedDateOnly.split("-")[0]); + entry.withField(new UnknownField("scholarApiHasText"), String.valueOf(scholarJsonEntry.getBoolean("has_text"))); + entry.withField(new UnknownField("scholarApiHasPdf"), String.valueOf(scholarJsonEntry.getBoolean("has_pdf"))); if (scholarJsonEntry.has("id")) { - entry.setField(new UnknownField("scholarapi-id"), scholarJsonEntry.getString("id")); + entry.withField(new UnknownField("scholarapi"), scholarJsonEntry.getString("id")); } - // doi + if (scholarJsonEntry.has("doi")) { entry.withField(StandardField.DOI, scholarJsonEntry.getString("doi")); } - // Journal issue + if (scholarJsonEntry.has("journal_issue")) { entry.withField(StandardField.NUMBER, scholarJsonEntry.getString("journal_issue")); } - // Journal pages + if (scholarJsonEntry.has("journal_pages")) { entry.withField(StandardField.PAGES, scholarJsonEntry.getString("journal_pages")); } - // ISSN + Optional.ofNullable(scholarJsonEntry.optJSONArray("journal_issn")).filter(arr -> !arr.isEmpty()).ifPresent(arr -> entry.withField(StandardField.ISSN, arr.getString(0))); // Journal if (scholarJsonEntry.has("journal")) { entry.withField(StandardField.JOURNAL, scholarJsonEntry.getString("journal")); } - // Url + + if (scholarJsonEntry.has("journal_issue")) { + String journalIssue = scholarJsonEntry.getString("journal_issue"); + Matcher volume = JOURNAL_VOLUME.matcher(journalIssue); + Matcher issue = JOURNAL_ISSUE_NUMBER.matcher(journalIssue); + boolean matchedVolume = volume.find(); + boolean matchedIssue = issue.find(); + + if (matchedVolume) { + entry.withField(StandardField.VOLUME, volume.group(1).trim()); + } + if (matchedIssue) { + entry.withField(StandardField.NUMBER, issue.group(1).trim()); + } + if (!matchedVolume && !matchedIssue) { + entry.withField(StandardField.NUMBER, journalIssue); + } + } + if (scholarJsonEntry.has("url")) { entry.withField(StandardField.URL, scholarJsonEntry.getString("url")); } - // Abstract + if (scholarJsonEntry.has("abstract")) { entry.withField(StandardField.ABSTRACT, scholarJsonEntry.getString("abstract")); } - // Journal publisher + if (scholarJsonEntry.has("journal_publisher")) { entry.withField(StandardField.PUBLISHER, scholarJsonEntry.getString("journal_publisher")); } @@ -135,6 +163,15 @@ public Page performRawSearchQueryPaged(String rawQuery, int pageNumber } private Page fetchPage(String query, int pageNumber, Optional startYear, Optional endYear) throws FetcherException { + if (query.isBlank() && startYear.isEmpty() && endYear.isEmpty()) { + return new Page<>(query, pageNumber, List.of()); + } + if (pageNumber == 0) { + int keyStartYear = startYear.orElse(NO_YEAR_BOUND); + int keyEndYear = endYear.orElse(NO_YEAR_BOUND); + cursorCacheMap.keySet().removeIf(key -> + key.query().equals(query) && key.startYear() == keyStartYear && key.endYear() == keyEndYear); + } URL url; try { url = buildSearchUrl(query, pageNumber, startYear, endYear); @@ -145,15 +182,17 @@ private Page fetchPage(String query, int pageNumber, Optional JSONObject response = callListApi(url); try { - int count = response.optInt("count", 0); - boolean isLastPage = count < getPageSize(); + JSONArray results = response.optJSONArray("results"); + int resultCount = results == null ? 0 : results.length(); + boolean isLastPage = resultCount < getPageSize(); if (!isLastPage) { - String nextIndexedAfter = response.getString("next_indexed_after"); - cursorCacheMap.put(new PageKey(query, startYear, endYear, pageNumber + 1), nextIndexedAfter); + Optional.ofNullable(response.optString("next_indexed_after", null)) + .ifPresent(cursor -> cursorCacheMap.put( + new PageKey(query, startYear.orElse(NO_YEAR_BOUND), endYear.orElse(NO_YEAR_BOUND), pageNumber + 1), + cursor)); } - JSONArray results = response.optJSONArray("results"); List entries = new ArrayList<>(); if (results != null) { for (int i = 0; i < results.length(); i++) { @@ -170,7 +209,9 @@ private Page fetchPage(String query, int pageNumber, Optional private JSONObject callListApi(URL url) throws FetcherException { URLDownload urlDownload = new URLDownload(url); - importerPreferences.getApiKey(getName()).ifPresent(key -> urlDownload.addHeader("X-API-Key", key)); + importerPreferences.getApiKey(getName()) + .filter(key -> !key.isBlank()) + .ifPresent(key -> urlDownload.addHeader("X-API-Key", key)); try (InputStream stream = urlDownload.asInputStream()) { return JsonReader.toJsonObject(stream); @@ -184,8 +225,8 @@ public boolean isValidKey(@NonNull String apiKey) { try { URLDownload urlDownload = new URLDownload(getTestUrl()); urlDownload.addHeader("X-API-Key", apiKey); - urlDownload.asInputStream().close(); - return true; + int statusCode = ((HttpURLConnection) urlDownload.openConnection()).getResponseCode(); + return (statusCode >= 200) && (statusCode < 300); } catch (IOException | FetcherException e) { return false; } @@ -201,7 +242,7 @@ public String getName() { } private URL buildSearchUrl(String query, int pageNumber, Optional startYear, Optional endYear) - throws URISyntaxException, MalformedURLException { + throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(LIST_URL); if (StringUtil.isNotBlank(query)) { uriBuilder.setParameter("q", query); @@ -211,9 +252,9 @@ private URL buildSearchUrl(String query, int pageNumber, Optional start endYear.ifPresent(year -> uriBuilder.addParameter("published_before", (year + 1) + "-01-01")); if (pageNumber > 0) { - String cursor = cursorCacheMap.get(new PageKey(query, startYear, endYear, pageNumber)); + String cursor = cursorCacheMap.get(new PageKey(query, startYear.orElse(NO_YEAR_BOUND), endYear.orElse(NO_YEAR_BOUND), pageNumber)); if (cursor == null) { - throw new URISyntaxException(LIST_URL, + throw new FetcherException( "Page " + pageNumber + " was requested before its cursor was available; pages must be fetched sequentially"); } uriBuilder.addParameter("indexed_after", cursor); @@ -221,6 +262,6 @@ private URL buildSearchUrl(String query, int pageNumber, Optional start return uriBuilder.build().toURL(); } - private record PageKey(String query, Optional startYear, Optional endYear, int pageNumber) { + private record PageKey(String query, int startYear, int endYear, int pageNumber) { } } diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformer.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformer.java index 8f8c443fdfef..8cd1da5ff052 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformer.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformer.java @@ -2,9 +2,11 @@ import org.jabref.logic.util.strings.StringUtil; +import org.jspecify.annotations.NullMarked; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@NullMarked public class ScholarApiQueryTransformer extends YearAndYearRangeByFilteringQueryTransformer { private static final Logger LOGGER = LoggerFactory.getLogger(ScholarApiQueryTransformer.class); @@ -37,7 +39,7 @@ protected String handleTitle(String title) { @Override protected String handleJournal(String journalTitle) { - LOGGER.warn("ScholarAPI has no journal scoped search"); - return ""; + LOGGER.debug("ScholarAPI has no journal scoped search"); + return StringUtil.quoteStringIfSpaceIsContained(journalTitle); } } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarApiFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java similarity index 97% rename from jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarApiFetcherTest.java rename to jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java index 04e83ddd81e6..6530dba2bd19 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarApiFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java @@ -30,7 +30,7 @@ import static org.mockito.Mockito.when; @FetcherTest -public class ScholarApiFetcherTest implements SearchBasedFetcherCapabilityTest, PagedSearchFetcherTest { +public class ScholarFetcherTest implements SearchBasedFetcherCapabilityTest, PagedSearchFetcherTest { ImporterPreferences importerPreferences = mock(ImporterPreferences.class); ScholarFetcher fetcher = new ScholarFetcher(importerPreferences); @@ -67,7 +67,7 @@ void scholarApiJsonToBibtex() throws ParseException { JSONObject jsonObject = new JSONObject(jsonString); BibEntry bibEntry = ScholarFetcher.jsonItemToBibEntry(jsonObject); - assertEquals(Optional.of("846a45f"), bibEntry.getField(new UnknownField("scholarapi-id"))); + assertEquals(Optional.of("846a45f"), bibEntry.getField(new UnknownField("scholarapi"))); assertEquals(Optional.of("2023-09-14"), bibEntry.getField(StandardField.DATE)); assertEquals(Optional.of("2023"), bibEntry.getField(StandardField.YEAR)); assertEquals(Optional.of("Paraneoplastic pemphigus case study"), bibEntry.getField(StandardField.TITLE)); From f67ad2f68fd9b3fd3507191f841397e6bc03cb64 Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Mon, 27 Jul 2026 22:26:31 +0530 Subject: [PATCH 07/16] fixed checkstyle errors --- .../org/jabref/logic/importer/fetcher/ScholarFetcher.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java index ae2fdf992a0f..612b12038d23 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java @@ -30,7 +30,6 @@ import org.jabref.model.paging.Page; import org.jabref.model.search.query.BaseQueryNode; -import kong.unirest.core.UnirestException; import kong.unirest.core.json.JSONArray; import kong.unirest.core.json.JSONException; import kong.unirest.core.json.JSONObject; @@ -48,13 +47,14 @@ public class ScholarFetcher implements PagedSearchBasedFetcher, CustomizableKeyF private static final int NO_YEAR_BOUND = Integer.MIN_VALUE; + private static final Pattern JOURNAL_VOLUME = Pattern.compile("Volume\\s+([^,]+)", Pattern.CASE_INSENSITIVE); + + private static final Pattern JOURNAL_ISSUE_NUMBER = Pattern.compile("Issue\\s+([^,]+)", Pattern.CASE_INSENSITIVE); + private final Map cursorCacheMap = new ConcurrentHashMap<>(); private final ImporterPreferences importerPreferences; - private static final Pattern JOURNAL_VOLUME = Pattern.compile("Volume\\s+([^,]+)", Pattern.CASE_INSENSITIVE); - private static final Pattern JOURNAL_ISSUE_NUMBER = Pattern.compile("Issue\\s+([^,]+)", Pattern.CASE_INSENSITIVE); - public ScholarFetcher(ImporterPreferences importerPreferences) { this.importerPreferences = importerPreferences; } From 23396434641c1e422a1d9dd162c886947d4fd27c Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Tue, 28 Jul 2026 16:15:44 +0530 Subject: [PATCH 08/16] fixed all jabgui related errors --- .../ManageStudyDefinitionViewModelTest.java | 4 +- .../importer/fetcher/ScholarFetcher.java | 10 +-- .../importer/fetcher/ScholarFetcherTest.java | 66 +++++++++++-------- 3 files changed, 43 insertions(+), 37 deletions(-) diff --git a/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java b/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java index 75d973abdbb5..760daa2cd305 100644 --- a/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java +++ b/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java @@ -62,7 +62,7 @@ void emptyStudyConstructorFillsDatabasesCorrectly() { new StudyCatalogItem("OpenAlex", false), new StudyCatalogItem("ResearchGate", false), new StudyCatalogItem("SAO/NASA ADS", false), - new StudyCatalogItem("Scholar", false), + new StudyCatalogItem("ScholarAPI", false), new StudyCatalogItem("ScholarArchive", false), new StudyCatalogItem("Scopus", false), new StudyCatalogItem("SemanticScholar", false), @@ -96,7 +96,7 @@ void studyConstructorFillsDatabasesCorrectly(@TempDir Path tempDir) { new StudyCatalogItem("OpenAlex", false), new StudyCatalogItem("ResearchGate", false), new StudyCatalogItem("SAO/NASA ADS", false), - new StudyCatalogItem("Scholar", false), + new StudyCatalogItem("ScholarAPI", false), new StudyCatalogItem("ScholarArchive", false), new StudyCatalogItem("Scopus", false), new StudyCatalogItem("SemanticScholar", false), diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java index 612b12038d23..ce88b9376c89 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java @@ -7,10 +7,10 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; +import java.util.Hashtable; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -51,7 +51,7 @@ public class ScholarFetcher implements PagedSearchBasedFetcher, CustomizableKeyF private static final Pattern JOURNAL_ISSUE_NUMBER = Pattern.compile("Issue\\s+([^,]+)", Pattern.CASE_INSENSITIVE); - private final Map cursorCacheMap = new ConcurrentHashMap<>(); + private final Map cursorCacheMap = new Hashtable<>(); private final ImporterPreferences importerPreferences; @@ -91,17 +91,13 @@ public static BibEntry jsonItemToBibEntry(JSONObject scholarJsonEntry) throws Pa entry.withField(new UnknownField("scholarApiHasPdf"), String.valueOf(scholarJsonEntry.getBoolean("has_pdf"))); if (scholarJsonEntry.has("id")) { - entry.withField(new UnknownField("scholarapi"), scholarJsonEntry.getString("id")); + entry.withField(new UnknownField("scholarAPiId"), scholarJsonEntry.getString("id")); } if (scholarJsonEntry.has("doi")) { entry.withField(StandardField.DOI, scholarJsonEntry.getString("doi")); } - if (scholarJsonEntry.has("journal_issue")) { - entry.withField(StandardField.NUMBER, scholarJsonEntry.getString("journal_issue")); - } - if (scholarJsonEntry.has("journal_pages")) { entry.withField(StandardField.PAGES, scholarJsonEntry.getString("journal_pages")); } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java index 6530dba2bd19..0056db411a35 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java @@ -47,38 +47,48 @@ void setUp() { void scholarApiJsonToBibtex() throws ParseException { String jsonString = """ {\r - "id": "846a45f",\r - "title": "Paraneoplastic pemphigus case study",\r - "authors": ["E. R. Novak"],\r - "abstract": "Case report on autoimmune blistering disorders...",\r - "journal": "Clinical Immunology",\r - "journal_issn": ["1521-6616"],\r - "journal_issue": "3",\r - "journal_pages": "12-18",\r - "doi": "10.1016/j.clim.2023.109245",\r - "published_date": "2023-09-14",\r - "published_date_raw": "2023-09-14",\r - "indexed_at": "2024-03-01T12:30:45.123Z",\r - "has_text": true,\r - "has_pdf": true,\r - "url": "https://clinical.example.com/paper/846a45f"\r - }"""; + "id": "7184",\r + "title": "Methylated N-(4-N,N-Dimethylaminobenzyl) Chitosan, a Novel Chitosan Derivative, Enhances Paracellular Permeability Across Intestinal Epithelial Cells (Caco-2)",\r + "authors": [ + "Jariya Kowapradit", + "Praneet Opanasopit", + "Tanasait Ngawhiranpat" + ],\r + "abstract": "The aim of this study was to investigate the effect of methylated N-(4-N,N-dimethylaminobenzyl) chitosan, TM-Bz-CS, on the paracellular permeability of Caco-2 cell monolayers and its toxicity towards the cell lines. The factors affecting epithelial permeability, e.g., degree of quaternization (DQ) and extent of dimethylaminobenzyl substitution (ES), were evaluated in intestinal cell monolayers of Caco-2 cells using the transepithelial electrical resistance and permeability of Caco-2 cell monolayers, with fluorescein isothiocyanate dextran 4,400 (FD-4) as a model compound for paracellular tight-junction transport. Cytotoxicity was evaluated with the 3-(4,5-dimethylthiazol-2-yl)-2,5-diphenyl tetrazolium bromide viability assay. The results revealed that, at pH 7.4, TM-Bz-CS appeared to increase cell permeability in a concentration-dependent manner, and this effect was relatively reversible at lower doses of 0.05–0.5 mM. Higher DQ and the ES caused the permeability of FD-4 to be higher. The cytotoxicity of TM-Bz-CS depended on concentration, %DQ, and %ES. These studies demonstrated that this novel modified chitosan has potential as an absorption enhancer.",\r + "journal": "AAPS PharmSciTech",\r + "journal_publisher": "Springer International Publishing",\r + "journal_issn": [ + "1530-9932" + ],\r + "journal_issue": "Volume 9, Issue 4",\r + "journal_pages": "1143-1152",\r + "doi": "10.1208/s12249-008-9160-7"\r, + "published_date": "2008-12-01T00:00:00Z"\r, + "published_date_raw": "2008-12-01T00:00:00Z",\r + "indexed_at": "2012-10-01T18:58:11.184Z",\r + "url": "https://link.springer.com/article/10.1208/s12249-008-9160-7",\r + "has_text": true,\r + "has_pdf": true\r + }"""; JSONObject jsonObject = new JSONObject(jsonString); BibEntry bibEntry = ScholarFetcher.jsonItemToBibEntry(jsonObject); - assertEquals(Optional.of("846a45f"), bibEntry.getField(new UnknownField("scholarapi"))); - assertEquals(Optional.of("2023-09-14"), bibEntry.getField(StandardField.DATE)); - assertEquals(Optional.of("2023"), bibEntry.getField(StandardField.YEAR)); - assertEquals(Optional.of("Paraneoplastic pemphigus case study"), bibEntry.getField(StandardField.TITLE)); - assertEquals(Optional.of("E. R. Novak"), bibEntry.getField(StandardField.AUTHOR)); - assertEquals(Optional.of("Case report on autoimmune blistering disorders..."), bibEntry.getField(StandardField.ABSTRACT)); - assertEquals(Optional.of("Clinical Immunology"), bibEntry.getField(StandardField.JOURNAL)); - assertEquals(Optional.of("1521-6616"), bibEntry.getField(StandardField.ISSN)); - assertEquals(Optional.of("3"), bibEntry.getField(StandardField.NUMBER)); - assertEquals(Optional.of("12-18"), bibEntry.getField(StandardField.PAGES)); - assertEquals(Optional.of("10.1016/j.clim.2023.109245"), bibEntry.getField(StandardField.DOI)); - assertEquals(Optional.of("https://clinical.example.com/paper/846a45f"), bibEntry.getField(StandardField.URL)); + assertEquals(Optional.of("7184"), bibEntry.getField(new UnknownField("scholarAPiId"))); + assertEquals(Optional.of("2008-12-01"), bibEntry.getField(StandardField.DATE)); + assertEquals(Optional.of("2008"), bibEntry.getField(StandardField.YEAR)); + assertEquals(Optional.of("Methylated N-(4-N,N-Dimethylaminobenzyl) Chitosan, a Novel Chitosan Derivative, Enhances Paracellular Permeability Across Intestinal Epithelial Cells (Caco-2)"), bibEntry.getField(StandardField.TITLE)); + assertEquals(Optional.of("Jariya Kowapradit and Praneet Opanasopit and Tanasait Ngawhiranpat"), bibEntry.getField(StandardField.AUTHOR)); + assertEquals(Optional.of("AAPS PharmSciTech"), bibEntry.getField(StandardField.JOURNAL)); + assertEquals(Optional.of("1530-9932"), bibEntry.getField(StandardField.ISSN)); + assertEquals(Optional.of("9"), bibEntry.getField(StandardField.VOLUME)); + assertEquals(Optional.of("4"), bibEntry.getField(StandardField.NUMBER)); + assertEquals(Optional.of("1143-1152"), bibEntry.getField(StandardField.PAGES)); + assertEquals(Optional.of("10.1208/s12249-008-9160-7"), bibEntry.getField(StandardField.DOI)); + assertEquals(Optional.of("https://link.springer.com/article/10.1208/s12249-008-9160-7"), bibEntry.getField(StandardField.URL)); + assertEquals(Optional.of("Springer International Publishing"), bibEntry.getField(StandardField.PUBLISHER)); + assertEquals(Optional.of("true"), bibEntry.getField(new UnknownField("scholarApiHasText"))); + assertEquals(Optional.of("true"), bibEntry.getField(new UnknownField("scholarApiHasPdf"))); } @Test From b8de12f1a8d3e1b04c9ba56847561d66a53fe6fb Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Wed, 29 Jul 2026 11:48:38 +0530 Subject: [PATCH 09/16] fixes modernizer error --- .../java/org/jabref/logic/importer/fetcher/ScholarFetcher.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java index ce88b9376c89..73350ffa088e 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java @@ -11,6 +11,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -51,7 +52,7 @@ public class ScholarFetcher implements PagedSearchBasedFetcher, CustomizableKeyF private static final Pattern JOURNAL_ISSUE_NUMBER = Pattern.compile("Issue\\s+([^,]+)", Pattern.CASE_INSENSITIVE); - private final Map cursorCacheMap = new Hashtable<>(); + private final Map cursorCacheMap = new ConcurrentHashMap<>(); private final ImporterPreferences importerPreferences; From 7d11565198937ebe7d2e63987e085196f4a96d1f Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Thu, 30 Jul 2026 08:05:10 +0530 Subject: [PATCH 10/16] fixes checkstyle errors --- .../java/org/jabref/logic/importer/fetcher/ScholarFetcher.java | 1 - 1 file changed, 1 deletion(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java index 73350ffa088e..b0017d05e091 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java @@ -7,7 +7,6 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; -import java.util.Hashtable; import java.util.List; import java.util.Map; import java.util.Optional; From fb8967235c17b189bef6d4d61753339668c65798 Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Sun, 9 Aug 2026 11:42:28 +0530 Subject: [PATCH 11/16] did the asked chnages --- .../logic/importer/fetcher/ScholarFetcher.java | 6 +++++- .../importer/fetcher/ScholarFetcherTest.java | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java index b0017d05e091..50bb8cf3f76a 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java @@ -35,9 +35,11 @@ import kong.unirest.core.json.JSONObject; import org.apache.hc.core5.net.URIBuilder; import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +@NullMarked public class ScholarFetcher implements PagedSearchBasedFetcher, CustomizableKeyFetcher { public static final String FETCHER_NAME = "ScholarAPI"; @@ -87,11 +89,13 @@ public static BibEntry jsonItemToBibEntry(JSONObject scholarJsonEntry) throws Pa String publishedDateOnly = publishedDate.split("T")[0]; entry.withField(StandardField.DATE, publishedDateOnly); entry.withField(StandardField.YEAR, publishedDateOnly.split("-")[0]); + + // ScholarAPI's has_text/has_pdf flags for future fulltext-fetcher integration without needing to re fetch metadata to check availability first entry.withField(new UnknownField("scholarApiHasText"), String.valueOf(scholarJsonEntry.getBoolean("has_text"))); entry.withField(new UnknownField("scholarApiHasPdf"), String.valueOf(scholarJsonEntry.getBoolean("has_pdf"))); if (scholarJsonEntry.has("id")) { - entry.withField(new UnknownField("scholarAPiId"), scholarJsonEntry.getString("id")); + entry.withField(new UnknownField("scholarapi"), scholarJsonEntry.getString("id")); } if (scholarJsonEntry.has("doi")) { diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java index 0056db411a35..861bd7a7e52e 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java @@ -16,7 +16,6 @@ import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; import org.jabref.model.paging.Page; -import org.jabref.support.DisabledOnCIServer; import org.jabref.testutils.category.FetcherTest; import com.airhacks.afterburner.injection.Injector; @@ -26,6 +25,7 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -74,7 +74,7 @@ void scholarApiJsonToBibtex() throws ParseException { JSONObject jsonObject = new JSONObject(jsonString); BibEntry bibEntry = ScholarFetcher.jsonItemToBibEntry(jsonObject); - assertEquals(Optional.of("7184"), bibEntry.getField(new UnknownField("scholarAPiId"))); + assertEquals(Optional.of("7184"), bibEntry.getField(new UnknownField("scholarapi"))); assertEquals(Optional.of("2008-12-01"), bibEntry.getField(StandardField.DATE)); assertEquals(Optional.of("2008"), bibEntry.getField(StandardField.YEAR)); assertEquals(Optional.of("Methylated N-(4-N,N-Dimethylaminobenzyl) Chitosan, a Novel Chitosan Derivative, Enhances Paracellular Permeability Across Intestinal Epithelial Cells (Caco-2)"), bibEntry.getField(StandardField.TITLE)); @@ -136,8 +136,16 @@ public String getTestJournal() { @Test @Override - @DisabledOnCIServer("Unstable on CI") - public void pageSearchReturnsUniqueResultsPerPage() { - // Implementation is done in the interface + public void pageSearchReturnsUniqueResultsPerPage() throws FetcherException { + String query = queryForUniqueResultsPerPage(); + Page firstPage = getPagedFetcher().performSearchPaged(query, 0); + Page secondPage = getPagedFetcher().performSearchPaged(query, 1); + + assertEquals(20, firstPage.getSize()); + assertEquals(20, secondPage.getSize()); + + for(BibEntry bibEntry : firstPage.getContent()) { + assertFalse(secondPage.getContent().contains(bibEntry), "%s contained in %s".formatted(bibEntry, secondPage.getContent())); + } } } From 1d7fa9bfe0a616f5df800dc0ed6fd0502a208ecf Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Sun, 9 Aug 2026 11:51:54 +0530 Subject: [PATCH 12/16] fixed formatting error --- .../org/jabref/logic/importer/fetcher/ScholarFetcherTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java index 861bd7a7e52e..1bc9fd8747e9 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java @@ -144,7 +144,7 @@ public void pageSearchReturnsUniqueResultsPerPage() throws FetcherException { assertEquals(20, firstPage.getSize()); assertEquals(20, secondPage.getSize()); - for(BibEntry bibEntry : firstPage.getContent()) { + for (BibEntry bibEntry : firstPage.getContent()) { assertFalse(secondPage.getContent().contains(bibEntry), "%s contained in %s".formatted(bibEntry, secondPage.getContent())); } } From 475f1f5285c98120e8a5c6d44b46a82f1ff96305 Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Sun, 9 Aug 2026 15:46:41 +0530 Subject: [PATCH 13/16] add deleted texts in fetchers.md --- docs/code-howtos/fetchers.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/code-howtos/fetchers.md b/docs/code-howtos/fetchers.md index 7ee190cc68d8..a67877a2b35c 100644 --- a/docs/code-howtos/fetchers.md +++ b/docs/code-howtos/fetchers.md @@ -8,8 +8,6 @@ Fetchers are the implementation of the [search using online services](https://do | Service | Key Source | Environment Variable | Rate Limit | |:--------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------|--------------------------------|---------------------------------------------------------------------------------------| | [Biodiversity Heritage Library](https://www.biodiversitylibrary.org/) | [Biodiversitylibrary](https://about.biodiversitylibrary.org/tools-and-services/developer-and-data-tools/#APIs) | `BiodiversityHeritageApiKey` | - | -| [arXiv](https://info.arxiv.org/help/api/index.html) | (none) | (none) | [One request per three seconds](https://info.arxiv.org/help/api/tou.html) | -| [DOAJ](https://doaj.org/) | (none) | (none) | [2 requests/second](https://doaj.org/api/docs) | | [Elsevier (ScienceDirect / Scopus)](https://dev.elsevier.com/) | [Elsevier Dev Portal](https://dev.elsevier.com/) | `ScopusApiKey` | [20.000 calls/week](https://dev.elsevier.com/api_key_settings.html) | | [IEEEXplore](https://docs.jabref.org/collect/import-using-online-bibliographic-database#ieeexplore) | [IEEE Xplore API portal](https://developer.ieee.org) | `IEEEAPIKey` | 200 calls/day | | [Medline/Pubmed](https://pubmed.ncbi.nlm.nih.gov/) | [NCBI User account](https://account.ncbi.nlm.nih.gov/settings/) | `medlineApiKey` | 10 requests/seconds | @@ -20,7 +18,9 @@ Fetchers are the implementation of the [search using online services](https://do | [Springer Nature](https://docs.jabref.org/collect/import-using-online-bibliographic-database#springer) | [Springer Nature API portal](https://dev.springernature.com). Use the "Meta API" API key. | `SpringerNatureAPIKey` | 5000 calls/day | | [Wiley (TDM)](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) | [Wiley TDM portal](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) | `WileyTdmApiKey` | 3 articles/second **AND** 60 requests/10min | | [Zentralblatt Math](https://www.zbmath.org) | (none) | (none) | Depending on the current network | -| [ScholarApi](https://scholarapi.net/) | [ScholarApi User Account](https://scholarapi.net/auth/register) | `ScholarApiKey` | 1000 free credits per key | +| [ScholarApi](https://scholarapi.net/) | [ScholarApi User Account](https://scholarapi.net/auth/register) | `ScholarApiKey` | 1000 free credits per key | | | | | + +"Depending on the current network" means that it depends on whether your request is routed through a network having paid access. For instance, some universities have subscriptions to MathSciNet. On Windows, you have to log off and log on to let IntelliJ know about the environment variable change. Execute the gradle task `processResources` in the group "others" within IntelliJ to ensure the values have been correctly written. Now, the fetcher tests should run without issues. @@ -32,8 +32,6 @@ JabRef supports different kinds of fetchers: There are more fetchers supported by JabRef. Investigate the package `org.jabref.logic.importer`. Another possibility is to investigate the inheritance relation of `WebFetcher` (Ctrl+H in IntelliJ). -Fetchers with a documented request limit use `FetcherRateLimiter`. Express limits with `ofRequestsPerInterval` when a provider specifies requests per time window; use `ofRequestsPerSecond` for limits returned dynamically by an API. A limiter is shared across all instances of a fetcher so parallel searches respect the provider-wide request rate. - ## Fulltext Fetchers * all fulltext fetchers run in parallel From 3995e090a4f35a47f9457cf59fe0f3797a25b1d2 Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Mon, 10 Aug 2026 15:15:40 +0530 Subject: [PATCH 14/16] did all asked chnages and fixed fetchers.md --- docs/code-howtos/fetchers.md | 4 +++- .../logic/importer/fetcher/ScholarFetcher.java | 14 +++++++++----- .../importer/fetcher/ScholarFetcherTest.java | 16 ---------------- .../ScholarApiQueryTransformerTest.java | 9 --------- 4 files changed, 12 insertions(+), 31 deletions(-) diff --git a/docs/code-howtos/fetchers.md b/docs/code-howtos/fetchers.md index a67877a2b35c..882304157660 100644 --- a/docs/code-howtos/fetchers.md +++ b/docs/code-howtos/fetchers.md @@ -18,7 +18,7 @@ Fetchers are the implementation of the [search using online services](https://do | [Springer Nature](https://docs.jabref.org/collect/import-using-online-bibliographic-database#springer) | [Springer Nature API portal](https://dev.springernature.com). Use the "Meta API" API key. | `SpringerNatureAPIKey` | 5000 calls/day | | [Wiley (TDM)](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) | [Wiley TDM portal](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) | `WileyTdmApiKey` | 3 articles/second **AND** 60 requests/10min | | [Zentralblatt Math](https://www.zbmath.org) | (none) | (none) | Depending on the current network | -| [ScholarApi](https://scholarapi.net/) | [ScholarApi User Account](https://scholarapi.net/auth/register) | `ScholarApiKey` | 1000 free credits per key | | | | | +| [ScholarApi](https://scholarapi.net/) | [ScholarApi User Account](https://scholarapi.net/auth/register) | `ScholarApiKey` | 1000 free credits per key | "Depending on the current network" means that it depends on whether your request is routed through a network having paid access. For instance, some universities have subscriptions to MathSciNet. @@ -32,6 +32,8 @@ JabRef supports different kinds of fetchers: There are more fetchers supported by JabRef. Investigate the package `org.jabref.logic.importer`. Another possibility is to investigate the inheritance relation of `WebFetcher` (Ctrl+H in IntelliJ). +Fetchers with a documented request limit use `FetcherRateLimiter`. Express limits with `ofRequestsPerInterval` when a provider specifies requests per time window; use `ofRequestsPerSecond` for limits returned dynamically by an API. A limiter is shared across all instances of a fetcher so parallel searches respect the provider-wide request rate. + ## Fulltext Fetchers * all fulltext fetchers run in parallel diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java index 50bb8cf3f76a..586a5df09eea 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java @@ -49,7 +49,7 @@ public class ScholarFetcher implements PagedSearchBasedFetcher, CustomizableKeyF private static final int NO_YEAR_BOUND = Integer.MIN_VALUE; - private static final Pattern JOURNAL_VOLUME = Pattern.compile("Volume\\s+([^,]+)", Pattern.CASE_INSENSITIVE); + private static final Pattern JOURNAL_VOLUME = Pattern.compile("Volume\\s+([^,\s]+)", Pattern.CASE_INSENSITIVE); private static final Pattern JOURNAL_ISSUE_NUMBER = Pattern.compile("Issue\\s+([^,]+)", Pattern.CASE_INSENSITIVE); @@ -187,10 +187,14 @@ private Page fetchPage(String query, int pageNumber, Optional boolean isLastPage = resultCount < getPageSize(); if (!isLastPage) { - Optional.ofNullable(response.optString("next_indexed_after", null)) - .ifPresent(cursor -> cursorCacheMap.put( - new PageKey(query, startYear.orElse(NO_YEAR_BOUND), endYear.orElse(NO_YEAR_BOUND), pageNumber + 1), - cursor)); + Optional cursor = Optional.ofNullable(response.optString("next_indexed_after", null)) + .filter(StringUtil::isNotBlank); + if (cursor.isEmpty()) { + throw new FetcherException(url, "More results are available but returned no pagination cursor", null); + } + cursorCacheMap.put( + new PageKey(query, startYear.orElse(NO_YEAR_BOUND), endYear.orElse(NO_YEAR_BOUND), pageNumber + 1), + cursor.get()); } List entries = new ArrayList<>(); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java index 1bc9fd8747e9..655eca220216 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java @@ -25,7 +25,6 @@ import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -133,19 +132,4 @@ public List getTestAuthors() { public String getTestJournal() { return ""; } - - @Test - @Override - public void pageSearchReturnsUniqueResultsPerPage() throws FetcherException { - String query = queryForUniqueResultsPerPage(); - Page firstPage = getPagedFetcher().performSearchPaged(query, 0); - Page secondPage = getPagedFetcher().performSearchPaged(query, 1); - - assertEquals(20, firstPage.getSize()); - assertEquals(20, secondPage.getSize()); - - for (BibEntry bibEntry : firstPage.getContent()) { - assertFalse(secondPage.getContent().contains(bibEntry), "%s contained in %s".formatted(bibEntry, secondPage.getContent())); - } - } } diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformerTest.java index 5ad60156ca04..4e0d32be7c0d 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformerTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformerTest.java @@ -1,18 +1,9 @@ package org.jabref.logic.importer.fetcher.transformers; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - class ScholarApiQueryTransformerTest extends YearAndYearRangeByFilteringQueryTransformerTest { @Override public ScholarApiQueryTransformer getTransformer() { return new ScholarApiQueryTransformer(); } - - @Test - @Disabled("ScholarAPI has no journal scoped search") - @Override - public void convertJournalFieldPrefix() { - } } From 97470eb5864d2774b4dbc09208935e2805b7783c Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Mon, 10 Aug 2026 15:54:36 +0530 Subject: [PATCH 15/16] fixed fetchers.md --- docs/code-howtos/fetchers.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/code-howtos/fetchers.md b/docs/code-howtos/fetchers.md index 882304157660..7f8fdc6d7c21 100644 --- a/docs/code-howtos/fetchers.md +++ b/docs/code-howtos/fetchers.md @@ -8,6 +8,8 @@ Fetchers are the implementation of the [search using online services](https://do | Service | Key Source | Environment Variable | Rate Limit | |:--------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------|--------------------------------|---------------------------------------------------------------------------------------| | [Biodiversity Heritage Library](https://www.biodiversitylibrary.org/) | [Biodiversitylibrary](https://about.biodiversitylibrary.org/tools-and-services/developer-and-data-tools/#APIs) | `BiodiversityHeritageApiKey` | - | +| [arXiv](https://info.arxiv.org/help/api/index.html) | (none) | (none) | [One request per three seconds](https://info.arxiv.org/help/api/tou.html) | +| [DOAJ](https://doaj.org/) | (none) | (none) | [2 requests/second](https://doaj.org/api/docs) | | [Elsevier (ScienceDirect / Scopus)](https://dev.elsevier.com/) | [Elsevier Dev Portal](https://dev.elsevier.com/) | `ScopusApiKey` | [20.000 calls/week](https://dev.elsevier.com/api_key_settings.html) | | [IEEEXplore](https://docs.jabref.org/collect/import-using-online-bibliographic-database#ieeexplore) | [IEEE Xplore API portal](https://developer.ieee.org) | `IEEEAPIKey` | 200 calls/day | | [Medline/Pubmed](https://pubmed.ncbi.nlm.nih.gov/) | [NCBI User account](https://account.ncbi.nlm.nih.gov/settings/) | `medlineApiKey` | 10 requests/seconds | @@ -16,9 +18,9 @@ Fetchers are the implementation of the [search using online services](https://do | [SAO/NASA Astrophysics Data System](https://docs.jabref.org/collect/import-using-online-bibliographic-database#sao-nasa-astrophysics-data-system) | [ADS UI](https://ui.adsabs.harvard.edu/user/settings/token) | `AstrophysicsDataSystemAPIKey` | 5000 calls/day | | [SemanticScholar](https://www.semanticscholar.org/) | | `SemanticScholarApiKey` | | | [Springer Nature](https://docs.jabref.org/collect/import-using-online-bibliographic-database#springer) | [Springer Nature API portal](https://dev.springernature.com). Use the "Meta API" API key. | `SpringerNatureAPIKey` | 5000 calls/day | +| [ScholarApi](https://scholarapi.net/) | [ScholarApi User Account](https://scholarapi.net/auth/register) | `ScholarApiKey` | 1000 free credits per key | | [Wiley (TDM)](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) | [Wiley TDM portal](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) | `WileyTdmApiKey` | 3 articles/second **AND** 60 requests/10min | | [Zentralblatt Math](https://www.zbmath.org) | (none) | (none) | Depending on the current network | -| [ScholarApi](https://scholarapi.net/) | [ScholarApi User Account](https://scholarapi.net/auth/register) | `ScholarApiKey` | 1000 free credits per key | "Depending on the current network" means that it depends on whether your request is routed through a network having paid access. For instance, some universities have subscriptions to MathSciNet. @@ -30,6 +32,8 @@ JabRef supports different kinds of fetchers: * `FulltextFetcher`: Searches for a PDF for an exiting bibliography entry * `SearchBasedFetcher`: Searches providers using a given query and returns a set of (new) bibliography entry. The user-facing side is implemented in the UI described at [https://docs.jabref.org/collect/import-using-online-bibliographic-database](https://docs.jabref.org/collect/import-using-online-bibliographic-database). +`JournalInformationFetcher` supplies the Entry Editor's journal-information popup. It uses Crossref for journal identity (title, publisher, ISSNs) and OpenAlex for the h-index and yearly work and citation counts. If enabled in the preferences, it uses the configured OpenAlex API key. + There are more fetchers supported by JabRef. Investigate the package `org.jabref.logic.importer`. Another possibility is to investigate the inheritance relation of `WebFetcher` (Ctrl+H in IntelliJ). Fetchers with a documented request limit use `FetcherRateLimiter`. Express limits with `ofRequestsPerInterval` when a provider specifies requests per time window; use `ofRequestsPerSecond` for limits returned dynamically by an API. A limiter is shared across all instances of a fetcher so parallel searches respect the provider-wide request rate. From 8545923da9fd629727271092b5d858de4f8f62ef Mon Sep 17 00:00:00 2001 From: thesauravpoddar Date: Thu, 13 Aug 2026 10:18:32 +0530 Subject: [PATCH 16/16] did all asked changes --- .../importer/fetcher/ScholarFetcher.java | 20 +++++++++---------- .../importer/fetcher/ScholarFetcherTest.java | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java index 586a5df09eea..52a5a1a1bfe8 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java @@ -23,6 +23,7 @@ import org.jabref.logic.net.URLDownload; import org.jabref.logic.util.URLUtil; import org.jabref.logic.util.strings.StringUtil; +import org.jabref.model.entry.AuthorList; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; @@ -49,7 +50,7 @@ public class ScholarFetcher implements PagedSearchBasedFetcher, CustomizableKeyF private static final int NO_YEAR_BOUND = Integer.MIN_VALUE; - private static final Pattern JOURNAL_VOLUME = Pattern.compile("Volume\\s+([^,\s]+)", Pattern.CASE_INSENSITIVE); + private static final Pattern JOURNAL_VOLUME = Pattern.compile("Volume\\s+([^,\\s]+)", Pattern.CASE_INSENSITIVE); private static final Pattern JOURNAL_ISSUE_NUMBER = Pattern.compile("Issue\\s+([^,]+)", Pattern.CASE_INSENSITIVE); @@ -76,7 +77,9 @@ public static BibEntry jsonItemToBibEntry(JSONObject scholarJsonEntry) throws Pa authorsList.add(authors.getString(i)); } if (!authorsList.isEmpty()) { - entry.withField(StandardField.AUTHOR, String.join(" and ", authorsList)); + String rawAuthors = String.join(" and ", authorsList); + AuthorList parsedAuthors = AuthorList.parse(rawAuthors); + entry.withField(StandardField.AUTHOR, parsedAuthors.getAsFirstLastNamesWithAnd()); } else { LOGGER.debug("Empty authors array."); } @@ -187,14 +190,11 @@ private Page fetchPage(String query, int pageNumber, Optional boolean isLastPage = resultCount < getPageSize(); if (!isLastPage) { - Optional cursor = Optional.ofNullable(response.optString("next_indexed_after", null)) - .filter(StringUtil::isNotBlank); - if (cursor.isEmpty()) { - throw new FetcherException(url, "More results are available but returned no pagination cursor", null); - } - cursorCacheMap.put( - new PageKey(query, startYear.orElse(NO_YEAR_BOUND), endYear.orElse(NO_YEAR_BOUND), pageNumber + 1), - cursor.get()); + Optional.ofNullable(response.optString("next_indexed_after", null)) + .filter(StringUtil::isNotBlank) + .ifPresent(cursor -> cursorCacheMap.put( + new PageKey(query, startYear.orElse(NO_YEAR_BOUND), endYear.orElse(NO_YEAR_BOUND), pageNumber + 1), + cursor)); } List entries = new ArrayList<>(); diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java index 655eca220216..d404efca6c98 100644 --- a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java @@ -125,11 +125,11 @@ public SearchBasedFetcher getFetcher() { @Override public List getTestAuthors() { - return List.of(); + return List.of("unsupported"); } @Override public String getTestJournal() { - return ""; + return "unsupported"; } }