diff --git a/.github/workflows/tests-code-fetchers.yml b/.github/workflows/tests-code-fetchers.yml index 08f7771228a8..fd1e43c64658 100644 --- a/.github/workflows/tests-code-fetchers.yml +++ b/.github/workflows/tests-code-fetchers.yml @@ -36,6 +36,7 @@ env: IEEEAPIKey: ${{ secrets.IEEEAPIKey_FOR_TESTS }} MedlineApiKey: ${{ secrets.MedlineApiKey_FOR_TESTS }} OpenAlexApiKey: ${{ secrets.OpenAlexApiKey_FOR_TESTS }} + ScholarApiKey: ${{ secrets.SCHOLARAPIKEY_FOR_TESTS }} SpringerNatureAPIKey: ${{ secrets.SPRINGERNATUREAPIKEY_FOR_TESTS }} concurrency: diff --git a/.github/workflows/tests-code.yml b/.github/workflows/tests-code.yml index 936927e3f894..8ddadd610227 100644 --- a/.github/workflows/tests-code.yml +++ b/.github/workflows/tests-code.yml @@ -16,6 +16,7 @@ env: MedlineAPiKey: ${{ secrets.MedlineApiKey_FOR_TESTS }} OpenAlexApiKey: ${{ secrets.OpenAlexApiKey_FOR_TESTS }} SpringerNatureAPIKey: ${{ secrets.SPRINGERNATUREAPIKEY_FOR_TESTS }} + ScholarApiKey: ${{ secrets.SCHOLARAPIKEY_FOR_TESTS }} GRADLE_OPTS: -Xmx4g JAVA_OPTS: -Xmx4g diff --git a/docs/code-howtos/fetchers.md b/docs/code-howtos/fetchers.md index 552683496133..7f8fdc6d7c21 100644 --- a/docs/code-howtos/fetchers.md +++ b/docs/code-howtos/fetchers.md @@ -18,6 +18,7 @@ Fetchers are the implementation of the [search using online services](https://do | [SAO/NASA Astrophysics Data System](https://docs.jabref.org/collect/import-using-online-bibliographic-database#sao-nasa-astrophysics-data-system) | [ADS UI](https://ui.adsabs.harvard.edu/user/settings/token) | `AstrophysicsDataSystemAPIKey` | 5000 calls/day | | [SemanticScholar](https://www.semanticscholar.org/) | | `SemanticScholarApiKey` | | | [Springer Nature](https://docs.jabref.org/collect/import-using-online-bibliographic-database#springer) | [Springer Nature API portal](https://dev.springernature.com). Use the "Meta API" API key. | `SpringerNatureAPIKey` | 5000 calls/day | +| [ScholarApi](https://scholarapi.net/) | [ScholarApi User Account](https://scholarapi.net/auth/register) | `ScholarApiKey` | 1000 free credits per key | | [Wiley (TDM)](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) | [Wiley TDM portal](https://onlinelibrary.wiley.com/library-info/resources/text-and-datamining) | `WileyTdmApiKey` | 3 articles/second **AND** 60 requests/10min | | [Zentralblatt Math](https://www.zbmath.org) | (none) | (none) | Depending on the current network | diff --git a/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java b/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java index 24b4a08d5499..0c0cc53b070b 100644 --- a/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java +++ b/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java @@ -61,6 +61,7 @@ void emptyStudyConstructorFillsDatabasesCorrectly() { new StudyCatalogItem("OpenAlex", false), new StudyCatalogItem("ResearchGate", false), new StudyCatalogItem("SAO/NASA ADS", false), + new StudyCatalogItem("ScholarAPI", false), new StudyCatalogItem("ScholarArchive", false), new StudyCatalogItem("Scopus", false), new StudyCatalogItem("SemanticScholar", false), @@ -93,6 +94,7 @@ void studyConstructorFillsDatabasesCorrectly(@TempDir Path tempDir) { new StudyCatalogItem("OpenAlex", false), new StudyCatalogItem("ResearchGate", false), new StudyCatalogItem("SAO/NASA ADS", false), + new StudyCatalogItem("ScholarAPI", false), new StudyCatalogItem("ScholarArchive", false), new StudyCatalogItem("Scopus", false), new StudyCatalogItem("SemanticScholar", false), diff --git a/jablib/build.gradle.kts b/jablib/build.gradle.kts index f199ae445669..c3191c28608c 100644 --- a/jablib/build.gradle.kts +++ b/jablib/build.gradle.kts @@ -193,6 +193,7 @@ val medlineApiKey = providers.environmentVariable("MedlineApiKey").orElse("") val openAlexApiKey = providers.environmentVariable("OpenAlexApiKey").orElse("") val scopusApiKey = providers.environmentVariable("ScopusApiKey").orElse("") val semanticScholarApiKey = providers.environmentVariable("SemanticScholarApiKey").orElse("") +val scholarApiKey = providers.environmentVariable("ScholarApiKey").orElse("") val springerNatureAPIKey = providers.environmentVariable("SpringerNatureAPIKey").orElse("") val unpaywallEmail = providers.environmentVariable("UNPAYWALL_EMAIL").orElse("") val wileyTdmApiKey = providers.environmentVariable("WileyTdmApiKey").orElse("") @@ -215,6 +216,7 @@ tasks.named("processResources") { inputs.property("medlineApiKey", medlineApiKey) inputs.property("openAlexApiKey", openAlexApiKey) inputs.property("springerNatureAPIKey", springerNatureAPIKey) + inputs.property("scholarApiKey", scholarApiKey) inputs.property("scopusApiKey", scopusApiKey) inputs.property("semanticScholarApiKey", semanticScholarApiKey) inputs.property("unpaywallEmail", unpaywallEmail) @@ -235,6 +237,7 @@ tasks.named("processResources") { "openAlexApiKey" to inputs.properties["openAlexApiKey"], "scopusApiKey" to inputs.properties["scopusApiKey"], "semanticScholarApiKey" to inputs.properties["semanticScholarApiKey"], + "scholarApiKey" to inputs.properties["scholarApiKey"], "springerNatureAPIKey" to inputs.properties["springerNatureAPIKey"], "unpaywallEmail" to inputs.properties["unpaywallEmail"], "wileyTdmApiKey" to inputs.properties["wileyTdmApiKey"], diff --git a/jablib/src/main/java/org/jabref/logic/importer/ImporterPreferences.java b/jablib/src/main/java/org/jabref/logic/importer/ImporterPreferences.java index 5ad00ec3c3b9..737e031dcaa9 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/ImporterPreferences.java +++ b/jablib/src/main/java/org/jabref/logic/importer/ImporterPreferences.java @@ -23,6 +23,7 @@ import org.jabref.logic.importer.fetcher.BiodiversityLibrary; import org.jabref.logic.importer.fetcher.DBLPFetcher; import org.jabref.logic.importer.fetcher.IEEE; +import org.jabref.logic.importer.fetcher.ScholarFetcher; import org.jabref.logic.importer.fetcher.Scopus; import org.jabref.logic.importer.fetcher.SpringerNatureWebFetcher; import org.jabref.logic.importer.fetcher.WileyFetcher; @@ -111,6 +112,7 @@ private static Map getDefaultFetcherKeys() { Scopus.FETCHER_NAME, buildInfo.scopusApiKey, SemanticScholarCitationFetcher.FETCHER_NAME, buildInfo.semanticScholarApiKey, // SpringerLink uses the same key and fetcher name as SpringerFetcher + ScholarFetcher.FETCHER_NAME, buildInfo.scholarApiKey, SpringerNatureWebFetcher.FETCHER_NAME, buildInfo.springerNatureAPIKey, WileyFetcher.FETCHER_NAME, buildInfo.wileyTdmApiKey ); diff --git a/jablib/src/main/java/org/jabref/logic/importer/WebFetchers.java b/jablib/src/main/java/org/jabref/logic/importer/WebFetchers.java index d1164e746741..1b3d05695759 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/WebFetchers.java +++ b/jablib/src/main/java/org/jabref/logic/importer/WebFetchers.java @@ -42,6 +42,7 @@ import org.jabref.logic.importer.fetcher.ResearchGate; import org.jabref.logic.importer.fetcher.RfcFetcher; import org.jabref.logic.importer.fetcher.ScholarArchiveFetcher; +import org.jabref.logic.importer.fetcher.ScholarFetcher; import org.jabref.logic.importer.fetcher.ScienceDirect; import org.jabref.logic.importer.fetcher.Scopus; import org.jabref.logic.importer.fetcher.SemanticScholar; @@ -78,7 +79,7 @@ public class WebFetchers { private WebFetchers() { } - /// @implNote Needs to be consistent with [#getIdBasedFetcherFoIdentifier(Identifier, ImportFormatPreferences) ] + /// @implNote Needs to be consistent with [#getIdBasedFetcherFoIdentifier(Identifier, ImportFormatPreferences)] public static Optional getIdBasedFetcherForField(Field field, ImportFormatPreferences importFormatPreferences) { IdBasedFetcher fetcher; @@ -99,7 +100,7 @@ public static Optional getIdBasedFetcherForField(Field field, Im return Optional.of(fetcher); } - /// @implNote Needs to be consistent with [#getIdBasedFetcherForField(Field, ImportFormatPreferences) ] + /// @implNote Needs to be consistent with [#getIdBasedFetcherForField(Field, ImportFormatPreferences)] public static Optional getIdBasedFetcherForIdentifier(Identifier identifier, ImportFormatPreferences importFormatPreferences) { IdBasedFetcher fetcher; @@ -172,6 +173,7 @@ public static synchronized SortedSet getSearchBasedFetchers( // set.add(new CollectionOfComputerScienceBibliographiesFetcher(importFormatPreferences)); searchBasedFetchers.add(new DOABFetcher()); // set.add(new JstorFetcher(importFormatPreferences)); + searchBasedFetchers.add(new ScholarFetcher(importerPreferences)); searchBasedFetchers.add(new SemanticScholar(importerPreferences)); searchBasedFetchers.add(new ResearchGate(importFormatPreferences)); searchBasedFetchers.add(new BiodiversityLibrary(importerPreferences)); @@ -293,6 +295,7 @@ public static Set getCustomizableKeyFetchers(ImportForma new IEEE(importFormatPreferences, importerPreferences), new MedlineFetcher(importerPreferences), new OpenAlex(importerPreferences), + new ScholarFetcher(importerPreferences), new SemanticScholar(importerPreferences), new Scopus(importerPreferences), new SpringerNatureWebFetcher(importerPreferences), diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java new file mode 100644 index 000000000000..52a5a1a1bfe8 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/ScholarFetcher.java @@ -0,0 +1,271 @@ +package org.jabref.logic.importer.fetcher; + +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.ImporterPreferences; +import org.jabref.logic.importer.PagedSearchBasedFetcher; +import org.jabref.logic.importer.ParseException; +import org.jabref.logic.importer.fetcher.transformers.ScholarApiQueryTransformer; +import org.jabref.logic.importer.util.JsonReader; +import org.jabref.logic.net.URLDownload; +import org.jabref.logic.util.URLUtil; +import org.jabref.logic.util.strings.StringUtil; +import org.jabref.model.entry.AuthorList; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.paging.Page; +import org.jabref.model.search.query.BaseQueryNode; + +import kong.unirest.core.json.JSONArray; +import kong.unirest.core.json.JSONException; +import kong.unirest.core.json.JSONObject; +import org.apache.hc.core5.net.URIBuilder; +import org.jspecify.annotations.NonNull; +import org.jspecify.annotations.NullMarked; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@NullMarked +public class ScholarFetcher implements PagedSearchBasedFetcher, CustomizableKeyFetcher { + public static final String FETCHER_NAME = "ScholarAPI"; + + private static final Logger LOGGER = LoggerFactory.getLogger(ScholarFetcher.class); + + private static final String LIST_URL = "https://scholarapi.net/api/v1/list"; + + private static final int NO_YEAR_BOUND = Integer.MIN_VALUE; + + private static final Pattern JOURNAL_VOLUME = Pattern.compile("Volume\\s+([^,\\s]+)", Pattern.CASE_INSENSITIVE); + + private static final Pattern JOURNAL_ISSUE_NUMBER = Pattern.compile("Issue\\s+([^,]+)", Pattern.CASE_INSENSITIVE); + + private final Map cursorCacheMap = new ConcurrentHashMap<>(); + + private final ImporterPreferences importerPreferences; + + public ScholarFetcher(ImporterPreferences importerPreferences) { + this.importerPreferences = importerPreferences; + } + + /// Convert a JSONObject obtained from the Scholar API to a BibEntry + /// + /// @param scholarJsonEntry the JSONObject from search results + /// @return the converted BibEntry + public static BibEntry jsonItemToBibEntry(JSONObject scholarJsonEntry) throws ParseException { + try { + BibEntry entry = new BibEntry(StandardEntryType.Article); + + if (scholarJsonEntry.has("authors")) { + JSONArray authors = scholarJsonEntry.getJSONArray("authors"); + List authorsList = new ArrayList<>(); + for (int i = 0; i < authors.length(); i++) { + authorsList.add(authors.getString(i)); + } + if (!authorsList.isEmpty()) { + String rawAuthors = String.join(" and ", authorsList); + AuthorList parsedAuthors = AuthorList.parse(rawAuthors); + entry.withField(StandardField.AUTHOR, parsedAuthors.getAsFirstLastNamesWithAnd()); + } else { + LOGGER.debug("Empty authors array."); + } + } else { + LOGGER.debug("No authors found."); + } + + entry.withField(StandardField.TITLE, scholarJsonEntry.getString("title")); + String publishedDate = scholarJsonEntry.getString("published_date"); + String publishedDateOnly = publishedDate.split("T")[0]; + entry.withField(StandardField.DATE, publishedDateOnly); + entry.withField(StandardField.YEAR, publishedDateOnly.split("-")[0]); + + // ScholarAPI's has_text/has_pdf flags for future fulltext-fetcher integration without needing to re fetch metadata to check availability first + entry.withField(new UnknownField("scholarApiHasText"), String.valueOf(scholarJsonEntry.getBoolean("has_text"))); + entry.withField(new UnknownField("scholarApiHasPdf"), String.valueOf(scholarJsonEntry.getBoolean("has_pdf"))); + + if (scholarJsonEntry.has("id")) { + entry.withField(new UnknownField("scholarapi"), scholarJsonEntry.getString("id")); + } + + if (scholarJsonEntry.has("doi")) { + entry.withField(StandardField.DOI, scholarJsonEntry.getString("doi")); + } + + if (scholarJsonEntry.has("journal_pages")) { + entry.withField(StandardField.PAGES, scholarJsonEntry.getString("journal_pages")); + } + + Optional.ofNullable(scholarJsonEntry.optJSONArray("journal_issn")).filter(arr -> !arr.isEmpty()).ifPresent(arr -> entry.withField(StandardField.ISSN, arr.getString(0))); + // Journal + if (scholarJsonEntry.has("journal")) { + entry.withField(StandardField.JOURNAL, scholarJsonEntry.getString("journal")); + } + + if (scholarJsonEntry.has("journal_issue")) { + String journalIssue = scholarJsonEntry.getString("journal_issue"); + Matcher volume = JOURNAL_VOLUME.matcher(journalIssue); + Matcher issue = JOURNAL_ISSUE_NUMBER.matcher(journalIssue); + boolean matchedVolume = volume.find(); + boolean matchedIssue = issue.find(); + + if (matchedVolume) { + entry.withField(StandardField.VOLUME, volume.group(1).trim()); + } + if (matchedIssue) { + entry.withField(StandardField.NUMBER, issue.group(1).trim()); + } + if (!matchedVolume && !matchedIssue) { + entry.withField(StandardField.NUMBER, journalIssue); + } + } + + if (scholarJsonEntry.has("url")) { + entry.withField(StandardField.URL, scholarJsonEntry.getString("url")); + } + + if (scholarJsonEntry.has("abstract")) { + entry.withField(StandardField.ABSTRACT, scholarJsonEntry.getString("abstract")); + } + + if (scholarJsonEntry.has("journal_publisher")) { + entry.withField(StandardField.PUBLISHER, scholarJsonEntry.getString("journal_publisher")); + } + return entry; + } catch (JSONException exception) { + throw new ParseException("ScholarAPI JSON format has changed", exception); + } + } + + @Override + public Page performSearchPaged(BaseQueryNode queryNode, int pageNumber) throws FetcherException { + ScholarApiQueryTransformer transformer = new ScholarApiQueryTransformer(); + String transformedQuery = transformer.transformSearchQuery(queryNode).orElse(""); + return fetchPage(transformedQuery, pageNumber, transformer.getStartYear(), transformer.getEndYear()); + } + + @Override + public Page performRawSearchQueryPaged(String rawQuery, int pageNumber) throws FetcherException { + if (rawQuery.isBlank()) { + return new Page<>(rawQuery, pageNumber, List.of()); + } + return fetchPage(rawQuery, pageNumber, Optional.empty(), Optional.empty()); + } + + private Page fetchPage(String query, int pageNumber, Optional startYear, Optional endYear) throws FetcherException { + if (query.isBlank() && startYear.isEmpty() && endYear.isEmpty()) { + return new Page<>(query, pageNumber, List.of()); + } + if (pageNumber == 0) { + int keyStartYear = startYear.orElse(NO_YEAR_BOUND); + int keyEndYear = endYear.orElse(NO_YEAR_BOUND); + cursorCacheMap.keySet().removeIf(key -> + key.query().equals(query) && key.startYear() == keyStartYear && key.endYear() == keyEndYear); + } + URL url; + try { + url = buildSearchUrl(query, pageNumber, startYear, endYear); + } catch (URISyntaxException | MalformedURLException e) { + throw new FetcherException("Invalid URL", e); + } + + JSONObject response = callListApi(url); + + try { + JSONArray results = response.optJSONArray("results"); + int resultCount = results == null ? 0 : results.length(); + boolean isLastPage = resultCount < getPageSize(); + + if (!isLastPage) { + Optional.ofNullable(response.optString("next_indexed_after", null)) + .filter(StringUtil::isNotBlank) + .ifPresent(cursor -> cursorCacheMap.put( + new PageKey(query, startYear.orElse(NO_YEAR_BOUND), endYear.orElse(NO_YEAR_BOUND), pageNumber + 1), + cursor)); + } + + List entries = new ArrayList<>(); + if (results != null) { + for (int i = 0; i < results.length(); i++) { + entries.add(jsonItemToBibEntry(results.getJSONObject(i))); + } + } + return new Page<>(query, pageNumber, entries); + } catch (JSONException e) { + throw new FetcherException(url, "ScholarAPI response was not in the expected format", e); + } catch (ParseException e) { + throw new FetcherException(url, "ScholarAPI response could not be parsed", e); + } + } + + private JSONObject callListApi(URL url) throws FetcherException { + URLDownload urlDownload = new URLDownload(url); + importerPreferences.getApiKey(getName()) + .filter(key -> !key.isBlank()) + .ifPresent(key -> urlDownload.addHeader("X-API-Key", key)); + + try (InputStream stream = urlDownload.asInputStream()) { + return JsonReader.toJsonObject(stream); + } catch (IOException | ParseException e) { + throw new FetcherException(url, "ScholarAPI request failed", e); + } + } + + @Override + public boolean isValidKey(@NonNull String apiKey) { + try { + URLDownload urlDownload = new URLDownload(getTestUrl()); + urlDownload.addHeader("X-API-Key", apiKey); + int statusCode = ((HttpURLConnection) urlDownload.openConnection()).getResponseCode(); + return (statusCode >= 200) && (statusCode < 300); + } catch (IOException | FetcherException e) { + return false; + } + } + + private URL getTestUrl() throws MalformedURLException { + return URLUtil.create(LIST_URL + "?limit=1"); + } + + @Override + public String getName() { + return FETCHER_NAME; + } + + private URL buildSearchUrl(String query, int pageNumber, Optional startYear, Optional endYear) + throws URISyntaxException, MalformedURLException, FetcherException { + URIBuilder uriBuilder = new URIBuilder(LIST_URL); + if (StringUtil.isNotBlank(query)) { + uriBuilder.setParameter("q", query); + } + uriBuilder.setParameter("limit", String.valueOf(getPageSize())); + startYear.ifPresent(year -> uriBuilder.addParameter("published_after", year + "-01-01")); + endYear.ifPresent(year -> uriBuilder.addParameter("published_before", (year + 1) + "-01-01")); + + if (pageNumber > 0) { + String cursor = cursorCacheMap.get(new PageKey(query, startYear.orElse(NO_YEAR_BOUND), endYear.orElse(NO_YEAR_BOUND), pageNumber)); + if (cursor == null) { + throw new FetcherException( + "Page " + pageNumber + " was requested before its cursor was available; pages must be fetched sequentially"); + } + uriBuilder.addParameter("indexed_after", cursor); + } + return uriBuilder.build().toURL(); + } + + private record PageKey(String query, int startYear, int endYear, int pageNumber) { + } +} diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformer.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformer.java new file mode 100644 index 000000000000..8cd1da5ff052 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformer.java @@ -0,0 +1,45 @@ +package org.jabref.logic.importer.fetcher.transformers; + +import org.jabref.logic.util.strings.StringUtil; + +import org.jspecify.annotations.NullMarked; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@NullMarked +public class ScholarApiQueryTransformer extends YearAndYearRangeByFilteringQueryTransformer { + private static final Logger LOGGER = LoggerFactory.getLogger(ScholarApiQueryTransformer.class); + + @Override + protected String getLogicalAndOperator() { + return " AND "; + } + + @Override + protected String getLogicalOrOperator() { + return " OR "; + } + + @Override + protected String getLogicalNotOperator() { + return " NOT "; + } + + @Override + protected String handleAuthor(String author) { + // ScholarApi does not support explicit author field search + return StringUtil.quoteStringIfSpaceIsContained(author); + } + + @Override + protected String handleTitle(String title) { + // ScholarApi does not support explicit title field search + return StringUtil.quoteStringIfSpaceIsContained(title); + } + + @Override + protected String handleJournal(String journalTitle) { + LOGGER.debug("ScholarAPI has no journal scoped search"); + return StringUtil.quoteStringIfSpaceIsContained(journalTitle); + } +} diff --git a/jablib/src/main/java/org/jabref/logic/util/BuildInfo.java b/jablib/src/main/java/org/jabref/logic/util/BuildInfo.java index 00273e72229d..8489b451e813 100644 --- a/jablib/src/main/java/org/jabref/logic/util/BuildInfo.java +++ b/jablib/src/main/java/org/jabref/logic/util/BuildInfo.java @@ -44,6 +44,7 @@ public final class BuildInfo { public final String openAlexApiKey; public final String scopusApiKey; public final String semanticScholarApiKey; + public final String scholarApiKey; public final String springerNatureAPIKey; public final String unpaywallEmail; public final String wileyTdmApiKey; @@ -76,6 +77,7 @@ public BuildInfo(String path) { openAlexApiKey = BuildInfo.getValue(properties, "openAlexApiKey", ""); scopusApiKey = BuildInfo.getValue(properties, "scopusApiKey", "fb82f2e692b3c72dafe5f4f1fa0ac00b"); semanticScholarApiKey = BuildInfo.getValue(properties, "semanticScholarApiKey", ""); + scholarApiKey = BuildInfo.getValue(properties, "scholarApiKey", ""); springerNatureAPIKey = BuildInfo.getValue(properties, "springerNatureAPIKey", "118d90a519d0fc2a01ee9715400054d4"); unpaywallEmail = BuildInfo.getValue(properties, "unpaywallEmail", ""); wileyTdmApiKey = BuildInfo.getValue(properties, "wileyTdmApiKey", ""); diff --git a/jablib/src/main/resources/build.properties b/jablib/src/main/resources/build.properties index 6e606abeb651..f4c230614c33 100644 --- a/jablib/src/main/resources/build.properties +++ b/jablib/src/main/resources/build.properties @@ -9,5 +9,6 @@ ieeeAPIKey=${ieeeAPIKey} medlineApiKey=${medlineApiKey} openAlexApiKey=${openAlexApiKey} semanticScholarApiKey=${semanticScholarApiKey} +scholarApiKey=${scholarApiKey} springerNatureAPIKey=${springerNatureAPIKey} wileyTdmApiKey=${wileyTdmApiKey} diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java new file mode 100644 index 000000000000..d404efca6c98 --- /dev/null +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/ScholarFetcherTest.java @@ -0,0 +1,135 @@ +package org.jabref.logic.importer.fetcher; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import javafx.collections.FXCollections; + +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.ImporterPreferences; +import org.jabref.logic.importer.PagedSearchBasedFetcher; +import org.jabref.logic.importer.ParseException; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.util.BuildInfo; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.paging.Page; +import org.jabref.testutils.category.FetcherTest; + +import com.airhacks.afterburner.injection.Injector; +import kong.unirest.core.json.JSONObject; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@FetcherTest +public class ScholarFetcherTest implements SearchBasedFetcherCapabilityTest, PagedSearchFetcherTest { + + ImporterPreferences importerPreferences = mock(ImporterPreferences.class); + ScholarFetcher fetcher = new ScholarFetcher(importerPreferences); + + @BeforeEach + void setUp() { + BuildInfo buildInfo = Injector.instantiateModelOrService(BuildInfo.class); + fetcher = new ScholarFetcher(importerPreferences); + when(importerPreferences.getApiKeys()).thenReturn(FXCollections.emptyObservableSet()); + when(importerPreferences.getApiKey(fetcher.getName())).thenReturn(Optional.of(buildInfo.scholarApiKey)); + } + + @Test + void scholarApiJsonToBibtex() throws ParseException { + String jsonString = """ + {\r + "id": "7184",\r + "title": "Methylated N-(4-N,N-Dimethylaminobenzyl) Chitosan, a Novel Chitosan Derivative, Enhances Paracellular Permeability Across Intestinal Epithelial Cells (Caco-2)",\r + "authors": [ + "Jariya Kowapradit", + "Praneet Opanasopit", + "Tanasait Ngawhiranpat" + ],\r + "abstract": "The aim of this study was to investigate the effect of methylated N-(4-N,N-dimethylaminobenzyl) chitosan, TM-Bz-CS, on the paracellular permeability of Caco-2 cell monolayers and its toxicity towards the cell lines. The factors affecting epithelial permeability, e.g., degree of quaternization (DQ) and extent of dimethylaminobenzyl substitution (ES), were evaluated in intestinal cell monolayers of Caco-2 cells using the transepithelial electrical resistance and permeability of Caco-2 cell monolayers, with fluorescein isothiocyanate dextran 4,400 (FD-4) as a model compound for paracellular tight-junction transport. Cytotoxicity was evaluated with the 3-(4,5-dimethylthiazol-2-yl)-2,5-diphenyl tetrazolium bromide viability assay. The results revealed that, at pH 7.4, TM-Bz-CS appeared to increase cell permeability in a concentration-dependent manner, and this effect was relatively reversible at lower doses of 0.05–0.5 mM. Higher DQ and the ES caused the permeability of FD-4 to be higher. The cytotoxicity of TM-Bz-CS depended on concentration, %DQ, and %ES. These studies demonstrated that this novel modified chitosan has potential as an absorption enhancer.",\r + "journal": "AAPS PharmSciTech",\r + "journal_publisher": "Springer International Publishing",\r + "journal_issn": [ + "1530-9932" + ],\r + "journal_issue": "Volume 9, Issue 4",\r + "journal_pages": "1143-1152",\r + "doi": "10.1208/s12249-008-9160-7"\r, + "published_date": "2008-12-01T00:00:00Z"\r, + "published_date_raw": "2008-12-01T00:00:00Z",\r + "indexed_at": "2012-10-01T18:58:11.184Z",\r + "url": "https://link.springer.com/article/10.1208/s12249-008-9160-7",\r + "has_text": true,\r + "has_pdf": true\r + }"""; + + JSONObject jsonObject = new JSONObject(jsonString); + BibEntry bibEntry = ScholarFetcher.jsonItemToBibEntry(jsonObject); + + assertEquals(Optional.of("7184"), bibEntry.getField(new UnknownField("scholarapi"))); + assertEquals(Optional.of("2008-12-01"), bibEntry.getField(StandardField.DATE)); + assertEquals(Optional.of("2008"), bibEntry.getField(StandardField.YEAR)); + assertEquals(Optional.of("Methylated N-(4-N,N-Dimethylaminobenzyl) Chitosan, a Novel Chitosan Derivative, Enhances Paracellular Permeability Across Intestinal Epithelial Cells (Caco-2)"), bibEntry.getField(StandardField.TITLE)); + assertEquals(Optional.of("Jariya Kowapradit and Praneet Opanasopit and Tanasait Ngawhiranpat"), bibEntry.getField(StandardField.AUTHOR)); + assertEquals(Optional.of("AAPS PharmSciTech"), bibEntry.getField(StandardField.JOURNAL)); + assertEquals(Optional.of("1530-9932"), bibEntry.getField(StandardField.ISSN)); + assertEquals(Optional.of("9"), bibEntry.getField(StandardField.VOLUME)); + assertEquals(Optional.of("4"), bibEntry.getField(StandardField.NUMBER)); + assertEquals(Optional.of("1143-1152"), bibEntry.getField(StandardField.PAGES)); + assertEquals(Optional.of("10.1208/s12249-008-9160-7"), bibEntry.getField(StandardField.DOI)); + assertEquals(Optional.of("https://link.springer.com/article/10.1208/s12249-008-9160-7"), bibEntry.getField(StandardField.URL)); + assertEquals(Optional.of("Springer International Publishing"), bibEntry.getField(StandardField.PUBLISHER)); + assertEquals(Optional.of("true"), bibEntry.getField(new UnknownField("scholarApiHasText"))); + assertEquals(Optional.of("true"), bibEntry.getField(new UnknownField("scholarApiHasPdf"))); + } + + @Test + void performRawSearchQueryPagedWithBlankQueryReturnsEmptyPage() throws FetcherException { + Page result = fetcher.performRawSearchQueryPaged("", 0); + assertEquals(List.of(), new ArrayList<>(result.getContent())); + } + + @Test + void searchByEmptyQueryFindsNothing() throws FetcherException { + assertEquals(List.of(), fetcher.performSearch("")); + } + + @Test + @Disabled("ScholarAPI has no journal scoped search") + @Override + public void supportsJournalSearch() { + } + + @Test + @Disabled("ScholarAPI has no author scoped search") + @Override + public void supportsAuthorSearch() { + } + + @Override + public PagedSearchBasedFetcher getPagedFetcher() { + return fetcher; + } + + @Override + public SearchBasedFetcher getFetcher() { + return fetcher; + } + + @Override + public List getTestAuthors() { + return List.of("unsupported"); + } + + @Override + public String getTestJournal() { + return "unsupported"; + } +} diff --git a/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformerTest.java b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformerTest.java new file mode 100644 index 000000000000..4e0d32be7c0d --- /dev/null +++ b/jablib/src/test/java/org/jabref/logic/importer/fetcher/transformers/ScholarApiQueryTransformerTest.java @@ -0,0 +1,9 @@ +package org.jabref.logic.importer.fetcher.transformers; + +class ScholarApiQueryTransformerTest extends YearAndYearRangeByFilteringQueryTransformerTest { + + @Override + public ScholarApiQueryTransformer getTransformer() { + return new ScholarApiQueryTransformer(); + } +}