From bf80faac60e11bc7a1d3254f7e2bc0bf25e9a6ad Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 10 Aug 2026 16:19:37 +0530 Subject: [PATCH 1/4] Add skeleton BaseSearchFetcher and query transformer for BASE (WIP, refs #15016) --- .../importer/fetcher/BaseSearchFetcher.java | 155 ++++++++++++++++++ .../BaseSearchQueryTransformer.java | 39 +++++ 2 files changed, 194 insertions(+) create mode 100644 jablib/src/main/java/org/jabref/logic/importer/fetcher/BaseSearchFetcher.java create mode 100644 jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/BaseSearchQueryTransformer.java diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/BaseSearchFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/BaseSearchFetcher.java new file mode 100644 index 000000000000..83f30aea73a7 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/BaseSearchFetcher.java @@ -0,0 +1,155 @@ +package org.jabref.logic.importer.fetcher; + +import java.net.MalformedURLException; +import java.net.URISyntaxException; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import org.jabref.logic.help.HelpFile; +import org.jabref.logic.importer.ImporterPreferences; +import org.jabref.logic.importer.PagedSearchBasedParserFetcher; +import org.jabref.logic.importer.ParseException; +import org.jabref.logic.importer.Parser; +import org.jabref.logic.importer.fetcher.transformers.BaseSearchQueryTransformer; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.search.query.BaseQueryNode; + +import kong.unirest.core.json.JSONArray; +import kong.unirest.core.json.JSONObject; +import org.apache.hc.core5.net.URIBuilder; +import org.jspecify.annotations.NonNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class BaseSearchFetcher implements PagedSearchBasedParserFetcher, CustomizableKeyFetcher { + + public static final String FETCHER_NAME = "BASE"; + + private static final Logger LOGGER = LoggerFactory.getLogger(BaseSearchFetcher.class); + + private static final String API_URL = "https://api.base-search.net/cgi-bin/BaseHttpSearchInterface.fcgi"; + + private static final FetcherRateLimiter RATE_LIMITER = + FetcherRateLimiter.ofRequestsPerSecond(FETCHER_NAME, 1.0); + + private final ImporterPreferences importerPreferences; + + public BaseSearchFetcher(@NonNull ImporterPreferences importerPreferences) { + this.importerPreferences = importerPreferences; + } + + @Override + public String getName() { + return FETCHER_NAME; + } + + @Override + public Optional getHelpPage() { + return Optional.empty(); + } + + @Override + public URL getURLForQuery(BaseQueryNode queryNode, int pageNumber) throws URISyntaxException, MalformedURLException { + BaseSearchQueryTransformer transformer = new BaseSearchQueryTransformer(); + String query = transformer.transformSearchQuery(queryNode).orElse(""); + + URIBuilder uriBuilder = new URIBuilder(API_URL); + uriBuilder.addParameter("func", "PerformSearch"); + uriBuilder.addParameter("format", "json"); + uriBuilder.addParameter("query", query); + uriBuilder.addParameter("hits", String.valueOf(getPageSize())); + uriBuilder.addParameter("offset", String.valueOf(getPageSize() * pageNumber)); + importerPreferences.getApiKey(FETCHER_NAME).ifPresent(apiKey -> uriBuilder.addParameter("apikey", apiKey)); + + return uriBuilder.build().toURL(); + } + + @Override + public Parser getParser() { + return inputStream -> { + String response; + try { + response = new String(inputStream.readAllBytes()); + } catch (java.io.IOException e) { + throw new ParseException("Could not read response from BASE", e); + } + JSONObject jsonObject = new JSONObject(response); + + if (jsonObject.has("error")) { + LOGGER.warn("BASE API returned error: {}", jsonObject.optString("error")); + return List.of(); + } + + List entries = new ArrayList<>(); + JSONObject result = jsonObject.optJSONObject("response").optJSONObject("result"); + if (result != null) { + JSONArray docs = result.optJSONArray("docs"); // TODO: confirm actual key name in real JSON response + for (int i = 0; i < docs.length(); i++) { + entries.add(parseEntry(docs.getJSONObject(i))); + } + } + return entries; + }; + } + + private BibEntry parseEntry(JSONObject doc) { + BibEntry entry = new BibEntry(); + + entry.setType(mapEntryType(doc)); + + entry.setField(StandardField.TITLE, doc.optString("dctitle")); + entry.setField(StandardField.YEAR, doc.optString("dcyear")); + entry.setField(StandardField.PUBLISHER, doc.optString("dcpublisher")); + entry.setField(StandardField.DOI, doc.optString("dcdoi")); + entry.setField(StandardField.URL, doc.optString("dclink")); + + JSONArray creators = doc.optJSONArray("dccreator"); + if (creators != null) { + List authorList = new ArrayList<>(); + for (int i = 0; i < creators.length(); i++) { + authorList.add(creators.getString(i)); + } + entry.setField(StandardField.AUTHOR, String.join(" and ", authorList)); + } + + JSONArray subjects = doc.optJSONArray("dcsubject"); + if (subjects != null) { + for (int i = 0; i < subjects.length(); i++) { + entry.addKeyword(subjects.getString(i), ','); + } + } + + return entry; + } + + private StandardEntryType mapEntryType(JSONObject doc) { + JSONArray typeNorm = doc.optJSONArray("dctypenorm"); + if (typeNorm == null || typeNorm.isEmpty()) { + return StandardEntryType.Misc; + } + String code = typeNorm.getString(0); + + if (code.startsWith("18")) { + return StandardEntryType.PhdThesis; + } else if (code.startsWith("13")) { + return StandardEntryType.InProceedings; + } else if (code.startsWith("14")) { + return StandardEntryType.TechReport; + } else if (code.startsWith("11")) { + return StandardEntryType.Book; + } else if (code.startsWith("121")) { + return StandardEntryType.Article; + } else { + return StandardEntryType.Misc; + } + } + + @Override + public boolean isValidKey(String apiKey) { + return false; + } +} diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/BaseSearchQueryTransformer.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/BaseSearchQueryTransformer.java new file mode 100644 index 000000000000..156bf0e8d812 --- /dev/null +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/transformers/BaseSearchQueryTransformer.java @@ -0,0 +1,39 @@ +package org.jabref.logic.importer.fetcher.transformers; + +public class BaseSearchQueryTransformer extends AbstractQueryTransformer { + + @Override + protected String getLogicalAndOperator() { + return " AND "; + } + + @Override + protected String getLogicalOrOperator() { + return " OR "; + } + + @Override + protected String getLogicalNotOperator() { + return "NOT "; + } + + @Override + protected String handleAuthor(String author) { + return createKeyValuePair("dccreator", author); + } + + @Override + protected String handleTitle(String title) { + return createKeyValuePair("dctitle", title); + } + + @Override + protected String handleJournal(String journalTitle) { + return createKeyValuePair("dcsource", journalTitle); + } + + @Override + protected String handleYear(String year) { + return createKeyValuePair("dcyear", year); + } +} From 4ffd08cd04bf45fe5508ea352b5ed31047fccf50 Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 11 Aug 2026 15:30:18 +0530 Subject: [PATCH 2/4] Register BaseSearchFetcher in WebFetchers --- jablib/src/main/java/org/jabref/logic/importer/WebFetchers.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jablib/src/main/java/org/jabref/logic/importer/WebFetchers.java b/jablib/src/main/java/org/jabref/logic/importer/WebFetchers.java index d1164e746741..f5b09a480686 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/WebFetchers.java +++ b/jablib/src/main/java/org/jabref/logic/importer/WebFetchers.java @@ -14,6 +14,7 @@ import org.jabref.logic.importer.fetcher.ApsFetcher; import org.jabref.logic.importer.fetcher.ArXivFetcher; import org.jabref.logic.importer.fetcher.AstrophysicsDataSystem; +import org.jabref.logic.importer.fetcher.BaseSearchFetcher; import org.jabref.logic.importer.fetcher.BiodiversityLibrary; import org.jabref.logic.importer.fetcher.BvbFetcher; import org.jabref.logic.importer.fetcher.CompositeSearchBasedFetcher; @@ -169,6 +170,7 @@ public static synchronized SortedSet getSearchBasedFetchers( searchBasedFetchers.add(new OpenAlex(importerPreferences)); searchBasedFetchers.add(new DOAJFetcher(importFormatPreferences)); searchBasedFetchers.add(new IEEE(importFormatPreferences, importerPreferences)); + searchBasedFetchers.add(new BaseSearchFetcher(importerPreferences)); // set.add(new CollectionOfComputerScienceBibliographiesFetcher(importFormatPreferences)); searchBasedFetchers.add(new DOABFetcher()); // set.add(new JstorFetcher(importFormatPreferences)); From eb040da639952bb64915f20bbc870fd70ab271ef Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 11 Aug 2026 16:04:22 +0530 Subject: [PATCH 3/4] Update ManageStudyDefinitionViewModelTest expectations for new BASE fetcher --- .../org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java b/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java index 24b4a08d5499..e7e8be993ef0 100644 --- a/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java +++ b/jabgui/src/test/java/org/jabref/gui/slr/ManageStudyDefinitionViewModelTest.java @@ -44,6 +44,7 @@ void emptyStudyConstructorFillsDatabasesCorrectly() { assertEquals(List.of( new StudyCatalogItem("ACM Portal", true), new StudyCatalogItem("arXiv", false), + new StudyCatalogItem("BASE", false), new StudyCatalogItem("Bibliotheksverbund Bayern (Experimental)", false), new StudyCatalogItem("Biodiversity Heritage", false), new StudyCatalogItem("Crossref", false), @@ -76,6 +77,7 @@ void studyConstructorFillsDatabasesCorrectly(@TempDir Path tempDir) { assertEquals(List.of( new StudyCatalogItem("ACM Portal", true), new StudyCatalogItem("arXiv", false), + new StudyCatalogItem("BASE", false), new StudyCatalogItem("Bibliotheksverbund Bayern (Experimental)", false), new StudyCatalogItem("Biodiversity Heritage", false), new StudyCatalogItem("Crossref", false), From 252fc1db67b4299edac68919ae008e01be3e10ad Mon Sep 17 00:00:00 2001 From: sam Date: Thu, 13 Aug 2026 09:46:04 +0530 Subject: [PATCH 4/4] Implement isValidKey and clean up leftover TODO comments --- .../importer/fetcher/BaseSearchFetcher.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/jablib/src/main/java/org/jabref/logic/importer/fetcher/BaseSearchFetcher.java b/jablib/src/main/java/org/jabref/logic/importer/fetcher/BaseSearchFetcher.java index 83f30aea73a7..a8bc3c4f36a0 100644 --- a/jablib/src/main/java/org/jabref/logic/importer/fetcher/BaseSearchFetcher.java +++ b/jablib/src/main/java/org/jabref/logic/importer/fetcher/BaseSearchFetcher.java @@ -87,7 +87,7 @@ public Parser getParser() { List entries = new ArrayList<>(); JSONObject result = jsonObject.optJSONObject("response").optJSONObject("result"); if (result != null) { - JSONArray docs = result.optJSONArray("docs"); // TODO: confirm actual key name in real JSON response + JSONArray docs = result.optJSONArray("docs"); for (int i = 0; i < docs.length(); i++) { entries.add(parseEntry(docs.getJSONObject(i))); } @@ -150,6 +150,21 @@ private StandardEntryType mapEntryType(JSONObject doc) { @Override public boolean isValidKey(String apiKey) { - return false; + try { + URIBuilder uriBuilder = new URIBuilder(API_URL); + uriBuilder.addParameter("func", "PerformSearch"); + uriBuilder.addParameter("format", "json"); + uriBuilder.addParameter("query", "test"); + uriBuilder.addParameter("hits", "0"); + uriBuilder.addParameter("apikey", apiKey); + URL testUrl = uriBuilder.build().toURL(); + + org.jabref.logic.net.URLDownload urlDownload = new org.jabref.logic.net.URLDownload(testUrl); + String response = urlDownload.asString(); + JSONObject jsonObject = new JSONObject(response); + return !jsonObject.has("error"); + } catch (Exception e) { + return false; + } } }